From 4f685879983f42191ecef55097f88ca00e9a0917 Mon Sep 17 00:00:00 2001 From: Aliaksandr Valialkin Date: Thu, 4 Sep 2025 13:56:48 +0200 Subject: [PATCH 01/82] app/vlinsert/syslog: add support for accepting Syslog messages from Unix sockets See https://github.com/VictoriaMetrics/VictoriaLogs/issues/570 --- app/vlinsert/syslog/syslog.go | 280 +++++++++++++-------- docs/victorialogs/CHANGELOG.md | 1 + docs/victorialogs/README.md | 40 ++- docs/victorialogs/data-ingestion/syslog.md | 39 +-- docs/victorialogs/vlagent.md | 50 +++- 5 files changed, 284 insertions(+), 126 deletions(-) diff --git a/app/vlinsert/syslog/syslog.go b/app/vlinsert/syslog/syslog.go index 46b2df8db3..f10c983075 100644 --- a/app/vlinsert/syslog/syslog.go +++ b/app/vlinsert/syslog/syslog.go @@ -35,65 +35,84 @@ var ( syslogTimezone = flag.String("syslog.timezone", "Local", "Timezone to use when parsing timestamps in RFC3164 syslog messages. Timezone must be a valid IANA Time Zone. "+ "For example: America/New_York, Europe/Berlin, Etc/GMT+3 . See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/") + listenAddrTCP = flagutil.NewArrayString("syslog.listenAddr.tcp", "Comma-separated list of TCP addresses to listen to for Syslog messages. "+ + "See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/") + listenAddrUDP = flagutil.NewArrayString("syslog.listenAddr.udp", "Comma-separated list of UDP addresses to listen to for Syslog messages. "+ + "See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/") + listenAddrUnix = flagutil.NewArrayString("syslog.listenAddr.unix", "Comma-separated list of Unix socket filepaths to listen to for Syslog messages. "+ + "Filepaths may be prepended with 'unixpacket:' for listening for SOCK_DGRAM sockets. By default SOCK_STREAM sockets are used. "+ + "See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/") + + tlsEnable = flagutil.NewArrayBool("syslog.tls", "Whether to enable TLS for receiving syslog messages at the corresponding -syslog.listenAddr.tcp. "+ + "The corresponding -syslog.tlsCertFile and -syslog.tlsKeyFile must be set if -syslog.tls is set. See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/#security") + tlsCertFile = flagutil.NewArrayString("syslog.tlsCertFile", "Path to file with TLS certificate for the corresponding -syslog.listenAddr.tcp if the corresponding -syslog.tls is set. "+ + "Prefer ECDSA certs instead of RSA certs as RSA certs are slower. The provided certificate file is automatically re-read every second, so it can be dynamically updated. "+ + "See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/#security") + tlsKeyFile = flagutil.NewArrayString("syslog.tlsKeyFile", "Path to file with TLS key for the corresponding -syslog.listenAddr.tcp if the corresponding -syslog.tls is set. "+ + "The provided key file is automatically re-read every second, so it can be dynamically updated. "+ + "See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/#security") + tlsCipherSuites = flagutil.NewArrayString("syslog.tlsCipherSuites", "Optional list of TLS cipher suites for -syslog.listenAddr.tcp if -syslog.tls is set. "+ + "See the list of supported cipher suites at https://pkg.go.dev/crypto/tls#pkg-constants . "+ + "See also https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/#security") + tlsMinVersion = flag.String("syslog.tlsMinVersion", "TLS13", "The minimum TLS version to use for -syslog.listenAddr.tcp if -syslog.tls is set. "+ + "Supported values: TLS10, TLS11, TLS12, TLS13. "+ + "See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/#security") + streamFieldsTCP = flagutil.NewArrayString("syslog.streamFields.tcp", "Fields to use as log stream labels for logs ingested via the corresponding -syslog.listenAddr.tcp. "+ `See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/#stream-fields`) streamFieldsUDP = flagutil.NewArrayString("syslog.streamFields.udp", "Fields to use as log stream labels for logs ingested via the corresponding -syslog.listenAddr.udp. "+ `See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/#stream-fields`) + streamFieldsUnix = flagutil.NewArrayString("syslog.streamFields.unix", "Fields to use as log stream labels for logs ingested via the corresponding -syslog.listenAddr.unix. "+ + `See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/#stream-fields`) ignoreFieldsTCP = flagutil.NewArrayString("syslog.ignoreFields.tcp", "Fields to ignore at logs ingested via the corresponding -syslog.listenAddr.tcp. "+ `See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/#dropping-fields`) ignoreFieldsUDP = flagutil.NewArrayString("syslog.ignoreFields.udp", "Fields to ignore at logs ingested via the corresponding -syslog.listenAddr.udp. "+ `See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/#dropping-fields`) + ignoreFieldsUnix = flagutil.NewArrayString("syslog.ignoreFields.unix", "Fields to ignore at logs ingested via the corresponding -syslog.listenAddr.unix. "+ + `See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/#dropping-fields`) decolorizeFieldsTCP = flagutil.NewArrayString("syslog.decolorizeFields.tcp", "Fields to remove ANSI color codes across logs ingested via the corresponding -syslog.listenAddr.tcp. "+ `See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/#decolorizing-fields`) decolorizeFieldsUDP = flagutil.NewArrayString("syslog.decolorizeFields.udp", "Fields to remove ANSI color codes across logs ingested via the corresponding -syslog.listenAddr.udp. "+ `See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/#decolorizing-fields`) + decolorizeFieldsUnix = flagutil.NewArrayString("syslog.decolorizeFields.unix", "Fields to remove ANSI color codes across logs ingested via the corresponding -syslog.listenAddr.unix. "+ + `See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/#decolorizing-fields`) extraFieldsTCP = flagutil.NewArrayString("syslog.extraFields.tcp", "Fields to add to logs ingested via the corresponding -syslog.listenAddr.tcp. "+ `See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/#adding-extra-fields`) extraFieldsUDP = flagutil.NewArrayString("syslog.extraFields.udp", "Fields to add to logs ingested via the corresponding -syslog.listenAddr.udp. "+ `See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/#adding-extra-fields`) + extraFieldsUnix = flagutil.NewArrayString("syslog.extraFields.unix", "Fields to add to logs ingested via the corresponding -syslog.listenAddr.unix. "+ + `See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/#adding-extra-fields`) tenantIDTCP = flagutil.NewArrayString("syslog.tenantID.tcp", "TenantID for logs ingested via the corresponding -syslog.listenAddr.tcp. "+ "See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/#multitenancy") tenantIDUDP = flagutil.NewArrayString("syslog.tenantID.udp", "TenantID for logs ingested via the corresponding -syslog.listenAddr.udp. "+ "See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/#multitenancy") - - listenAddrTCP = flagutil.NewArrayString("syslog.listenAddr.tcp", "Comma-separated list of TCP addresses to listen to for Syslog messages. "+ - "See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/") - listenAddrUDP = flagutil.NewArrayString("syslog.listenAddr.udp", "Comma-separated list of UDP address to listen to for Syslog messages. "+ - "See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/") - - tlsEnable = flagutil.NewArrayBool("syslog.tls", "Whether to enable TLS for receiving syslog messages at the corresponding -syslog.listenAddr.tcp. "+ - "The corresponding -syslog.tlsCertFile and -syslog.tlsKeyFile must be set if -syslog.tls is set. See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/#security") - tlsCertFile = flagutil.NewArrayString("syslog.tlsCertFile", "Path to file with TLS certificate for the corresponding -syslog.listenAddr.tcp if the corresponding -syslog.tls is set. "+ - "Prefer ECDSA certs instead of RSA certs as RSA certs are slower. The provided certificate file is automatically re-read every second, so it can be dynamically updated. "+ - "See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/#security") - tlsKeyFile = flagutil.NewArrayString("syslog.tlsKeyFile", "Path to file with TLS key for the corresponding -syslog.listenAddr.tcp if the corresponding -syslog.tls is set. "+ - "The provided key file is automatically re-read every second, so it can be dynamically updated. "+ - "See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/#security") - tlsCipherSuites = flagutil.NewArrayString("syslog.tlsCipherSuites", "Optional list of TLS cipher suites for -syslog.listenAddr.tcp if -syslog.tls is set. "+ - "See the list of supported cipher suites at https://pkg.go.dev/crypto/tls#pkg-constants . "+ - "See also https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/#security") - tlsMinVersion = flag.String("syslog.tlsMinVersion", "TLS13", "The minimum TLS version to use for -syslog.listenAddr.tcp if -syslog.tls is set. "+ - "Supported values: TLS10, TLS11, TLS12, TLS13. "+ - "See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/#security") + tenantIDUnix = flagutil.NewArrayString("syslog.tenantID.unix", "TenantID for logs ingested via the corresponding -syslog.listenAddr.unix. "+ + "See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/#multitenancy") compressMethodTCP = flagutil.NewArrayString("syslog.compressMethod.tcp", "Compression method for syslog messages received at the corresponding -syslog.listenAddr.tcp. "+ "Supported values: none, gzip, deflate. See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/#compression") compressMethodUDP = flagutil.NewArrayString("syslog.compressMethod.udp", "Compression method for syslog messages received at the corresponding -syslog.listenAddr.udp. "+ "Supported values: none, gzip, deflate. See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/#compression") + compressMethodUnix = flagutil.NewArrayString("syslog.compressMethod.unix", "Compression method for syslog messages received at the corresponding -syslog.listenAddr.unix. "+ + "Supported values: none, gzip, deflate. See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/#compression") useLocalTimestampTCP = flagutil.NewArrayBool("syslog.useLocalTimestamp.tcp", "Whether to use local timestamp instead of the original timestamp for the ingested syslog messages "+ "at the corresponding -syslog.listenAddr.tcp. See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/#log-timestamps") useLocalTimestampUDP = flagutil.NewArrayBool("syslog.useLocalTimestamp.udp", "Whether to use local timestamp instead of the original timestamp for the ingested syslog messages "+ "at the corresponding -syslog.listenAddr.udp. See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/#log-timestamps") + useLocalTimestampUnix = flagutil.NewArrayBool("syslog.useLocalTimestamp.unix", "Whether to use local timestamp instead of the original timestamp for the ingested syslog messages "+ + "at the corresponding -syslog.listenAddr.unix. See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/#log-timestamps") useRemoteIPTCP = flagutil.NewArrayBool("syslog.useRemoteIP.tcp", "Whether to add remote ip address as 'remote_ip' log field for syslog messages ingested "+ "via the corresponding -syslog.listenAddr.tcp. See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/#capturing-remote-ip-address") useRemoteIPUDP = flagutil.NewArrayBool("syslog.useRemoteIP.udp", "Whether to add remote ip address as 'remote_ip' log field for syslog messages ingested "+ "via the corresponding -syslog.listenAddr.udp. See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/#capturing-remote-ip-address") + useRemoteIPUnix = flagutil.NewArrayBool("syslog.useRemoteIP.unix", "Whether to add remote ip address as 'remote_ip' log field for syslog messages ingested "+ + "via the corresponding -syslog.listenAddr.unix. See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/#capturing-remote-ip-address") ) // MustInit initializes syslog parser at the given -syslog.listenAddr.tcp and -syslog.listenAddr.udp ports @@ -123,6 +142,14 @@ func MustInit() { }(addr, argIdx) } + for argIdx, addr := range *listenAddrUnix { + workersWG.Add(1) + go func(addr string, argIdx int) { + runUnixListener(addr, argIdx) + workersWG.Done() + }(addr, argIdx) + } + currentYear := time.Now().Year() globalCurrentYear.Store(int64(currentYear)) workersWG.Add(1) @@ -169,51 +196,56 @@ func MustStop() { workersStopCh = nil } -func runUDPListener(addr string, argIdx int) { - ln, err := net.ListenPacket(netutil.GetUDPNetwork(), addr) +func runUnixListener(addr string, argIdx int) { + network, path := getUnixSocketNetworkAndPath(addr) + ln, err := net.Listen(network, path) if err != nil { - logger.Fatalf("cannot start UDP syslog server at %q: %s", addr, err) + logger.Fatalf("cannot start Unix socket syslog server at %q: %s", addr, err) } - tenantIDStr := tenantIDUDP.GetOptionalArg(argIdx) - tenantID, err := logstorage.ParseTenantID(tenantIDStr) + cfg, err := getConfigs("unix", argIdx, streamFieldsUnix, ignoreFieldsUnix, decolorizeFieldsUnix, extraFieldsUnix, tenantIDUnix, compressMethodUnix, useLocalTimestampUnix, useRemoteIPUnix) if err != nil { - logger.Fatalf("cannot parse -syslog.tenantID.udp=%q for -syslog.listenAddr.udp=%q: %s", tenantIDStr, addr, err) + logger.Fatalf("cannot parse configs for -syslog.listenAddr.unix=%q: %s", addr, err) } - compressMethod := compressMethodUDP.GetOptionalArg(argIdx) - checkCompressMethod(compressMethod, addr, "udp") - - useLocalTimestamp := useLocalTimestampUDP.GetOptionalArg(argIdx) - useRemoteIP := useRemoteIPUDP.GetOptionalArg(argIdx) + doneCh := make(chan struct{}) + go func() { + serveUnixListener(ln, cfg) + close(doneCh) + }() - streamFieldsStr := streamFieldsUDP.GetOptionalArg(argIdx) - streamFields, err := parseFieldsList(streamFieldsStr) - if err != nil { - logger.Fatalf("cannot parse -syslog.streamFields.udp=%q for -syslog.listenAddr.udp=%q: %s", streamFieldsStr, addr, err) + logger.Infof("started accepting syslog messages at -syslog.listenAddr.unix=%q", addr) + <-workersStopCh + if err := ln.Close(); err != nil { + logger.Fatalf("syslog: cannot close UDP listener at %s: %s", addr, err) } + <-doneCh + logger.Infof("finished accepting syslog messages at -syslog.listenAddr.unix=%q", addr) +} - ignoreFieldsStr := ignoreFieldsUDP.GetOptionalArg(argIdx) - ignoreFields, err := parseFieldsList(ignoreFieldsStr) - if err != nil { - logger.Fatalf("cannot parse -syslog.ignoreFields.udp=%q for -syslog.listenAddr.udp=%q: %s", ignoreFieldsStr, addr, err) +func getUnixSocketNetworkAndPath(addr string) (string, string) { + // An optional network such as unix, unixpacket or unixgram can be specified in front of addr and followed by ':' + n := strings.IndexByte(addr, ':') + if n < 0 { + return "unix", addr } + return addr[:n], addr[n+1:] +} - decolorizeFieldsStr := decolorizeFieldsUDP.GetOptionalArg(argIdx) - decolorizeFields, err := parseFieldsList(decolorizeFieldsStr) +func runUDPListener(addr string, argIdx int) { + ln, err := net.ListenPacket(netutil.GetUDPNetwork(), addr) if err != nil { - logger.Fatalf("cannot parse -syslog.decolorizeFields.udp=%q for -syslog.listenAddr.udp=%q: %s", decolorizeFieldsStr, addr, err) + logger.Fatalf("cannot start UDP syslog server at %q: %s", addr, err) } - extraFieldsStr := extraFieldsUDP.GetOptionalArg(argIdx) - extraFields, err := parseExtraFields(extraFieldsStr) + cfg, err := getConfigs("udp", argIdx, streamFieldsUDP, ignoreFieldsUDP, decolorizeFieldsUDP, extraFieldsUDP, tenantIDUDP, compressMethodUDP, useLocalTimestampUDP, useRemoteIPUDP) if err != nil { - logger.Fatalf("cannot parse -syslog.extraFields.udp=%q for -syslog.listenAddr.udp=%q: %s", extraFieldsStr, addr, err) + logger.Fatalf("cannot parse configs for -syslog.listenAddr.udp=%q: %s", addr, err) } doneCh := make(chan struct{}) go func() { - serveUDP(ln, tenantID, compressMethod, useLocalTimestamp, useRemoteIP, streamFields, ignoreFields, decolorizeFields, extraFields) + servePacketListener(ln, cfg) close(doneCh) }() @@ -243,45 +275,14 @@ func runTCPListener(addr string, argIdx int) { logger.Fatalf("syslog: cannot start TCP listener at %s: %s", addr, err) } - tenantIDStr := tenantIDTCP.GetOptionalArg(argIdx) - tenantID, err := logstorage.ParseTenantID(tenantIDStr) - if err != nil { - logger.Fatalf("cannot parse -syslog.tenantID.tcp=%q for -syslog.listenAddr.tcp=%q: %s", tenantIDStr, addr, err) - } - - compressMethod := compressMethodTCP.GetOptionalArg(argIdx) - checkCompressMethod(compressMethod, addr, "tcp") - - useLocalTimestamp := useLocalTimestampTCP.GetOptionalArg(argIdx) - useRemoteIP := useRemoteIPTCP.GetOptionalArg(argIdx) - - streamFieldsStr := streamFieldsTCP.GetOptionalArg(argIdx) - streamFields, err := parseFieldsList(streamFieldsStr) - if err != nil { - logger.Fatalf("cannot parse -syslog.streamFields.tcp=%q for -syslog.listenAddr.tcp=%q: %s", streamFieldsStr, addr, err) - } - - ignoreFieldsStr := ignoreFieldsTCP.GetOptionalArg(argIdx) - ignoreFields, err := parseFieldsList(ignoreFieldsStr) - if err != nil { - logger.Fatalf("cannot parse -syslog.ignoreFields.tcp=%q for -syslog.listenAddr.tcp=%q: %s", ignoreFieldsStr, addr, err) - } - - decolorizeFieldsStr := decolorizeFieldsTCP.GetOptionalArg(argIdx) - decolorizeFields, err := parseFieldsList(decolorizeFieldsStr) - if err != nil { - logger.Fatalf("cannot parse -syslog.decolorizeFields.tcp=%q for -syslog.listenAddr.tcp=%q: %s", decolorizeFieldsStr, addr, err) - } - - extraFieldsStr := extraFieldsTCP.GetOptionalArg(argIdx) - extraFields, err := parseExtraFields(extraFieldsStr) + cfg, err := getConfigs("tcp", argIdx, streamFieldsTCP, ignoreFieldsTCP, decolorizeFieldsTCP, extraFieldsTCP, tenantIDTCP, compressMethodTCP, useLocalTimestampTCP, useRemoteIPTCP) if err != nil { - logger.Fatalf("cannot parse -syslog.extraFields.tcp=%q for -syslog.listenAddr.tcp=%q: %s", extraFieldsStr, addr, err) + logger.Fatalf("cannot parse configs for -syslog.listenAddr.tcp=%q: %s", addr, err) } doneCh := make(chan struct{}) go func() { - serveTCP(ln, tenantID, compressMethod, useLocalTimestamp, useRemoteIP, streamFields, ignoreFields, decolorizeFields, extraFields) + serveStreamListener(ln, cfg) close(doneCh) }() @@ -294,16 +295,15 @@ func runTCPListener(addr string, argIdx int) { logger.Infof("finished accepting syslog messages at -syslog.listenAddr.tcp=%q", addr) } -func checkCompressMethod(compressMethod, addr, protocol string) { - switch compressMethod { - case "", "none", "zstd", "gzip", "deflate": - return - default: - logger.Fatalf("unsupported -syslog.compressMethod.%s=%q for -syslog.listenAddr.%s=%q; supported values: 'none', 'zstd', 'gzip', 'deflate'", protocol, compressMethod, protocol, addr) +func serveUnixListener(ln net.Listener, cfg *configs) { + if pc, ok := ln.(net.PacketConn); ok { + servePacketListener(pc, cfg) + } else { + serveStreamListener(ln, cfg) } } -func serveUDP(ln net.PacketConn, tenantID logstorage.TenantID, encoding string, useLocalTimestamp bool, useRemoteIP bool, streamFields, ignoreFields, decolorizeFields []string, extraFields []logstorage.Field) { +func servePacketListener(ln net.PacketConn, cfg *configs) { gomaxprocs := cgroup.AvailableCPUs() var wg sync.WaitGroup localAddr := ln.LocalAddr() @@ -311,7 +311,7 @@ func serveUDP(ln net.PacketConn, tenantID logstorage.TenantID, encoding string, wg.Add(1) go func() { defer wg.Done() - cp := insertutil.GetCommonParamsForSyslog(tenantID, streamFields, ignoreFields, decolorizeFields, extraFields) + cp := insertutil.GetCommonParamsForSyslog(cfg.tenantID, cfg.streamFields, cfg.ignoreFields, cfg.decolorizeFields, cfg.extraFields) var bb bytesutil.ByteBuffer bb.B = bytesutil.ResizeNoCopyNoOverallocate(bb.B, 64*1024) for { @@ -323,7 +323,7 @@ func serveUDP(ln net.PacketConn, tenantID logstorage.TenantID, encoding string, var ne net.Error if errors.As(err, &ne) { if ne.Temporary() { - logger.Errorf("syslog: temporary error when listening for UDP at %q: %s", localAddr, err) + logger.Errorf("syslog: temporary error when listening for %s at %q: %s", cfg.typ, localAddr, err) time.Sleep(time.Second) continue } @@ -331,16 +331,16 @@ func serveUDP(ln net.PacketConn, tenantID logstorage.TenantID, encoding string, break } } - logger.Errorf("syslog: cannot read UDP data from %s at %s: %s", remoteAddr, localAddr, err) + logger.Errorf("syslog: cannot read %s data from %s at %s: %s", cfg.typ, remoteAddr, localAddr, err) continue } bb.B = bb.B[:n] udpRequestsTotal.Inc() - remoteIP := getRemoteIP(remoteAddr, useRemoteIP) + remoteIP := getRemoteIP(remoteAddr, cfg.useRemoteIP) - if err := processStream("udp", bb.NewReader(), encoding, useLocalTimestamp, remoteIP, cp); err != nil { - logger.Errorf("syslog: cannot process UDP data from %s at %s: %s", remoteAddr, localAddr, err) + if err := processStream(cfg.typ, bb.NewReader(), cfg.compressMethod, cfg.useLocalTimestamp, remoteIP, cp); err != nil { + logger.Errorf("syslog: cannot process %s data from %s at %s: %s", cfg.typ, remoteAddr, localAddr, err) } } }() @@ -348,7 +348,7 @@ func serveUDP(ln net.PacketConn, tenantID logstorage.TenantID, encoding string, wg.Wait() } -func serveTCP(ln net.Listener, tenantID logstorage.TenantID, encoding string, useLocalTimestamp bool, useRemoteIP bool, streamFields, ignoreFields, decolorizeFields []string, extraFields []logstorage.Field) { +func serveStreamListener(ln net.Listener, cfg *configs) { var cm ingestserver.ConnsMap cm.Init("syslog") @@ -360,16 +360,16 @@ func serveTCP(ln net.Listener, tenantID logstorage.TenantID, encoding string, us var ne net.Error if errors.As(err, &ne) { if ne.Temporary() { - logger.Errorf("syslog: temporary error when listening for TCP addr %q: %s", addr, err) + logger.Errorf("syslog: temporary error when listening for %s addr %q: %s", cfg.typ, addr, err) time.Sleep(time.Second) continue } if strings.Contains(err.Error(), "use of closed network connection") { break } - logger.Fatalf("syslog: unrecoverable error when accepting TCP connections at %q: %s", addr, err) + logger.Fatalf("syslog: unrecoverable error when accepting %s connections at %q: %s", cfg.typ, addr, err) } - logger.Fatalf("syslog: unexpected error when accepting TCP connections at %q: %s", addr, err) + logger.Fatalf("syslog: unexpected error when accepting %s connections at %q: %s", cfg.typ, addr, err) } if !cm.Add(c) { _ = c.Close() @@ -378,12 +378,12 @@ func serveTCP(ln net.Listener, tenantID logstorage.TenantID, encoding string, us wg.Add(1) go func() { - cp := insertutil.GetCommonParamsForSyslog(tenantID, streamFields, ignoreFields, decolorizeFields, extraFields) + cp := insertutil.GetCommonParamsForSyslog(cfg.tenantID, cfg.streamFields, cfg.ignoreFields, cfg.decolorizeFields, cfg.extraFields) remoteAddr := c.RemoteAddr() - remoteIP := getRemoteIP(remoteAddr, useRemoteIP) - if err := processStream("tcp", c, encoding, useLocalTimestamp, remoteIP, cp); err != nil { - logger.Errorf("syslog: cannot process TCP data at %q: %s", addr, err) + remoteIP := getRemoteIP(remoteAddr, cfg.useRemoteIP) + if err := processStream(cfg.typ, c, cfg.compressMethod, cfg.useLocalTimestamp, remoteIP, cp); err != nil { + logger.Errorf("syslog: cannot process %s data at %q: %s", cfg.typ, addr, err) } cm.Delete(c) @@ -397,20 +397,20 @@ func serveTCP(ln net.Listener, tenantID logstorage.TenantID, encoding string, us } // processStream parses a stream of syslog messages from r and ingests them into vlstorage. -func processStream(protocol string, r io.Reader, encoding string, useLocalTimestamp bool, remoteIP string, cp *insertutil.CommonParams) error { +func processStream(protocol string, r io.Reader, compressMethod string, useLocalTimestamp bool, remoteIP string, cp *insertutil.CommonParams) error { if err := insertutil.CanWriteData(); err != nil { return err } lmp := cp.NewLogMessageProcessor("syslog_"+protocol, true) - err := processStreamInternal(r, encoding, useLocalTimestamp, remoteIP, lmp) + err := processStreamInternal(r, compressMethod, useLocalTimestamp, remoteIP, lmp) lmp.MustClose() return err } -func processStreamInternal(r io.Reader, encoding string, useLocalTimestamp bool, remoteIP string, lmp insertutil.LogMessageProcessor) error { - reader, err := protoparserutil.GetUncompressedReader(r, encoding) +func processStreamInternal(r io.Reader, compressMethod string, useLocalTimestamp bool, remoteIP string, lmp insertutil.LogMessageProcessor) error { + reader, err := protoparserutil.GetUncompressedReader(r, compressMethod) if err != nil { return fmt.Errorf("cannot decode syslog data: %w", err) } @@ -647,3 +647,73 @@ func parseExtraFields(s string) ([]logstorage.Field, error) { }) return fields, nil } + +type configs struct { + typ string + + streamFields []string + ignoreFields []string + decolorizeFields []string + extraFields []logstorage.Field + tenantID logstorage.TenantID + compressMethod string + useLocalTimestamp bool + useRemoteIP bool +} + +func getConfigs(typ string, argIdx int, streamFieldsArg, ignoreFieldsArg, decolorizeFieldsArg, extraFieldsArg, tenantIDArg, compressMethodArg *flagutil.ArrayString, + useLocalTimestampArg, useRemoteIPArg *flagutil.ArrayBool) (*configs, error) { + + streamFieldsStr := streamFieldsArg.GetOptionalArg(argIdx) + streamFields, err := parseFieldsList(streamFieldsStr) + if err != nil { + return nil, fmt.Errorf("cannot parse -syslog.streamFields.%s=%q: %w", typ, streamFieldsStr, err) + } + + ignoreFieldsStr := ignoreFieldsArg.GetOptionalArg(argIdx) + ignoreFields, err := parseFieldsList(ignoreFieldsStr) + if err != nil { + return nil, fmt.Errorf("cannot parse -syslog.ignoreFields.%s=%q: %w", typ, ignoreFieldsStr, err) + } + + decolorizeFieldsStr := decolorizeFieldsArg.GetOptionalArg(argIdx) + decolorizeFields, err := parseFieldsList(decolorizeFieldsStr) + if err != nil { + return nil, fmt.Errorf("cannot parse -syslog.decolorizeFields.%s=%q: %w", typ, decolorizeFieldsStr, err) + } + + extraFieldsStr := extraFieldsArg.GetOptionalArg(argIdx) + extraFields, err := parseExtraFields(extraFieldsStr) + if err != nil { + return nil, fmt.Errorf("cannot parse -syslog.extraFields.%s=%q: %w", typ, extraFieldsStr, err) + } + + tenantIDStr := tenantIDArg.GetOptionalArg(argIdx) + tenantID, err := logstorage.ParseTenantID(tenantIDStr) + if err != nil { + return nil, fmt.Errorf("cannot parse -syslog.tenantID.%s=%q: %w", typ, tenantIDStr, err) + } + + compressMethod := compressMethodArg.GetOptionalArg(argIdx) + switch compressMethod { + case "", "none", "zstd", "gzip", "deflate": + // These methods are supported + default: + return nil, fmt.Errorf("unsupported -syslog.compressMethod.%s=%q; supported values: 'none', 'zstd', 'gzip', 'deflate'", typ, compressMethod) + } + + useLocalTimestamp := useLocalTimestampArg.GetOptionalArg(argIdx) + useRemoteIP := useRemoteIPArg.GetOptionalArg(argIdx) + + return &configs{ + typ: typ, + streamFields: streamFields, + ignoreFields: ignoreFields, + decolorizeFields: decolorizeFields, + extraFields: extraFields, + tenantID: tenantID, + compressMethod: compressMethod, + useLocalTimestamp: useLocalTimestamp, + useRemoteIP: useRemoteIP, + }, nil +} diff --git a/docs/victorialogs/CHANGELOG.md b/docs/victorialogs/CHANGELOG.md index e1eb3801a6..32cfffbf0e 100644 --- a/docs/victorialogs/CHANGELOG.md +++ b/docs/victorialogs/CHANGELOG.md @@ -19,6 +19,7 @@ according to [these docs](https://docs.victoriametrics.com/victorialogs/quicksta ## tip * FEATURE: [LogsQL](https://docs.victoriametrics.com/victorialogs/logsql/): add [pattern match filter](https://docs.victoriametrics.com/victorialogs/logsql/#pattern-match-filter) for searching logs by the given patterns such as `: user_id=, ip=, trace_id=`. These filters are needed for [#518](https://github.com/VictoriaMetrics/VictoriaLogs/issues/518). +* FEATURE: [Syslog data ingestion](https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/): support for receiving Syslog messages from Unix sockets of `SOCK_STREAM` and `SOCK_DGRAM` types via `-syslog.listenAddr.unix=/path/to/socket` and `-syslog.listenAddr.unix=unixpacket:/path/to/socket` command-line flags. See [#570](https://github.com/VictoriaMetrics/VictoriaLogs/issues/570). ## [v1.32.0](https://github.com/VictoriaMetrics/VictoriaLogs/releases/tag/v1.32.0) diff --git a/docs/victorialogs/README.md b/docs/victorialogs/README.md index 75caa8d4e0..98cae91b70 100644 --- a/docs/victorialogs/README.md +++ b/docs/victorialogs/README.md @@ -691,7 +691,7 @@ Pass `-help` to VictoriaLogs in order to see the list of supported command-line -search.maxQueryDuration duration The maximum duration for query execution. It can be overridden to a smaller value on a per-query basis via 'timeout' query arg (default 30s) -search.maxQueryTimeRange duration - The maximum time range, which can be set in the query sent to querying APIs. Queries with bigger time ranges are rejected. See https://docs.victoriametrics.com/victorialogs/querying/#http-api + The maximum time range, which can be set in the query sent to querying APIs. Queries with bigger time ranges are rejected. See https://docs.victoriametrics.com/victorialogs/querying/#resource-usage-limits -search.maxQueueDuration duration The maximum time the search request waits for execution when -search.maxConcurrentRequests limit is reached; see also -search.maxQueryDuration (default 10s) -select.disable @@ -759,6 +759,10 @@ Pass `-help` to VictoriaLogs in order to see the list of supported command-line Compression method for syslog messages received at the corresponding -syslog.listenAddr.udp. Supported values: none, gzip, deflate. See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/#compression Supports an array of values separated by comma or specified via multiple flags. Value can contain comma inside single-quoted or double-quoted string, {}, [] and () braces. + -syslog.compressMethod.unix array + Compression method for syslog messages received at the corresponding -syslog.listenAddr.unix. Supported values: none, gzip, deflate. See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/#compression + Supports an array of values separated by comma or specified via multiple flags. + Value can contain comma inside single-quoted or double-quoted string, {}, [] and () braces. -syslog.decolorizeFields.tcp array Fields to remove ANSI color codes across logs ingested via the corresponding -syslog.listenAddr.tcp. See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/#decolorizing-fields Supports an array of values separated by comma or specified via multiple flags. @@ -767,6 +771,10 @@ Pass `-help` to VictoriaLogs in order to see the list of supported command-line Fields to remove ANSI color codes across logs ingested via the corresponding -syslog.listenAddr.udp. See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/#decolorizing-fields Supports an array of values separated by comma or specified via multiple flags. Value can contain comma inside single-quoted or double-quoted string, {}, [] and () braces. + -syslog.decolorizeFields.unix array + Fields to remove ANSI color codes across logs ingested via the corresponding -syslog.listenAddr.unix. See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/#decolorizing-fields + Supports an array of values separated by comma or specified via multiple flags. + Value can contain comma inside single-quoted or double-quoted string, {}, [] and () braces. -syslog.extraFields.tcp array Fields to add to logs ingested via the corresponding -syslog.listenAddr.tcp. See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/#adding-extra-fields Supports an array of values separated by comma or specified via multiple flags. @@ -775,6 +783,10 @@ Pass `-help` to VictoriaLogs in order to see the list of supported command-line Fields to add to logs ingested via the corresponding -syslog.listenAddr.udp. See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/#adding-extra-fields Supports an array of values separated by comma or specified via multiple flags. Value can contain comma inside single-quoted or double-quoted string, {}, [] and () braces. + -syslog.extraFields.unix array + Fields to add to logs ingested via the corresponding -syslog.listenAddr.unix. See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/#adding-extra-fields + Supports an array of values separated by comma or specified via multiple flags. + Value can contain comma inside single-quoted or double-quoted string, {}, [] and () braces. -syslog.ignoreFields.tcp array Fields to ignore at logs ingested via the corresponding -syslog.listenAddr.tcp. See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/#dropping-fields Supports an array of values separated by comma or specified via multiple flags. @@ -783,12 +795,20 @@ Pass `-help` to VictoriaLogs in order to see the list of supported command-line Fields to ignore at logs ingested via the corresponding -syslog.listenAddr.udp. See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/#dropping-fields Supports an array of values separated by comma or specified via multiple flags. Value can contain comma inside single-quoted or double-quoted string, {}, [] and () braces. + -syslog.ignoreFields.unix array + Fields to ignore at logs ingested via the corresponding -syslog.listenAddr.unix. See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/#dropping-fields + Supports an array of values separated by comma or specified via multiple flags. + Value can contain comma inside single-quoted or double-quoted string, {}, [] and () braces. -syslog.listenAddr.tcp array Comma-separated list of TCP addresses to listen to for Syslog messages. See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/ Supports an array of values separated by comma or specified via multiple flags. Value can contain comma inside single-quoted or double-quoted string, {}, [] and () braces. -syslog.listenAddr.udp array - Comma-separated list of UDP address to listen to for Syslog messages. See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/ + Comma-separated list of UDP addresses to listen to for Syslog messages. See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/ + Supports an array of values separated by comma or specified via multiple flags. + Value can contain comma inside single-quoted or double-quoted string, {}, [] and () braces. + -syslog.listenAddr.unix array + Comma-separated list of Unix socket filepaths to listen to for Syslog messages. Filepaths may be prepended with 'unixpacket:' for listening for SOCK_DGRAM sockets. By default SOCK_STREAM sockets are used. See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/ Supports an array of values separated by comma or specified via multiple flags. Value can contain comma inside single-quoted or double-quoted string, {}, [] and () braces. -syslog.mtls array @@ -807,6 +827,10 @@ Pass `-help` to VictoriaLogs in order to see the list of supported command-line Fields to use as log stream labels for logs ingested via the corresponding -syslog.listenAddr.udp. See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/#stream-fields Supports an array of values separated by comma or specified via multiple flags. Value can contain comma inside single-quoted or double-quoted string, {}, [] and () braces. + -syslog.streamFields.unix array + Fields to use as log stream labels for logs ingested via the corresponding -syslog.listenAddr.unix. See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/#stream-fields + Supports an array of values separated by comma or specified via multiple flags. + Value can contain comma inside single-quoted or double-quoted string, {}, [] and () braces. -syslog.tenantID.tcp array TenantID for logs ingested via the corresponding -syslog.listenAddr.tcp. See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/#multitenancy Supports an array of values separated by comma or specified via multiple flags. @@ -815,6 +839,10 @@ Pass `-help` to VictoriaLogs in order to see the list of supported command-line TenantID for logs ingested via the corresponding -syslog.listenAddr.udp. See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/#multitenancy Supports an array of values separated by comma or specified via multiple flags. Value can contain comma inside single-quoted or double-quoted string, {}, [] and () braces. + -syslog.tenantID.unix array + TenantID for logs ingested via the corresponding -syslog.listenAddr.unix. See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/#multitenancy + Supports an array of values separated by comma or specified via multiple flags. + Value can contain comma inside single-quoted or double-quoted string, {}, [] and () braces. -syslog.timezone string Timezone to use when parsing timestamps in RFC3164 syslog messages. Timezone must be a valid IANA Time Zone. For example: America/New_York, Europe/Berlin, Etc/GMT+3 . See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/ (default "Local") -syslog.tls array @@ -843,6 +871,10 @@ Pass `-help` to VictoriaLogs in order to see the list of supported command-line Whether to use local timestamp instead of the original timestamp for the ingested syslog messages at the corresponding -syslog.listenAddr.udp. See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/#log-timestamps Supports array of values separated by comma or specified via multiple flags. Empty values are set to false. + -syslog.useLocalTimestamp.unix array + Whether to use local timestamp instead of the original timestamp for the ingested syslog messages at the corresponding -syslog.listenAddr.unix. See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/#log-timestamps + Supports array of values separated by comma or specified via multiple flags. + Empty values are set to false. -syslog.useRemoteIP.tcp array Whether to add remote ip address as 'remote_ip' log field for syslog messages ingested via the corresponding -syslog.listenAddr.tcp. See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/#capturing-remote-ip-address Supports array of values separated by comma or specified via multiple flags. @@ -851,6 +883,10 @@ Pass `-help` to VictoriaLogs in order to see the list of supported command-line Whether to add remote ip address as 'remote_ip' log field for syslog messages ingested via the corresponding -syslog.listenAddr.udp. See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/#capturing-remote-ip-address Supports array of values separated by comma or specified via multiple flags. Empty values are set to false. + -syslog.useRemoteIP.unix array + Whether to add remote ip address as 'remote_ip' log field for syslog messages ingested via the corresponding -syslog.listenAddr.unix. See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/#capturing-remote-ip-address + Supports array of values separated by comma or specified via multiple flags. + Empty values are set to false. -tls array Whether to enable TLS for incoming HTTP requests at the given -httpListenAddr (aka https). -tlsCertFile and -tlsKeyFile must be set if -tls is set. See also -mtls Supports array of values separated by comma or specified via multiple flags. diff --git a/docs/victorialogs/data-ingestion/syslog.md b/docs/victorialogs/data-ingestion/syslog.md index e52c9bb2c5..bed4bc19d7 100644 --- a/docs/victorialogs/data-ingestion/syslog.md +++ b/docs/victorialogs/data-ingestion/syslog.md @@ -11,8 +11,12 @@ tags: aliases: - /victorialogs/data-ingestion/syslog.html --- -[VictoriaLogs](https://docs.victoriametrics.com/victorialogs/) can accept logs in [Syslog formats](https://en.wikipedia.org/wiki/Syslog) at the specified TCP and UDP addresses -via `-syslog.listenAddr.tcp` and `-syslog.listenAddr.udp` command-line flags. The following syslog formats are supported: + +[VictoriaLogs](https://docs.victoriametrics.com/victorialogs/) can accept logs in [Syslog formats](https://en.wikipedia.org/wiki/Syslog) at the specified TCP, UDP or Unix socket addresses +via `-syslog.listenAddr.tcp`, `-syslog.listenAddr.udp` and `-syslog.listenAddr.unix` command-line flags. VictoriaLogs listens for `SOCK_STREAM` unix sockets by default. +Prepend the unix socket path passed to `-syslog.listenAddr.unix` with `unixpacket:` for `SOCK_DGRAM` sockets. + +The following syslog formats are supported: - [RFC3164](https://datatracker.ietf.org/doc/html/rfc3164) aka `MMM DD hh:mm:ss HOSTNAME APP-NAME[PROCID]: MESSAGE` - [RFC5424](https://datatracker.ietf.org/doc/html/rfc5424) aka `1 TIMESTAMP HOSTNAME APP-NAME PROCID MSGID [STRUCTURED-DATA] MESSAGE` @@ -105,6 +109,13 @@ via the corresponding `-syslog.listenAddr.udp` address: ./victoria-logs -syslog.listenAddr.udp=:514 -syslog.useLocalTimestamp.udp ``` +The `-syslog.useLocalTimestamp.unix` command-line flag can be used for instructing VictoriaLogs to use local timestamps for the ingested logs +via the corresponding `-syslog.listenAddr.unix` address: + +```sh +./victoria-logs -syslog.listenAddr.unix=/dev/log -syslog.useLocalTimestamp.unix +``` + ## Security By default VictoriaLogs accepts plaintext data at `-syslog.listenAddr.tcp` address. Run VictoriaLogs with `-syslog.tls` command-line flag @@ -133,8 +144,8 @@ from [the releases page](https://github.com/VictoriaMetrics/VictoriaLogs/release ## Compression -By default VictoriaLogs accepts uncompressed log messages in Syslog format at `-syslog.listenAddr.tcp` and `-syslog.listenAddr.udp` addresses. -It is possible configuring VictoriaLogs to accept compressed log messages via `-syslog.compressMethod.tcp` and `-syslog.compressMethod.udp` command-line flags. +By default VictoriaLogs accepts uncompressed log messages in Syslog format at `-syslog.listenAddr.tcp`, `-syslog.listenAddr.udp` and `-syslog.listenAddr.unix` addresses. +It is possible configuring VictoriaLogs to accept compressed log messages via `-syslog.compressMethod.tcp`, `-syslog.compressMethod.udp` and `-syslog.compressMethod.unix` command-line flags. The following compression methods are supported: - `none` - no compression @@ -151,8 +162,8 @@ For example, the following command starts VictoriaLogs, which accepts gzip-compr ## Multitenancy By default, the ingested logs are stored in the `(AccountID=0, ProjectID=0)` [tenant](https://docs.victoriametrics.com/victorialogs/#multitenancy). -If you need storing logs in other tenant, then specify the needed tenant via `-syslog.tenantID.tcp` or `-syslog.tenantID.udp` command-line flags -depending on whether TCP or UDP ports are listened for syslog messages. +If you need storing logs in other tenant, then specify the needed tenant via `-syslog.tenantID.tcp`, `-syslog.tenantID.udp` or `-syslog.tenantID.unix` command-line flags +depending on whether TCP, UDP or Unix sockets listened for syslog messages. For example, the following command starts VictoriaLogs, which writes syslog messages received at TCP port 514, to `(AccountID=12, ProjectID=34)` tenant: ```sh @@ -162,8 +173,8 @@ For example, the following command starts VictoriaLogs, which writes syslog mess ## Stream fields VictoriaLogs uses `(hostname, app_name, proc_id)` fields as labels for [log streams](https://docs.victoriametrics.com/victorialogs/keyconcepts/#stream-fields) by default. -It is possible setting other set of labels via `-syslog.streamFields.tcp` and `-syslog.streamFields.udp` command-line flags -for logs instead via the corresponding `-syslog.listenAddr.tcp` and `-syslog.listenAddr.dup` addresses. +It is possible setting other set of labels via `-syslog.streamFields.tcp`, `-syslog.streamFields.udp` and `-syslog.streamFields.unix` command-line flags +for logs instead via the corresponding `-syslog.listenAddr.tcp`, `-syslog.listenAddr.udp` and `-syslog.listenAddr.unix` addresses. For example, the following command starts VictoriaLogs, which uses `(hostname, app_name)` fields as log stream labels for logs received at TCP port 514: @@ -173,9 +184,9 @@ for logs received at TCP port 514: ## Dropping fields -VictoriaLogs supports `-syslog.ignoreFields.tcp` and `-syslog.ignoreFields.udp` command-line flags for skipping +VictoriaLogs supports `-syslog.ignoreFields.tcp`, `-syslog.ignoreFields.udp` and `-syslog.ignoreFields.unix` command-line flags for skipping the given [log fields](https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model) during ingestion -of Syslog logs into `-syslog.listenAddr.tcp` and `-syslog.listenAddr.udp` addresses. +of Syslog logs into `-syslog.listenAddr.tcp`, `-syslog.listenAddr.udp` and `-syslog.listenAddr.unix` addresses. For example, the following command starts VictoriaLogs, which drops `proc_id` and `msg_id` fields from logs received at TCP port 514: ```sh @@ -187,9 +198,9 @@ are ignored during data ingestion. ## Decolorizing fields -VictoriaLogs supports `-syslog.decolorizeFields.tcp` and `-syslog.decolorizeFields.udp` command-line flags, +VictoriaLogs supports `-syslog.decolorizeFields.tcp`, `-syslog.decolorizeFields.udp` and `-syslog.decolorizeFields.unix` command-line flags, which can be used for removing ANSI color codes from the provided list fields during ingestion of Syslog logs -into `-syslog.listenAddr.tcp` and `-syslog.listenAddr.upd` addresses. +into `-syslog.listenAddr.tcp`, `-syslog.listenAddr.udp` and `-syslog.listenAddr.unix` addresses. For example, the following command starts VictoriaLogs, which removes ANSI color codes from [`_msg` field](https://docs.victoriametrics.com/victorialogs/keyconcepts/#message-field) at logs received via TCP port 514: @@ -199,9 +210,9 @@ at logs received via TCP port 514: ## Adding extra fields -VictoriaLogs supports -`syslog.extraFields.tcp` and `-syslog.extraFields.udp` command-line flags for adding +VictoriaLogs supports -`syslog.extraFields.tcp`, `-syslog.extraFields.udp` and `-syslog.extraFields.unix` command-line flags for adding the given [log fields](https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model) during data ingestion -of Syslog logs into `-syslog.listenAddr.tcp` and `-syslog.listenAddr.udp` addresses. +of Syslog logs into `-syslog.listenAddr.tcp`, `-syslog.listenAddr.udp` and `-syslog.listenAddr.unix` addresses. For example, the following command starts VictoriaLogs, which adds `source=foo` and `abc=def` fields to logs received at TCP port 514: ```sh diff --git a/docs/victorialogs/vlagent.md b/docs/victorialogs/vlagent.md index 3b22aa0bf0..103c022416 100644 --- a/docs/victorialogs/vlagent.md +++ b/docs/victorialogs/vlagent.md @@ -155,7 +155,8 @@ See the docs at https://docs.victoriametrics.com/victorialogs/vlagent/ . Whether to disable fadvise() syscall when reading large data files. The fadvise() syscall prevents from eviction of recently accessed data from OS page cache during background merges and backups. In some rare cases it is better to disable the syscall if it uses too much CPU -flagsAuthKey value Auth key for /flags endpoint. It must be passed via authKey query arg. It overrides -httpAuth.* - Flag value can be read from the given file when using -flagsAuthKey=file:///abs/path/to/file or -flagsAuthKey=file://./relative/path/to/file . Flag value can be read from the given http/https url when using -flagsAuthKey=http://host/path or -flagsAuthKey=https://host/path + Flag value can be read from the given file when using -flagsAuthKey=file:///abs/path/to/file or -flagsAuthKey=file://./relative/path/to/file. + Flag value can be read from the given http/https url when using -flagsAuthKey=http://host/path or -flagsAuthKey=https://host/path -fs.disableMmap Whether to use pread() instead of mmap() for reading data files. By default, mmap() is used for 64-bit arches and pread() is used for 32-bit arches, since they cannot read data files bigger than 2^32 bytes in memory. mmap() is usually faster for reading small data chunks than pread() -http.connTimeout duration @@ -182,7 +183,8 @@ See the docs at https://docs.victoriametrics.com/victorialogs/vlagent/ . Optional delay before http server shutdown. During this delay, the server returns non-OK responses from /health page, so load balancers can route new requests to other servers -httpAuth.password value Password for HTTP server's Basic Auth. The authentication is disabled if -httpAuth.username is empty - Flag value can be read from the given file when using -httpAuth.password=file:///abs/path/to/file or -httpAuth.password=file://./relative/path/to/file . Flag value can be read from the given http/https url when using -httpAuth.password=http://host/path or -httpAuth.password=https://host/path + Flag value can be read from the given file when using -httpAuth.password=file:///abs/path/to/file or -httpAuth.password=file://./relative/path/to/file. + Flag value can be read from the given http/https url when using -httpAuth.password=http://host/path or -httpAuth.password=https://host/path -httpAuth.username string Username for HTTP server's Basic Auth. The authentication is disabled if empty. See also -httpAuth.password -httpListenAddr array @@ -269,7 +271,8 @@ See the docs at https://docs.victoriametrics.com/victorialogs/vlagent/ . Whether to expose TYPE and HELP metadata at the /metrics page, which is exposed at -httpListenAddr . The metadata may be needed when the /metrics page is consumed by systems, which require this information. For example, Managed Prometheus in Google Cloud - https://cloud.google.com/stackdriver/docs/managed-prometheus/troubleshooting#missing-metric-type -metricsAuthKey value Auth key for /metrics endpoint. It must be passed via authKey query arg. It overrides -httpAuth.* - Flag value can be read from the given file when using -metricsAuthKey=file:///abs/path/to/file or -metricsAuthKey=file://./relative/path/to/file . Flag value can be read from the given http/https url when using -metricsAuthKey=http://host/path or -metricsAuthKey=https://host/path + Flag value can be read from the given file when using -metricsAuthKey=file:///abs/path/to/file or -metricsAuthKey=file://./relative/path/to/file. + Flag value can be read from the given http/https url when using -metricsAuthKey=http://host/path or -metricsAuthKey=https://host/path -mtls array Whether to require valid client certificate for https requests to the corresponding -httpListenAddr . This flag works only if -tls flag is set. See also -mtlsCAFile . This flag is available only in Enterprise binaries. See https://docs.victoriametrics.com/victoriametrics/enterprise/ Supports array of values separated by comma or specified via multiple flags. @@ -283,7 +286,8 @@ See the docs at https://docs.victoriametrics.com/victorialogs/vlagent/ . Supports the following optional suffixes for size values: KB, MB, GB, TB, KiB, MiB, GiB, TiB (default 67108864) -pprofAuthKey value Auth key for /debug/pprof/* endpoints. It must be passed via authKey query arg. It overrides -httpAuth.* - Flag value can be read from the given file when using -pprofAuthKey=file:///abs/path/to/file or -pprofAuthKey=file://./relative/path/to/file . Flag value can be read from the given http/https url when using -pprofAuthKey=http://host/path or -pprofAuthKey=https://host/path + Flag value can be read from the given file when using -pprofAuthKey=file:///abs/path/to/file or -pprofAuthKey=file://./relative/path/to/file. + Flag value can be read from the given http/https url when using -pprofAuthKey=http://host/path or -pprofAuthKey=https://host/path -pushmetrics.disableCompression Whether to disable request body compression when pushing metrics to every -pushmetrics.url -pushmetrics.extraLabel array @@ -420,6 +424,10 @@ See the docs at https://docs.victoriametrics.com/victorialogs/vlagent/ . Compression method for syslog messages received at the corresponding -syslog.listenAddr.udp. Supported values: none, gzip, deflate. See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/#compression Supports an array of values separated by comma or specified via multiple flags. Value can contain comma inside single-quoted or double-quoted string, {}, [] and () braces. + -syslog.compressMethod.unix array + Compression method for syslog messages received at the corresponding -syslog.listenAddr.unix. Supported values: none, gzip, deflate. See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/#compression + Supports an array of values separated by comma or specified via multiple flags. + Value can contain comma inside single-quoted or double-quoted string, {}, [] and () braces. -syslog.decolorizeFields.tcp array Fields to remove ANSI color codes across logs ingested via the corresponding -syslog.listenAddr.tcp. See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/#decolorizing-fields Supports an array of values separated by comma or specified via multiple flags. @@ -428,6 +436,10 @@ See the docs at https://docs.victoriametrics.com/victorialogs/vlagent/ . Fields to remove ANSI color codes across logs ingested via the corresponding -syslog.listenAddr.udp. See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/#decolorizing-fields Supports an array of values separated by comma or specified via multiple flags. Value can contain comma inside single-quoted or double-quoted string, {}, [] and () braces. + -syslog.decolorizeFields.unix array + Fields to remove ANSI color codes across logs ingested via the corresponding -syslog.listenAddr.unix. See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/#decolorizing-fields + Supports an array of values separated by comma or specified via multiple flags. + Value can contain comma inside single-quoted or double-quoted string, {}, [] and () braces. -syslog.extraFields.tcp array Fields to add to logs ingested via the corresponding -syslog.listenAddr.tcp. See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/#adding-extra-fields Supports an array of values separated by comma or specified via multiple flags. @@ -436,6 +448,10 @@ See the docs at https://docs.victoriametrics.com/victorialogs/vlagent/ . Fields to add to logs ingested via the corresponding -syslog.listenAddr.udp. See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/#adding-extra-fields Supports an array of values separated by comma or specified via multiple flags. Value can contain comma inside single-quoted or double-quoted string, {}, [] and () braces. + -syslog.extraFields.unix array + Fields to add to logs ingested via the corresponding -syslog.listenAddr.unix. See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/#adding-extra-fields + Supports an array of values separated by comma or specified via multiple flags. + Value can contain comma inside single-quoted or double-quoted string, {}, [] and () braces. -syslog.ignoreFields.tcp array Fields to ignore at logs ingested via the corresponding -syslog.listenAddr.tcp. See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/#dropping-fields Supports an array of values separated by comma or specified via multiple flags. @@ -444,12 +460,20 @@ See the docs at https://docs.victoriametrics.com/victorialogs/vlagent/ . Fields to ignore at logs ingested via the corresponding -syslog.listenAddr.udp. See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/#dropping-fields Supports an array of values separated by comma or specified via multiple flags. Value can contain comma inside single-quoted or double-quoted string, {}, [] and () braces. + -syslog.ignoreFields.unix array + Fields to ignore at logs ingested via the corresponding -syslog.listenAddr.unix. See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/#dropping-fields + Supports an array of values separated by comma or specified via multiple flags. + Value can contain comma inside single-quoted or double-quoted string, {}, [] and () braces. -syslog.listenAddr.tcp array Comma-separated list of TCP addresses to listen to for Syslog messages. See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/ Supports an array of values separated by comma or specified via multiple flags. Value can contain comma inside single-quoted or double-quoted string, {}, [] and () braces. -syslog.listenAddr.udp array - Comma-separated list of UDP address to listen to for Syslog messages. See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/ + Comma-separated list of UDP addresses to listen to for Syslog messages. See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/ + Supports an array of values separated by comma or specified via multiple flags. + Value can contain comma inside single-quoted or double-quoted string, {}, [] and () braces. + -syslog.listenAddr.unix array + Comma-separated list of Unix socket filepaths to listen to for Syslog messages. Filepaths may be prepended with 'unixpacket:' for listening for SOCK_DGRAM sockets. By default SOCK_STREAM sockets are used. See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/ Supports an array of values separated by comma or specified via multiple flags. Value can contain comma inside single-quoted or double-quoted string, {}, [] and () braces. -syslog.mtls array @@ -468,6 +492,10 @@ See the docs at https://docs.victoriametrics.com/victorialogs/vlagent/ . Fields to use as log stream labels for logs ingested via the corresponding -syslog.listenAddr.udp. See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/#stream-fields Supports an array of values separated by comma or specified via multiple flags. Value can contain comma inside single-quoted or double-quoted string, {}, [] and () braces. + -syslog.streamFields.unix array + Fields to use as log stream labels for logs ingested via the corresponding -syslog.listenAddr.unix. See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/#stream-fields + Supports an array of values separated by comma or specified via multiple flags. + Value can contain comma inside single-quoted or double-quoted string, {}, [] and () braces. -syslog.tenantID.tcp array TenantID for logs ingested via the corresponding -syslog.listenAddr.tcp. See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/#multitenancy Supports an array of values separated by comma or specified via multiple flags. @@ -476,6 +504,10 @@ See the docs at https://docs.victoriametrics.com/victorialogs/vlagent/ . TenantID for logs ingested via the corresponding -syslog.listenAddr.udp. See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/#multitenancy Supports an array of values separated by comma or specified via multiple flags. Value can contain comma inside single-quoted or double-quoted string, {}, [] and () braces. + -syslog.tenantID.unix array + TenantID for logs ingested via the corresponding -syslog.listenAddr.unix. See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/#multitenancy + Supports an array of values separated by comma or specified via multiple flags. + Value can contain comma inside single-quoted or double-quoted string, {}, [] and () braces. -syslog.timezone string Timezone to use when parsing timestamps in RFC3164 syslog messages. Timezone must be a valid IANA Time Zone. For example: America/New_York, Europe/Berlin, Etc/GMT+3 . See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/ (default "Local") -syslog.tls array @@ -504,6 +536,10 @@ See the docs at https://docs.victoriametrics.com/victorialogs/vlagent/ . Whether to use local timestamp instead of the original timestamp for the ingested syslog messages at the corresponding -syslog.listenAddr.udp. See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/#log-timestamps Supports array of values separated by comma or specified via multiple flags. Empty values are set to false. + -syslog.useLocalTimestamp.unix array + Whether to use local timestamp instead of the original timestamp for the ingested syslog messages at the corresponding -syslog.listenAddr.unix. See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/#log-timestamps + Supports array of values separated by comma or specified via multiple flags. + Empty values are set to false. -syslog.useRemoteIP.tcp array Whether to add remote ip address as 'remote_ip' log field for syslog messages ingested via the corresponding -syslog.listenAddr.tcp. See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/#capturing-remote-ip-address Supports array of values separated by comma or specified via multiple flags. @@ -512,6 +548,10 @@ See the docs at https://docs.victoriametrics.com/victorialogs/vlagent/ . Whether to add remote ip address as 'remote_ip' log field for syslog messages ingested via the corresponding -syslog.listenAddr.udp. See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/#capturing-remote-ip-address Supports array of values separated by comma or specified via multiple flags. Empty values are set to false. + -syslog.useRemoteIP.unix array + Whether to add remote ip address as 'remote_ip' log field for syslog messages ingested via the corresponding -syslog.listenAddr.unix. See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/#capturing-remote-ip-address + Supports array of values separated by comma or specified via multiple flags. + Empty values are set to false. -tls array Whether to enable TLS for incoming HTTP requests at the given -httpListenAddr (aka https). -tlsCertFile and -tlsKeyFile must be set if -tls is set. See also -mtls Supports array of values separated by comma or specified via multiple flags. From 00f0d5d9d87b5d9731c61862a4c813528f9e6010 Mon Sep 17 00:00:00 2001 From: Phuong Le <39565248+func25@users.noreply.github.com> Date: Thu, 4 Sep 2025 19:13:55 +0700 Subject: [PATCH 02/82] logsql: allow the -search.maxQueryTimeRange flag with d, w, and y suffixes (#630) Co-authored-by: Aliaksandr Valialkin --- app/vlselect/logsql/logsql.go | 10 +- docs/victorialogs/CHANGELOG.md | 2 + go.mod | 2 +- go.sum | 4 +- .../VictoriaMetrics/lib/flagutil/duration.go | 91 +++++++++++++++++++ vendor/modules.txt | 2 +- 6 files changed, 102 insertions(+), 9 deletions(-) diff --git a/app/vlselect/logsql/logsql.go b/app/vlselect/logsql/logsql.go index 0c4ebb656f..18879d469b 100644 --- a/app/vlselect/logsql/logsql.go +++ b/app/vlselect/logsql/logsql.go @@ -2,7 +2,6 @@ package logsql import ( "context" - "flag" "fmt" "io" "math" @@ -17,6 +16,7 @@ import ( "github.com/VictoriaMetrics/VictoriaMetrics/lib/atomicutil" "github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil" + "github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil" "github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver" "github.com/VictoriaMetrics/VictoriaMetrics/lib/httputil" "github.com/VictoriaMetrics/VictoriaMetrics/lib/logger" @@ -29,7 +29,7 @@ import ( ) var ( - maxQueryTimeRange = flag.Duration("search.maxQueryTimeRange", 0, "The maximum time range, which can be set in the query sent to querying APIs. "+ + maxQueryTimeRange = flagutil.NewExtendedDuration("search.maxQueryTimeRange", "0", "The maximum time range, which can be set in the query sent to querying APIs. "+ "Queries with bigger time ranges are rejected. See https://docs.victoriametrics.com/victorialogs/querying/#resource-usage-limits") ) @@ -1200,14 +1200,14 @@ func parseCommonArgs(r *http.Request) (*commonArgs, error) { } } - if *maxQueryTimeRange > 0 { + if maxRange := maxQueryTimeRange.Duration(); maxRange > 0 { start, end := q.GetFilterTimeRange() if end > start { queryTimeRange := end - start - if queryTimeRange < 0 || queryTimeRange > maxQueryTimeRange.Nanoseconds() { + if queryTimeRange < 0 || queryTimeRange > maxRange.Nanoseconds() { return nil, fmt.Errorf("too big time range selected: [%s, %s]; it cannot exceed -search.maxQueryTimeRange=%s; "+ "see https://docs.victoriametrics.com/victorialogs/querying/#resource-usage-limits", - timestampToString(start), timestampToString(end), *maxQueryTimeRange) + timestampToString(start), timestampToString(end), maxRange) } } } diff --git a/docs/victorialogs/CHANGELOG.md b/docs/victorialogs/CHANGELOG.md index 32cfffbf0e..56bb9ad59c 100644 --- a/docs/victorialogs/CHANGELOG.md +++ b/docs/victorialogs/CHANGELOG.md @@ -21,6 +21,8 @@ according to [these docs](https://docs.victoriametrics.com/victorialogs/quicksta * FEATURE: [LogsQL](https://docs.victoriametrics.com/victorialogs/logsql/): add [pattern match filter](https://docs.victoriametrics.com/victorialogs/logsql/#pattern-match-filter) for searching logs by the given patterns such as `: user_id=, ip=, trace_id=`. These filters are needed for [#518](https://github.com/VictoriaMetrics/VictoriaLogs/issues/518). * FEATURE: [Syslog data ingestion](https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/): support for receiving Syslog messages from Unix sockets of `SOCK_STREAM` and `SOCK_DGRAM` types via `-syslog.listenAddr.unix=/path/to/socket` and `-syslog.listenAddr.unix=unixpacket:/path/to/socket` command-line flags. See [#570](https://github.com/VictoriaMetrics/VictoriaLogs/issues/570). +* BUGFIX: [querying](https://docs.victoriametrics.com/victorialogs/querying) `-search.maxQueryTimeRange` command-line flag now supports day (`d`), week (`w`) and year (`y`) suffixes additionally to the supported hour (`h`), minute (`m`) and second (`s`) suffixes. See [#50](https://github.com/VictoriaMetrics/VictoriaLogs/issues/50#issuecomment-3244097676) + ## [v1.32.0](https://github.com/VictoriaMetrics/VictoriaLogs/releases/tag/v1.32.0) Released at 2025-09-03 diff --git a/go.mod b/go.mod index 860cb082ae..90bba34443 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/VictoriaMetrics/VictoriaLogs go 1.25.0 require ( - github.com/VictoriaMetrics/VictoriaMetrics v0.0.0-20250829235354-7967ad661e76 + github.com/VictoriaMetrics/VictoriaMetrics v0.0.0-20250903201027-a0a33f0ce1c2 github.com/VictoriaMetrics/easyproto v0.1.4 github.com/VictoriaMetrics/metrics v1.39.1 github.com/cespare/xxhash/v2 v2.3.0 diff --git a/go.sum b/go.sum index 07e4870d4c..6e16563954 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,5 @@ -github.com/VictoriaMetrics/VictoriaMetrics v0.0.0-20250829235354-7967ad661e76 h1:HyLFTlZN3UY84HQPWIZT4hSS7537618ALvMla3bV2hI= -github.com/VictoriaMetrics/VictoriaMetrics v0.0.0-20250829235354-7967ad661e76/go.mod h1:mRRUzTxEuernAU9h20/UeZs11dJA2J8LaP9DxbtCag0= +github.com/VictoriaMetrics/VictoriaMetrics v0.0.0-20250903201027-a0a33f0ce1c2 h1:XKK5/XxMw1HdlKic12Wh9rJRKxxpy0MTHBdWi+80MAs= +github.com/VictoriaMetrics/VictoriaMetrics v0.0.0-20250903201027-a0a33f0ce1c2/go.mod h1:mRRUzTxEuernAU9h20/UeZs11dJA2J8LaP9DxbtCag0= github.com/VictoriaMetrics/easyproto v0.1.4 h1:r8cNvo8o6sR4QShBXQd1bKw/VVLSQma/V2KhTBPf+Sc= github.com/VictoriaMetrics/easyproto v0.1.4/go.mod h1:QlGlzaJnDfFd8Lk6Ci/fuLxfTo3/GThPs2KH23mv710= github.com/VictoriaMetrics/metrics v1.39.1 h1:AT7jz7oSpAK9phDl5O5Tmy06nXnnzALwqVnf4ros3Ow= diff --git a/vendor/github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil/duration.go b/vendor/github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil/duration.go index e63940e9a8..64d6ff34eb 100644 --- a/vendor/github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil/duration.go +++ b/vendor/github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil/duration.go @@ -106,5 +106,96 @@ func (d *RetentionDuration) Set(value string) error { return nil } +// NewExtendedDuration returns new `duration` flag with the given name, defaultValue and description. +// +// Requires explicit unit suffixes (s, m, h, d, w, y). Bare numbers without units will cause errors (except 0). +func NewExtendedDuration(name string, defaultValue string, description string) *ExtendedDuration { + description += "\nThe following unit suffixes are required: s (second), m (minute), h (hour), d (day), w (week), y (year). " + + "Bare numbers without units are not allowed (except 0)" + d := &ExtendedDuration{} + if err := d.Set(defaultValue); err != nil { + panic(fmt.Sprintf("BUG: can not parse default value %s for flag %s", defaultValue, name)) + } + flag.Var(d, name, description) + return d +} + +// ExtendedDuration is a flag for specifying time durations with explicit unit suffixes. +// Unlike RetentionDuration, it requires explicit unit suffixes and doesn't treat bare numbers as months. +// Supported units: s (seconds), m (minutes), h (hours), d (days), w (weeks), y (years). +type ExtendedDuration struct { + // msecs contains parsed duration in milliseconds. + msecs int64 + + valueString string +} + +var ( + _ json.Marshaler = (*ExtendedDuration)(nil) + _ json.Unmarshaler = (*ExtendedDuration)(nil) +) + +// MarshalJSON implements json.Marshaler interface +func (d *ExtendedDuration) MarshalJSON() ([]byte, error) { + return json.Marshal(d.valueString) +} + +// UnmarshalJSON implements json.Unmarshaler interface +func (d *ExtendedDuration) UnmarshalJSON(data []byte) error { + var s string + if err := json.Unmarshal(data, &s); err != nil { + return err + } + return d.Set(s) +} + +// Duration returns d as time.Duration +func (d *ExtendedDuration) Duration() time.Duration { + return time.Millisecond * time.Duration(d.msecs) +} + +// Milliseconds returns d in milliseconds +func (d *ExtendedDuration) Milliseconds() int64 { + return d.msecs +} + +// String implements flag.Value interface +func (d *ExtendedDuration) String() string { + return d.valueString +} + +// Set implements flag.Value interface +// It requires explicit unit suffixes and rejects bare numbers (except 0). +func (d *ExtendedDuration) Set(value string) error { + if value == "" { + d.msecs = 0 + d.valueString = "" + return nil + } + + // Check for bare numbers first + if f, err := strconv.ParseFloat(value, 64); err == nil { + // It's a bare number + if f != 0 { + return fmt.Errorf("duration value must have a unit suffix (s, m, h, d, w, y); got bare number %q (0 is allowed)", value) + } + // Allow 0 as it's unambiguous + d.msecs = 0 + d.valueString = value + return nil + } + + // Parse duration with units using metricsql + value = strings.ToLower(value) + msecs, err := metricsql.PositiveDurationValue(value, 0) + if err != nil { + return fmt.Errorf("cannot parse duration %q: %w", value, err) + } + + d.msecs = msecs + d.valueString = value + return nil +} + const maxMonths = 12 * 100 const msecsPer31Days = 31 * 24 * 3600 * 1000 diff --git a/vendor/modules.txt b/vendor/modules.txt index 9d427cdc02..4eb4cbdae8 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -1,4 +1,4 @@ -# github.com/VictoriaMetrics/VictoriaMetrics v0.0.0-20250829235354-7967ad661e76 +# github.com/VictoriaMetrics/VictoriaMetrics v0.0.0-20250903201027-a0a33f0ce1c2 ## explicit; go 1.25.0 github.com/VictoriaMetrics/VictoriaMetrics/lib/appmetrics github.com/VictoriaMetrics/VictoriaMetrics/lib/atomicutil From 1089aa791208d5012d99f51d5b9d1e69de584865 Mon Sep 17 00:00:00 2001 From: Phuong Le <39565248+func25@users.noreply.github.com> Date: Thu, 4 Sep 2025 19:25:54 +0700 Subject: [PATCH 03/82] app/vlinsert: fix leaking json parser on error (#624) --- app/vlinsert/elasticsearch/elasticsearch.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/app/vlinsert/elasticsearch/elasticsearch.go b/app/vlinsert/elasticsearch/elasticsearch.go index bcf1390711..56b6d3993c 100644 --- a/app/vlinsert/elasticsearch/elasticsearch.go +++ b/app/vlinsert/elasticsearch/elasticsearch.go @@ -187,7 +187,10 @@ func readBulkLine(lr *insertutil.LineReader, timeFields, msgFields []string, lmp // Continue parsing next lines. return true, nil } + p := logstorage.GetJSONParser() + defer logstorage.PutJSONParser(p) + if err := p.ParseLogMessage(line); err != nil { return false, fmt.Errorf("cannot parse json-encoded log entry: %w", err) } @@ -201,7 +204,6 @@ func readBulkLine(lr *insertutil.LineReader, timeFields, msgFields []string, lmp } logstorage.RenameField(p.Fields, msgFields, "_msg") lmp.AddRow(ts, p.Fields, nil) - logstorage.PutJSONParser(p) return true, nil } From f4096527754dae1618c12fa69a195699d2c007ab Mon Sep 17 00:00:00 2001 From: Aliaksandr Valialkin Date: Thu, 4 Sep 2025 17:50:31 +0200 Subject: [PATCH 04/82] docs/victorialogs/CHANGELOG.md: add missing ":" after 00f0d5d9d87b5d9731c61862a4c813528f9e6010 --- docs/victorialogs/CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/victorialogs/CHANGELOG.md b/docs/victorialogs/CHANGELOG.md index 56bb9ad59c..f0c72c1293 100644 --- a/docs/victorialogs/CHANGELOG.md +++ b/docs/victorialogs/CHANGELOG.md @@ -21,7 +21,7 @@ according to [these docs](https://docs.victoriametrics.com/victorialogs/quicksta * FEATURE: [LogsQL](https://docs.victoriametrics.com/victorialogs/logsql/): add [pattern match filter](https://docs.victoriametrics.com/victorialogs/logsql/#pattern-match-filter) for searching logs by the given patterns such as `: user_id=, ip=, trace_id=`. These filters are needed for [#518](https://github.com/VictoriaMetrics/VictoriaLogs/issues/518). * FEATURE: [Syslog data ingestion](https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/): support for receiving Syslog messages from Unix sockets of `SOCK_STREAM` and `SOCK_DGRAM` types via `-syslog.listenAddr.unix=/path/to/socket` and `-syslog.listenAddr.unix=unixpacket:/path/to/socket` command-line flags. See [#570](https://github.com/VictoriaMetrics/VictoriaLogs/issues/570). -* BUGFIX: [querying](https://docs.victoriametrics.com/victorialogs/querying) `-search.maxQueryTimeRange` command-line flag now supports day (`d`), week (`w`) and year (`y`) suffixes additionally to the supported hour (`h`), minute (`m`) and second (`s`) suffixes. See [#50](https://github.com/VictoriaMetrics/VictoriaLogs/issues/50#issuecomment-3244097676) +* BUGFIX: [querying](https://docs.victoriametrics.com/victorialogs/querying): `-search.maxQueryTimeRange` command-line flag now supports day (`d`), week (`w`) and year (`y`) suffixes additionally to the supported hour (`h`), minute (`m`) and second (`s`) suffixes. See [#50](https://github.com/VictoriaMetrics/VictoriaLogs/issues/50#issuecomment-3244097676) ## [v1.32.0](https://github.com/VictoriaMetrics/VictoriaLogs/releases/tag/v1.32.0) From 76940bb8b7bef7fb00dc677d548f46dbab53a75f Mon Sep 17 00:00:00 2001 From: Aliaksandr Valialkin Date: Mon, 8 Sep 2025 17:13:13 +0200 Subject: [PATCH 05/82] app/vlinsert/syslog: properly accept Syslog logs over SOCK_DGRAM Unix sockets SOCK_DGRAM Unix sockets must have unixgram network type instead of unixpacket network type. Updates https://github.com/VictoriaMetrics/VictoriaLogs/issues/570 --- app/vlinsert/syslog/syslog.go | 71 +++++++++++++++------- docs/victorialogs/CHANGELOG.md | 2 +- docs/victorialogs/README.md | 2 +- docs/victorialogs/data-ingestion/syslog.md | 2 +- docs/victorialogs/vlagent.md | 2 +- 5 files changed, 53 insertions(+), 26 deletions(-) diff --git a/app/vlinsert/syslog/syslog.go b/app/vlinsert/syslog/syslog.go index f10c983075..3fefd4241f 100644 --- a/app/vlinsert/syslog/syslog.go +++ b/app/vlinsert/syslog/syslog.go @@ -40,7 +40,7 @@ var ( listenAddrUDP = flagutil.NewArrayString("syslog.listenAddr.udp", "Comma-separated list of UDP addresses to listen to for Syslog messages. "+ "See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/") listenAddrUnix = flagutil.NewArrayString("syslog.listenAddr.unix", "Comma-separated list of Unix socket filepaths to listen to for Syslog messages. "+ - "Filepaths may be prepended with 'unixpacket:' for listening for SOCK_DGRAM sockets. By default SOCK_STREAM sockets are used. "+ + "Filepaths may be prepended with 'unixgram:' for listening for SOCK_DGRAM sockets. By default SOCK_STREAM sockets are used. "+ "See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/") tlsEnable = flagutil.NewArrayBool("syslog.tls", "Whether to enable TLS for receiving syslog messages at the corresponding -syslog.listenAddr.tcp. "+ @@ -197,39 +197,74 @@ func MustStop() { } func runUnixListener(addr string, argIdx int) { - network, path := getUnixSocketNetworkAndPath(addr) - ln, err := net.Listen(network, path) + cfg, err := getConfigs("unix", argIdx, streamFieldsUnix, ignoreFieldsUnix, decolorizeFieldsUnix, extraFieldsUnix, tenantIDUnix, compressMethodUnix, useLocalTimestampUnix, useRemoteIPUnix) if err != nil { - logger.Fatalf("cannot start Unix socket syslog server at %q: %s", addr, err) + logger.Fatalf("cannot parse configs for -syslog.listenAddr.unix=%q: %s", addr, err) } - cfg, err := getConfigs("unix", argIdx, streamFieldsUnix, ignoreFieldsUnix, decolorizeFieldsUnix, extraFieldsUnix, tenantIDUnix, compressMethodUnix, useLocalTimestampUnix, useRemoteIPUnix) + laddr := getUnixSocketNetworkAndPath(addr) + if laddr.Net == "unix" { + runUnixStreamListener(laddr, cfg) + } else { + runUnixPacketListener(laddr, cfg) + } +} + +func runUnixStreamListener(laddr *net.UnixAddr, cfg *configs) { + ln, err := net.ListenUnix(laddr.Net, laddr) if err != nil { - logger.Fatalf("cannot parse configs for -syslog.listenAddr.unix=%q: %s", addr, err) + logger.Fatalf("cannot start Unix socket syslog server at %q: %s", laddr, err) } doneCh := make(chan struct{}) go func() { - serveUnixListener(ln, cfg) + serveStreamListener(ln, cfg) close(doneCh) }() - logger.Infof("started accepting syslog messages at -syslog.listenAddr.unix=%q", addr) + logger.Infof("started accepting syslog messages at %q", laddr) <-workersStopCh if err := ln.Close(); err != nil { - logger.Fatalf("syslog: cannot close UDP listener at %s: %s", addr, err) + logger.Fatalf("syslog: cannot close UDP listener at %s: %s", laddr, err) + } + <-doneCh + logger.Infof("finished accepting syslog messages at -syslog.listenAddr.unix=%q", laddr) +} + +func runUnixPacketListener(laddr *net.UnixAddr, cfg *configs) { + ln, err := net.ListenUnixgram(laddr.Net, laddr) + if err != nil { + logger.Fatalf("cannot start Unix socket syslog server at %q: %s", laddr, err) + } + + doneCh := make(chan struct{}) + go func() { + servePacketListener(ln, cfg) + close(doneCh) + }() + + logger.Infof("started accepting syslog messages at %q", laddr) + <-workersStopCh + if err := ln.Close(); err != nil { + logger.Fatalf("syslog: cannot close UDP listener at %s: %s", laddr, err) } <-doneCh - logger.Infof("finished accepting syslog messages at -syslog.listenAddr.unix=%q", addr) + logger.Infof("finished accepting syslog messages at %q", laddr) } -func getUnixSocketNetworkAndPath(addr string) (string, string) { - // An optional network such as unix, unixpacket or unixgram can be specified in front of addr and followed by ':' +func getUnixSocketNetworkAndPath(addr string) *net.UnixAddr { + // An optional network such as unix or unixgram can be specified in front of addr and followed by ':' n := strings.IndexByte(addr, ':') if n < 0 { - return "unix", addr + return &net.UnixAddr{ + Net: "unix", + Name: addr, + } + } + return &net.UnixAddr{ + Net: addr[:n], + Name: addr[n+1:], } - return addr[:n], addr[n+1:] } func runUDPListener(addr string, argIdx int) { @@ -295,14 +330,6 @@ func runTCPListener(addr string, argIdx int) { logger.Infof("finished accepting syslog messages at -syslog.listenAddr.tcp=%q", addr) } -func serveUnixListener(ln net.Listener, cfg *configs) { - if pc, ok := ln.(net.PacketConn); ok { - servePacketListener(pc, cfg) - } else { - serveStreamListener(ln, cfg) - } -} - func servePacketListener(ln net.PacketConn, cfg *configs) { gomaxprocs := cgroup.AvailableCPUs() var wg sync.WaitGroup diff --git a/docs/victorialogs/CHANGELOG.md b/docs/victorialogs/CHANGELOG.md index f0c72c1293..44dbeff7b3 100644 --- a/docs/victorialogs/CHANGELOG.md +++ b/docs/victorialogs/CHANGELOG.md @@ -19,7 +19,7 @@ according to [these docs](https://docs.victoriametrics.com/victorialogs/quicksta ## tip * FEATURE: [LogsQL](https://docs.victoriametrics.com/victorialogs/logsql/): add [pattern match filter](https://docs.victoriametrics.com/victorialogs/logsql/#pattern-match-filter) for searching logs by the given patterns such as `: user_id=, ip=, trace_id=`. These filters are needed for [#518](https://github.com/VictoriaMetrics/VictoriaLogs/issues/518). -* FEATURE: [Syslog data ingestion](https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/): support for receiving Syslog messages from Unix sockets of `SOCK_STREAM` and `SOCK_DGRAM` types via `-syslog.listenAddr.unix=/path/to/socket` and `-syslog.listenAddr.unix=unixpacket:/path/to/socket` command-line flags. See [#570](https://github.com/VictoriaMetrics/VictoriaLogs/issues/570). +* FEATURE: [Syslog data ingestion](https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/): support for receiving Syslog messages from Unix sockets of `SOCK_STREAM` and `SOCK_DGRAM` types via `-syslog.listenAddr.unix=/path/to/socket` and `-syslog.listenAddr.unix=unixgram:/path/to/socket` command-line flags. See [#570](https://github.com/VictoriaMetrics/VictoriaLogs/issues/570). * BUGFIX: [querying](https://docs.victoriametrics.com/victorialogs/querying): `-search.maxQueryTimeRange` command-line flag now supports day (`d`), week (`w`) and year (`y`) suffixes additionally to the supported hour (`h`), minute (`m`) and second (`s`) suffixes. See [#50](https://github.com/VictoriaMetrics/VictoriaLogs/issues/50#issuecomment-3244097676) diff --git a/docs/victorialogs/README.md b/docs/victorialogs/README.md index 98cae91b70..79edca8db8 100644 --- a/docs/victorialogs/README.md +++ b/docs/victorialogs/README.md @@ -808,7 +808,7 @@ Pass `-help` to VictoriaLogs in order to see the list of supported command-line Supports an array of values separated by comma or specified via multiple flags. Value can contain comma inside single-quoted or double-quoted string, {}, [] and () braces. -syslog.listenAddr.unix array - Comma-separated list of Unix socket filepaths to listen to for Syslog messages. Filepaths may be prepended with 'unixpacket:' for listening for SOCK_DGRAM sockets. By default SOCK_STREAM sockets are used. See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/ + Comma-separated list of Unix socket filepaths to listen to for Syslog messages. Filepaths may be prepended with 'unixgram:' for listening for SOCK_DGRAM sockets. By default SOCK_STREAM sockets are used. See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/ Supports an array of values separated by comma or specified via multiple flags. Value can contain comma inside single-quoted or double-quoted string, {}, [] and () braces. -syslog.mtls array diff --git a/docs/victorialogs/data-ingestion/syslog.md b/docs/victorialogs/data-ingestion/syslog.md index bed4bc19d7..8cbf997f76 100644 --- a/docs/victorialogs/data-ingestion/syslog.md +++ b/docs/victorialogs/data-ingestion/syslog.md @@ -14,7 +14,7 @@ aliases: [VictoriaLogs](https://docs.victoriametrics.com/victorialogs/) can accept logs in [Syslog formats](https://en.wikipedia.org/wiki/Syslog) at the specified TCP, UDP or Unix socket addresses via `-syslog.listenAddr.tcp`, `-syslog.listenAddr.udp` and `-syslog.listenAddr.unix` command-line flags. VictoriaLogs listens for `SOCK_STREAM` unix sockets by default. -Prepend the unix socket path passed to `-syslog.listenAddr.unix` with `unixpacket:` for `SOCK_DGRAM` sockets. +Prepend the unix socket path passed to `-syslog.listenAddr.unix` with `unixgram:` for `SOCK_DGRAM` sockets. The following syslog formats are supported: diff --git a/docs/victorialogs/vlagent.md b/docs/victorialogs/vlagent.md index 103c022416..b6c7dfb79b 100644 --- a/docs/victorialogs/vlagent.md +++ b/docs/victorialogs/vlagent.md @@ -473,7 +473,7 @@ See the docs at https://docs.victoriametrics.com/victorialogs/vlagent/ . Supports an array of values separated by comma or specified via multiple flags. Value can contain comma inside single-quoted or double-quoted string, {}, [] and () braces. -syslog.listenAddr.unix array - Comma-separated list of Unix socket filepaths to listen to for Syslog messages. Filepaths may be prepended with 'unixpacket:' for listening for SOCK_DGRAM sockets. By default SOCK_STREAM sockets are used. See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/ + Comma-separated list of Unix socket filepaths to listen to for Syslog messages. Filepaths may be prepended with 'unixgram:' for listening for SOCK_DGRAM sockets. By default SOCK_STREAM sockets are used. See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/ Supports an array of values separated by comma or specified via multiple flags. Value can contain comma inside single-quoted or double-quoted string, {}, [] and () braces. -syslog.mtls array From c15434db7a86b08c1db9e49d85020c8fc623c070 Mon Sep 17 00:00:00 2001 From: Aliaksandr Valialkin Date: Mon, 8 Sep 2025 20:12:31 +0200 Subject: [PATCH 06/82] lib/logstorage: add `` placeholder for `pattern_match(...)` filter This placeholder matches any word (see https://docs.victoriametrics.com/victorialogs/logsql/#word for word definition) or any quoted string put into single quoted, double quotes or backticks. --- docs/victorialogs/CHANGELOG.md | 1 + docs/victorialogs/LogsQL.md | 3 +- lib/logstorage/parser.go | 2 +- lib/logstorage/pattern.go | 37 ++++++++++---- lib/logstorage/pattern_matcher.go | 69 +++++++++++++++++++++++++- lib/logstorage/pattern_matcher_test.go | 27 +++++++--- lib/logstorage/pattern_test.go | 1 + lib/logstorage/pipe_collapse_nums.go | 4 +- 8 files changed, 122 insertions(+), 22 deletions(-) diff --git a/docs/victorialogs/CHANGELOG.md b/docs/victorialogs/CHANGELOG.md index 44dbeff7b3..7477d510ae 100644 --- a/docs/victorialogs/CHANGELOG.md +++ b/docs/victorialogs/CHANGELOG.md @@ -18,6 +18,7 @@ according to [these docs](https://docs.victoriametrics.com/victorialogs/quicksta ## tip +* FEATURE: [`extract` pipe](https://docs.victoriametrics.com/victorialogs/logsql/#extract-pipe): the `<...>` placeholder now matchs quoted strings in single quotes additionally to strings in double quotes and backticks. For example, the `` placeholder at the `... | extact "login=,"` now matches `foo,bar` for the log message with the text `login='foo,bar'`. * FEATURE: [LogsQL](https://docs.victoriametrics.com/victorialogs/logsql/): add [pattern match filter](https://docs.victoriametrics.com/victorialogs/logsql/#pattern-match-filter) for searching logs by the given patterns such as `: user_id=, ip=, trace_id=`. These filters are needed for [#518](https://github.com/VictoriaMetrics/VictoriaLogs/issues/518). * FEATURE: [Syslog data ingestion](https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/): support for receiving Syslog messages from Unix sockets of `SOCK_STREAM` and `SOCK_DGRAM` types via `-syslog.listenAddr.unix=/path/to/socket` and `-syslog.listenAddr.unix=unixgram:/path/to/socket` command-line flags. See [#570](https://github.com/VictoriaMetrics/VictoriaLogs/issues/570). diff --git a/docs/victorialogs/LogsQL.md b/docs/victorialogs/LogsQL.md index b76f2cf512..68f997b498 100644 --- a/docs/victorialogs/LogsQL.md +++ b/docs/victorialogs/LogsQL.md @@ -739,6 +739,7 @@ The `"pattern"` must contain the text to match, plus arbitrary number of the fol - `