From ef0d199f9a9059dfc2bcf55537c3bc30a69879e5 Mon Sep 17 00:00:00 2001 From: Arash Deshmeh Date: Thu, 17 Jul 2025 08:52:45 -0400 Subject: [PATCH 1/3] Fix: skip JSONRPC response validation in Cosmos QoS --- qos/cosmos/request_validator.go | 10 ++++++---- qos/cosmos/response.go | 1 - 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/qos/cosmos/request_validator.go b/qos/cosmos/request_validator.go index beffb4d08..4ed3b9880 100644 --- a/qos/cosmos/request_validator.go +++ b/qos/cosmos/request_validator.go @@ -47,16 +47,16 @@ type cosmosSDKRequestValidator struct { // If validation fails, an errorContext is returned along with false. // If validation succeeds, a fully initialized requestContext is returned along with true. func (crv *cosmosSDKRequestValidator) validateHTTPRequest(req *http.Request) (gateway.RequestQoSContext, bool) { - logger := crv.logger.With( + crv.logger = crv.logger.With( "qos", "CosmosSDK", - "method", "validateHTTPRequest", + "http_method", req.Method, ) // For POST requests, we need to distinguish between: // 1. REST API calls: POST /cosmos/tx/v1beta1/txs with transaction data // 2. CometBFT RPC calls: POST with JSON-RPC payload like {"jsonrpc":"2.0","method":"abci_query",...} if req.Method == http.MethodPost { - return crv.validatePOSTRequest(req, logger) + return crv.validatePOSTRequest(req) } // All other HTTP methods (GET, PUT, DELETE, etc.) are REST API calls @@ -75,7 +75,9 @@ func (crv *cosmosSDKRequestValidator) validateHTTPRequest(req *http.Request) (ga // - Attempt JSON-RPC parsing // - If JSON-RPC parsing succeeds, treat as JSON-RPC // - If JSON-RPC parsing fails for any reason, treat as REST (CosmosSDK supports POST for REST) -func (crv *cosmosSDKRequestValidator) validatePOSTRequest(req *http.Request, logger polylog.Logger) (gateway.RequestQoSContext, bool) { +func (crv *cosmosSDKRequestValidator) validatePOSTRequest(req *http.Request) (gateway.RequestQoSContext, bool) { + logger := crv.logger.With("method", "validatePOSTRequest") + // Read the HTTP request body body, err := io.ReadAll(req.Body) if err != nil { diff --git a/qos/cosmos/response.go b/qos/cosmos/response.go index ad32abddf..7e3a7c828 100644 --- a/qos/cosmos/response.go +++ b/qos/cosmos/response.go @@ -72,7 +72,6 @@ func unmarshalResponse( return responseUnmarshallerGeneric(logger, jsonrpcResponse, data) } - // Validate the JSON-RPC response. // TODO_NEXT(@adshmh): Use proper JSON-RPC ID response validation that works for all CosmosSDK chains. // NOTE: We intentionally skip checking whether the JSON-RPC response indicates an error. From 979d869388e28e7c729893d2119ab71215c6a07c Mon Sep 17 00:00:00 2001 From: Arash Deshmeh Date: Thu, 17 Jul 2025 12:02:21 -0400 Subject: [PATCH 2/3] Enhance E2E tests to output a preview of any malformed payloads --- e2e/vegeta_test.go | 96 +++++++++++++++++++++++++++++++++++++++------- 1 file changed, 83 insertions(+), 13 deletions(-) diff --git a/e2e/vegeta_test.go b/e2e/vegeta_test.go index 1fec44c6b..d609cc9e4 100644 --- a/e2e/vegeta_test.go +++ b/e2e/vegeta_test.go @@ -168,6 +168,9 @@ func initMethodMetrics(method string, totalRequests int) *methodMetrics { statusCodes: make(map[int]int), errors: make(map[string]int), results: make([]*vegeta.Result, 0, totalRequests), + // Initialize the new error tracking fields + jsonRPCParseErrors: make(map[string]int), + jsonRPCValidationErrors: make(map[string]int), } } @@ -261,6 +264,33 @@ attackLoop: } } +// createResponsePreview creates a sanitized preview of the response body for error logging +func createResponsePreview(body []byte, maxLen int) string { + if len(body) == 0 { + return "(empty)" + } + + // Convert to string and normalize whitespace using strings lib + bodyStr := string(body) + + // Replace all whitespace characters with single spaces + bodyStr = strings.ReplaceAll(bodyStr, "\n", " ") + bodyStr = strings.ReplaceAll(bodyStr, "\r", " ") + bodyStr = strings.ReplaceAll(bodyStr, "\t", " ") + + // Collapse multiple spaces into single spaces + bodyStr = strings.Join(strings.Fields(bodyStr), " ") + + // Truncate if needed + if len(bodyStr) <= maxLen { + return bodyStr + } + if maxLen <= 3 { + return bodyStr[:maxLen] + } + return bodyStr[:maxLen-3] + "..." +} + // processResult // • Updates metrics based on a single result func processResult(m *methodMetrics, result *vegeta.Result, serviceType serviceType) { @@ -278,24 +308,47 @@ func processResult(m *methodMetrics, result *vegeta.Result, serviceType serviceT } // Update status code counts m.statusCodes[int(result.Code)]++ + // Process JSON-RPC validation if we have a successful HTTP response var rpcResponse jsonrpc.Response if err := json.Unmarshal(result.Body, &rpcResponse); err != nil { m.jsonRPCUnmarshalErrors++ + + // Create response preview for parse errors + preview := createResponsePreview(result.Body, 100) + errorMsg := fmt.Sprintf("JSON parse error: %v (response preview: %s)", err, preview) + m.jsonRPCParseErrors[errorMsg]++ + m.errors[errorMsg]++ } else { m.jsonRPCResponses++ + + // Validate the response first + validationErr := rpcResponse.Validate(getExpectedID(serviceType)) + // Check if Error field is nil (good) if rpcResponse.Error != nil { m.jsonRPCErrorField++ - m.errors[rpcResponse.Error.Message]++ + // Only track the error field message if there's no validation error + // (to avoid duplicate tracking when validation fails due to error field) + if validationErr == nil { + m.errors[rpcResponse.Error.Message]++ + } } + // Check if Result field is not nil (good) if rpcResponse.Result == nil { m.jsonRPCNilResult++ } - // Validate the response - if err := rpcResponse.Validate(getExpectedID(serviceType)); err != nil { + + // Process validation error + if validationErr != nil { m.jsonRPCValidateErrors++ + + // Create response preview for validation errors + preview := createResponsePreview(result.Body, 100) + errorMsg := fmt.Sprintf("JSON-RPC validation error: %v (response preview: %s)", validationErr, preview) + m.jsonRPCValidationErrors[errorMsg]++ + m.errors[errorMsg]++ } } } @@ -328,6 +381,10 @@ type methodMetrics struct { jsonRPCNilResult int // Count of responses with nil Result field jsonRPCValidateErrors int // Count of responses that fail validation + // New fields for detailed error tracking with response previews + jsonRPCParseErrors map[string]int // Parse errors with response previews + jsonRPCValidationErrors map[string]int // Validation errors with response previews + // Success rates for specific checks jsonRPCSuccessRate float64 // Success rate for JSON-RPC unmarshaling jsonRPCErrorFieldRate float64 // Error field absent rate (success = no error) @@ -618,21 +675,34 @@ func validateResults(t *testing.T, serviceId protocol.ServiceID, m *methodMetric errorColor = RED // Red for critical errors (test failed) } - // Log top errors with appropriate color + // Log top errors with appropriate color and include detailed JSON-RPC errors if len(m.errors) > 0 { fmt.Println("") // Add a new line before logging errors fmt.Printf("%sTop errors:%s\n", errorColor, RESET) - count := 0 - num := 1 - for err, errCount := range m.errors { - if count < 5 { - fmt.Printf(" %d. %s%s%s: %d\n", num, errorColor, err, RESET, errCount) - count++ - num++ + + // Sort errors by count (descending) to show most frequent first + type errorEntry struct { + message string + count int + } + var sortedErrors []errorEntry + for errMsg, count := range m.errors { + sortedErrors = append(sortedErrors, errorEntry{message: errMsg, count: count}) + } + sort.Slice(sortedErrors, func(i, j int) bool { + return sortedErrors[i].count > sortedErrors[j].count + }) + + // Display top 5 errors + for i, err := range sortedErrors { + if i >= 5 { + break } + fmt.Printf(" %d. %s%s%s: %d\n", i+1, errorColor, err.message, RESET, err.count) } - if len(m.errors) > 5 { - fmt.Printf(" ... and %s%d%s more error types\n", errorColor, len(m.errors)-5, RESET) + + if len(sortedErrors) > 5 { + fmt.Printf(" ... and %s%d%s more error types\n", errorColor, len(sortedErrors)-5, RESET) } } From 3622169fabb4379510fb0fb99c91f32dbf371d6b Mon Sep 17 00:00:00 2001 From: Arash Deshmeh Date: Thu, 17 Jul 2025 14:30:58 -0400 Subject: [PATCH 3/3] Make response preview appear for all validation errors --- e2e/vegeta_test.go | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/e2e/vegeta_test.go b/e2e/vegeta_test.go index d609cc9e4..558f508a2 100644 --- a/e2e/vegeta_test.go +++ b/e2e/vegeta_test.go @@ -328,11 +328,6 @@ func processResult(m *methodMetrics, result *vegeta.Result, serviceType serviceT // Check if Error field is nil (good) if rpcResponse.Error != nil { m.jsonRPCErrorField++ - // Only track the error field message if there's no validation error - // (to avoid duplicate tracking when validation fails due to error field) - if validationErr == nil { - m.errors[rpcResponse.Error.Message]++ - } } // Check if Result field is not nil (good) @@ -340,7 +335,7 @@ func processResult(m *methodMetrics, result *vegeta.Result, serviceType serviceT m.jsonRPCNilResult++ } - // Process validation error + // Process validation error - this takes priority over error field messages if validationErr != nil { m.jsonRPCValidateErrors++ @@ -349,6 +344,13 @@ func processResult(m *methodMetrics, result *vegeta.Result, serviceType serviceT errorMsg := fmt.Sprintf("JSON-RPC validation error: %v (response preview: %s)", validationErr, preview) m.jsonRPCValidationErrors[errorMsg]++ m.errors[errorMsg]++ + } else if rpcResponse.Error != nil { + // Only track error field message if validation passed + // (meaning the response structure is valid but contains an API error) + // Add response preview to API errors too for consistency - use longer preview for API errors + preview := createResponsePreview(result.Body, 200) + errorMsg := fmt.Sprintf("API error: %s (response preview: %s)", rpcResponse.Error.Message, preview) + m.errors[errorMsg]++ } } } @@ -706,6 +708,8 @@ func validateResults(t *testing.T, serviceId protocol.ServiceID, m *methodMetric } } + // TODO_TECHDEBT(@adshmh): Output the most frequently occurring malformed payloads. + // Collect assertion failures failures = append(failures, collectHTTPSuccessRateFailures(m, serviceConfig.SuccessRate)...) failures = append(failures, collectJSONRPCRatesFailures(m, serviceConfig.SuccessRate)...)