[pull] master from pocketbase:master

CHANGELOG.md

@@ -1,3 +1,25 @@
+## v0.40.3
+
+- Write the status header for JSON responses only if the fields picker succeed or has acceptable fallback.
+    _This is to allow custom response status code for failed json writes._
+
+- Fixed collection index validator to allow expressions with parenthesis in the optional `WHERE` clause.
+
+- Clamped arccosine to [-1,1] in the Harvesine formula for the `geoDistance()` filter function to workaround edge case related to float rounding errors for some coordinates.
+
+- Prevent unnecessary body chunk read if we already known that we are beyond the allowed limit.
+
+- Updated the `json` field validator to check the `encoding/json/v2` semantics and allow duplicated keys on record marshalize for compliance with old jsonv1 data.
+
+- Fixed nested cascade delete of self-referenced relation records.
+
+- Minor UI fixes (updated dark primary btn color contrast, force reload the records list if the deleted record has self-referenced cascade relation field, etc.).
+
+- Changed JSVM `$app` variable definition from TS type to interface ([#7834](https://github.com/pocketbase/pocketbase/issues/7834)).
+
+- Bumped `golang.org/x/*` dependencies to silence security scanners ([#7829](https://github.com/pocketbase/pocketbase/discussions/7829)).
+
+
 ## v0.40.2
 
 - Return an error when filter params fallback fails to json serialize and optimized params replacement to execute in a single pass.

CHANGELOG_16_22.md

@@ -2,6 +2,15 @@
 > For the most recent versions, please refer to [CHANGELOG.md](./CHANGELOG.md)
 ---
 
+## v0.22.55
+
+- (_Backported from v0.40.3_) Fixed collection index validator to allow expressions with parenthesis in the optional `WHERE` clause.
+
+- (_Backported from v0.40.3_) Fixed nested cascade delete of self-referenced relation records.
+
+- (_Backported from v0.40.3_) Bumped `golang.org/x/*` dependencies to silence security scanners ([#7829](https://github.com/pocketbase/pocketbase/discussions/7829)).
+
+
 ## v0.22.54
 
 - (_Backported from v0.40.2_) Bumped goja and its related dependencies _(regex unescaped dash error fix and base64 optimizations)_.

apis/health.go

@@ -37,7 +37,7 @@ func healthCheck(e *core.RequestEvent) error {
 		headersToCheck := append(
 			slices.Clone(e.App.Settings().TrustedProxy.Headers),
 			// common proxy headers
-			"CF-Connecting-IP", "Fly-Client-IP", "X‑Forwarded-For",
+			"CF-Connecting-IP", "Fly-Client-IP", "X-Forwarded-For",
 		)
 		for _, header := range headersToCheck {
 			if e.Request.Header.Get(header) != "" {

apis/middlewares_body_limit.go

@@ -84,49 +84,73 @@ func applyBodyLimit(e *core.RequestEvent, limitBytes int64) error {
 	}
 
 	// replace the request body
-	//
-	// note: we don't use sync.Pool since the size of the elements could vary too much
-	// and it might not be efficient (see https://github.com/golang/go/issues/23199)
-	e.Request.Body = &limitedReader{ReadCloser: e.Request.Body, limit: limitBytes}
+	e.Request.Body = newMaxBytesReader(e.Request.Body, limitBytes)
 
 	return nil
 }
 
-type limitedReader struct {
+func newMaxBytesReader(body io.ReadCloser, limitBytes int64) *maxBytesReader {
+	return &maxBytesReader{
+		ReadCloser: body,
+		limit:      limitBytes,
+		remaining:  limitBytes,
+	}
+}
+
+// maxBytesReader is very similar to the http.MaxBytesReader but support
+// rereads and doesn't try to prematurely close the related response
+// to allow consequent middlewares to operate correctly.
+type maxBytesReader struct {
 	io.ReadCloser
 	limit     int64
-	totalRead int64
+	remaining int64
+	stickyErr error
 }
 
-func (r *limitedReader) Read(b []byte) (int, error) {
+func (r *maxBytesReader) Read(b []byte) (int, error) {
+	if r.stickyErr != nil {
+		return 0, r.stickyErr
+	}
+
+	if len(b) == 0 {
+		return 0, nil
+	}
+
+	// if possible no need to read the entire chunk since
+	// remaining+1 is enough to determine whether it exceed the limit
+	if int64(len(b))-1 > r.remaining {
+		b = b[:r.remaining+1]
+	}
+
 	n, err := r.ReadCloser.Read(b)
-	if err != nil {
+
+	if int64(n) <= r.remaining {
+		r.remaining -= int64(n)
+		r.stickyErr = err
 		return n, err
 	}
 
-	r.totalRead += int64(n)
-	if r.totalRead > r.limit {
-		return n, ErrRequestEntityTooLarge
-	}
+	n = int(r.remaining)
+
+	r.remaining = 0
+	r.stickyErr = ErrRequestEntityTooLarge
 
-	return n, nil
+	return n, r.stickyErr
 }
 
 // explicit casts to ensure that the main struct methods will be invoked
 // (extra precautions in case of nested interface wrapping erasure)
 // ---
 
-func (r *limitedReader) Reread() {
+func (r *maxBytesReader) Reread() {
 	rereader, ok := r.ReadCloser.(router.Rereader)
 	if ok {
 		rereader.Reread()
+		r.remaining = r.limit
+		r.stickyErr = nil
 	}
 }
 
-func (r *limitedReader) Close() error {
-	closer, ok := r.ReadCloser.(io.Closer)
-	if ok {
-		return closer.Close()
-	}
-	return nil
+func (r *maxBytesReader) Close() error {
+	return r.ReadCloser.Close()
 }

apis/middlewares_body_limit_test.go

@@ -1,9 +1,10 @@
 package apis_test
 
 import (
-	"bytes"
-	"fmt"
+	"io"
+	"net/http"
 	"net/http/httptest"
+	"strings"
 	"testing"
 
 	"github.com/pocketbase/pocketbase/apis"
@@ -19,34 +20,126 @@ func TestBodyLimitMiddleware(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	pbRouter.POST("/a", func(e *core.RequestEvent) error {
-		return e.String(200, "a")
-	}) // default global BodyLimit check
 
-	pbRouter.POST("/b", func(e *core.RequestEvent) error {
-		return e.String(200, "b")
-	}).Bind(apis.BodyLimit(20))
+	testHandler := func(e *core.RequestEvent) error {
+		// read the body multiple times to ensure that the limited
+		// reader guards and rereads are invoked
+		var result any
+		if err := e.BindBody(&result); err != nil {
+			return err
+		}
+
+		if err := e.BindBody(&result); err != nil {
+			return err
+		}
+
+		return e.JSON(200, result)
+	}
+
+	const customLimit = 20
+
+	pbRouter.POST("/a", testHandler) // default global BodyLimit check
+	pbRouter.POST("/b", testHandler).Bind(apis.BodyLimit(customLimit))
+	pbRouter.POST("/iof", func(e *core.RequestEvent) error {
+		// ensure that normal io methods still operate correctly
+		b, err := io.ReadAll(e.Request.Body)
+		if err != nil {
+			return err
+		}
+
+		return e.String(http.StatusOK, string(b))
+	}).Bind(apis.BodyLimit(customLimit))
 
 	mux, err := pbRouter.BuildMux()
 	if err != nil {
 		t.Fatal(err)
 	}
 
 	scenarios := []struct {
-		url            string
-		size           int64
-		expectedStatus int
+		name              string
+		url               string
+		body              string
+		lazyContentLength bool
+		expectedStatus    int
 	}{
-		{"/a", 21, 200},
-		{"/a", apis.DefaultMaxBodySize + 1, 413},
-		{"/b", 20, 200},
-		{"/b", 21, 413},
+		{
+			"(eager content-length check) with body = default limit",
+			"/a",
+			`"` + strings.Repeat("a", int(apis.DefaultMaxBodySize-2)) + `"`,
+			false,
+			http.StatusOK,
+		},
+		{
+			"(eager content-length check) with body > default limit",
+			"/a",
+			`"` + strings.Repeat("a", int(apis.DefaultMaxBodySize)) + `"`,
+			false,
+			http.StatusRequestEntityTooLarge,
+		},
+		{
+			"(lazy content-length check) with body = default limit",
+			"/a",
+			`"` + strings.Repeat("a", int(apis.DefaultMaxBodySize-2)) + `"`,
+			true,
+			http.StatusOK,
+		},
+		{
+			"(lazy content-length check) with body > default limit",
+			"/a",
+			`"` + strings.Repeat("a", int(apis.DefaultMaxBodySize)) + `"`,
+			true,
+			http.StatusRequestEntityTooLarge,
+		},
+		// ---
+		{
+			"(eager content-length check) with body = custom limit",
+			"/b",
+			`"` + strings.Repeat("a", customLimit-2) + `"`,
+			false,
+			http.StatusOK,
+		},
+		{
+			"(eager content-length check) with body > custom limit",
+			"/b",
+			`"` + strings.Repeat("a", customLimit) + `"`,
+			false,
+			http.StatusRequestEntityTooLarge,
+		},
+		{
+			"(lazy content-length check) with body = custom limit",
+			"/b",
+			`"` + strings.Repeat("a", customLimit-2) + `"`,
+			true,
+			http.StatusOK,
+		},
+		{
+			"(lazy content-length check) with body > custom limit",
+			"/b",
+			`"` + strings.Repeat("a", customLimit) + `"`,
+			true,
+			http.StatusRequestEntityTooLarge,
+		},
+		// ---
+		{
+			"io.ReadAll io.EOF exact limit check",
+			"/iof",
+			`"` + strings.Repeat("a", customLimit-2) + `"`,
+			true,
+			http.StatusOK,
+		},
 	}
 
 	for _, s := range scenarios {
-		t.Run(fmt.Sprintf("%s_%d", s.url, s.size), func(t *testing.T) {
+		t.Run(s.name, func(t *testing.T) {
 			rec := httptest.NewRecorder()
-			req := httptest.NewRequest("POST", s.url, bytes.NewReader(make([]byte, s.size)))
+
+			req := httptest.NewRequest("POST", s.url, strings.NewReader(s.body))
+			req.Header.Set("Content-Type", "application/json")
+
+			if s.lazyContentLength {
+				req.ContentLength = -1
+			}
+
 			mux.ServeHTTP(rec, req)
 
 			result := rec.Result()

core/field_json.go

@@ -2,6 +2,8 @@ package core
 
 import (
 	"context"
+	"encoding/json/jsontext"
+	"encoding/json/v2"
 	"slices"
 	"strconv"
 	"strings"
@@ -171,6 +173,18 @@ func (f *JSONField) ValidateValue(ctx context.Context, app App, record *Record)
 		return validation.NewError("validation_invalid_json", "Must be a valid json value")
 	}
 
+	// temp extra encoding/json/v2 check since the above validator is
+	// still using the v1 semantics
+	//
+	// @todo remove after updating the string validator
+	if len(raw) > 0 {
+		var dummy any
+		err := json.Unmarshal(raw, &dummy, jsontext.AllowInvalidUTF8(true))
+		if err != nil {
+			return validation.NewError("validation_invalid_json", "Must be a valid json value")
+		}
+	}
+
 	rawStr := strings.TrimSpace(raw.String())
 
 	if f.Required && slices.Contains(emptyJSONValues, rawStr) {

core/field_json_test.go

@@ -101,6 +101,16 @@ func TestJSONFieldValidateValue(t *testing.T) {
 			},
 			true,
 		},
+		{
+			"invalid raw value (v2 semantics)",
+			&core.JSONField{Name: "test"},
+			func() *core.Record {
+				record := core.NewRecord(collection)
+				record.SetRaw("test", types.JSONRaw(`{"a": 1, "a": 2}`))
+				return record
+			},
+			true,
+		},
 		{
 			"zero field value (not required)",
 			&core.JSONField{Name: "test"},