fix: preserve UTF-8 semantics through read-only scalars

dev/design/issue-queue.md

@@ -1,6 +1,6 @@
 # Reviewed issue queues
 
-Snapshot: 2026-11-02, from live open issues, PRs and the #1649 phase contract.
+Snapshot: 2026-11-04, from live open issues, PRs and the #1649 phase contract.
 This is a bounded selection aid, not the complete backlog or fresh runtime
 validation. Follow the [triage and claim protocol](../../docs/guides/issue-triage.md)
 and recheck live issue comments/PRs before starting. Unlisted work is unreviewed
@@ -26,6 +26,7 @@ resolved before implementation. No new pass or failure is asserted here.
 
 | Code/test boundary | Issue | Priority / state | Evidence, next action and blocker |
 |---|---|---|---|
+| Scalar string metadata | [#1970](https://github.com/fglock/PerlOnJava/issues/1970) `Internals::SvREADONLY` UTF-8 flag | Normal / Active | Runtime read-only wrapping hides the underlying string type from UTF-8 queries and interpolation. The Encode-Wide 0.07 system-Perl oracle passes; PerlOnJava fails its sequential Unicode conversion integration case. [Current claim](https://github.com/fglock/PerlOnJava/issues/1970#issuecomment-6633863980). |
 | Hash dereference assignment | [#1958](https://github.com/fglock/PerlOnJava/issues/1958) Undefined-reference autovivification | Normal / Active | Shared-runtime fix and focused regression pass on both backends; full `make` passes in [PR #1960](https://github.com/fglock/PerlOnJava/pull/1960). [Current claim](https://github.com/fglock/PerlOnJava/issues/1958#issuecomment-6626463414). Object-Hybrid 0.07's follow-on generated `can` dispatch failure is also addressed in that PR under #1959. |
 | Dynamic class method lookup | [#1959](https://github.com/fglock/PerlOnJava/issues/1959) Object::Hybrid `tieable` promotion | Normal / Active | Generated `HASH_STATIC` declares a bodyless `can`; route it through that class's `AUTOLOAD` when present. Focused reproducer and full `make` pass on both backends. The full upstream suite advances beyond this failure but later exits at `IO::File::FILENO` (separate behavior). Candidate fix is in [PR #1960](https://github.com/fglock/PerlOnJava/pull/1960). |
 | Scalar introspection | [#1954](https://github.com/fglock/PerlOnJava/issues/1954) `Scalar::Util::isdual` booleans | Normal / Active | Mars 0.05 passes on system Perl and failed assertions 10–11 on both PerlOnJava backends (run `20261103-123832-91742`). [Draft PR #1956](https://github.com/fglock/PerlOnJava/pull/1956) recognizes BOOLEAN values; focused boolean, explicit-dualvar and plain-string coverage plus all 13 Mars assertions pass on both backends, and full `make` passes. |

docs/about/changelog.md

@@ -6,6 +6,8 @@ priorities and future plans.
 
 ## Work in progress
 
+- Preserve wide-string UTF-8 metadata through runtime read-only marking and string interpolation (#1970).
+
 - Make an explicit `-n` run sample new and previously tested CPAN targets while maintaining a rolling window of target and dependency results.
 
 - Preserve callable CODE slots and captured values when copying stash globs returned by `each`.

src/main/java/org/perlonjava/runtime/operators/StringOperators.java

@@ -624,15 +624,15 @@ public static RuntimeScalar stringConcatAssign(RuntimeScalar runtimeScalar, Runt
                 || b.getClass() != RuntimeScalar.class
                 || (runtimeScalar.type != RuntimeScalarType.STRING
                     && runtimeScalar.type != RuntimeScalarType.BYTE_STRING)
-                || (b.type != RuntimeScalarType.STRING
-                    && b.type != RuntimeScalarType.BYTE_STRING)) {
+                || (RuntimeScalarType.effectiveType(b) != RuntimeScalarType.STRING
+                    && RuntimeScalarType.effectiveType(b) != RuntimeScalarType.BYTE_STRING)) {
             return stringConcat(runtimeScalar, b, false, true);
         }
 
         String bStr = b.toString();
         int resultType = runtimeScalar.type;
         if (resultType == RuntimeScalarType.BYTE_STRING
-                && (b.type == RuntimeScalarType.STRING || !isLatin1(bStr))) {
+                && (RuntimeScalarType.effectiveType(b) == RuntimeScalarType.STRING || !isLatin1(bStr))) {
             resultType = RuntimeScalarType.STRING;
         }
         return runtimeScalar.appendedStringAssignmentResult(bStr, resultType, b);
@@ -679,8 +679,8 @@ private static RuntimeScalar stringConcat(RuntimeScalar runtimeScalar, RuntimeSc
         // In Perl, concatenation produces a UTF-8 string only if at least one
         // operand has the UTF-8 flag on (STRING type). Non-STRING types
         // (BYTE_STRING, INTEGER, DOUBLE, UNDEF) are all byte-compatible.
-        boolean aIsUtf8 = aResolved.type == RuntimeScalarType.STRING;
-        boolean bIsUtf8 = bResolved.type == RuntimeScalarType.STRING;
+        boolean aIsUtf8 = RuntimeScalarType.effectiveType(aResolved) == RuntimeScalarType.STRING;
+        boolean bIsUtf8 = RuntimeScalarType.effectiveType(bResolved) == RuntimeScalarType.STRING;
 
         if (aIsUtf8 || bIsUtf8) {
             return propagateTaint(new RuntimeScalar(aStr + bStr), aResolved, bResolved);
@@ -779,7 +779,8 @@ public static RuntimeScalar stringConcatWarnUninitialized(RuntimeScalar runtimeS
         // scalars whose toString() delegates to the current capture; treating
         // every non-BYTE_STRING proxy as UTF-8 upgraded byte captures during
         // s///e replacement evaluation.
-        if (aResolved.type == RuntimeScalarType.STRING || bResolved.type == RuntimeScalarType.STRING) {
+        if (RuntimeScalarType.effectiveType(aResolved) == RuntimeScalarType.STRING
+                || RuntimeScalarType.effectiveType(bResolved) == RuntimeScalarType.STRING) {
             return propagateTaint(new RuntimeScalar(aStr + bStr), aResolved, bResolved);
         }
 
@@ -1100,7 +1101,7 @@ private static RuntimeScalar joinInternal(RuntimeScalar runtimeScalar, RuntimeBa
             }
             resolved = stringifyForStringContext(resolved);
             RuntimeScalar res = new RuntimeScalar(resolved.toString());
-            if (resolved.type != RuntimeScalarType.STRING) {
+            if (RuntimeScalarType.effectiveType(resolved) != RuntimeScalarType.STRING) {
                 res.type = BYTE_STRING;
             }
             res.tainted = resolved.isTainted();
@@ -1120,7 +1121,7 @@ private static RuntimeScalar joinInternal(RuntimeScalar runtimeScalar, RuntimeBa
         // In Perl, join produces a byte-string unless one of the inputs has
         // the UTF-8 flag on. Only STRING type has the flag; INTEGER, DOUBLE,
         // UNDEF, and BYTE_STRING are all byte-compatible.
-        boolean hasUtf8 = separatorResolved.type == RuntimeScalarType.STRING;
+        boolean hasUtf8 = RuntimeScalarType.effectiveType(separatorResolved) == RuntimeScalarType.STRING;
         boolean tainted = separatorResolved.isTainted();
 
         // Join the elements
@@ -1153,7 +1154,7 @@ private static RuntimeScalar joinInternal(RuntimeScalar runtimeScalar, RuntimeBa
                 resolvedElements.add(materialized);
                 hasExecutableValue |= RuntimeRegexTemplate.hasExecutableValue(materialized);
             }
-            if (resolved.type == RuntimeScalarType.STRING) {
+            if (RuntimeScalarType.effectiveType(resolved) == RuntimeScalarType.STRING) {
                 hasUtf8 = true;
             }
             tainted |= resolved.isTainted();
@@ -1248,17 +1249,20 @@ public static RuntimeScalar stringConcatNoOverload(RuntimeScalar runtimeScalar,
             return propagateTaint(stringConcatBytes(aStr, aResolved, bStr, bResolved), aResolved, bResolved);
         }
 
-        if (aResolved.type == RuntimeScalarType.STRING || bResolved.type == RuntimeScalarType.STRING) {
+        if (RuntimeScalarType.effectiveType(aResolved) == RuntimeScalarType.STRING
+                || RuntimeScalarType.effectiveType(bResolved) == RuntimeScalarType.STRING) {
             return propagateTaint(new RuntimeScalar(aStr + bStr), aResolved, bResolved);
         }
 
-        if (aResolved.type == BYTE_STRING || bResolved.type == BYTE_STRING) {
-            boolean aIsByte = aResolved.type == BYTE_STRING
-                    || aResolved.type == RuntimeScalarType.UNDEF
-                    || (aStr.isEmpty() && aResolved.type != RuntimeScalarType.STRING);
-            boolean bIsByte = bResolved.type == BYTE_STRING
-                    || bResolved.type == RuntimeScalarType.UNDEF
-                    || (bStr.isEmpty() && bResolved.type != RuntimeScalarType.STRING);
+        int aType = RuntimeScalarType.effectiveType(aResolved);
+        int bType = RuntimeScalarType.effectiveType(bResolved);
+        if (aType == BYTE_STRING || bType == BYTE_STRING) {
+            boolean aIsByte = aType == BYTE_STRING
+                    || aType == RuntimeScalarType.UNDEF
+                    || (aStr.isEmpty() && aType != RuntimeScalarType.STRING);
+            boolean bIsByte = bType == BYTE_STRING
+                    || bType == RuntimeScalarType.UNDEF
+                    || (bStr.isEmpty() && bType != RuntimeScalarType.STRING);
             if (aIsByte && bIsByte) {
                 boolean safe = true;
                 for (int i = 0; safe && i < aStr.length(); i++) {
@@ -1289,9 +1293,9 @@ public static RuntimeScalar stringConcatNoOverload(RuntimeScalar runtimeScalar,
 
     private static RuntimeScalar stringConcatBytes(String aStr, RuntimeScalar a,
                                                   String bStr, RuntimeScalar b) {
-        byte[] aBytes = a.type == RuntimeScalarType.STRING
+        byte[] aBytes = RuntimeScalarType.effectiveType(a) == RuntimeScalarType.STRING
                 ? aStr.getBytes(StandardCharsets.UTF_8) : stringBytesForBytesPragma(aStr);
-        byte[] bBytes = b.type == RuntimeScalarType.STRING
+        byte[] bBytes = RuntimeScalarType.effectiveType(b) == RuntimeScalarType.STRING
                 ? bStr.getBytes(StandardCharsets.UTF_8) : stringBytesForBytesPragma(bStr);
         byte[] out = new byte[aBytes.length + bBytes.length];
         System.arraycopy(aBytes, 0, out, 0, aBytes.length);

src/main/java/org/perlonjava/runtime/perlmodule/Utf8.java

@@ -391,9 +391,10 @@ public static boolean isUtf8(RuntimeScalar scalar) {
         if (scalar instanceof ScalarSpecialVariable sv) {
             scalar = sv.getValueAsScalar();
         }
-        // Only STRING type has the UTF-8 flag set.
-        // INTEGER, DOUBLE, UNDEF, REFERENCE etc. don't have the UTF-8 flag in Perl.
-        return scalar.type == STRING;
+        // Only STRING type has the UTF-8 flag set. Runtime SvREADONLY wraps
+        // the original scalar in READONLY_SCALAR, so inspect the preserved
+        // value type instead of treating the wrapper as a byte string.
+        return RuntimeScalarType.effectiveType(scalar) == STRING;
     }
 
     /**

src/main/java/org/perlonjava/runtime/runtimetypes/RuntimeScalarType.java

@@ -36,6 +36,15 @@ public class RuntimeScalarType {
     private RuntimeScalarType() {
     } // Prevent instantiation
 
+    /** Return the value type stored under one or more runtime read-only wrappers. */
+    public static int effectiveType(RuntimeScalar runtimeScalar) {
+        while (runtimeScalar != null && runtimeScalar.type == READONLY_SCALAR
+                && runtimeScalar.value instanceof RuntimeScalar inner) {
+            runtimeScalar = inner;
+        }
+        return runtimeScalar == null ? UNDEF : runtimeScalar.type;
+    }
+
     // Get blessing ID as an integer
     public static int blessedId(RuntimeScalar runtimeScalar) {
         int type = runtimeScalar.type;

src/test/resources/unit/issue_1970_readonly_utf8_flag.t

@@ -0,0 +1,18 @@
+use strict;
+use warnings;
+use Test::More;
+
+my $wide = "\x{2122}";
+Internals::SvREADONLY($wide, 1);
+
+ok(utf8::is_utf8($wide), 'read-only wide scalar retains its UTF-8 flag');
+
+my $interpolate = sub { return "$wide" };
+my $copy = $interpolate->();
+my $concatenated = $wide . '';
+
+ok(utf8::is_utf8($copy), 'closure interpolation preserves the UTF-8 flag');
+ok(utf8::is_utf8($concatenated), 'concatenation preserves the UTF-8 flag');
+is(ord($copy), 0x2122, 'closure interpolation preserves the wide character');
+
+done_testing();