diff --git a/dev/design/issue-queue.md b/dev/design/issue-queue.md index 376bc326b8..09463cc893 100644 --- a/dev/design/issue-queue.md +++ b/dev/design/issue-queue.md @@ -1,6 +1,6 @@ # Reviewed issue queues -Snapshot: 2026-11-02, from live open issues, PRs and the #1649 phase contract. +Snapshot: 2026-11-04, from live open issues, PRs and the #1649 phase contract. This is a bounded selection aid, not the complete backlog or fresh runtime validation. Follow the [triage and claim protocol](../../docs/guides/issue-triage.md) and recheck live issue comments/PRs before starting. Unlisted work is unreviewed @@ -26,6 +26,7 @@ resolved before implementation. No new pass or failure is asserted here. | Code/test boundary | Issue | Priority / state | Evidence, next action and blocker | |---|---|---|---| +| Scalar string metadata | [#1970](https://github.com/fglock/PerlOnJava/issues/1970) `Internals::SvREADONLY` UTF-8 flag | Normal / Active | Runtime read-only wrapping hides the underlying string type from UTF-8 queries and interpolation. The Encode-Wide 0.07 system-Perl oracle passes; PerlOnJava fails its sequential Unicode conversion integration case. [Current claim](https://github.com/fglock/PerlOnJava/issues/1970#issuecomment-6633863980). | | Hash dereference assignment | [#1958](https://github.com/fglock/PerlOnJava/issues/1958) Undefined-reference autovivification | Normal / Active | Shared-runtime fix and focused regression pass on both backends; full `make` passes in [PR #1960](https://github.com/fglock/PerlOnJava/pull/1960). [Current claim](https://github.com/fglock/PerlOnJava/issues/1958#issuecomment-6626463414). Object-Hybrid 0.07's follow-on generated `can` dispatch failure is also addressed in that PR under #1959. | | Dynamic class method lookup | [#1959](https://github.com/fglock/PerlOnJava/issues/1959) Object::Hybrid `tieable` promotion | Normal / Active | Generated `HASH_STATIC` declares a bodyless `can`; route it through that class's `AUTOLOAD` when present. Focused reproducer and full `make` pass on both backends. The full upstream suite advances beyond this failure but later exits at `IO::File::FILENO` (separate behavior). Candidate fix is in [PR #1960](https://github.com/fglock/PerlOnJava/pull/1960). | | Scalar introspection | [#1954](https://github.com/fglock/PerlOnJava/issues/1954) `Scalar::Util::isdual` booleans | Normal / Active | Mars 0.05 passes on system Perl and failed assertions 10–11 on both PerlOnJava backends (run `20261103-123832-91742`). [Draft PR #1956](https://github.com/fglock/PerlOnJava/pull/1956) recognizes BOOLEAN values; focused boolean, explicit-dualvar and plain-string coverage plus all 13 Mars assertions pass on both backends, and full `make` passes. | diff --git a/docs/about/changelog.md b/docs/about/changelog.md index aaf0298604..e2e5df89e7 100644 --- a/docs/about/changelog.md +++ b/docs/about/changelog.md @@ -6,6 +6,8 @@ priorities and future plans. ## Work in progress +- Preserve wide-string UTF-8 metadata through runtime read-only marking and string interpolation (#1970). + - Make an explicit `-n` run sample new and previously tested CPAN targets while maintaining a rolling window of target and dependency results. - Preserve callable CODE slots and captured values when copying stash globs returned by `each`. diff --git a/src/main/java/org/perlonjava/runtime/operators/StringOperators.java b/src/main/java/org/perlonjava/runtime/operators/StringOperators.java index 52b95613bb..88414faafa 100644 --- a/src/main/java/org/perlonjava/runtime/operators/StringOperators.java +++ b/src/main/java/org/perlonjava/runtime/operators/StringOperators.java @@ -624,15 +624,15 @@ public static RuntimeScalar stringConcatAssign(RuntimeScalar runtimeScalar, Runt || b.getClass() != RuntimeScalar.class || (runtimeScalar.type != RuntimeScalarType.STRING && runtimeScalar.type != RuntimeScalarType.BYTE_STRING) - || (b.type != RuntimeScalarType.STRING - && b.type != RuntimeScalarType.BYTE_STRING)) { + || (RuntimeScalarType.effectiveType(b) != RuntimeScalarType.STRING + && RuntimeScalarType.effectiveType(b) != RuntimeScalarType.BYTE_STRING)) { return stringConcat(runtimeScalar, b, false, true); } String bStr = b.toString(); int resultType = runtimeScalar.type; if (resultType == RuntimeScalarType.BYTE_STRING - && (b.type == RuntimeScalarType.STRING || !isLatin1(bStr))) { + && (RuntimeScalarType.effectiveType(b) == RuntimeScalarType.STRING || !isLatin1(bStr))) { resultType = RuntimeScalarType.STRING; } return runtimeScalar.appendedStringAssignmentResult(bStr, resultType, b); @@ -679,8 +679,8 @@ private static RuntimeScalar stringConcat(RuntimeScalar runtimeScalar, RuntimeSc // In Perl, concatenation produces a UTF-8 string only if at least one // operand has the UTF-8 flag on (STRING type). Non-STRING types // (BYTE_STRING, INTEGER, DOUBLE, UNDEF) are all byte-compatible. - boolean aIsUtf8 = aResolved.type == RuntimeScalarType.STRING; - boolean bIsUtf8 = bResolved.type == RuntimeScalarType.STRING; + boolean aIsUtf8 = RuntimeScalarType.effectiveType(aResolved) == RuntimeScalarType.STRING; + boolean bIsUtf8 = RuntimeScalarType.effectiveType(bResolved) == RuntimeScalarType.STRING; if (aIsUtf8 || bIsUtf8) { return propagateTaint(new RuntimeScalar(aStr + bStr), aResolved, bResolved); @@ -779,7 +779,8 @@ public static RuntimeScalar stringConcatWarnUninitialized(RuntimeScalar runtimeS // scalars whose toString() delegates to the current capture; treating // every non-BYTE_STRING proxy as UTF-8 upgraded byte captures during // s///e replacement evaluation. - if (aResolved.type == RuntimeScalarType.STRING || bResolved.type == RuntimeScalarType.STRING) { + if (RuntimeScalarType.effectiveType(aResolved) == RuntimeScalarType.STRING + || RuntimeScalarType.effectiveType(bResolved) == RuntimeScalarType.STRING) { return propagateTaint(new RuntimeScalar(aStr + bStr), aResolved, bResolved); } @@ -1100,7 +1101,7 @@ private static RuntimeScalar joinInternal(RuntimeScalar runtimeScalar, RuntimeBa } resolved = stringifyForStringContext(resolved); RuntimeScalar res = new RuntimeScalar(resolved.toString()); - if (resolved.type != RuntimeScalarType.STRING) { + if (RuntimeScalarType.effectiveType(resolved) != RuntimeScalarType.STRING) { res.type = BYTE_STRING; } res.tainted = resolved.isTainted(); @@ -1120,7 +1121,7 @@ private static RuntimeScalar joinInternal(RuntimeScalar runtimeScalar, RuntimeBa // In Perl, join produces a byte-string unless one of the inputs has // the UTF-8 flag on. Only STRING type has the flag; INTEGER, DOUBLE, // UNDEF, and BYTE_STRING are all byte-compatible. - boolean hasUtf8 = separatorResolved.type == RuntimeScalarType.STRING; + boolean hasUtf8 = RuntimeScalarType.effectiveType(separatorResolved) == RuntimeScalarType.STRING; boolean tainted = separatorResolved.isTainted(); // Join the elements @@ -1153,7 +1154,7 @@ private static RuntimeScalar joinInternal(RuntimeScalar runtimeScalar, RuntimeBa resolvedElements.add(materialized); hasExecutableValue |= RuntimeRegexTemplate.hasExecutableValue(materialized); } - if (resolved.type == RuntimeScalarType.STRING) { + if (RuntimeScalarType.effectiveType(resolved) == RuntimeScalarType.STRING) { hasUtf8 = true; } tainted |= resolved.isTainted(); @@ -1248,17 +1249,20 @@ public static RuntimeScalar stringConcatNoOverload(RuntimeScalar runtimeScalar, return propagateTaint(stringConcatBytes(aStr, aResolved, bStr, bResolved), aResolved, bResolved); } - if (aResolved.type == RuntimeScalarType.STRING || bResolved.type == RuntimeScalarType.STRING) { + if (RuntimeScalarType.effectiveType(aResolved) == RuntimeScalarType.STRING + || RuntimeScalarType.effectiveType(bResolved) == RuntimeScalarType.STRING) { return propagateTaint(new RuntimeScalar(aStr + bStr), aResolved, bResolved); } - if (aResolved.type == BYTE_STRING || bResolved.type == BYTE_STRING) { - boolean aIsByte = aResolved.type == BYTE_STRING - || aResolved.type == RuntimeScalarType.UNDEF - || (aStr.isEmpty() && aResolved.type != RuntimeScalarType.STRING); - boolean bIsByte = bResolved.type == BYTE_STRING - || bResolved.type == RuntimeScalarType.UNDEF - || (bStr.isEmpty() && bResolved.type != RuntimeScalarType.STRING); + int aType = RuntimeScalarType.effectiveType(aResolved); + int bType = RuntimeScalarType.effectiveType(bResolved); + if (aType == BYTE_STRING || bType == BYTE_STRING) { + boolean aIsByte = aType == BYTE_STRING + || aType == RuntimeScalarType.UNDEF + || (aStr.isEmpty() && aType != RuntimeScalarType.STRING); + boolean bIsByte = bType == BYTE_STRING + || bType == RuntimeScalarType.UNDEF + || (bStr.isEmpty() && bType != RuntimeScalarType.STRING); if (aIsByte && bIsByte) { boolean safe = true; for (int i = 0; safe && i < aStr.length(); i++) { @@ -1289,9 +1293,9 @@ public static RuntimeScalar stringConcatNoOverload(RuntimeScalar runtimeScalar, private static RuntimeScalar stringConcatBytes(String aStr, RuntimeScalar a, String bStr, RuntimeScalar b) { - byte[] aBytes = a.type == RuntimeScalarType.STRING + byte[] aBytes = RuntimeScalarType.effectiveType(a) == RuntimeScalarType.STRING ? aStr.getBytes(StandardCharsets.UTF_8) : stringBytesForBytesPragma(aStr); - byte[] bBytes = b.type == RuntimeScalarType.STRING + byte[] bBytes = RuntimeScalarType.effectiveType(b) == RuntimeScalarType.STRING ? bStr.getBytes(StandardCharsets.UTF_8) : stringBytesForBytesPragma(bStr); byte[] out = new byte[aBytes.length + bBytes.length]; System.arraycopy(aBytes, 0, out, 0, aBytes.length); diff --git a/src/main/java/org/perlonjava/runtime/perlmodule/Utf8.java b/src/main/java/org/perlonjava/runtime/perlmodule/Utf8.java index 198ffee60b..14e35d011f 100644 --- a/src/main/java/org/perlonjava/runtime/perlmodule/Utf8.java +++ b/src/main/java/org/perlonjava/runtime/perlmodule/Utf8.java @@ -391,9 +391,10 @@ public static boolean isUtf8(RuntimeScalar scalar) { if (scalar instanceof ScalarSpecialVariable sv) { scalar = sv.getValueAsScalar(); } - // Only STRING type has the UTF-8 flag set. - // INTEGER, DOUBLE, UNDEF, REFERENCE etc. don't have the UTF-8 flag in Perl. - return scalar.type == STRING; + // Only STRING type has the UTF-8 flag set. Runtime SvREADONLY wraps + // the original scalar in READONLY_SCALAR, so inspect the preserved + // value type instead of treating the wrapper as a byte string. + return RuntimeScalarType.effectiveType(scalar) == STRING; } /** diff --git a/src/main/java/org/perlonjava/runtime/runtimetypes/RuntimeScalarType.java b/src/main/java/org/perlonjava/runtime/runtimetypes/RuntimeScalarType.java index fd108662a0..10ae3a8cdc 100644 --- a/src/main/java/org/perlonjava/runtime/runtimetypes/RuntimeScalarType.java +++ b/src/main/java/org/perlonjava/runtime/runtimetypes/RuntimeScalarType.java @@ -36,6 +36,15 @@ public class RuntimeScalarType { private RuntimeScalarType() { } // Prevent instantiation + /** Return the value type stored under one or more runtime read-only wrappers. */ + public static int effectiveType(RuntimeScalar runtimeScalar) { + while (runtimeScalar != null && runtimeScalar.type == READONLY_SCALAR + && runtimeScalar.value instanceof RuntimeScalar inner) { + runtimeScalar = inner; + } + return runtimeScalar == null ? UNDEF : runtimeScalar.type; + } + // Get blessing ID as an integer public static int blessedId(RuntimeScalar runtimeScalar) { int type = runtimeScalar.type; diff --git a/src/test/resources/unit/issue_1970_readonly_utf8_flag.t b/src/test/resources/unit/issue_1970_readonly_utf8_flag.t new file mode 100644 index 0000000000..e70018a042 --- /dev/null +++ b/src/test/resources/unit/issue_1970_readonly_utf8_flag.t @@ -0,0 +1,18 @@ +use strict; +use warnings; +use Test::More; + +my $wide = "\x{2122}"; +Internals::SvREADONLY($wide, 1); + +ok(utf8::is_utf8($wide), 'read-only wide scalar retains its UTF-8 flag'); + +my $interpolate = sub { return "$wide" }; +my $copy = $interpolate->(); +my $concatenated = $wide . ''; + +ok(utf8::is_utf8($copy), 'closure interpolation preserves the UTF-8 flag'); +ok(utf8::is_utf8($concatenated), 'concatenation preserves the UTF-8 flag'); +is(ord($copy), 0x2122, 'closure interpolation preserves the wide character'); + +done_testing();