Unicode data

meson.build

@@ -86,11 +86,31 @@ runtime_init_once_embedded = custom_target(
         depend_files : files('tools/embed_file_as_c.py')
 )
 
+unicode_data_source = custom_target(
+        'unicode_data.c',
+        input : [
+                'data/UnicodeData.txt',
+                'data/CaseFolding.txt',
+                'data/SpecialCasing.txt'
+        ],
+        output : 'unicode_data.c',
+        command : [
+                python3,
+                files('tools/generate_unicode_data.py'),
+                '@INPUT0@',
+                '@INPUT1@',
+                '@INPUT2@',
+                '@OUTPUT@'
+        ],
+        depend_files : files('tools/generate_unicode_data.py')
+)
+
 vm_lib = library(
                 'ListTalkVM', 
         [
                 runtime_init_embedded,
                 runtime_init_once_embedded,
+                unicode_data_source,
                 'src/vm/eval.c',
                 'src/vm/loader.c',
                 'src/vm/quasiquote.c',

src/classes/Character.c

@@ -14,6 +14,7 @@
 #include <ListTalk/classes/Class.h>
 #include <ListTalk/macros/decl_macros.h>
 #include <ListTalk/utils/utf8.h>
+#include "src/utils/unicode_data.h"
 
 #include <ctype.h>
 #include <inttypes.h>
@@ -90,8 +91,107 @@ LT_DEFINE_PRIMITIVE(
     return LT_SmallInteger_new((int64_t)width);
 }
 
+#define CHARACTER_CASE_METHOD(name, selector, function, description) \
+    LT_DEFINE_PRIMITIVE_FLAGS( \
+        name, "Character>>" selector, "(self)", description, \
+        LT_PRIMITIVE_FLAG_PURE \
+    ){ \
+        LT_Value cursor = arguments; \
+        LT_Value self; \
+        (void)tail_call_unwind_marker; \
+        LT_OBJECT_ARG(cursor, self); \
+        LT_ARG_END(cursor); \
+        return LT_Character_new(function(LT_Character_value(self))); \
+    }
+
+CHARACTER_CASE_METHOD(character_method_lower_case, "lowerCase",
+    LT_unicode_lowercase, "Return the Unicode simple lowercase mapping.")
+CHARACTER_CASE_METHOD(character_method_upper_case, "upperCase",
+    LT_unicode_uppercase, "Return the Unicode simple uppercase mapping.")
+CHARACTER_CASE_METHOD(character_method_title_case, "titleCase",
+    LT_unicode_titlecase, "Return the Unicode simple titlecase mapping.")
+
+LT_DEFINE_PRIMITIVE_FLAGS(
+    character_method_category,
+    "Character>>category",
+    "(self)",
+    "Return the two-letter Unicode general category.",
+    LT_PRIMITIVE_FLAG_PURE
+){
+    LT_Value cursor = arguments;
+    LT_Value self;
+    (void)tail_call_unwind_marker;
+
+    LT_OBJECT_ARG(cursor, self);
+    LT_ARG_END(cursor);
+    return (LT_Value)(uintptr_t)LT_String_new_cstr(
+        (char*)LT_unicode_category(LT_Character_value(self))
+    );
+}
+
+static int character_alphabetic_p(uint32_t codepoint){
+    return LT_unicode_category(codepoint)[0] == 'L';
+}
+static int character_numeric_p(uint32_t codepoint){
+    return LT_unicode_category(codepoint)[0] == 'N';
+}
+static int character_decimal_p(uint32_t codepoint){
+    const char* category = LT_unicode_category(codepoint);
+    return category[0] == 'N' && category[1] == 'd';
+}
+static int character_upper_case_p(uint32_t codepoint){
+    const char* category = LT_unicode_category(codepoint);
+    return category[0] == 'L' && category[1] == 'u';
+}
+static int character_lower_case_p(uint32_t codepoint){
+    const char* category = LT_unicode_category(codepoint);
+    return category[0] == 'L' && category[1] == 'l';
+}
+static int character_mark_p(uint32_t codepoint){
+    return LT_unicode_category(codepoint)[0] == 'M';
+}
+
+#define CHARACTER_PROPERTY_METHOD(name, selector, predicate, description) \
+    LT_DEFINE_PRIMITIVE_FLAGS( \
+        name, "Character>>" selector, "(self)", description, \
+        LT_PRIMITIVE_FLAG_PURE \
+    ){ \
+        LT_Value cursor = arguments; \
+        LT_Value self; \
+        (void)tail_call_unwind_marker; \
+        LT_OBJECT_ARG(cursor, self); \
+        LT_ARG_END(cursor); \
+        return predicate(LT_Character_value(self)) ? LT_TRUE : LT_FALSE; \
+    }
+
+CHARACTER_PROPERTY_METHOD(character_method_alphabetic_p, "alphabetic?",
+    character_alphabetic_p, "Return true for a Unicode letter.")
+CHARACTER_PROPERTY_METHOD(character_method_numeric_p, "numeric?",
+    character_numeric_p, "Return true for a Unicode number.")
+CHARACTER_PROPERTY_METHOD(character_method_whitespace_p, "whitespace?",
+    LT_unicode_whitespace_p, "Return true for Unicode whitespace.")
+CHARACTER_PROPERTY_METHOD(character_method_decimal_p, "decimal?",
+    character_decimal_p, "Return true for a Unicode decimal digit.")
+CHARACTER_PROPERTY_METHOD(character_method_upper_case_p, "upperCase?",
+    character_upper_case_p, "Return true for an uppercase Unicode letter.")
+CHARACTER_PROPERTY_METHOD(character_method_lower_case_p, "lowerCase?",
+    character_lower_case_p, "Return true for a lowercase Unicode letter.")
+CHARACTER_PROPERTY_METHOD(character_method_mark_p, "mark?",
+    character_mark_p, "Return true for a Unicode combining mark.")
+
 static LT_Method_Descriptor Character_methods[] = {
     {"asString", &character_method_as_string},
+    {"lowerCase", &character_method_lower_case},
+    {"upperCase", &character_method_upper_case},
+    {"titleCase", &character_method_title_case},
+    {"category", &character_method_category},
+    {"alphabetic?", &character_method_alphabetic_p},
+    {"numeric?", &character_method_numeric_p},
+    {"whitespace?", &character_method_whitespace_p},
+    {"decimal?", &character_method_decimal_p},
+    {"upperCase?", &character_method_upper_case_p},
+    {"lowerCase?", &character_method_lower_case_p},
+    {"mark?", &character_method_mark_p},
     {"width", &character_method_width},
     LT_NULL_NATIVE_CLASS_METHOD_DESCRIPTOR
 };

src/classes/String.c

@@ -32,6 +32,7 @@
 #include <ListTalk/utils/base64.h>
 #include <ListTalk/utils/hex.h>
 #include <ListTalk/utils/utf8.h>
+#include "src/utils/unicode_data.h"
 
 #include <ctype.h>
 #include <errno.h>
@@ -1790,6 +1791,152 @@ LT_DEFINE_PRIMITIVE(
     return self;
 }
 
+LT_DEFINE_PRIMITIVE_FLAGS(
+    string_method_case_fold,
+    "String>>caseFold",
+    "(self)",
+    "Return the Unicode default full case folding of receiver.",
+    LT_PRIMITIVE_FLAG_PURE
+){
+    LT_Value cursor = arguments;
+    LT_String* string;
+    LT_StringBuilder* builder;
+    const char* input;
+    const char* end;
+    (void)tail_call_unwind_marker;
+
+    LT_GENERIC_ARG(cursor, string, LT_String*, LT_String_from_value);
+    LT_ARG_END(cursor);
+    builder = LT_StringBuilder_new();
+    input = LT_String_value_cstr(string);
+    end = input + LT_String_byte_length(string);
+    while (input < end){
+        uint32_t codepoint = LT_utf8_codepoint_at_bounded(
+            input,
+            (size_t)(end - input)
+        );
+        const uint32_t* mapping;
+        size_t mapping_length;
+        size_t index;
+
+        mapping = LT_unicode_casefold(codepoint, &mapping_length);
+        if (mapping == NULL){
+            StringBuilder_append_codepoint(builder, codepoint);
+        } else {
+            for (index = 0; index < mapping_length; index++){
+                StringBuilder_append_codepoint(builder, mapping[index]);
+            }
+        }
+        input = LT_utf8_next_bounded(input, (size_t)(end - input));
+    }
+    return (LT_Value)(uintptr_t)LT_String_new(
+        LT_StringBuilder_value(builder),
+        LT_StringBuilder_length(builder)
+    );
+}
+
+typedef enum {
+    STRING_CASE_LOWER,
+    STRING_CASE_TITLE,
+    STRING_CASE_UPPER
+} StringCase;
+
+static const uint32_t* String_full_case_mapping(StringCase mode,
+                                                uint32_t codepoint,
+                                                size_t* length_out){
+    switch (mode){
+        case STRING_CASE_LOWER:
+            return LT_unicode_full_lowercase(codepoint, length_out);
+        case STRING_CASE_TITLE:
+            return LT_unicode_full_titlecase(codepoint, length_out);
+        case STRING_CASE_UPPER:
+            return LT_unicode_full_uppercase(codepoint, length_out);
+    }
+    LT_error("Invalid string case mapping");
+    return NULL;
+}
+
+static uint32_t String_simple_case_mapping(StringCase mode,
+                                           uint32_t codepoint){
+    switch (mode){
+        case STRING_CASE_LOWER:
+            return LT_unicode_lowercase(codepoint);
+        case STRING_CASE_TITLE:
+            return LT_unicode_titlecase(codepoint);
+        case STRING_CASE_UPPER:
+            return LT_unicode_uppercase(codepoint);
+    }
+    LT_error("Invalid string case mapping");
+    return codepoint;
+}
+
+static LT_Value String_case(LT_String* string,
+                            StringCase mode,
+                            int title_each_word){
+    LT_StringBuilder* builder = LT_StringBuilder_new();
+    const char* input = LT_String_value_cstr(string);
+    const char* end = input + LT_String_byte_length(string);
+    int first = 1;
+
+    while (input < end){
+        uint32_t codepoint = LT_utf8_codepoint_at_bounded(
+            input,
+            (size_t)(end - input)
+        );
+        StringCase character_mode = mode;
+        const uint32_t* mapping;
+        size_t mapping_length;
+        size_t index;
+
+        if (mode == STRING_CASE_TITLE && !first){
+            character_mode = STRING_CASE_LOWER;
+        }
+        mapping = String_full_case_mapping(
+            character_mode,
+            codepoint,
+            &mapping_length
+        );
+        if (mapping == NULL){
+            StringBuilder_append_codepoint(
+                builder,
+                String_simple_case_mapping(character_mode, codepoint)
+            );
+        } else {
+            for (index = 0; index < mapping_length; index++){
+                StringBuilder_append_codepoint(builder, mapping[index]);
+            }
+        }
+        first = title_each_word && LT_unicode_whitespace_p(codepoint);
+        input = LT_utf8_next_bounded(input, (size_t)(end - input));
+    }
+    return (LT_Value)(uintptr_t)LT_String_new(
+        LT_StringBuilder_value(builder),
+        LT_StringBuilder_length(builder)
+    );
+}
+
+#define STRING_CASE_METHOD(name, selector, mode, title_each_word, description) \
+    LT_DEFINE_PRIMITIVE_FLAGS( \
+        name, "String>>" selector, "(self)", description, \
+        LT_PRIMITIVE_FLAG_PURE \
+    ){ \
+        LT_Value cursor = arguments; \
+        LT_String* string; \
+        (void)tail_call_unwind_marker; \
+        LT_GENERIC_ARG(cursor, string, LT_String*, LT_String_from_value); \
+        LT_ARG_END(cursor); \
+        return String_case(string, mode, title_each_word); \
+    }
+
+STRING_CASE_METHOD(string_method_lower, "lower", STRING_CASE_LOWER, 0,
+    "Return the locale-invariant Unicode lowercase mapping.")
+STRING_CASE_METHOD(string_method_upper, "upper", STRING_CASE_UPPER, 0,
+    "Return the locale-invariant Unicode uppercase mapping.")
+STRING_CASE_METHOD(string_method_capitalize, "capitalize", STRING_CASE_TITLE, 0,
+    "Titlecase the first code point and lowercase the remainder.")
+STRING_CASE_METHOD(string_method_title, "title", STRING_CASE_TITLE, 1,
+    "Titlecase each whitespace-delimited word.")
+
 LT_DEFINE_PRIMITIVE(
     string_method_as_list,
     "String>>asList",
@@ -1906,6 +2053,11 @@ static LT_Method_Descriptor String_methods[] = {
     {"asHexByteVector", &string_method_as_hex_bytevector},
     {"asPortableFilename", &string_method_as_portable_filename},
     {"asString", &string_method_as_string},
+    {"caseFold", &string_method_case_fold},
+    {"lower", &string_method_lower},
+    {"upper", &string_method_upper},
+    {"capitalize", &string_method_capitalize},
+    {"title", &string_method_title},
     {"asList", &string_method_as_list},
     {"asIterator", &string_method_as_iterator},
     {"writeToFile:", &string_method_write_to_file},

src/utils/unicode_data.h

@@ -0,0 +1,28 @@
+/*
+ * SPDX-License-Identifier: MIT
+ * Copyright (c) 2026 Ales Hakl
+ */
+
+#ifndef H__ListTalk__utils__unicode_data__
+#define H__ListTalk__utils__unicode_data__
+
+#include <stdint.h>
+#include <stddef.h>
+
+const char* LT_unicode_category(uint32_t codepoint);
+
+static inline int LT_unicode_whitespace_p(uint32_t codepoint){
+    return (codepoint >= UINT32_C(9) && codepoint <= UINT32_C(13))
+        || codepoint == UINT32_C(0x85)
+        || LT_unicode_category(codepoint)[0] == 'Z';
+}
+
+uint32_t LT_unicode_lowercase(uint32_t codepoint);
+uint32_t LT_unicode_uppercase(uint32_t codepoint);
+uint32_t LT_unicode_titlecase(uint32_t codepoint);
+const uint32_t* LT_unicode_casefold(uint32_t codepoint, size_t* length_out);
+const uint32_t* LT_unicode_full_lowercase(uint32_t codepoint, size_t* length_out);
+const uint32_t* LT_unicode_full_titlecase(uint32_t codepoint, size_t* length_out);
+const uint32_t* LT_unicode_full_uppercase(uint32_t codepoint, size_t* length_out);
+
+#endif

tests/eval-objects.lt

@@ -616,7 +616,39 @@
        "Number>>asString formats big integers")
 (check (equal? [#true asString] "#true") "Object>>asString uses debug printing")
 (check (equal? ["aλ" asString] "aλ") "String>>asString returns receiver")
+(check (equal? ["Straße" caseFold] "strasse")
+       "String>>caseFold applies expanding full folds")
+(check (equal? ["Σςσ" caseFold] "σσσ")
+       "String>>caseFold normalizes Greek sigma variants")
+(check (equal? ["İ" caseFold] "i̇")
+       "String>>caseFold supports multi-code-point mappings")
+(check (equal? ["Straße" upper] "STRASSE")
+       "String>>upper applies full locale-invariant mappings")
+(check (equal? ["İ" lower] "i̇")
+       "String>>lower applies full locale-invariant mappings")
+(check (equal? ["dzON" capitalize] "Dzon")
+       "String>>capitalize titlecases first code point and lowers the remainder")
+(check (equal? ["ßTRAẞE" capitalize] "Sstraße")
+       "String>>capitalize supports expanding mappings")
+(check (equal? ["dzON ßTRAẞE" title] "Dzon Sstraße")
+       "String>>title titlecases every word")
+(check (equal? ["MULTIPLE\tWORDS\nHERE" title] "Multiple\tWords\nHere")
+       "String>>title recognizes whitespace word boundaries")
+(check (equal? ["NO-BREAK\u00A0SPACE\u2003EM SPACE" title]
+               "No-break\u00A0Space\u2003Em Space")
+       "String>>title recognizes Unicode whitespace word boundaries")
 (check (equal? [#\λ asString] "λ") "Character>>asString returns UTF-8 string")
+(check (equal? [#\A lowerCase] #\a) "Character>>lowerCase maps ASCII")
+(check (equal? [#\λ upperCase] #\Λ) "Character>>upperCase maps Unicode")
+(check (equal? [#\dz titleCase] #\Dz) "Character>>titleCase maps digraphs")
+(check (equal? [#\λ category] "Ll") "Character>>category")
+(check [#\λ alphabetic?] "Character>>alphabetic?")
+(check [#\Ⅷ numeric?] "Character>>numeric?")
+(check [#\space whitespace?] "Character>>whitespace?")
+(check [#\٣ decimal?] "Character>>decimal?")
+(check [#\Λ upperCase?] "Character>>upperCase?")
+(check [#\λ lowerCase?] "Character>>lowerCase?")
+(check [#\u+0301 mark?] "Character>>mark?")
 (check (= [#\a width] 1) "Character>>width returns wcwidth")
 (check (= ["abc" width] 3) "String>>width returns wcswidth")
 (check (= [3 asFloat] 3.0) "RealNumber>>asFloat converts integers")