Don't fault on an unaligned lock cmpxchg8b

asbestos/gadgets-aarch64/misc.S

@@ -128,7 +128,23 @@
     gret 1
     write_bullshit 64, atomic_cmpxchg8b
 
-3:  # All unaligned paths
+3:  # Unaligned. The exclusive load above needs 8-byte alignment, so do it in C.
+    str eax, [_cpu, CPU_eax]
+    str ebx, [_cpu, CPU_ebx]
+    str ecx, [_cpu, CPU_ecx]
+    str edx, [_cpu, CPU_edx]
+    save_c
+    mov x0, _cpu
+    sub x1, _tlb, TLB_entries
+    mov w2, _addr
+    bl NAME(helper_cmpxchg8b_unaligned)
+    mov x19, x0
+    restore_c
+    cbnz x19, 4f
+    ldr eax, [_cpu, CPU_eax]
+    ldr edx, [_cpu, CPU_edx]
+    gret 1
+4:
     b segfault_write
 
 

asbestos/helpers.c

@@ -1,6 +1,8 @@
 #include <time.h>
 #include "emu/cpu.h"
 #include "emu/cpuid.h"
+#include "emu/tlb.h"
+#include "util/sync.h"
 
 void helper_cpuid(dword_t *a, dword_t *b, dword_t *c, dword_t *d) {
     do_cpuid(a, b, c, d);
@@ -21,3 +23,27 @@ void helper_expand_flags(struct cpu_state *cpu) {
 void helper_collapse_flags(struct cpu_state *cpu) {
     collapse_flags(cpu);
 }
+
+// lock cmpxchg8b on an address that isn't 8-byte aligned, which x86 allows and
+// the i386 ABI produces (it aligns 64-bit struct fields to 4 bytes). The
+// gadget's exclusive load needs 8-byte alignment, so this does the exchange
+// under a lock instead. Returns nonzero on a fault, with tlb->segfault_addr set.
+static lock_t cmpxchg8b_lock = LOCK_INITIALIZER;
+int helper_cmpxchg8b_unaligned(struct cpu_state *cpu, struct tlb *tlb, addr_t addr) {
+    uint64_t expected = ((uint64_t) cpu->edx << 32) | cpu->eax;
+    uint64_t desired = ((uint64_t) cpu->ecx << 32) | cpu->ebx;
+    uint64_t value;
+    bool ok;
+    lock(&cmpxchg8b_lock);
+    ok = tlb_read(tlb, addr, &value, sizeof(value));
+    if (ok && value == expected)
+        ok = tlb_write(tlb, addr, &desired, sizeof(desired));
+    unlock(&cmpxchg8b_lock);
+    if (!ok)
+        return 1;
+    cpu->eax = (dword_t) value;
+    cpu->edx = (dword_t) (value >> 32);
+    cpu->zf = value == expected;
+    cpu->zf_res = 0;
+    return 0;
+}

tests/e2e/cmpxchg8b/cmpxchg8b.c

@@ -0,0 +1,32 @@
+// lock cmpxchg8b on 64-bit values that are only 4-byte aligned, the way the
+// i386 ABI lays out uint64_t struct fields: one mid-page and one straddling a
+// page boundary, each incremented from several threads at once.
+#include <pthread.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <sys/mman.h>
+
+#define THREADS 4
+#define ITERS 20000
+
+static void *bump(void *p) {
+    for (int i = 0; i < ITERS; i++)
+        __atomic_fetch_add((uint64_t *) p, 1, __ATOMIC_SEQ_CST);
+    return NULL;
+}
+
+static unsigned long long run(char *where) {
+    pthread_t threads[THREADS];
+    for (int i = 0; i < THREADS; i++)
+        pthread_create(&threads[i], NULL, bump, where);
+    for (int i = 0; i < THREADS; i++)
+        pthread_join(threads[i], NULL);
+    return *(uint64_t *) where;
+}
+
+int main(void) {
+    char *pages = mmap(NULL, 8192, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
+    printf("mid-page: %llu\n", run(pages + 100));
+    printf("page boundary: %llu\n", run(pages + 4092));
+    return 0;
+}

tests/e2e/cmpxchg8b/expected.txt

@@ -0,0 +1,2 @@
+mid-page: 80000
+page boundary: 80000

tests/e2e/cmpxchg8b/test.sh

@@ -0,0 +1,3 @@
+#!/bin/sh
+gcc -O2 -pthread cmpxchg8b.c -o cmpxchg8b
+./cmpxchg8b