Add compile_executable (#56)

* Allow changeing filename base in generate_shlib, though leave obj as default

* Add `generate_executable`

* Add `compile_executable`

* Update docstring for compile_executable

* Add tests for compile_executable

* llvmcall used in standalone test only works on Julia 1.8+

* Looks like clang_jll ain't gonna cut it for the standalone executables even on linux

* Try annotating compiled function with Base.@ccallable

* Let's try some inlining

* Ah, so `@ccallable` requires a return type

* Ok, so no `@ccallable`

* Potential workaround on non-apple systems: use minimal wrapper

* We can probably use clang_jll after all

* Probably about time for a version bump

* Add more minimal test for v1.7, fix specifier on advanced test

* Implement changes requested in code review

Author: brenhinkeller

src/StaticCompiler.jl

@@ -7,20 +7,20 @@ using Base: RefValue
 using Serialization: serialize, deserialize
 using Clang_jll: clang
 
-export compile, load_function
+export compile, load_function, compile_executable
 export native_code_llvm, native_code_typed, native_llvm_module
 
 """
     compile(f, types, path::String = tempname()) --> (compiled_f, path)
 
-   !!! Warning: this will fail on programs that heap allocate any memory, or have dynamic dispatch !!!
+   !!! Warning: this will fail on programs that heap allocate any memory tracked by the GC, or have dynamic dispatch !!!
 
 Statically compile the method of a function `f` specialized to arguments of the type given by `types`.
 
 This will create a directory at the specified path (or in a temporary directory if you exclude that argument)
 that contains the files needed for your static compiled function. `compile` will return a
-`StaticCompiledFunction` object and `obj_path` which is the absolute path of the directory containing the 
-compilation artifacts. The `StaticCompiledFunction` can be treated as if it is a function with a single 
+`StaticCompiledFunction` object and `obj_path` which is the absolute path of the directory containing the
+compilation artifacts. The `StaticCompiledFunction` can be treated as if it is a function with a single
 method corresponding to the types you specified when it was compiled.
 
 To deserialize and instantiate a previously compiled function, simply execute `load_function(path)`, which
@@ -68,14 +68,14 @@ path
 
 0 directories, 3 files
 ````
-* `obj.so` (or `.dylib` on MacOS) is a shared object file that can be linked to in order to execute your 
-compiled julia function. 
+* `obj.so` (or `.dylib` on MacOS) is a shared object file that can be linked to in order to execute your
+compiled julia function.
 * `obj.cjl` is a serialized `LazyStaticCompiledFunction` object which will be deserialized and instantiated
-with `load_function(path)`. `LazyStaticcompiledfunction`s contain the requisite information needed to link to the 
-`obj.so` inside a julia session. Once it is instantiated in a julia session (i.e. by 
-`instantiate(::LazyStaticCompiledFunction)`, this happens automatically in `load_function`), it will be of type 
-`StaticCompiledFunction` and may be called with arguments of type `types` as if it were a function with a 
-single method (the method determined by `types`).  
+with `load_function(path)`. `LazyStaticcompiledfunction`s contain the requisite information needed to link to the
+`obj.so` inside a julia session. Once it is instantiated in a julia session (i.e. by
+`instantiate(::LazyStaticCompiledFunction)`, this happens automatically in `load_function`), it will be of type
+`StaticCompiledFunction` and may be called with arguments of type `types` as if it were a function with a
+single method (the method determined by `types`).
 """
 function compile(f, _tt, path::String = tempname();  name = GPUCompiler.safe_name(repr(f)), kwargs...)
     tt = Base.to_tuple_type(_tt)
@@ -134,6 +134,81 @@ end
 
 instantiate(f::StaticCompiledFunction) = f
 
+
+"""
+```julia
+compile_executable(f, types::Tuple, path::String, name::String=repr(f); filename::String=name, kwargs...)
+```
+Attempt to compile a standalone executable that runs function `f` with a type signature given by the tuple of `types`.
+
+### Examples
+```julia
+julia> using StaticCompiler
+
+julia> function puts(s::Ptr{UInt8}) # Can't use Base.println because it allocates.
+           # Note, this `llvmcall` requires Julia 1.8+
+           Base.llvmcall((\"""
+           ; External declaration of the puts function
+           declare i32 @puts(i8* nocapture) nounwind
+
+           define i32 @main(i8*) {
+           entry:
+               %call = call i32 (i8*) @puts(i8* %0)
+               ret i32 0
+           }
+           \""", "main"), Int32, Tuple{Ptr{UInt8}}, s)
+       end
+puts (generic function with 1 method)
+
+julia> function print_args(argc::Int, argv::Ptr{Ptr{UInt8}})
+           for i=1:argc
+               # Get pointer
+               p = unsafe_load(argv, i)
+               # Print string at pointer location (which fortunately already exists isn't tracked by the GC)
+               puts(p)
+           end
+           return 0
+       end
+
+julia> compile_executable(print_args, (Int, Ptr{Ptr{UInt8}}))
+"/Users/foo/code/StaticCompiler.jl/print_args"
+
+shell> ./print_args 1 2 3 4 Five
+./print_args
+1
+2
+3
+4
+Five
+```
+```julia
+julia> using StaticTools # So you don't have to define `puts` and friends every time
+
+julia> hello() = println(c"Hello, world!") # c"..." makes a stack-allocated StaticString
+
+julia> compile_executable(hello)
+"/Users/foo/code/StaticCompiler.jl/hello"
+
+shell> ./hello
+Hello, world!
+```
+"""
+function compile_executable(f, _tt=(), path::String="./", name=GPUCompiler.safe_name(repr(f)); filename=name, kwargs...)
+    tt = Base.to_tuple_type(_tt)
+    tt == Tuple{} || tt == Tuple{Int, Ptr{Ptr{UInt8}}} || error("input type signature $_tt must be either () or (Int, Ptr{Ptr{UInt8}})")
+
+    rt = only(native_code_typed(f, tt))[2]
+    isconcretetype(rt) || error("$f$_tt did not infer to a concrete type. Got $rt")
+
+    # Would be nice to use a compiler pass or something to check if there are any heap allocations or references to globals
+    # Keep an eye on https://github.com/JuliaLang/julia/pull/43747 for this
+
+    generate_executable(f, tt, path, name, filename; kwargs...)
+
+    joinpath(abspath(path), filename)
+end
+
+
 module TestRuntime
     # dummy methods
     signal_exception() = return
@@ -160,7 +235,7 @@ end
 
 """
 ```julia
-generate_shlib(f, tt, path::String, name::String; kwargs...)
+generate_shlib(f, tt, path::String, name::String, filenamebase::String="obj"; kwargs...)
 ```
 Low level interface for compiling a shared object / dynamically loaded library
  (`.so` / `.dylib`) for function `f` given a tuple type `tt` characterizing
@@ -196,10 +271,10 @@ julia> ccall(StaticCompiler.generate_shlib_fptr(path, name), Float64, (Int64,),
 5.256496109495593
 ```
 """
-function generate_shlib(f, tt, path::String = tempname(), name = GPUCompiler.safe_name(repr(f)); kwargs...)
+function generate_shlib(f, tt, path::String = tempname(), name = GPUCompiler.safe_name(repr(f)), filenamebase::String="obj"; kwargs...)
     mkpath(path)
-    obj_path = joinpath(path, "obj.o")
-    lib_path = joinpath(path, "obj.$(Libdl.dlext)")
+    obj_path = joinpath(path, "$filenamebase.o")
+    lib_path = joinpath(path, "$filenamebase.$(Libdl.dlext)")
     open(obj_path, "w") do io
         job, kwargs = native_job(f, tt; name, kwargs...)
         obj, _ = GPUCompiler.codegen(:obj, job; strip=true, only_entry=false, validate=false)
@@ -216,9 +291,9 @@ function generate_shlib(f, tt, path::String = tempname(), name = GPUCompiler.saf
 end
 
 
-function generate_shlib_fptr(f, tt, path::String=tempname(), name = GPUCompiler.safe_name(repr(f)); temp::Bool=true, kwargs...)
+function generate_shlib_fptr(f, tt, path::String=tempname(), name = GPUCompiler.safe_name(repr(f)), filenamebase::String="obj"; temp::Bool=true, kwargs...)
     generate_shlib(f, tt, path, name; kwargs...)
-    lib_path = joinpath(abspath(path), "obj.$(Libdl.dlext)")
+    lib_path = joinpath(abspath(path), "$filenamebase.$(Libdl.dlext)")
     ptr = Libdl.dlopen(lib_path, Libdl.RTLD_LOCAL)
     fptr = Libdl.dlsym(ptr, "julia_$name")
     @assert fptr != C_NULL
@@ -236,7 +311,7 @@ Low level interface for obtaining a function pointer by `dlopen`ing a shared
 library given the `path` and `name` of a `.so`/`.dylib` already compiled by
 `generate_shlib`.
 
-See also `StaticCompiler.enerate_shlib`.
+See also `StaticCompiler.generate_shlib`.
 
 ### Examples
 ```julia
@@ -264,14 +339,70 @@ julia> test(100_000)
 5.256496109495593
 ```
 """
-function generate_shlib_fptr(path::String, name)
-    lib_path = joinpath(abspath(path), "obj.$(Libdl.dlext)")
+function generate_shlib_fptr(path::String, name, filenamebase::String="obj")
+    lib_path = joinpath(abspath(path), "$filenamebase.$(Libdl.dlext)")
     ptr = Libdl.dlopen(lib_path, Libdl.RTLD_LOCAL)
     fptr = Libdl.dlsym(ptr, "julia_$name")
     @assert fptr != C_NULL
     fptr
 end
 
+"""
+```julia
+generate_executable(f, tt, path::String, name, filename=string(name); kwargs...)
+```
+Attempt to compile a standalone executable that runs `f`.
+
+### Examples
+```julia
+julia> function test(n)
+           r = 0.0
+           for i=1:n
+               r += log(sqrt(i))
+           end
+           return r/n
+       end
+test (generic function with 1 method)
+
+julia> path, name = StaticCompiler.generate_executable(test, Tuple{Int64}, "./scratch")
+```
+"""
+function generate_executable(f, tt, path::String = tempname(), name = GPUCompiler.safe_name(repr(f)), filename::String=string(name); kwargs...)
+    mkpath(path)
+    obj_path = joinpath(path, "$filename.o")
+    exec_path = joinpath(path, filename)
+    open(obj_path, "w") do io
+        job, kwargs = native_job(f, tt; name, kwargs...)
+        obj, _ = GPUCompiler.codegen(:obj, job; strip=true, only_entry=false, validate=false)
+
+        write(io, obj)
+        flush(io)
+
+        # Pick a compiler
+        cc = Sys.isapple() ? `cc` : clang()
+        # Compile!
+        if Sys.isapple()
+            # Apple no longer uses _start, so we can just specify a custom entry
+            entry = "_julia_$name"
+            run(`$cc -e $entry $obj_path -o $exec_path`)
+        else
+            # Write a minimal wrapper to avoid having to specify a custom entry
+            wrapper_path = joinpath(path, "wrapper.c")
+            f = open(wrapper_path, "w")
+            print(f, """int main(int argc, char** argv)
+            {
+                julia_$name(argc, argv);
+                return 0;
+            }""")
+            close(f)
+            run(`$cc $wrapper_path $obj_path -o $exec_path`)
+            # Clean up
+            run(`rm $wrapper_path`)
+        end
+    end
+    path, name
+end
+
 function native_code_llvm(@nospecialize(func), @nospecialize(types); kwargs...)
     job, kwargs = native_job(func, types; kwargs...)
     GPUCompiler.code_llvm(stdout, job; kwargs...)

test/runtests.jl

@@ -33,16 +33,16 @@ fib(n) = n <= 1 ? n : fib(n - 1) + fib(n - 2) # This needs to be defined globall
     # Trick to work around #40990
     _fib2(_fib2, n) = n <= 1 ? n : _fib2(_fib2, n-1) + _fib2(_fib2, n-2)
     fib2(n) = _fib2(_fib2, n)
-    
+
     _, path = compile(fib2, (Int,))
     @test remote_load_call(path, 20) == fib(20)
-    #@test compile(fib2, (Int,))[1](20) == fib(20)    
+    #@test compile(fib2, (Int,))[1](20) == fib(20)
 end
 
 # Call binaries for testing
 # @testset "Generate binary" begin
 #     fib(n) = n <= 1 ? n : fib(n - 1) + fib(n - 2)
-#     libname = tempname() 
+#     libname = tempname()
 #     generate_shlib(fib, (Int,), libname)
 #     ptr = Libdl.dlopen(libname * "." * Libdl.dlext, Libdl.RTLD_LOCAL)
 #     fptr = Libdl.dlsym(ptr, "julia_fib")
@@ -62,7 +62,7 @@ end
     end
     _, path = compile(sum_first_N_int, (Int,))
     @test remote_load_call(path, 10) == 55
-    
+
     function sum_first_N_float64(N)
         s = Float64(0)
         for a in 1:N
@@ -116,10 +116,10 @@ end
 
 # Julia wants to treat Tuple (and other things like it) as plain bits, but LLVM wants to treat it as something with a pointer.
 # We need to be careful to not send, nor receive an unwrapped Tuple to a compiled function.
-# The interface made in `compile` should handle this fine. 
+# The interface made in `compile` should handle this fine.
 @testset "Send and receive Tuple" begin
     foo(u::Tuple) = 2 .* reverse(u) .- 1
-    
+
     _, path = compile(foo, (NTuple{3, Int},))
     @test remote_load_call(path, (1, 2, 3)) == (5, 3, 1)
 end
@@ -172,7 +172,7 @@ end
 end
 
 # This is a trick to get stack allocated arrays inside a function body (so long as they don't escape).
-# This lets us have intermediate, mutable stack allocated arrays inside our 
+# This lets us have intermediate, mutable stack allocated arrays inside our
 @testset "Alloca" begin
     function f(N)
         # this can hold at most 100 Int values, if you use it for more, you'll segfault
@@ -186,7 +186,54 @@ end
     end
     _, path = compile(f, (Int,))
     @test remote_load_call(path, 20) == 20
-end 
+end
+
+@testset "Standalone Executables" begin
+    # Minimal test with no `llvmcall`
+    @inline function foo()
+        v = 0.0
+        n = 1000
+        for i=1:n
+            v += sqrt(n)
+        end
+        return 0
+    end
+
+    filepath = compile_executable(foo, (), tempdir())
+
+    r = run(`$filepath`);
+    @test isa(r, Base.Process)
+    @test r.exitcode == 0
 
+    @static if VERSION>v"1.8.0-DEV" # The llvmcall here only works on 1.8+
+        @inline function puts(s::Ptr{UInt8}) # Can't use Base.println because it allocates
+            Base.llvmcall(("""
+            ; External declaration of the puts function
+            declare i32 @puts(i8* nocapture) nounwind
 
+            define i32 @main(i8*) {
+            entry:
+               %call = call i32 (i8*) @puts(i8* %0)
+               ret i32 0
+            }
+            """, "main"), Int32, Tuple{Ptr{UInt8}}, s)
+        end
+
+        @inline function print_args(argc::Int, argv::Ptr{Ptr{UInt8}})
+            for i=1:argc
+                # Get pointer
+                p = unsafe_load(argv, i)
+                # Print string at pointer location (which fortunately already exists isn't tracked by the GC)
+                puts(p)
+            end
+            return 0
+        end
+
+        filepath = compile_executable(print_args, (Int, Ptr{Ptr{UInt8}}), tempdir())
+
+        r = run(`$filepath Hello, world!`);
+        @test isa(r, Base.Process)
+        @test r.exitcode == 0
+    end
+end
 # data structures, dictionaries, tuples, named tuples