diff --git a/edge0-infer/src/app/sampling.cpp b/edge0-infer/src/app/sampling.cpp index 1d7f4a4d..c5128a59 100644 --- a/edge0-infer/src/app/sampling.cpp +++ b/edge0-infer/src/app/sampling.cpp @@ -90,12 +90,24 @@ TokenLogprob raw_logprob(const std::vector& logits,std::int32_t token,std const double normalizer=std::log(sum); TokenLogprob result;result.token=token; result.logprob=(static_cast(logits.at(static_cast(token)))-maximum)-normalizer; - std::vector order(logits.size()); - for(std::size_t i=0;i(i); - count=std::min(count,order.size()); - std::partial_sort(order.begin(),order.begin()+count,order.end(),[&](auto a,auto b) { + count=std::min(count,logits.size()); + if (!count) return result; + const auto better=[&](auto a,auto b) { return logits[a]>logits[b] || (logits[a]==logits[b] && a order; + order.reserve(count); + for (std::size_t i=0;i(i); + if (order.size()(logits[order[i]])-maximum)-normalizer}); return result; diff --git a/edge0-infer/tests/unit/app/sampling_test.cpp b/edge0-infer/tests/unit/app/sampling_test.cpp index 2fd0c1fc..2ac6c390 100644 --- a/edge0-infer/tests/unit/app/sampling_test.cpp +++ b/edge0-infer/tests/unit/app/sampling_test.cpp @@ -3,8 +3,55 @@ #include #include #include +#include +#include +#include +namespace allocation_probe { +bool active=false; +std::size_t maximum=0,large=0; +constexpr std::size_t threshold=32768; +// Keep the deallocator out of line so GCC does not diagnose an inlined free +// against the matching replacement operator new used by this test. +#if defined(__GNUC__) +__attribute__((noinline)) +#elif defined(_MSC_VER) +__declspec(noinline) +#endif +void release(void* p) noexcept {std::free(p);} +} +void* operator new(std::size_t bytes) { + if (allocation_probe::active) { + allocation_probe::maximum=std::max(allocation_probe::maximum,bytes); + if (bytes>=allocation_probe::threshold) ++allocation_probe::large; + } + if (auto* p=std::malloc(bytes?bytes:1)) return p; + throw std::bad_alloc(); +} +void operator delete(void* p) noexcept {allocation_probe::release(p);} +void operator delete(void* p,std::size_t) noexcept {allocation_probe::release(p);} +void* operator new[](std::size_t bytes) {return ::operator new(bytes);} +void operator delete[](void* p) noexcept {allocation_probe::release(p);} +void operator delete[](void* p,std::size_t) noexcept {allocation_probe::release(p);} + using namespace edge0; int main() { + { + std::vector row(262144); + std::vector expected(row.size()); + for (std::size_t i=0;i(i);} + std::partial_sort(expected.begin(),expected.begin()+20,expected.end(),[&](auto a,auto b) { + return row[a]>row[b] || (row[a]==row[b] && a