Add support for user-provided job system

encoder/basisu_astc_hdr_6x6_enc.cpp

@@ -6569,7 +6569,7 @@ void global_init()
 	//fmt_printf("astc_6x6_hdr::global_init() total time: {}\n", tm.get_elapsed_secs());
 }
 
-bool compress_photo(const basisu::imagef &orig_src_img, const astc_hdr_6x6_global_config &orig_global_cfg, job_pool *pJob_pool,
+bool compress_photo(const basisu::imagef &orig_src_img, const astc_hdr_6x6_global_config &orig_global_cfg, job_pool_base *pJob_pool,
 	basisu::uint8_vec& intermediate_tex_data, basisu::uint8_vec& astc_tex_data, result_metrics& metrics)
 {
 	assert(g_initialized);

encoder/basisu_astc_hdr_6x6_enc.h

@@ -133,7 +133,7 @@ namespace astc_6x6_hdr
 	};
 
 	// The input image should be unpadded to 6x6 boundaries, i.e. the original unexpanded image.
-	bool compress_photo(const basisu::imagef& orig_src_img, const astc_hdr_6x6_global_config& global_cfg, basisu::job_pool* pJob_pool,
+	bool compress_photo(const basisu::imagef& orig_src_img, const astc_hdr_6x6_global_config& global_cfg, basisu::job_pool_base* pJob_pool,
 		basisu::uint8_vec& intermediate_tex_data, basisu::uint8_vec& astc_tex_data, result_metrics& metrics);
 
 } // namespace uastc_6x6_hdr

encoder/basisu_astc_ldr_encode.cpp

@@ -6460,7 +6460,7 @@ struct ldr_astc_block_encode_image_high_level_config
 
 	std::string m_debug_file_prefix;
 
-	job_pool* m_pJob_pool;
+	job_pool_base* m_pJob_pool;
 
 	astc_ldr::cem_encode_params m_cem_enc_params;
 };
@@ -7804,7 +7804,7 @@ static bool ldr_astc_block_encode_image(
 	memset(packed_blocks.get_ptr(), 0, packed_blocks.size_in_bytes());
 
 	assert(enc_cfg.m_pJob_pool);
-	job_pool& job_pool = *enc_cfg.m_pJob_pool;
+	job_pool_base& job_pool = *enc_cfg.m_pJob_pool;
 
 	std::atomic<bool> encoder_failed_flag;
 	encoder_failed_flag.store(false);
@@ -9703,7 +9703,7 @@ bool ldr_astc_block_encode_image_fast_4x4(
 		grid_coder.init(block_width, block_height);
 
 	assert(enc_cfg.m_pJob_pool);
-	job_pool& job_pool = *enc_cfg.m_pJob_pool;
+	job_pool_base& job_pool = *enc_cfg.m_pJob_pool;
 
 	const uint32_t num_threads = (uint32_t)job_pool.get_total_threads();
 	assert(num_threads);
@@ -10007,7 +10007,7 @@ static bool comp_astc_image_u8(
 	uint32_t block_w,
 	uint32_t block_h,
 	basisu::vector2D<astc_helpers::astc_block>& out_blocks,
-	job_pool &job_pool)
+	job_pool_base &job_pool)
 {
 	static const astcenc_swizzle swizzle{
 		ASTCENC_SWZ_R, ASTCENC_SWZ_G, ASTCENC_SWZ_B, ASTCENC_SWZ_A
@@ -10125,7 +10125,7 @@ static bool ldr_astc_block_encode_image_astcenc(
 	const uint32_t block_size_index = astc_helpers::get_block_size_index(block_width, block_height);
 
 	assert(enc_cfg.m_pJob_pool);
-	job_pool& job_pool = *enc_cfg.m_pJob_pool;
+	job_pool_base& job_pool = *enc_cfg.m_pJob_pool;
 	const uint32_t num_threads = (uint32_t)job_pool.get_total_threads();
 
 	float astcenc_quality = ASTCENC_PRE_FASTEST;
@@ -10674,7 +10674,7 @@ static bool ldr_astc_block_encode_image_astcf(
 		grid_coder.init(block_width, block_height);
 
 	assert(enc_cfg.m_pJob_pool);
-	job_pool& job_pool = *enc_cfg.m_pJob_pool;
+	job_pool_base& job_pool = *enc_cfg.m_pJob_pool;
 	const uint32_t num_threads = (uint32_t)job_pool.get_total_threads();
 
 	std::atomic<int> cur_row;
@@ -11843,7 +11843,7 @@ static uint32_t encode_values(bitwise_coder& coder, uint32_t total_values, const
 static bool compress_image_full_zstd(
 	const image& orig_img, uint8_vec& comp_data, vector2D<astc_helpers::log_astc_block>& coded_blocks,
 	const astc_ldr_encode_config& global_cfg,
-	job_pool& job_pool,
+	job_pool_base& job_pool,
 	ldr_astc_block_encode_image_high_level_config& enc_cfg,	const ldr_astc_block_encode_image_output& enc_out)
 {
 	BASISU_NOTE_UNUSED(job_pool);
@@ -13604,7 +13604,7 @@ static bool mutate_candidates(
 	const uint32_t max_new_blocks, const uint_vec* const pJob_block_list, const vector2D<float> &block_std_dev,
 	const uint32_t NUM_SIMILAR_PATS, const vector2D<uint16_t> similar_pats[2], const vector2D<uint8_t> similar_pat_perm_index[2],
 	const image& orig_img, const image& candidate_img,
-	uint32_t num_threads, job_pool &jp, 
+	uint32_t num_threads, job_pool_base &jp, 
 	const astc_ldr_encode_config& global_cfg, const ldr_astc_block_encode_image_high_level_config& enc_cfg, ldr_astc_block_encode_image_output& enc_out)
 {
 	assert(num_threads);
@@ -14331,7 +14331,7 @@ static bool refine_output_for_deblocking(
 	const uint32_t max_new_blocks = ((width * height) >= (2048 * 2048)) ? ((g_astc_refine_max_new_blocks.get_int() + 1) / 2) : g_astc_refine_max_new_blocks.get_int();
 
 	assert(enc_cfg.m_pJob_pool);
-	job_pool& job_pool = *enc_cfg.m_pJob_pool;
+	job_pool_base& job_pool = *enc_cfg.m_pJob_pool;
 
 	const uint32_t num_threads = (uint32_t)job_pool.get_total_threads();
 	assert(num_threads);
@@ -14867,7 +14867,7 @@ static bool cross_check_dct(
 bool compress_image(
 	const image& actual_orig_img, uint8_vec& comp_data, vector2D<astc_helpers::log_astc_block>& coded_blocks,
 	const astc_ldr_encode_config& orig_global_cfg,
-	job_pool& job_pool)
+	job_pool_base& job_pool)
 {
 	assert(g_initialized);
 

encoder/basisu_astc_ldr_encode.h

@@ -143,7 +143,7 @@ namespace astc_ldr {
 	bool compress_image(
 		const image& orig_img, uint8_vec &comp_data, vector2D<astc_helpers::log_astc_block>& coded_blocks,
 		const astc_ldr_encode_config& global_cfg,
-		job_pool& job_pool);
+		job_pool_base& job_pool);
 
 	void downsample_weight_residual_grid(
 		const float* pMatrix_weights,

encoder/basisu_comp.h

@@ -934,7 +934,7 @@ namespace basisu
 		param<int> m_xubc7_bc7e_scalar_level; // bc7e_scalar quality level, clamped to [BC7E_SCALAR_MIN_LEVEL, BC7E_SCALAR_MAX_LEVEL]
 
 		// Job pool, MUST not be nullptr;
-		job_pool *m_pJob_pool;
+		job_pool_base *m_pJob_pool;
 
 		// Returns the current format mode as set by set_format_mode() above.
 		// Because of backwards API compatibility we don't use this directly yet, it's just here to aid the transition to the new API.

encoder/basisu_enc.h

@@ -818,22 +818,44 @@ namespace basisu
 
 #undef BASISU_GET_KEY
 
+	// Abstract job pool interface: lets the embedding application supply its own task
+	// scheduler to the encoders. add_job() queues a job for asynchronous execution,
+	// wait_for_all() blocks until every queued job has completed, and
+	// get_total_threads() returns the pool's parallelism hint (the TOTAL number of
+	// threads, including the calling thread, matching job_pool's constructor).
+	// job_pool below is the default implementation.
+	class job_pool_base
+	{
+		BASISU_NO_EQUALS_OR_COPY_CONSTRUCT(job_pool_base);
+
+	public:
+		job_pool_base() { }
+		virtual ~job_pool_base() { }
+
+		virtual void add_job(const std::function<void()>& job) = 0;
+		virtual void add_job(std::function<void()>&& job) = 0;
+
+		virtual void wait_for_all() = 0;
+
+		virtual size_t get_total_threads() const = 0;
+	};
+
 	// Very simple job pool with no dependencies.
-	class job_pool
+	class job_pool : public job_pool_base
 	{
 		BASISU_NO_EQUALS_OR_COPY_CONSTRUCT(job_pool);
 
 	public:
 		// num_threads is the TOTAL number of job pool threads, including the calling thread! So 2=1 new thread, 3=2 new threads, etc.
 		job_pool(uint32_t num_threads);
-		~job_pool();
+		virtual ~job_pool();
 
-		void add_job(const std::function<void()>& job);
-		void add_job(std::function<void()>&& job);
+		virtual void add_job(const std::function<void()>& job) override;
+		virtual void add_job(std::function<void()>&& job) override;
 
-		void wait_for_all();
+		virtual void wait_for_all() override;
 
-		size_t get_total_threads() const { return 1 + m_threads.size(); }
+		virtual size_t get_total_threads() const override { return 1 + m_threads.size(); }
 
 	private:
 		std::vector<std::thread> m_threads;
@@ -2088,7 +2110,7 @@ namespace basisu
 		uint32_t max_codebook_size, uint32_t max_parent_codebook_size,
 		basisu::vector<uint_vec>& codebook,
 		basisu::vector<uint_vec>& parent_codebook,
-		uint32_t max_threads, bool limit_clusterizers, job_pool *pJob_pool)
+		uint32_t max_threads, bool limit_clusterizers, job_pool_base *pJob_pool)
 	{
 		codebook.resize(0);
 		parent_codebook.resize(0);
@@ -2220,7 +2242,7 @@ namespace basisu
 		uint32_t max_codebook_size, uint32_t max_parent_codebook_size,
 		basisu::vector<uint_vec>& codebook,
 		basisu::vector<uint_vec>& parent_codebook,
-		uint32_t max_threads, job_pool *pJob_pool,
+		uint32_t max_threads, job_pool_base *pJob_pool,
 		bool even_odd_input_pairs_equal)
 	{
 		//typedef bit_hasher<typename Quantizer::training_vec_type> training_vec_bit_hasher;

encoder/basisu_frontend.h

@@ -109,7 +109,7 @@ namespace basisu
 
 			opencl_context_ptr m_pOpenCL_context;
 
-			job_pool *m_pJob_pool;
+			job_pool_base *m_pJob_pool;
 		};
 
 		bool init(const params &p);

encoder/basisu_uastc_enc.cpp

@@ -4105,7 +4105,7 @@ namespace basisu
 	// It only changes selectors and then updates the hints. It uses very approximate LZ bitprice estimation.
 	// There's A LOT that can be done better in here, but it's a start.
 	// One nice advantage of the method used here is that it works for any input, no matter which or how many modes it uses.
-	bool uastc_rdo(uint32_t num_blocks, basist::uastc_block* pBlocks, const color_rgba* pBlock_pixels, const uastc_rdo_params& params, uint32_t flags, job_pool* pJob_pool, uint32_t total_jobs)
+	bool uastc_rdo(uint32_t num_blocks, basist::uastc_block* pBlocks, const color_rgba* pBlock_pixels, const uastc_rdo_params& params, uint32_t flags, job_pool_base* pJob_pool, uint32_t total_jobs)
 	{
 		assert(params.m_max_allowed_rms_increase_ratio > 1.0f);
 		assert(params.m_lz_dict_size > 0);

encoder/basisu_uastc_enc.h

@@ -136,5 +136,5 @@ namespace basisu
 	// num_blocks, pBlocks: Number of blocks and pointer to UASTC blocks to process.
 	// pBlock_pixels: Pointer to an array of 4x4 blocks containing the original texture pixels. This is NOT a raster image, but a pointer to individual 4x4 blocks.
 	// flags: Pass in the same flags used to encode the UASTC blocks. The flags are used to reencode the transcode hints in the same way.
-	bool uastc_rdo(uint32_t num_blocks, basist::uastc_block* pBlocks, const color_rgba* pBlock_pixels, const uastc_rdo_params &params, uint32_t flags = cPackUASTCLevelDefault, job_pool* pJob_pool = nullptr, uint32_t total_jobs = 0);
+	bool uastc_rdo(uint32_t num_blocks, basist::uastc_block* pBlocks, const color_rgba* pBlock_pixels, const uastc_rdo_params &params, uint32_t flags = cPackUASTCLevelDefault, job_pool_base* pJob_pool = nullptr, uint32_t total_jobs = 0);
 } // namespace basisu

encoder/basisu_xbc7_encode.h

@@ -157,7 +157,7 @@ namespace xbc7 {
 		// the caller, basisu convention). The encoder always uses it; a pool of 1
 		// just runs serially. Pool size affects scheduling only, never the emitted
 		// bytes (the stripe count is independent of it).
-		job_pool* m_pJob_pool = nullptr;
+		job_pool_base* m_pJob_pool = nullptr;
 
 		// Stripe count for the main coding pass: 0 == auto (derived from image
 		// dimensions); else force exactly this many, clamped to