diff --git a/.github/workflows/linting.yml b/.github/workflows/linting.yml deleted file mode 100644 index 002eb4d..0000000 --- a/.github/workflows/linting.yml +++ /dev/null @@ -1,28 +0,0 @@ -name: Libact linting - -on: [push, pull_request] - -jobs: - lint: - runs-on: ${{ matrix.os }} - strategy: - fail-fast: false - matrix: - os: [ubuntu-latest] - python-version: ['3.9'] - - steps: - - uses: actions/checkout@v4 - - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 - with: - python-version: ${{ matrix.python-version }} - - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install pylint - - - name: Run pylint (errors only) - run: pylint --errors-only libact diff --git a/README.md b/README.md index 4d45839..7519c82 100644 --- a/README.md +++ b/README.md @@ -172,6 +172,7 @@ This ensures that `ninja`, `meson`, and other build tools remain available in yo | `VarianceReduction` | Variance | Minimizes output variance (requires C extension) | | `HintSVM` | SVM-based | SVM-guided active learning (requires C extension) | | `DensityWeightedMeta` | Density | Weights informativeness by density | +| `DiversityWeightedMeta` | Batch Diversity | Diversity-aware batch selection over any score-based strategy | | `DWUS` | Density + Uncertainty | Density-weighted uncertainty sampling | ## Available Models @@ -196,6 +197,34 @@ lb = lbr.label(X[ask_id]) # query the label of unlabeled data from labeler insta trn_ds.update(ask_id, lb) # update the dataset with newly queried data ``` +### Batch querying + +Sequential querying retrains the model once per label, which is +impractical for expensive models. Every score-based strategy also +supports querying a batch of distinct samples in one call: + +```python +ask_ids = qs.make_query_batch(10) # top-10 by acquisition score +labels = [lbr.label(X[i]) for i in ask_ids] +trn_ds.update_batch(ask_ids, labels) # one call, same per-entry callbacks +``` + +A plain top-k batch can contain redundant near-duplicate points. Wrap +any base strategy with `DiversityWeightedMeta` to make batches +diversity-aware: + +```python +from libact.models import LogisticRegression +from libact.query_strategies import DiversityWeightedMeta, UncertaintySampling + +qs = DiversityWeightedMeta( + trn_ds, + base_query_strategy=UncertaintySampling(trn_ds, model=LogisticRegression()), + lmbda=0.5, # 0 = pure top-k, 1 = pure farthest-point +) +ask_ids = qs.make_query_batch(10) +``` + ### Using CoreSet, BALD, and InformationDensity Strategies ```python @@ -254,6 +283,9 @@ Available examples: with other active learning algorithms. - [albl_new_strategies_benchmark](examples/albl_new_strategies_benchmark.py): Benchmarks CoreSet, BALD, and InformationDensity query strategies individually and combined via ALBL. + - [batch_query_plot](examples/batch_query_plot.py): This example compares sequential + active learning with batch-mode querying (`make_query_batch` / `update_batch`), + including diversity-aware batches via `DiversityWeightedMeta`. - [multilabel_plot](examples/multilabel_plot.py): This example compares the performance of algorithms under multilabel setting. - [alce_plot](examples/alce_plot.py): This example compares the performance of diff --git a/docs/libact.query_strategies.rst b/docs/libact.query_strategies.rst index 7d01f0e..1839b08 100644 --- a/docs/libact.query_strategies.rst +++ b/docs/libact.query_strategies.rst @@ -83,6 +83,14 @@ libact.query_strategies.coreset module :undoc-members: :show-inheritance: +libact.query_strategies.diversity_weighted_meta module +------------------------------------------------------- + +.. automodule:: libact.query_strategies.diversity_weighted_meta + :members: + :undoc-members: + :show-inheritance: + libact.query_strategies.epsilon_uncertainty_sampling module ----------------------------------------------------------- diff --git a/examples/batch_query_plot.py b/examples/batch_query_plot.py new file mode 100644 index 0000000..eb6a5be --- /dev/null +++ b/examples/batch_query_plot.py @@ -0,0 +1,136 @@ +#!/usr/bin/env python3 +""" +Batch-mode active learning example. + +Sequential active learning retrains the model after every single query +(make_query -> label -> update), which is impractical when training is +expensive. Batch querying selects batch_size points per round +(make_query_batch -> label -> update_batch), cutting the number of +training rounds by a factor of batch_size. + +This script compares, on the diabetes dataset: + +- sequential uncertainty sampling (one query per training round), +- plain top-k batch uncertainty sampling (one round per batch, but the + batch may contain redundant near-duplicate points), and +- DiversityWeightedMeta over the same base strategy (one round per + batch, with diversity-aware batches). +""" + +import copy +import os + +import numpy as np +import matplotlib.pyplot as plt +try: + from sklearn.model_selection import train_test_split +except ImportError: + from sklearn.cross_validation import train_test_split + +# libact classes +from libact.base.dataset import Dataset, import_libsvm_sparse +from libact.models import LogisticRegression +from libact.query_strategies import DiversityWeightedMeta, UncertaintySampling +from libact.labelers import IdealLabeler + + +def run_sequential(trn_ds, tst_ds, lbr, model, qs, quota): + """One query per round: quota training rounds in total.""" + n_labeled_axis, E_out = [], [] + + for _ in range(quota): + ask_id = qs.make_query() + lb = lbr.label(trn_ds.data[ask_id][0]) + trn_ds.update(ask_id, lb) + + model.train(trn_ds) + n_labeled_axis.append(trn_ds.len_labeled()) + E_out.append(1 - model.score(tst_ds)) + + return n_labeled_axis, E_out + + +def run_batch(trn_ds, tst_ds, lbr, model, qs, quota, batch_size): + """batch_size queries per round: quota/batch_size training rounds.""" + n_labeled_axis, E_out = [], [] + + for _ in range(quota // batch_size): + ask_ids = qs.make_query_batch(batch_size) + labels = [lbr.label(trn_ds.data[ask_id][0]) for ask_id in ask_ids] + trn_ds.update_batch(ask_ids, labels) + + model.train(trn_ds) + n_labeled_axis.append(trn_ds.len_labeled()) + E_out.append(1 - model.score(tst_ds)) + + return n_labeled_axis, E_out + + +def split_train_test(dataset_filepath, test_size, n_labeled): + X, y = import_libsvm_sparse(dataset_filepath).format_sklearn() + + X_train, X_test, y_train, y_test = \ + train_test_split(X, y, test_size=test_size) + trn_ds = Dataset(X_train, np.concatenate( + [y_train[:n_labeled], [None] * (len(y_train) - n_labeled)])) + tst_ds = Dataset(X_test, y_test) + fully_labeled_trn_ds = Dataset(X_train, y_train) + + return trn_ds, tst_ds, fully_labeled_trn_ds + + +def main(): + dataset_filepath = os.path.join( + os.path.dirname(os.path.realpath(__file__)), 'diabetes.txt') + test_size = 0.33 # fraction of samples assigned to the test set + n_labeled = 10 # number of samples that are initially labeled + quota = 120 # number of samples to query in total + batch_size = 10 # number of samples per batch query + + trn_ds, tst_ds, fully_labeled_trn_ds = \ + split_train_test(dataset_filepath, test_size, n_labeled) + trn_ds2 = copy.deepcopy(trn_ds) + trn_ds3 = copy.deepcopy(trn_ds) + lbr = IdealLabeler(fully_labeled_trn_ds) + + # 1) Sequential uncertainty sampling: one training round per label. + qs1 = UncertaintySampling(trn_ds, method='lc', model=LogisticRegression()) + n1, E1 = run_sequential( + trn_ds, tst_ds, lbr, LogisticRegression(), qs1, quota) + print('sequential US : %3d training rounds for %d labels' + % (quota, quota)) + + # 2) Plain top-k batch: batch_size labels per training round. The + # batch may contain redundant near-duplicates. + qs2 = UncertaintySampling(trn_ds2, method='lc', model=LogisticRegression()) + n2, E2 = run_batch( + trn_ds2, tst_ds, lbr, LogisticRegression(), qs2, quota, batch_size) + print('top-k batch US : %3d training rounds for %d labels' + % (quota // batch_size, quota)) + + # 3) Diversity-aware batches over the same base strategy. + qs3 = DiversityWeightedMeta( + trn_ds3, + base_query_strategy=UncertaintySampling( + trn_ds3, method='lc', model=LogisticRegression()), + lmbda=0.5, + random_state=1126, + ) + n3, E3 = run_batch( + trn_ds3, tst_ds, lbr, LogisticRegression(), qs3, quota, batch_size) + print('diversity batch US : %3d training rounds for %d labels' + % (quota // batch_size, quota)) + + plt.plot(n1, E1, 'g', label='sequential US') + plt.plot(n2, E2, 'b--o', label='top-k batch US') + plt.plot(n3, E3, 'r--s', label='DiversityWeightedMeta batch') + plt.xlabel('Number of labeled samples') + plt.ylabel('Test error') + plt.title('Sequential vs batch-mode active learning') + plt.legend(loc='upper center', bbox_to_anchor=(0.5, -0.05), + fancybox=True, shadow=True, ncol=3) + plt.show() + + +if __name__ == '__main__': + main() diff --git a/libact/base/dataset.py b/libact/base/dataset.py index 40aaa72..0ba9c16 100644 --- a/libact/base/dataset.py +++ b/libact/base/dataset.py @@ -147,6 +147,67 @@ def update(self, entry_id, new_label): for callback in self._update_callback: callback(entry_id, new_label) + def update_batch(self, entry_ids, labels): + """Update multiple entries with their labels in a single call. + + Labels are applied through :py:meth:`update` one entry at a time, + in the given order, so every registered callback observes exactly + the same incremental sequence of ``(entry_id, label)`` + notifications as the equivalent series of individual ``update()`` + calls. This keeps stateful observers correct (e.g. query + strategies that retrain models or maintain index bookkeeping in + their update hook). + + Note that some observers impose assumptions of their own on the + update stream; for instance ActiveLearningByLearning assumes each + update corresponds to an entry it has itself queried via + ``make_query()``, and updating other entries is unsupported — + exactly as with individual ``update()`` calls. + + Parameters + ---------- + entry_ids : array-like of int, shape (n_updates,) + Distinct entry ids of the samples to update. + + labels : sequence, shape (n_updates,) + Label for each entry. None marks an entry as unlabeled. + + Raises + ------ + ValueError + If entry_ids is not one-dimensional, labels is not a sequence, + their lengths differ, entry_ids contains duplicate entries, or + any entry id is out of range (not in ``[0, len(dataset))``). + """ + entry_ids = np.asarray(entry_ids) + if entry_ids.ndim != 1: + raise ValueError( + "entry_ids must be a one-dimensional array-like; for a " + "single entry use update(entry_id, label)") + try: + n_labels = len(labels) + except TypeError: + raise ValueError( + "labels must be a sequence of the same length as entry_ids") + if entry_ids.shape[0] != n_labels: + raise ValueError( + "entry_ids and labels must have the same length, got " + "%d and %d" % (entry_ids.shape[0], n_labels)) + if len(np.unique(entry_ids)) != entry_ids.shape[0]: + raise ValueError("entry_ids contains duplicate entries") + # Validate the ids are all in range *before* applying any update, so + # a bad id fails cleanly instead of leaving a partial update behind + # (and so negative ids are rejected rather than silently wrapping + # around to the wrong entry via numpy indexing). + if entry_ids.size and (entry_ids.min() < 0 or + entry_ids.max() >= len(self)): + raise ValueError( + "entry_ids must all be in [0, %d), got values in [%s, %s]" + % (len(self), entry_ids.min(), entry_ids.max())) + + for entry_id, label in zip(entry_ids, labels): + self.update(entry_id, label) + def on_update(self, callback): """ Add callback function to call when dataset updated. diff --git a/libact/base/interfaces.py b/libact/base/interfaces.py index 6924e50..ccb0fdc 100644 --- a/libact/base/interfaces.py +++ b/libact/base/interfaces.py @@ -2,10 +2,14 @@ Base interfaces for use in the package. The package works according to the interfaces defined below. """ +import numbers + from six import with_metaclass from abc import ABCMeta, abstractmethod +import numpy as np + class QueryStrategy(with_metaclass(ABCMeta, object)): @@ -61,6 +65,88 @@ def _get_scores(self): "This is required for batch mode and score-based composition." ) + @staticmethod + def _check_batch_size(batch_size, n_unlabeled): + """Validate make_query_batch arguments. + + Parameters + ---------- + batch_size : int + The requested batch size. + + n_unlabeled : int + Number of unlabeled samples currently in the pool. + + Raises + ------ + TypeError + If batch_size is not an integer (bool is rejected). + + ValueError + If batch_size < 1, the pool is empty, or batch_size exceeds the + pool size. + """ + if isinstance(batch_size, bool) or \ + not isinstance(batch_size, numbers.Integral): + raise TypeError( + "batch_size must be an integer, got %r" % (batch_size,)) + if batch_size < 1: + raise ValueError( + "batch_size must be at least 1, got %d" % batch_size) + if n_unlabeled == 0: + raise ValueError("No unlabeled samples available") + if batch_size > n_unlabeled: + raise ValueError( + "batch_size (%d) exceeds the number of unlabeled samples " + "(%d)" % (batch_size, n_unlabeled)) + + def make_query_batch(self, batch_size): + """Return a batch of distinct unlabeled samples to be queried. + + The default implementation ranks the unlabeled pool by the + acquisition scores from :py:meth:`_get_scores` and returns the + ``batch_size`` highest-scoring entry ids. Strategies override this + method when a faithful batch generalization differs from top-k + (e.g. iterative k-center for CoreSet, sampling without replacement + for RandomSampling). + + Unlike :py:meth:`make_query`, ties are broken deterministically + (stable sort, original pool order), so ``make_query_batch(1)`` may + differ from ``make_query()`` for strategies that randomize + tie-breaking. + + Parameters + ---------- + batch_size : int + Number of samples to query. Must satisfy + ``1 <= batch_size <= n_unlabeled``. No silent clamping is + performed. + + Returns + ------- + entry_ids : np.ndarray of int, shape (batch_size,) + Distinct entry ids of the samples to be queried, most preferred + first. + + Raises + ------ + TypeError + If batch_size is not an integer. + + ValueError + If batch_size < 1, batch_size exceeds the number of unlabeled + samples, or there are no unlabeled samples. + + NotImplementedError + If the strategy does not support per-sample scoring through + :py:meth:`_get_scores`. + """ + entry_ids, scores = self._get_scores() + self._check_batch_size(batch_size, len(entry_ids)) + + order = np.argsort(-np.asarray(scores, dtype=float), kind='stable') + return np.asarray(entry_ids)[order[:batch_size]] + @abstractmethod def make_query(self): """Return the index of the sample to be queried and labeled. Read-only. diff --git a/libact/base/tests/test_dataset.py b/libact/base/tests/test_dataset.py index e538696..9d162e9 100644 --- a/libact/base/tests/test_dataset.py +++ b/libact/base/tests/test_dataset.py @@ -107,5 +107,112 @@ def test_labeled_uniform_sample(self): dataset_s = dataset.labeled_uniform_sample(4, replace=False) +class TestUpdateBatchMethods(unittest.TestCase): + + initial_X = np.arange(15).reshape((5, 3)) + initial_y = np.array([1, 2, None, 1, None]) + + def setup_dataset(self): + return Dataset(np.copy(self.initial_X), np.copy(self.initial_y)) + + def test_callback_stream_matches_sequential(self): + # update_batch must produce exactly the same (entry_id, label) + # notification stream and final labels as the equivalent series + # of individual update() calls. + ds_batch = self.setup_dataset() + ds_seq = self.setup_dataset() + log_batch, log_seq = [], [] + ds_batch.on_update(lambda eid, lbl: log_batch.append((int(eid), lbl))) + ds_seq.on_update(lambda eid, lbl: log_seq.append((int(eid), lbl))) + + ds_batch.update_batch([2, 4], [3, 5]) + ds_seq.update(2, 3) + ds_seq.update(4, 5) + + self.assertEqual(log_batch, log_seq) + self.assertEqual(list(ds_batch.get_entries()[1]), + list(ds_seq.get_entries()[1])) + self.assertTrue(ds_batch.modified) + + def test_incremental_visibility(self): + # Each callback observes only the labels applied so far — exactly + # like sequential update() calls (labels are NOT pre-applied + # atomically, which would leak future labels to observers that + # read dataset state, e.g. committee retraining). + ds = self.setup_dataset() + seen = [] + ds.on_update(lambda eid, lbl: seen.append(ds.len_labeled())) + ds.update_batch([2, 4], [3, 5]) + self.assertEqual(seen, [4, 5]) + + def test_length_mismatch_raises(self): + ds = self.setup_dataset() + with self.assertRaises(ValueError): + ds.update_batch([2, 4], [3]) + + def test_scalar_entry_ids_raise_clearly(self): + # update(entry_id, label) takes scalars; update_batch must reject + # the analogous misuse with a clear error, not a cryptic one. + ds = self.setup_dataset() + with self.assertRaises(ValueError): + ds.update_batch(2, 3) + + def test_scalar_labels_raise_clearly(self): + ds = self.setup_dataset() + with self.assertRaises(ValueError): + ds.update_batch([2, 4], None) + + def test_multidimensional_entry_ids_raise(self): + ds = self.setup_dataset() + with self.assertRaises(ValueError): + ds.update_batch([[2, 4]], [3, 5]) + + def test_duplicate_entry_ids_raise(self): + ds = self.setup_dataset() + with self.assertRaises(ValueError): + ds.update_batch([2, 2], [3, 5]) + + def test_empty_is_noop(self): + ds = self.setup_dataset() + fired = [] + ds.on_update(lambda eid, lbl: fired.append(eid)) + ds.update_batch([], []) + self.assertEqual(fired, []) + self.assertEqual(ds.len_labeled(), 3) + + def test_none_label_unlabels(self): + ds = self.setup_dataset() + received = [] + ds.on_update(lambda eid, lbl: received.append((int(eid), lbl))) + ds.update_batch([0], [None]) + self.assertEqual(ds.len_labeled(), 2) + self.assertEqual(received, [(0, None)]) + + def test_numpy_array_inputs(self): + ds = self.setup_dataset() + ds.update_batch(np.array([2, 4]), np.array([3, 5])) + self.assertEqual(ds.len_labeled(), 5) + self.assertEqual(ds.get_entries()[1][2], 3) + self.assertEqual(ds.get_entries()[1][4], 5) + + def test_out_of_range_entry_ids_raise_before_any_update(self): + # An out-of-range id must be rejected up front, leaving the dataset + # untouched — not applied partially up to the offending entry. + ds = self.setup_dataset() + fired = [] + ds.on_update(lambda eid, lbl: fired.append(int(eid))) + with self.assertRaises(ValueError): + ds.update_batch([2, 4, 9999], [1, 0, 1]) + self.assertEqual(fired, []) + self.assertEqual(ds.len_labeled(), 3) + + def test_negative_entry_ids_raise(self): + # Negative ids must raise rather than silently wrapping around to + # the wrong entry via numpy indexing. + ds = self.setup_dataset() + with self.assertRaises(ValueError): + ds.update_batch([-1], [7]) + + if __name__ == '__main__': unittest.main() diff --git a/libact/models/multilabel/tests/test_binary_relevance.py b/libact/models/multilabel/tests/test_binary_relevance.py index 61403ae..4e526c4 100644 --- a/libact/models/multilabel/tests/test_binary_relevance.py +++ b/libact/models/multilabel/tests/test_binary_relevance.py @@ -25,8 +25,7 @@ def setUp(self): def test_binary_relevance_lr(self): br = BinaryRelevance( - base_clf=LogisticRegression(solver='liblinear', multi_class="ovr", - random_state=1126)) + base_clf=LogisticRegression(solver='liblinear', random_state=1126)) br.train(Dataset(self.X_train, self.Y_train)) br_pred_train = br.predict(self.X_train).astype(int) @@ -37,7 +36,7 @@ def test_binary_relevance_lr(self): for i in range(np.shape(self.Y_train)[1]): clf = sklearn.linear_model.LogisticRegression( - solver='liblinear', multi_class="ovr", random_state=1126) + solver='liblinear', random_state=1126) clf.fit(self.X_train, self.Y_train[:, i]) assert_array_almost_equal(clf.predict(self.X_train).astype(int), @@ -60,7 +59,7 @@ def test_binary_relevance_lr(self): def test_binary_relevance_parallel(self): br = BinaryRelevance(base_clf=LogisticRegression(solver='liblinear', - multi_class="ovr", random_state=1126), + random_state=1126), n_jobs=1) br.train(Dataset(self.X_train, self.Y_train)) br_par = BinaryRelevance( diff --git a/libact/models/tests/test_logistic_regression.py b/libact/models/tests/test_logistic_regression.py index 6d8bbff..6d63869 100644 --- a/libact/models/tests/test_logistic_regression.py +++ b/libact/models/tests/test_logistic_regression.py @@ -27,10 +27,9 @@ def setUp(self): train_test_split(X, y, test_size=0.3, random_state=1126) def test_logistic_regression(self): - clf = sklearn.linear_model.LogisticRegression( - solver='liblinear', multi_class="ovr") + clf = sklearn.linear_model.LogisticRegression(max_iter=1000) clf.fit(self.X_train, self.y_train) - lr = LogisticRegression(solver='liblinear', multi_class="ovr") + lr = LogisticRegression(max_iter=1000) lr.train(Dataset(self.X_train, self.y_train)) assert_array_equal( diff --git a/libact/models/tests/test_sklearn_adapter.py b/libact/models/tests/test_sklearn_adapter.py index 0737ef5..e215b6e 100644 --- a/libact/models/tests/test_sklearn_adapter.py +++ b/libact/models/tests/test_sklearn_adapter.py @@ -51,10 +51,8 @@ def check_proba(self, adapter, clf): def test_adapt_logistic_regression(self): adapter = SklearnProbaAdapter( - LogisticRegression(solver='liblinear', multi_class="ovr", - random_state=1126)) - clf = LogisticRegression(solver='liblinear', multi_class="ovr", - random_state=1126) + LogisticRegression(random_state=1126, max_iter=1000)) + clf = LogisticRegression(random_state=1126, max_iter=1000) self.check_functions(adapter, clf) def test_adapt_linear_svc(self): diff --git a/libact/query_strategies/__init__.py b/libact/query_strategies/__init__.py index 7004cf7..f8eb98f 100644 --- a/libact/query_strategies/__init__.py +++ b/libact/query_strategies/__init__.py @@ -20,6 +20,7 @@ from .information_density import InformationDensity # don't import c extentions when on readthedocs server from .density_weighted_meta import DensityWeightedMeta +from .diversity_weighted_meta import DiversityWeightedMeta if not ON_RTD: try: from ._variance_reduction import estVar @@ -54,4 +55,5 @@ 'UncertaintySampling', 'VarianceReduction', 'DensityWeightedMeta', + 'DiversityWeightedMeta', ] diff --git a/libact/query_strategies/active_learning_by_learning.py b/libact/query_strategies/active_learning_by_learning.py index 22b7bfa..c818545 100644 --- a/libact/query_strategies/active_learning_by_learning.py +++ b/libact/query_strategies/active_learning_by_learning.py @@ -214,6 +214,24 @@ def update(self, entry_id, label): self.W.append(1. / self.query_dist[ask_idx]) self.queried_hist_.append(entry_id) + def make_query_batch(self, batch_size): + """ALBL does not support batch queries. + + Raises + ------ + NotImplementedError + Always. ALBL is inherently sequential: each query requires the + reward feedback (the label) of the previous one to update its + multi-armed bandit. Call :py:meth:`make_query` and update the + dataset one sample at a time instead. + """ + raise NotImplementedError( + "ActiveLearningByLearning is inherently sequential: each query " + "requires the reward feedback of the previous one. Batch " + "querying is not supported; call make_query() and update the " + "dataset one sample at a time." + ) + @inherit_docstring_from(QueryStrategy) def make_query(self): dataset = self.dataset diff --git a/libact/query_strategies/coreset.py b/libact/query_strategies/coreset.py index 8093284..e5637fe 100644 --- a/libact/query_strategies/coreset.py +++ b/libact/query_strategies/coreset.py @@ -4,7 +4,7 @@ the unlabeled point farthest from all labeled points (greedy k-Center). """ import numpy as np -from scipy.spatial.distance import cdist +from sklearn.metrics.pairwise import pairwise_distances from libact.base.interfaces import QueryStrategy from libact.utils import inherit_docstring_from, seed_random_state @@ -24,7 +24,7 @@ class CoreSet(QueryStrategy): The dataset to query from. metric : str, optional (default='euclidean') - Distance metric passed to ``scipy.spatial.distance.cdist``. + Distance metric passed to ``sklearn.metrics.pairwise_distances``. Common options: 'euclidean', 'cosine', 'cityblock', 'minkowski'. transformer : object with transform method, optional (default=None) @@ -79,6 +79,14 @@ def __init__(self, dataset, **kwargs): random_state = kwargs.pop('random_state', None) self.random_state_ = seed_random_state(random_state) + def _transform(self, X): + """Apply the optional feature transformer.""" + if self.transformer is not None: + X = self.transformer.transform(X) + if isinstance(X, (list, tuple)): + X = np.asarray(X) + return X + def _get_scores(self): """Return min-distances to labeled set for all unlabeled samples. @@ -92,30 +100,82 @@ def _get_scores(self): """ dataset = self.dataset unlabeled_entry_ids, X_pool = dataset.get_unlabeled_entries() - X_pool = np.asarray(X_pool) if len(unlabeled_entry_ids) == 0: return np.array([], dtype=int), np.array([], dtype=float) - labeled_entries = dataset.get_labeled_entries() - X_labeled = np.asarray(labeled_entries[0]) + X_labeled, _ = dataset.get_labeled_entries() - if len(X_labeled) == 0: + if X_labeled.shape[0] == 0: return np.asarray(unlabeled_entry_ids), \ np.full(len(unlabeled_entry_ids), float('inf')) - if self.transformer is not None: - X_pool_t = np.asarray(self.transformer.transform(X_pool)) - X_labeled_t = np.asarray(self.transformer.transform(X_labeled)) - else: - X_pool_t = X_pool - X_labeled_t = X_labeled + X_pool_t = self._transform(X_pool) + X_labeled_t = self._transform(X_labeled) - dist_matrix = cdist(X_pool_t, X_labeled_t, metric=self.metric) + # pairwise_distances handles both dense and sparse feature matrices + dist_matrix = pairwise_distances( + X_pool_t, X_labeled_t, metric=self.metric) min_distances = np.min(dist_matrix, axis=1) return np.asarray(unlabeled_entry_ids), min_distances + def make_query_batch(self, batch_size): + """Select a batch with the true greedy k-Center algorithm. + + Unlike the default top-k of :py:meth:`_get_scores` (which can + return a cluster of mutually close points that are all far from + the labeled set), this recomputes the min-distance after every + pick: each selected point joins the covered set, so the next pick + maximizes the distance to the union of the labeled set and the + already-selected batch. This is the batch algorithm of Sener & + Savarese (2018). + + Parameters + ---------- + batch_size : int + Number of samples to query. Must satisfy + ``1 <= batch_size <= n_unlabeled``. + + Returns + ------- + entry_ids : np.ndarray of int, shape (batch_size,) + Distinct entry ids in selection order. + """ + dataset = self.dataset + unlabeled_entry_ids, X_pool = dataset.get_unlabeled_entries() + n_unlabeled = len(unlabeled_entry_ids) + self._check_batch_size(batch_size, n_unlabeled) + + X_labeled, _ = dataset.get_labeled_entries() + X_pool_t = self._transform(X_pool) + + selected = [] + if X_labeled.shape[0] == 0: + # No labeled data: seed the batch with a random pick, mirroring + # make_query's random fallback, then proceed greedily. + first = self.random_state_.randint(0, n_unlabeled) + selected.append(first) + min_distances = pairwise_distances( + X_pool_t, X_pool_t[[first]], metric=self.metric).ravel() + else: + X_labeled_t = self._transform(X_labeled) + min_distances = np.min(pairwise_distances( + X_pool_t, X_labeled_t, metric=self.metric), axis=1) + + while len(selected) < batch_size: + masked = min_distances.copy() + masked[selected] = -np.inf + candidates = np.where(np.isclose(masked, np.max(masked)))[0] + pick = self.random_state_.choice(candidates) + selected.append(pick) + # The picked point now covers its neighborhood. + new_distances = pairwise_distances( + X_pool_t, X_pool_t[[pick]], metric=self.metric).ravel() + min_distances = np.minimum(min_distances, new_distances) + + return np.asarray(unlabeled_entry_ids)[selected] + @inherit_docstring_from(QueryStrategy) def make_query(self): unlabeled_entry_ids, min_distances = self._get_scores() diff --git a/libact/query_strategies/diversity_weighted_meta.py b/libact/query_strategies/diversity_weighted_meta.py new file mode 100644 index 0000000..5bc005f --- /dev/null +++ b/libact/query_strategies/diversity_weighted_meta.py @@ -0,0 +1,286 @@ +"""Diversity Weighted Meta-Strategy + +This module implements a meta query strategy that turns any score-based +base strategy into a diversity-aware batch strategy: batches balance the +base strategy's own preference with within-batch diversity, so a batch is +not just the top-k points with near-duplicate redundancy. +""" +import numbers + +import numpy as np +from sklearn.metrics.pairwise import pairwise_distances + +from libact.base.interfaces import QueryStrategy +from libact.utils import inherit_docstring_from, seed_random_state + + +def _minmax_normalize(values): + """Monotone (direction-preserving) min-max rescale to [0, 1]. + + Constant input maps to all 0.5 (neutral). Non-finite values are + clipped into the finite range beforehand; if no finite value exists, + everything maps to 0.5. The transform never inverts or re-signs + values, so the input's ranking is preserved exactly. + """ + values = np.asarray(values, dtype=float) + finite = np.isfinite(values) + if not finite.any(): + return np.full(len(values), 0.5) + lo = values[finite].min() + hi = values[finite].max() + if np.isclose(hi, lo): + normalized = np.full(len(values), 0.5) + # keep -inf placeholders at the bottom of the scale + normalized[values == -np.inf] = 0.0 + return normalized + clipped = np.clip(values, lo, hi) + clipped = np.nan_to_num(clipped, nan=lo) + return (clipped - lo) / (hi - lo) + + +class DiversityWeightedMeta(QueryStrategy): + + r"""Diversity Weighted Meta-Strategy + + A meta algorithm that makes the batch queries of any score-based base + strategy diversity-aware. A plain top-k over acquisition scores tends + to select clusters of near-duplicate points that carry redundant + information; this wrapper greedily builds the batch so that each pick + balances the base strategy's preference against the distance to the + points already selected: + + .. math:: + + \mathbf{x}_{next} = \operatorname{argmax}_{\mathbf{x}} + (1 - \lambda) \cdot s(\mathbf{x}) + + \lambda \cdot d(\mathbf{x}) + + where :math:`s(\mathbf{x})` is the min-max normalized acquisition + score from the base strategy's ``_get_scores()`` and + :math:`d(\mathbf{x})` is the min-max normalized distance from + :math:`\mathbf{x}` to the nearest already-selected batch member. The + first pick is always the base strategy's argmax, so a batch of one + matches sequential behavior. + + The score normalization is monotone and direction-preserving: the + wrapper follows the base strategy's own preference ranking (higher + score = preferred, i.e. what ``make_query()`` would pick), whatever + the semantics of the raw scores are. In particular, strategies whose + scores are not uncertainty-flavored (e.g. HintSVM's absolute decision + values) are handled correctly by construction — scores are never + re-interpreted, re-signed, or inverted. + + Parameters + ---------- + dataset : Dataset object + The dataset to query from. Must be the same instance the base + strategy operates on. + + base_query_strategy : :py:mod:`libact.query_strategies` object instance + The base strategy whose scores are diversified. Has to support the + ``_get_scores()`` method and share this dataset instance. + + lmbda : float, optional (default=0.5) + Trade-off between the base score and diversity, in [0, 1]. + ``lmbda=0`` reproduces the plain top-k ranking of the base scores; + ``lmbda=1`` performs a pure farthest-point traversal after the + first pick. + + metric : str, optional (default='euclidean') + Distance metric passed to ``sklearn.metrics.pairwise_distances`` + (handles both dense and sparse feature matrices). + + transformer : object with transform method, optional (default=None) + Optional feature transformer (e.g. encoder, embedding model). If + provided, distances are computed in the transformed space. Must + have a ``transform(X)`` method. + + candidate_pool_size : int, optional (default=None) + Performance cap: if set, the greedy selection is restricted to the + ``candidate_pool_size`` highest-scoring candidates (at least + ``batch_size``). ``None`` uses the full unlabeled pool (exact). + Setting it trades diversity coverage for speed on very large + pools. + + random_state : {int, np.random.RandomState instance, None}, optional (default=None) + Random state for tie-breaking reproducibility. + + Attributes + ---------- + base_query_strategy : QueryStrategy + The wrapped base strategy. + + random_state\_ : np.random.RandomState instance + The random number generator using. + + Examples + -------- + Here is an example of how to use DiversityWeightedMeta to query a + diverse batch from an uncertainty sampling base: + + .. code-block:: python + + from libact.query_strategies import ( + DiversityWeightedMeta, UncertaintySampling) + from libact.models import LogisticRegression + + qs = DiversityWeightedMeta( + dataset, + base_query_strategy=UncertaintySampling( + dataset, model=LogisticRegression()), + lmbda=0.5, + ) + ask_ids = qs.make_query_batch(10) + + References + ---------- + .. [1] Brinker, Klaus. "Incorporating diversity in active learning + with support vector machines." ICML 2003. + + .. [2] Sener, Ozan, and Silvio Savarese. "Active learning for + convolutional neural networks: A core-set approach." ICLR 2018. + """ + + def __init__(self, dataset, base_query_strategy, lmbda=0.5, + metric='euclidean', transformer=None, + candidate_pool_size=None, random_state=None): + super(DiversityWeightedMeta, self).__init__(dataset=dataset) + if not isinstance(base_query_strategy, QueryStrategy): + raise TypeError( + "'base_query_strategy' has to be an instance of " + "'QueryStrategy'" + ) + if base_query_strategy.dataset != self.dataset: + raise ValueError("base_query_strategy should share the same" + "dataset instance with DiversityWeightedMeta") + if not 0 <= lmbda <= 1: + raise ValueError("lmbda must be in [0, 1], got %r" % (lmbda,)) + if transformer is not None and not hasattr(transformer, 'transform'): + raise TypeError("transformer must have a 'transform' method") + if candidate_pool_size is not None: + if isinstance(candidate_pool_size, bool) or \ + not isinstance(candidate_pool_size, numbers.Integral): + raise TypeError( + "candidate_pool_size must be an integer or None, got %r" + % (candidate_pool_size,)) + if candidate_pool_size < 1: + raise ValueError( + "candidate_pool_size must be at least 1, got %d" + % candidate_pool_size) + + self.base_query_strategy = base_query_strategy + self.lmbda = lmbda + self.metric = metric + self.transformer = transformer + self.candidate_pool_size = candidate_pool_size + self.random_state_ = seed_random_state(random_state) + + @inherit_docstring_from(QueryStrategy) + def update(self, entry_id, label): + # Stateless wrapper; the base strategy receives its own + # notification through the dataset's callback registry. + pass + + def _get_scores(self): + """Delegate to the base strategy (rank-faithful passthrough). + + Returns + ------- + entry_ids : np.ndarray, shape (n_unlabeled,) + Global entry IDs of unlabeled samples. + scores : np.ndarray, shape (n_unlabeled,) + The base strategy's acquisition scores, untouched. + """ + return self.base_query_strategy._get_scores() + + @inherit_docstring_from(QueryStrategy) + def make_query(self): + entry_ids, scores = self._get_scores() + + if len(entry_ids) == 0: + raise ValueError("No unlabeled samples available") + + candidates = np.where(np.isclose(scores, np.max(scores)))[0] + return entry_ids[self.random_state_.choice(candidates)] + + def make_query_batch(self, batch_size): + """Select a batch balancing base-strategy preference and diversity. + + Greedy selection: the first pick is the base strategy's argmax + (ties randomized); each subsequent pick maximizes + ``(1 - lmbda) * s + lmbda * d`` where ``s`` is the min-max + normalized base score and ``d`` the min-max normalized distance to + the nearest already-selected batch member. + + Parameters + ---------- + batch_size : int + Number of samples to query. Must satisfy + ``1 <= batch_size <= n_unlabeled``. + + Returns + ------- + entry_ids : np.ndarray of int, shape (batch_size,) + Distinct entry ids in selection order. + """ + entry_ids, scores = self._get_scores() + entry_ids = np.asarray(entry_ids) + scores = np.asarray(scores, dtype=float) + self._check_batch_size(batch_size, len(entry_ids)) + + # Optional performance cap: restrict the greedy selection to the + # highest-scoring candidates. + if self.candidate_pool_size is not None and \ + self.candidate_pool_size < len(entry_ids): + n_candidates = max(self.candidate_pool_size, batch_size) + keep = np.argsort(-scores, kind='stable')[:n_candidates] + entry_ids = entry_ids[keep] + scores = scores[keep] + + n_candidates = len(entry_ids) + if batch_size == n_candidates: + # Every candidate is selected; diversity cannot exclude anyone. + return entry_ids[np.argsort(-scores, kind='stable')] + + X, _ = self.dataset.get_entries() + X_candidates = X[entry_ids] + if self.transformer is not None: + X_candidates = self.transformer.transform(X_candidates) + if isinstance(X_candidates, (list, tuple)): + X_candidates = np.asarray(X_candidates) + + s_norm = _minmax_normalize(scores) + + # First pick: the base strategy's argmax, ties randomized + # (matches make_query). + candidates = np.where(np.isclose(scores, np.max(scores)))[0] + first = self.random_state_.choice(candidates) + selected = [first] + remaining = np.ones(n_candidates, dtype=bool) + remaining[first] = False + min_distances = self._distances_to(X_candidates, first) + + for _ in range(batch_size - 1): + d_norm = _minmax_normalize( + np.where(remaining, min_distances, -np.inf)) + utility = (1 - self.lmbda) * s_norm + self.lmbda * d_norm + utility[~remaining] = -np.inf + candidates = np.where( + remaining & np.isclose(utility, np.max(utility)))[0] + pick = self.random_state_.choice(candidates) + selected.append(pick) + remaining[pick] = False + min_distances = np.minimum( + min_distances, self._distances_to(X_candidates, pick)) + + return entry_ids[np.asarray(selected)] + + def _distances_to(self, X, idx): + """Distances from every candidate row of X to row idx.""" + distances = pairwise_distances( + X, X[[idx]], metric=self.metric).ravel() + # NaN can occur for degenerate inputs (e.g. cosine with a zero + # vector); treat it as zero distance so such points are never + # rewarded for being "far". + return np.nan_to_num(distances, nan=0.0, + posinf=np.inf, neginf=-np.inf) diff --git a/libact/query_strategies/epsilon_uncertainty_sampling.py b/libact/query_strategies/epsilon_uncertainty_sampling.py index 9baf455..a9c5324 100644 --- a/libact/query_strategies/epsilon_uncertainty_sampling.py +++ b/libact/query_strategies/epsilon_uncertainty_sampling.py @@ -215,6 +215,50 @@ def make_query(self, return_score=False): else: return ask_id + def make_query_batch(self, batch_size): + """Epsilon-greedy batch query. + + Each of the ``batch_size`` slots independently explores with + probability epsilon: the number of random picks is drawn from + ``Binomial(batch_size, epsilon)``. The remaining slots take the + highest-uncertainty samples, and the random picks are then drawn + without replacement from the rest of the pool, so the batch always + contains exactly ``batch_size`` distinct entries. + + Parameters + ---------- + batch_size : int + Number of samples to query. Must satisfy + ``1 <= batch_size <= n_unlabeled``. + + Returns + ------- + entry_ids : np.ndarray of int, shape (batch_size,) + Distinct entry ids; the exploitation picks come first in + descending uncertainty, followed by the exploration picks. + """ + dataset = self.dataset + unlabeled_entry_ids, X_pool = dataset.get_unlabeled_entries() + n_unlabeled = len(unlabeled_entry_ids) + self._check_batch_size(batch_size, n_unlabeled) + + n_random = self.random_state_.binomial(batch_size, self.epsilon) + n_exploit = batch_size - n_random + + self.model.train(dataset) + scores = self._get_uncertainty_scores(np.asarray(X_pool)) + + order = np.argsort(-scores, kind='stable') + selected = order[:n_exploit] + if n_random > 0: + # Exploration picks come from the complement of the + # exploitation picks, guaranteeing batch_size distinct ids. + explore = self.random_state_.choice( + order[n_exploit:], size=n_random, replace=False) + selected = np.concatenate([selected, explore]) + + return np.asarray(unlabeled_entry_ids)[selected] + @inherit_docstring_from(QueryStrategy) def update(self, entry_id, label): self.model.train(self.dataset) diff --git a/libact/query_strategies/meson.build b/libact/query_strategies/meson.build index 7d72fe3..1e6b3e0 100644 --- a/libact/query_strategies/meson.build +++ b/libact/query_strategies/meson.build @@ -5,6 +5,7 @@ py_src = [ 'coreset.py', 'density_weighted_meta.py', 'density_weighted_uncertainty_sampling.py', + 'diversity_weighted_meta.py', 'epsilon_uncertainty_sampling.py', 'hintsvm.py', 'information_density.py', diff --git a/libact/query_strategies/multiclass/tests/test_iris.py b/libact/query_strategies/multiclass/tests/test_iris.py index c279797..6d2257b 100644 --- a/libact/query_strategies/multiclass/tests/test_iris.py +++ b/libact/query_strategies/multiclass/tests/test_iris.py @@ -7,9 +7,11 @@ from sklearn import datasets from sklearn.neighbors import KNeighborsClassifier from sklearn.linear_model import LinearRegression +from sklearn.linear_model import LogisticRegression as SkLogisticRegression +from sklearn.multiclass import OneVsRestClassifier from libact.base.dataset import Dataset -from libact.models import LogisticRegression +from libact.models import SklearnProbaAdapter from libact.query_strategies.multiclass import ActiveLearningWithCostEmbedding as ALCE from libact.query_strategies.multiclass import EER from ...tests.utils import run_qs @@ -56,7 +58,8 @@ def test_eer(self): ds = Dataset(self.X + self.X_pool, self.y[:3] + [None for _ in range(len(self.X_pool))]) qs = EER(ds, - LogisticRegression(solver='liblinear', multi_class="ovr"), + SklearnProbaAdapter( + OneVsRestClassifier(SkLogisticRegression(solver='liblinear'))), random_state=1126) qseq = run_qs(ds, qs, self.y_truth, self.quota) assert_array_equal( @@ -66,7 +69,8 @@ def test_eer_01(self): ds = Dataset(self.X + self.X_pool, self.y[:3] + [None for _ in range(len(self.X_pool))]) qs = EER(ds, - LogisticRegression(solver='liblinear', multi_class="ovr"), + SklearnProbaAdapter( + OneVsRestClassifier(SkLogisticRegression(solver='liblinear'))), loss='01', random_state=1126) qseq = run_qs(ds, qs, self.y_truth, self.quota) assert_array_equal( diff --git a/libact/query_strategies/multilabel/cost_sensitive_reference_pair_encoding.py b/libact/query_strategies/multilabel/cost_sensitive_reference_pair_encoding.py index adcdc68..5148de2 100644 --- a/libact/query_strategies/multilabel/cost_sensitive_reference_pair_encoding.py +++ b/libact/query_strategies/multilabel/cost_sensitive_reference_pair_encoding.py @@ -53,9 +53,8 @@ class CostSensitiveReferencePairEncoding(QueryStrategy): from libact.utils.multilabel import pairwise_f1_score base_model = LogisticRegression( - solver='liblinear', multi_class="ovr") - model = BinaryRelevance(LogisticRegression(solver='liblinear', - multi_class="ovr")) + solver='liblinear') + model = BinaryRelevance(LogisticRegression(solver='liblinear')) qs = CostSensitiveReferencePairEncoding( dataset, scoring_fn=pairwise_f1_score, diff --git a/libact/query_strategies/multilabel/maximum_margin_reduction.py b/libact/query_strategies/multilabel/maximum_margin_reduction.py index c658d75..df77b7c 100644 --- a/libact/query_strategies/multilabel/maximum_margin_reduction.py +++ b/libact/query_strategies/multilabel/maximum_margin_reduction.py @@ -74,8 +74,7 @@ def __init__(self, *args, **kwargs): self.random_state_ = seed_random_state(random_state) self.logreg_param = kwargs.pop('logreg_param', - {'multi_class': 'multinomial', - 'solver': 'newton-cg', + {'solver': 'newton-cg', 'random_state': random_state}) self.logistic_regression_ = LogisticRegression(**self.logreg_param) diff --git a/libact/query_strategies/multilabel/tests/test_multilabel_realdata.py b/libact/query_strategies/multilabel/tests/test_multilabel_realdata.py index 3c14239..6098de9 100644 --- a/libact/query_strategies/multilabel/tests/test_multilabel_realdata.py +++ b/libact/query_strategies/multilabel/tests/test_multilabel_realdata.py @@ -55,7 +55,6 @@ def test_multilabel_with_auxiliary_learner_hlr(self): qs = MultilabelWithAuxiliaryLearner(trn_ds, major_learner=BinaryRelevance( LogisticRegression(solver='liblinear', - multi_class="ovr", random_state=1126)), auxiliary_learner=BinaryRelevance(SVM(gamma="auto")), criterion='hlr', @@ -68,8 +67,7 @@ def test_multilabel_with_auxiliary_learner_shlr(self): trn_ds = Dataset(self.X, self.y[:5] + [None] * (len(self.y) - 5)) qs = MultilabelWithAuxiliaryLearner(trn_ds, - major_learner=BinaryRelevance(LogisticRegression(solver='liblinear', - multi_class="ovr")), + major_learner=BinaryRelevance(LogisticRegression(solver='liblinear')), auxiliary_learner=BinaryRelevance(SVM(gamma="auto")), criterion='shlr', b=1., @@ -82,8 +80,7 @@ def test_multilabel_with_auxiliary_learner_mmr(self): trn_ds = Dataset(self.X, self.y[:5] + [None] * (len(self.y) - 5)) qs = MultilabelWithAuxiliaryLearner(trn_ds, - major_learner=BinaryRelevance(LogisticRegression(solver='liblinear', - multi_class="ovr")), + major_learner=BinaryRelevance(LogisticRegression(solver='liblinear')), auxiliary_learner=BinaryRelevance(SVM(gamma="auto")), criterion='mmr', random_state=1126) @@ -93,7 +90,7 @@ def test_multilabel_with_auxiliary_learner_mmr(self): def test_binary_minimization(self): trn_ds = Dataset(self.X, self.y[:5] + [None] * (len(self.y) - 5)) - qs = BinaryMinimization(trn_ds, LogisticRegression(solver='liblinear', multi_class="ovr"), + qs = BinaryMinimization(trn_ds, LogisticRegression(solver='liblinear'), random_state=1126) qseq = run_qs(trn_ds, qs, self.y, self.quota) assert_array_equal(qseq, @@ -102,7 +99,7 @@ def test_binary_minimization(self): def test_adaptive_active_learning(self): trn_ds = Dataset(self.X, self.y[:5] + [None] * (len(self.y) - 5)) qs = AdaptiveActiveLearning(trn_ds, - base_clf=LogisticRegression(solver='liblinear', multi_class="ovr"), n_jobs=-1, + base_clf=LogisticRegression(solver='liblinear'), n_jobs=-1, random_state=1126) qseq = run_qs(trn_ds, qs, self.y, self.quota) assert_array_equal(qseq, @@ -111,10 +108,9 @@ def test_adaptive_active_learning(self): def test_cost_sensitive_random_pair_encoding(self): trn_ds = Dataset(self.X, self.y[:5] + [None] * (len(self.y) - 5)) - model = BinaryRelevance(LogisticRegression(solver='liblinear', - multi_class="ovr")) + model = BinaryRelevance(LogisticRegression(solver='liblinear')) base_model = LogisticRegression( - solver='liblinear', multi_class="ovr", random_state=1126) + solver='liblinear', random_state=1126) qs = CostSensitiveReferencePairEncoding( trn_ds, scoring_fn=pairwise_f1_score, diff --git a/libact/query_strategies/random_sampling.py b/libact/query_strategies/random_sampling.py index 5691c78..52c198e 100644 --- a/libact/query_strategies/random_sampling.py +++ b/libact/query_strategies/random_sampling.py @@ -58,6 +58,29 @@ def _get_scores(self): scores = np.ones(len(unlabeled_entry_ids), dtype=float) return unlabeled_entry_ids, scores + def make_query_batch(self, batch_size): + """Return a uniformly random batch of distinct unlabeled samples. + + Overrides the default top-k behavior: with uniform scores a stable + top-k would always return the first ``batch_size`` pool entries, + which is not random sampling. + + Parameters + ---------- + batch_size : int + Number of samples to query. Must satisfy + ``1 <= batch_size <= n_unlabeled``. + + Returns + ------- + entry_ids : np.ndarray of int, shape (batch_size,) + Distinct entry ids sampled uniformly without replacement. + """ + unlabeled_entry_ids, _ = self.dataset.get_unlabeled_entries() + self._check_batch_size(batch_size, len(unlabeled_entry_ids)) + return self.random_state_.choice( + unlabeled_entry_ids, size=batch_size, replace=False) + @inherit_docstring_from(QueryStrategy) def make_query(self): dataset = self.dataset diff --git a/libact/query_strategies/tests/meson.build b/libact/query_strategies/tests/meson.build index 8c6bf3b..8dd12ce 100644 --- a/libact/query_strategies/tests/meson.build +++ b/libact/query_strategies/tests/meson.build @@ -3,11 +3,14 @@ py_src = [ 'test_bald.py', 'test_coreset.py', 'test_density_weighted_meta.py', + 'test_diversity_weighted_meta.py', 'test_epsilon_uncertainty_sampling.py', 'test_get_scores.py', 'test_hintsvm.py', 'test_information_density.py', + 'test_make_query_batch.py', 'test_quire.py', + 'test_update_batch.py', 'test_realdata.py', 'test_uncertainty_sampling.py', 'test_variance_reduction.py', diff --git a/libact/query_strategies/tests/test_density_weighted_meta.py b/libact/query_strategies/tests/test_density_weighted_meta.py index b528d58..c7d794b 100644 --- a/libact/query_strategies/tests/test_density_weighted_meta.py +++ b/libact/query_strategies/tests/test_density_weighted_meta.py @@ -28,7 +28,7 @@ def test_density_weighted_meta_uncertainty_lc(self): [self.y[:6], [None] * 14])) base_qs = UncertaintySampling( trn_ds, method='lc', - model=LogisticRegression(solver='liblinear', multi_class="ovr")) + model=LogisticRegression(solver='liblinear')) similarity_metric = cosine_similarity clustering_method = KMeans(n_clusters=3, random_state=1126) qs = DensityWeightedMeta( @@ -36,7 +36,7 @@ def test_density_weighted_meta_uncertainty_lc(self): similarity_metric=similarity_metric, clustering_method=clustering_method, beta=1.0, random_state=1126) - model = LogisticRegression(solver='liblinear', multi_class="ovr") + model = LogisticRegression(solver='liblinear') qseq = run_qs(trn_ds, qs, self.y, self.quota) assert_array_equal(qseq, np.array( [18, 13, 9, 12, 8, 16, 10, 19, 15, 7])) diff --git a/libact/query_strategies/tests/test_diversity_weighted_meta.py b/libact/query_strategies/tests/test_diversity_weighted_meta.py new file mode 100644 index 0000000..784415e --- /dev/null +++ b/libact/query_strategies/tests/test_diversity_weighted_meta.py @@ -0,0 +1,272 @@ +"""Tests for DiversityWeightedMeta. + +The fixtures pin two properties the wrapper exists to provide: +(1) within-batch diversity — near-duplicate points are not selected +together, with a control assertion proving the plain top-k of the base +strategy WOULD select them; and (2) rank-faithfulness — the base +strategy's preference ranking is followed exactly through a monotone +normalization, whatever the magnitude or sign of the raw scores +(HintSVM-style scores must never be re-interpreted). +""" +import unittest + +import numpy as np +import scipy.sparse as sp +from sklearn.linear_model import LogisticRegression + +from libact.base.dataset import Dataset +from libact.base.interfaces import QueryStrategy +from libact.models import SklearnProbaAdapter +from libact.query_strategies import DiversityWeightedMeta, UncertaintySampling +from libact.query_strategies.diversity_weighted_meta import _minmax_normalize + + +class MockScoreStrategy(QueryStrategy): + """Deterministic strategy with canned scores (unlabeled-pool order).""" + + def __init__(self, dataset, scores): + super(MockScoreStrategy, self).__init__(dataset) + self.scores = np.asarray(scores, dtype=float) + + def _get_scores(self): + entry_ids, _ = self.dataset.get_unlabeled_entries() + return np.asarray(entry_ids), self.scores[:len(entry_ids)].copy() + + def make_query(self): + entry_ids, scores = self._get_scores() + return entry_ids[int(np.argmax(scores))] + + +def _duplicate_cluster_fixture(sparse=False): + """Two near-duplicate top-scoring points plus a far third candidate. + + Entry 0 (10, 0) and entry 1 (10.01, 0) are near-duplicates holding the + two highest scores; entry 2 (0, 10) is high-scoring but far away; + entry 3 (0, 0) is a low-scoring filler. + """ + X = np.array([[10., 0.], [10.01, 0.], [0., 10.], [0., 0.]]) + if sparse: + X = sp.csr_matrix(X) + dataset = Dataset(X, [None, None, None, None]) + base = MockScoreStrategy(dataset, [10., 9.99, 9., 1.]) + return dataset, base + + +class TestDiversityWeightedMetaValidation(unittest.TestCase): + + def setUp(self): + X = np.arange(8).reshape(4, 2) + self.dataset = Dataset(X, [0, None, None, None]) + self.base = MockScoreStrategy(self.dataset, [3., 2., 1.]) + + def test_base_must_be_query_strategy(self): + with self.assertRaises(TypeError): + DiversityWeightedMeta(self.dataset, base_query_strategy=object()) + + def test_base_must_share_dataset(self): + other = Dataset(np.arange(8).reshape(4, 2), [0, None, None, None]) + with self.assertRaises(ValueError): + DiversityWeightedMeta(other, base_query_strategy=self.base) + + def test_lmbda_range(self): + for bad in [-0.1, 1.5]: + with self.assertRaises(ValueError): + DiversityWeightedMeta( + self.dataset, base_query_strategy=self.base, lmbda=bad) + + def test_transformer_must_have_transform(self): + with self.assertRaises(TypeError): + DiversityWeightedMeta( + self.dataset, base_query_strategy=self.base, + transformer=object()) + + def test_candidate_pool_size_type(self): + for bad in ['3', True, 2.0]: + with self.assertRaises(TypeError): + DiversityWeightedMeta( + self.dataset, base_query_strategy=self.base, + candidate_pool_size=bad) + with self.assertRaises(ValueError): + DiversityWeightedMeta( + self.dataset, base_query_strategy=self.base, + candidate_pool_size=0) + + +class TestRankFaithfulness(unittest.TestCase): + """The normalization must be monotone and direction-preserving.""" + + def test_minmax_preserves_ranking(self): + for scores in ([1000., 1., 0.5, 0.4], + [-5., -1., -0.5], + [3., 1., 2.], + [0.9, 0.1, 0.5]): + scores = np.asarray(scores) + np.testing.assert_array_equal( + np.argsort(_minmax_normalize(scores)), np.argsort(scores)) + + def test_minmax_constant_input(self): + np.testing.assert_array_equal( + _minmax_normalize([2., 2., 2.]), [0.5, 0.5, 0.5]) + + def test_inverted_semantics_scores_followed(self): + # HintSVM stand-in: raw scores are confidence-flavored, but the + # strategy's preference is still its argmax. The wrapper must + # follow that ranking without re-interpretation. + X = np.array([[0., 0.], [5., 5.], [9., 1.]]) + dataset = Dataset(X, [None, None, None]) + base = MockScoreStrategy(dataset, [0.9, 0.1, 0.5]) + qs = DiversityWeightedMeta( + dataset, base_query_strategy=base, lmbda=0.0, random_state=0) + + self.assertEqual(qs.make_query(), base.make_query()) + np.testing.assert_array_equal(qs.make_query_batch(3), [0, 2, 1]) + + def test_lmbda_zero_equals_base_topk(self): + X = np.arange(16, dtype=float).reshape(8, 2) + dataset = Dataset(X, [0, 1, 0, None, None, None, None, None]) + base = MockScoreStrategy(dataset, [5., 1., 4., 2., 3.]) + qs = DiversityWeightedMeta( + dataset, base_query_strategy=base, lmbda=0.0, random_state=0) + np.testing.assert_array_equal( + qs.make_query_batch(3), base.make_query_batch(3)) + + +class TestDiversity(unittest.TestCase): + + def test_avoids_near_duplicates_with_control(self): + dataset, base = _duplicate_cluster_fixture() + + # CONTROL: the plain top-k of the base strategy picks both + # near-duplicates. + np.testing.assert_array_equal(base.make_query_batch(2), [0, 1]) + + # The wrapper keeps the base argmax but replaces the duplicate + # with the far high-scoring point. + qs = DiversityWeightedMeta( + dataset, base_query_strategy=base, lmbda=0.5, random_state=0) + batch = qs.make_query_batch(2) + np.testing.assert_array_equal(batch, [0, 2]) + + def test_sparse_input(self): + dataset, base = _duplicate_cluster_fixture(sparse=True) + qs = DiversityWeightedMeta( + dataset, base_query_strategy=base, lmbda=0.5, random_state=0) + np.testing.assert_array_equal(qs.make_query_batch(2), [0, 2]) + + def test_constant_scores_spread_geometrically(self): + # With a constant-score base (e.g. RandomSampling), selection is + # driven purely by geometry: whatever the tie-randomized first + # pick, the second pick must be far from it. Looping over seeds + # exercises different tie-randomized first picks. + X = np.array([[0.], [0.01], [10.], [5.]]) + dataset = Dataset(X, [None] * 4) + for seed in range(8): + base = MockScoreStrategy(dataset, [1., 1., 1., 1.]) + qs = DiversityWeightedMeta( + dataset, base_query_strategy=base, lmbda=0.5, + random_state=seed) + batch = qs.make_query_batch(2) + self.assertGreater(abs(X[batch[0], 0] - X[batch[1], 0]), 4.9) + + def test_transformer_honored(self): + # A transformer collapsing all points to one location removes all + # geometric information, so the wrapper falls back to score order + # and selects the near-duplicate it would otherwise avoid. + class Collapse: + def transform(self, X): + n = X.shape[0] + return np.zeros((n, 2)) + + dataset, base = _duplicate_cluster_fixture() + qs = DiversityWeightedMeta( + dataset, base_query_strategy=base, lmbda=0.5, + transformer=Collapse(), random_state=0) + np.testing.assert_array_equal(qs.make_query_batch(2), [0, 1]) + + def test_candidate_pool_size_caps_greedy(self): + X = np.array([[0., 0.], [0.01, 0.], [0.02, 0.], [100., 0.]]) + dataset = Dataset(X, [None] * 4) + base = MockScoreStrategy(dataset, [10., 9., 8., 1.]) + + full = DiversityWeightedMeta( + dataset, base_query_strategy=base, lmbda=0.9, random_state=0) + np.testing.assert_array_equal(full.make_query_batch(2), [0, 3]) + + capped = DiversityWeightedMeta( + dataset, base_query_strategy=base, lmbda=0.9, + candidate_pool_size=3, random_state=0) + np.testing.assert_array_equal(capped.make_query_batch(2), [0, 2]) + + +class TestApiContract(unittest.TestCase): + + def test_make_query_returns_base_argmax_int(self): + X = np.arange(10, dtype=float).reshape(5, 2) + dataset = Dataset(X, [0, 1, None, None, None]) + base = MockScoreStrategy(dataset, [1., 5., 3.]) + qs = DiversityWeightedMeta( + dataset, base_query_strategy=base, random_state=0) + ask_id = qs.make_query() + self.assertIsInstance(ask_id, (int, np.integer)) + self.assertEqual(ask_id, 3) + + def test_get_scores_is_passthrough(self): + X = np.arange(10, dtype=float).reshape(5, 2) + dataset = Dataset(X, [0, 1, None, None, None]) + base = MockScoreStrategy(dataset, [1., 5., 3.]) + qs = DiversityWeightedMeta( + dataset, base_query_strategy=base, random_state=0) + ids_w, scores_w = qs._get_scores() + ids_b, scores_b = base._get_scores() + np.testing.assert_array_equal(ids_w, ids_b) + np.testing.assert_array_equal(scores_w, scores_b) + + def test_full_pool_batch_is_score_ordered(self): + dataset, base = _duplicate_cluster_fixture() + qs = DiversityWeightedMeta( + dataset, base_query_strategy=base, lmbda=0.5, random_state=0) + np.testing.assert_array_equal(qs.make_query_batch(4), [0, 1, 2, 3]) + + def test_error_paths(self): + dataset, base = _duplicate_cluster_fixture() + qs = DiversityWeightedMeta( + dataset, base_query_strategy=base, random_state=0) + with self.assertRaises(ValueError): + qs.make_query_batch(0) + with self.assertRaises(ValueError): + qs.make_query_batch(5) + with self.assertRaises(TypeError): + qs.make_query_batch(2.0) + + def test_empty_pool(self): + dataset = Dataset(np.arange(6).reshape(3, 2), [0, 1, 0]) + base = MockScoreStrategy(dataset, []) + qs = DiversityWeightedMeta( + dataset, base_query_strategy=base, random_state=0) + with self.assertRaises(ValueError): + qs.make_query() + with self.assertRaises(ValueError): + qs.make_query_batch(1) + + def test_with_real_base_strategy(self): + np.random.seed(1126) + X = np.random.randn(30, 5) + y = np.random.choice([0, 1], size=30) + dataset = Dataset(X, list(y[:10]) + [None] * 20) + base = UncertaintySampling( + dataset, + model=SklearnProbaAdapter( + LogisticRegression(max_iter=200, solver='liblinear') + ) + ) + qs = DiversityWeightedMeta( + dataset, base_query_strategy=base, random_state=42) + batch = qs.make_query_batch(5) + self.assertEqual(len(set(batch.tolist())), 5) + for eid in batch: + self.assertIsNone(dataset[eid][1]) + self.assertIsInstance(qs.make_query(), (int, np.integer)) + + +if __name__ == '__main__': + unittest.main() diff --git a/libact/query_strategies/tests/test_make_query_batch.py b/libact/query_strategies/tests/test_make_query_batch.py new file mode 100644 index 0000000..e03ffb4 --- /dev/null +++ b/libact/query_strategies/tests/test_make_query_batch.py @@ -0,0 +1,527 @@ +"""Tests for the make_query_batch() contract across all query strategies. + +Covers the default top-k implementation on the QueryStrategy base class, +the strategy-specific overrides (RandomSampling, CoreSet, +EpsilonUncertaintySampling), the explicit NotImplementedError overrides +(ActiveLearningByLearning, VarianceReduction), and the argument +validation / error paths. +""" +import unittest + +import numpy as np +import scipy.sparse as sp +from sklearn.linear_model import LogisticRegression + +from libact.base.dataset import Dataset +from libact.base.interfaces import QueryStrategy +from libact.models import SklearnProbaAdapter +from libact.query_strategies import ( + UncertaintySampling, + BALD, + CoreSet, + EpsilonUncertaintySampling, + InformationDensity, + DensityWeightedMeta, + DiversityWeightedMeta, + QueryByCommittee, + QUIRE, + RandomSampling, + ActiveLearningByLearning, +) + +# Try importing C-extension strategies +try: + from libact.query_strategies import HintSVM + HAS_HINTSVM = True +except (ImportError, ModuleNotFoundError): + HAS_HINTSVM = False + +try: + from libact.query_strategies import VarianceReduction + HAS_VARIANCE_REDUCTION = True +except (ImportError, ModuleNotFoundError): + HAS_VARIANCE_REDUCTION = False + + +class MockScoreStrategy(QueryStrategy): + """Deterministic strategy with canned scores for contract tests. + + Scores are given in unlabeled-pool order. make_query is a plain + argmax with no tie randomization, so make_query_batch(1) must equal + make_query() exactly. + """ + + def __init__(self, dataset, scores): + super(MockScoreStrategy, self).__init__(dataset) + self.scores = np.asarray(scores, dtype=float) + + def _get_scores(self): + entry_ids, _ = self.dataset.get_unlabeled_entries() + return np.asarray(entry_ids), self.scores[:len(entry_ids)].copy() + + def make_query(self): + entry_ids, scores = self._get_scores() + return entry_ids[int(np.argmax(scores))] + + +class TestMakeQueryBatchContract(unittest.TestCase): + """Verify the make_query_batch() contract across all strategies.""" + + def setUp(self): + np.random.seed(1126) + self.X = np.random.randn(30, 5) + self.y = np.random.choice([0, 1], size=30) + # First 10 labeled, rest unlabeled + y_partial = list(self.y[:10]) + [None] * 20 + self.dataset = Dataset(self.X, y_partial) + self.n_unlabeled = 20 + + def _make_dataset(self): + """Create a fresh dataset for strategies that need their own copy.""" + np.random.seed(1126) + X = np.random.randn(30, 5) + y = np.random.choice([0, 1], size=30) + y_partial = list(y[:10]) + [None] * 20 + return Dataset(X, y_partial) + + def _check_batch_contract(self, qs, batch_size=5): + """Verify the make_query_batch return format contract.""" + batch = qs.make_query_batch(batch_size) + + self.assertIsInstance(batch, np.ndarray) + self.assertTrue(np.issubdtype(batch.dtype, np.integer)) + self.assertEqual(len(batch), batch_size) + # All ids pairwise distinct + self.assertEqual(len(set(batch.tolist())), batch_size) + # All ids valid and unlabeled + for eid in batch: + self.assertTrue(0 <= eid < len(qs.dataset)) + self.assertIsNone(qs.dataset[eid][1]) + + # make_query must keep returning a single int (public contract) + ask_id = qs.make_query() + self.assertIsInstance(ask_id, (int, np.integer)) + self.assertNotIsInstance(ask_id, np.ndarray) + + def test_uncertainty_sampling(self): + qs = UncertaintySampling( + self.dataset, + model=SklearnProbaAdapter( + LogisticRegression(max_iter=200, solver='liblinear') + ) + ) + self._check_batch_contract(qs) + + def test_uncertainty_sampling_entropy(self): + qs = UncertaintySampling( + self.dataset, + model=SklearnProbaAdapter( + LogisticRegression(max_iter=200, solver='liblinear') + ), + method='entropy' + ) + self._check_batch_contract(qs) + + def test_bald(self): + qs = BALD( + self.dataset, + models=[ + SklearnProbaAdapter( + LogisticRegression(C=c, max_iter=200, solver='liblinear') + ) + for c in [0.01, 0.1, 1.0] + ], + random_state=42 + ) + self._check_batch_contract(qs) + + def test_coreset(self): + qs = CoreSet(self.dataset, random_state=42) + self._check_batch_contract(qs) + + def test_epsilon_uncertainty_sampling(self): + qs = EpsilonUncertaintySampling( + self.dataset, + model=SklearnProbaAdapter( + LogisticRegression(max_iter=200, solver='liblinear') + ), + epsilon=0.2, + random_state=42 + ) + self._check_batch_contract(qs) + + def test_information_density(self): + qs = InformationDensity( + self.dataset, + model=SklearnProbaAdapter( + LogisticRegression(max_iter=200, solver='liblinear') + ), + random_state=42 + ) + self._check_batch_contract(qs) + + def test_density_weighted_meta(self): + base_qs = UncertaintySampling( + self.dataset, + model=SklearnProbaAdapter( + LogisticRegression(max_iter=200, solver='liblinear') + ) + ) + qs = DensityWeightedMeta(self.dataset, base_qs, beta=1.0, + random_state=42) + self._check_batch_contract(qs) + + def test_diversity_weighted_meta(self): + base_qs = UncertaintySampling( + self.dataset, + model=SklearnProbaAdapter( + LogisticRegression(max_iter=200, solver='liblinear') + ) + ) + qs = DiversityWeightedMeta(self.dataset, base_qs, random_state=42) + self._check_batch_contract(qs) + + def test_query_by_committee(self): + qs = QueryByCommittee( + self.dataset, + models=[ + SklearnProbaAdapter( + LogisticRegression(C=c, max_iter=200, solver='liblinear') + ) + for c in [0.01, 0.1, 1.0] + ], + random_state=42 + ) + self._check_batch_contract(qs) + + def test_quire(self): + qs = QUIRE(self.dataset) + self._check_batch_contract(qs) + + def test_random_sampling(self): + qs = RandomSampling(self.dataset, random_state=42) + self._check_batch_contract(qs) + + @unittest.skipUnless(HAS_HINTSVM, "HintSVM C extension not compiled") + def test_hintsvm(self): + qs = HintSVM(self.dataset, random_state=42) + self._check_batch_contract(qs) + + def test_albl_raises(self): + """ALBL is inherently sequential and must reject batch queries.""" + ds = self._make_dataset() + qs1 = UncertaintySampling( + ds, + model=SklearnProbaAdapter( + LogisticRegression(C=1., max_iter=200, solver='liblinear') + ) + ) + albl = ActiveLearningByLearning( + ds, + query_strategies=[qs1], + T=20, + model=SklearnProbaAdapter( + LogisticRegression(max_iter=200, solver='liblinear') + ), + random_state=42 + ) + with self.assertRaises(NotImplementedError) as cm: + albl.make_query_batch(5) + self.assertIn("sequential", str(cm.exception)) + + @unittest.skipUnless(HAS_VARIANCE_REDUCTION, + "VarianceReduction C extension not compiled") + def test_variance_reduction_raises(self): + qs = VarianceReduction( + self.dataset, + model=SklearnProbaAdapter( + LogisticRegression(max_iter=200, solver='liblinear') + ) + ) + with self.assertRaises(NotImplementedError) as cm: + qs.make_query_batch(5) + self.assertIn("batch", str(cm.exception)) + + +class TestMakeQueryBatchDefaultImpl(unittest.TestCase): + """Pin the default implementation's exact ordering semantics with a + deterministic mock (no model training, no randomness).""" + + def setUp(self): + X = np.arange(16).reshape(8, 2) + # entries 0-2 labeled, 3-7 unlabeled + y = [0, 1, 0, None, None, None, None, None] + self.dataset = Dataset(X, y) + self.unlabeled_ids = [3, 4, 5, 6, 7] + + def test_descending_score_order(self): + qs = MockScoreStrategy(self.dataset, [5., 1., 4., 2., 3.]) + batch = qs.make_query_batch(3) + # ids by descending score: 5.0->3, 4.0->5, 3.0->7 + np.testing.assert_array_equal(batch, [3, 5, 7]) + + def test_stable_tie_order(self): + # Equal scores must keep original pool order (stable sort). + qs = MockScoreStrategy(self.dataset, [5., 5., 3., 3., 1.]) + batch = qs.make_query_batch(4) + np.testing.assert_array_equal(batch, [3, 4, 5, 6]) + + def test_negative_scores_rank_preserved(self): + # Direction-preserving for any magnitude/sign (HintSVM-style raw + # scores must never be re-interpreted). + qs = MockScoreStrategy(self.dataset, [-5., -1., -0.5, -2., -3.]) + batch = qs.make_query_batch(3) + # descending: -0.5->5, -1->4, -2->6 + np.testing.assert_array_equal(batch, [5, 4, 6]) + + def test_adversarial_magnitudes_rank_preserved(self): + qs = MockScoreStrategy(self.dataset, [1000., 1., 0.5, 0.4, 2.]) + batch = qs.make_query_batch(5) + np.testing.assert_array_equal(batch, [3, 7, 4, 5, 6]) + + def test_batch_of_one_equals_make_query(self): + # Holds exactly for strategies whose make_query is a plain argmax. + qs = MockScoreStrategy(self.dataset, [1., 9., 4., 2., 3.]) + self.assertEqual(qs.make_query_batch(1)[0], qs.make_query()) + + def test_full_pool_batch(self): + qs = MockScoreStrategy(self.dataset, [5., 1., 4., 2., 3.]) + batch = qs.make_query_batch(5) + np.testing.assert_array_equal(batch, [3, 5, 7, 6, 4]) + self.assertEqual(sorted(batch.tolist()), self.unlabeled_ids) + + +class TestMakeQueryBatchErrors(unittest.TestCase): + """Argument validation and error paths.""" + + def setUp(self): + X = np.arange(16).reshape(8, 2) + y = [0, 1, 0, None, None, None, None, None] + self.dataset = Dataset(X, y) + self.qs = MockScoreStrategy(self.dataset, [5., 1., 4., 2., 3.]) + + def test_zero_batch_size(self): + with self.assertRaises(ValueError): + self.qs.make_query_batch(0) + + def test_negative_batch_size(self): + with self.assertRaises(ValueError): + self.qs.make_query_batch(-3) + + def test_float_batch_size(self): + with self.assertRaises(TypeError): + self.qs.make_query_batch(2.0) + + def test_bool_batch_size(self): + with self.assertRaises(TypeError): + self.qs.make_query_batch(True) + + def test_string_batch_size(self): + with self.assertRaises(TypeError): + self.qs.make_query_batch('2') + + def test_none_batch_size(self): + with self.assertRaises(TypeError): + self.qs.make_query_batch(None) + + def test_numpy_integer_batch_size_accepted(self): + batch = self.qs.make_query_batch(np.int64(2)) + self.assertEqual(len(batch), 2) + + def test_batch_size_equal_to_pool_allowed(self): + batch = self.qs.make_query_batch(5) + self.assertEqual(len(batch), 5) + + def test_batch_size_exceeds_pool(self): + with self.assertRaises(ValueError) as cm: + self.qs.make_query_batch(6) + self.assertIn("exceeds", str(cm.exception)) + + def test_empty_pool(self): + full_ds = Dataset(np.arange(6).reshape(3, 2), [0, 1, 0]) + qs = MockScoreStrategy(full_ds, []) + with self.assertRaises(ValueError): + qs.make_query_batch(1) + + def test_no_get_scores_raises_not_implemented(self): + class NoScores(QueryStrategy): + def make_query(self): + return 0 + + qs = NoScores(self.dataset) + with self.assertRaises(NotImplementedError): + qs.make_query_batch(2) + + +class TestRandomSamplingBatch(unittest.TestCase): + """RandomSampling override: uniform sampling without replacement.""" + + def setUp(self): + X = np.arange(16).reshape(8, 2) + y = [0, 1, 0, None, None, None, None, None] + self.dataset = Dataset(X, y) + + def test_full_pool_is_permutation(self): + qs = RandomSampling(self.dataset, random_state=3) + batch = qs.make_query_batch(5) + self.assertEqual(sorted(batch.tolist()), [3, 4, 5, 6, 7]) + + def test_reproducible_with_seed(self): + qs1 = RandomSampling(self.dataset, random_state=7) + qs2 = RandomSampling(self.dataset, random_state=7) + np.testing.assert_array_equal( + qs1.make_query_batch(3), qs2.make_query_batch(3)) + + def test_not_always_first_k(self): + # A stable top-k of the uniform scores would always return the + # first pool entries; the override must actually randomize. + first_k = [3, 4] + differs = False + for seed in range(10): + qs = RandomSampling(self.dataset, random_state=seed) + if qs.make_query_batch(2).tolist() != first_k: + differs = True + break + self.assertTrue(differs) + + +class TestCoreSetBatch(unittest.TestCase): + """CoreSet override: true iterative k-center greedy.""" + + def test_greedy_diverges_from_static_topk(self): + # Labeled point at 0; unlabeled at 10, 10.1 (near-duplicates far + # from the label) and 5. Static top-2 of the min-distances picks + # both near-duplicates; true greedy covers 10.1 after picking + # 10.1's neighbor and takes 5 instead. + X = np.array([[0.], [10.], [10.1], [5.]]) + y = [0, None, None, None] + ds = Dataset(X, y) + qs = CoreSet(ds, random_state=0) + + entry_ids, scores = qs._get_scores() + static_top2 = np.asarray(entry_ids)[ + np.argsort(-scores, kind='stable')[:2]] + np.testing.assert_array_equal(static_top2, [2, 1]) + + batch = qs.make_query_batch(2) + np.testing.assert_array_equal(batch, [2, 3]) + + def test_greedy_sparse_input(self): + X = sp.csr_matrix(np.array([[0.], [10.], [10.1], [5.]])) + y = [0, None, None, None] + ds = Dataset(X, y) + qs = CoreSet(ds, random_state=0) + + batch = qs.make_query_batch(2) + np.testing.assert_array_equal(batch, [2, 3]) + + def test_get_scores_sparse_input(self): + # Regression: _get_scores used scipy cdist, which crashes on + # sparse input. pairwise_distances must give identical dense + # results. + X_dense = np.array([[0., 1.], [10., 0.], [3., 4.]]) + y = [0, None, None] + scores_dense = CoreSet(Dataset(X_dense, y))._get_scores()[1] + scores_sparse = CoreSet( + Dataset(sp.csr_matrix(X_dense), y))._get_scores()[1] + np.testing.assert_allclose(scores_dense, scores_sparse) + + def test_no_labeled_data_random_seed_then_greedy(self): + X = np.array([[0.], [1.], [100.]]) + ds = Dataset(X, [None, None, None]) + qs = CoreSet(ds, random_state=0) + + expected_first = np.random.RandomState(0).randint(0, 3) + batch = qs.make_query_batch(2) + self.assertEqual(batch[0], expected_first) + # The second pick maximizes the distance to the first. + distances = np.abs(X.ravel() - X[expected_first, 0]) + distances[expected_first] = -np.inf + self.assertEqual(batch[1], int(np.argmax(distances))) + + def test_transformer_honored_in_greedy_loop(self): + # Without the transformer the farthest point (euclidean, 2-D) is + # entry 1; projecting onto the first feature makes it entry 2. + X = np.array([[0., 0.], [1., 9.], [8., 0.], [7.5, 0.2]]) + y = [0, None, None, None] + + qs_plain = CoreSet(Dataset(X, y), random_state=0) + self.assertEqual(qs_plain.make_query_batch(1)[0], 1) + + class Project0: + def transform(self, X): + return np.asarray(X)[:, :1] + + qs_proj = CoreSet(Dataset(X, y), transformer=Project0(), + random_state=0) + self.assertEqual(qs_proj.make_query_batch(1)[0], 2) + + def test_metric_honored_in_greedy_loop(self): + # Euclidean-far but cosine-identical vs euclidean-near but + # cosine-orthogonal. + X = np.array([[1., 0.], [10., 0.], [0., 0.5]]) + y = [0, None, None] + + qs_euclid = CoreSet(Dataset(X, y), random_state=0) + self.assertEqual(qs_euclid.make_query_batch(1)[0], 1) + + qs_cosine = CoreSet(Dataset(X, y), metric='cosine', random_state=0) + self.assertEqual(qs_cosine.make_query_batch(1)[0], 2) + + +class TestEpsilonUncertaintySamplingBatch(unittest.TestCase): + """EpsilonUncertaintySampling override: binomial explore/exploit mix.""" + + def setUp(self): + np.random.seed(1126) + X = np.random.randn(30, 5) + y = np.random.choice([0, 1], size=30) + y_partial = list(y[:10]) + [None] * 20 + self.dataset = Dataset(X, y_partial) + + def _make_qs(self, epsilon, seed=42): + return EpsilonUncertaintySampling( + self.dataset, + model=SklearnProbaAdapter( + LogisticRegression(max_iter=200, solver='liblinear') + ), + epsilon=epsilon, + random_state=seed, + ) + + def test_epsilon_zero_is_exact_topk(self): + qs = self._make_qs(epsilon=0.0) + batch = qs.make_query_batch(5) + entry_ids, scores = qs._get_scores() + expected = np.asarray(entry_ids)[ + np.argsort(-scores, kind='stable')[:5]] + np.testing.assert_array_equal(batch, expected) + + def test_epsilon_one_is_valid_random_sample(self): + qs = self._make_qs(epsilon=1.0) + batch = qs.make_query_batch(5) + self.assertEqual(len(set(batch.tolist())), 5) + for eid in batch: + self.assertIsNone(self.dataset[eid][1]) + + def test_epsilon_one_reproducible(self): + b1 = self._make_qs(epsilon=1.0, seed=7).make_query_batch(5) + b2 = self._make_qs(epsilon=1.0, seed=7).make_query_batch(5) + np.testing.assert_array_equal(b1, b2) + + def test_always_exactly_batch_size_distinct(self): + for epsilon in [0.0, 0.3, 0.7, 1.0]: + qs = self._make_qs(epsilon=epsilon) + batch = qs.make_query_batch(8) + self.assertEqual(len(batch), 8) + self.assertEqual(len(set(batch.tolist())), 8) + + def test_full_pool_any_epsilon(self): + for epsilon in [0.0, 0.5, 1.0]: + qs = self._make_qs(epsilon=epsilon) + batch = qs.make_query_batch(20) + self.assertEqual(sorted(batch.tolist()), list(range(10, 30))) + + +if __name__ == '__main__': + unittest.main() diff --git a/libact/query_strategies/tests/test_realdata.py b/libact/query_strategies/tests/test_realdata.py index 066591f..039fb8c 100644 --- a/libact/query_strategies/tests/test_realdata.py +++ b/libact/query_strategies/tests/test_realdata.py @@ -73,9 +73,9 @@ def test_query_by_committee_vote(self): qs = QueryByCommittee( trn_ds, disagreement='vote', - models=[LogisticRegression(C=1.0, solver="liblinear", multi_class="ovr"), - LogisticRegression(C=0.01, solver="liblinear", multi_class="ovr"), - LogisticRegression(C=100, solver="liblinear", multi_class="ovr")], + models=[LogisticRegression(C=1.0, solver="liblinear"), + LogisticRegression(C=0.01, solver="liblinear"), + LogisticRegression(C=100, solver="liblinear")], random_state=1126) qseq = run_qs(trn_ds, qs, self.y, self.quota) assert_array_equal( @@ -88,9 +88,9 @@ def test_query_by_committee_kl_divergence(self): qs = QueryByCommittee( trn_ds, disagreement='kl_divergence', - models=[LogisticRegression(C=1.0, solver="liblinear", multi_class="ovr"), - LogisticRegression(C=0.01, solver="liblinear", multi_class="ovr"), - LogisticRegression(C=100, solver="liblinear", multi_class="ovr")], + models=[LogisticRegression(C=1.0, solver="liblinear"), + LogisticRegression(C=0.01, solver="liblinear"), + LogisticRegression(C=100, solver="liblinear")], random_state=1126) qseq = run_qs(trn_ds, qs, self.y, self.quota) assert_array_equal( @@ -102,7 +102,7 @@ def test_UcertaintySamplingLc(self): np.concatenate([self.y[:10], [None] * (len(self.y) - 10)])) qs = UncertaintySampling(trn_ds, method='lc', - model=LogisticRegression(solver="liblinear", multi_class="ovr")) + model=LogisticRegression(solver="liblinear")) qseq = run_qs(trn_ds, qs, self.y, self.quota) assert_array_equal( qseq, np.array([145, 66, 82, 37, 194, 60, 191, 211, 245, 131])) @@ -113,7 +113,7 @@ def test_UcertaintySamplingSm(self): np.concatenate([self.y[:10], [None] * (len(self.y) - 10)])) qs = UncertaintySampling(trn_ds, method='sm', - model=LogisticRegression(solver="liblinear", multi_class="ovr")) + model=LogisticRegression(solver="liblinear")) qseq = run_qs(trn_ds, qs, self.y, self.quota) assert_array_equal( qseq, np.array([145, 66, 82, 37, 194, 60, 191, 211, 245, 131])) @@ -124,7 +124,7 @@ def test_UcertaintySamplingEntropy(self): np.concatenate([self.y[:10], [None] * (len(self.y) - 10)])) qs = UncertaintySampling(trn_ds, method='entropy', - model=LogisticRegression(solver="liblinear", multi_class="ovr")) + model=LogisticRegression(solver="liblinear")) qseq = run_qs(trn_ds, qs, self.y, self.quota) assert_array_equal( qseq, np.array([145, 66, 82, 37, 194, 60, 191, 211, 245, 131])) @@ -138,9 +138,9 @@ def test_ActiveLearningByLearning(self): query_strategies=[ UncertaintySampling( trn_ds, - model=LogisticRegression(solver="liblinear", multi_class="ovr")), + model=LogisticRegression(solver="liblinear")), HintSVM(trn_ds, random_state=1126)], - model=LogisticRegression(solver="liblinear", multi_class="ovr"), + model=LogisticRegression(solver="liblinear"), random_state=1126) qseq = run_qs(trn_ds, qs, self.y, self.quota) assert_array_equal( diff --git a/libact/query_strategies/tests/test_uncertainty_sampling.py b/libact/query_strategies/tests/test_uncertainty_sampling.py index c57f131..cab5d18 100644 --- a/libact/query_strategies/tests/test_uncertainty_sampling.py +++ b/libact/query_strategies/tests/test_uncertainty_sampling.py @@ -39,8 +39,8 @@ def test_uncertainty_lc(self): trn_ds = init_toyexample(self.X, self.y) qs = UncertaintySampling( trn_ds, method='lc', - model=LogisticRegression(solver='liblinear', multi_class="ovr")) - model = LogisticRegression(solver='liblinear', multi_class="ovr") + model=LogisticRegression(solver='liblinear')) + model = LogisticRegression(solver='liblinear') qseq = run_qs(trn_ds, self.lbr, model, qs, self.quota) assert_array_equal(qseq, np.array([6, 7, 8, 9])) @@ -48,8 +48,8 @@ def test_uncertainty_sm(self): trn_ds = init_toyexample(self.X, self.y) qs = UncertaintySampling( trn_ds, method='sm', - model=LogisticRegression(solver='liblinear', multi_class="ovr")) - model = LogisticRegression(solver='liblinear', multi_class="ovr") + model=LogisticRegression(solver='liblinear')) + model = LogisticRegression(solver='liblinear') qseq = run_qs(trn_ds, self.lbr, model, qs, self.quota) assert_array_equal(qseq, np.array([6, 7, 8, 9])) @@ -57,8 +57,8 @@ def test_uncertainty_entropy(self): trn_ds = init_toyexample(self.X, self.y) qs = UncertaintySampling( trn_ds, method='entropy', - model=LogisticRegression(solver='liblinear', multi_class="ovr")) - model = LogisticRegression(solver='liblinear', multi_class="ovr") + model=LogisticRegression(solver='liblinear')) + model = LogisticRegression(solver='liblinear') qseq = run_qs(trn_ds, self.lbr, model, qs, self.quota) assert_array_equal(qseq, np.array([6, 7, 8, 9])) @@ -75,7 +75,7 @@ def test_uncertainty_entropy_exceptions(self): with self.assertRaises(TypeError): qs = UncertaintySampling( trn_ds, method='not_exist', - model=LogisticRegression(solver='liblinear', multi_class="ovr")) + model=LogisticRegression(solver='liblinear')) if __name__ == '__main__': diff --git a/libact/query_strategies/tests/test_update_batch.py b/libact/query_strategies/tests/test_update_batch.py new file mode 100644 index 0000000..1933fc0 --- /dev/null +++ b/libact/query_strategies/tests/test_update_batch.py @@ -0,0 +1,184 @@ +"""Integration tests for Dataset.update_batch with the real, stateful +observers that rely on the per-entry (entry_id, label) callback: QUIRE +(index bookkeeping), QueryByCommittee (committee retrain per label), +EpsilonUncertaintySampling (model retrain per label) and +ActiveLearningByLearning (bandit bookkeeping with preconditions). +""" +import unittest + +import numpy as np +from sklearn.linear_model import LogisticRegression + +from libact.base.dataset import Dataset +from libact.models import SklearnProbaAdapter +from libact.query_strategies import ( + ActiveLearningByLearning, + EpsilonUncertaintySampling, + QueryByCommittee, + QUIRE, + UncertaintySampling, +) + + +def _make_dataset(): + np.random.seed(1126) + X = np.random.randn(30, 5) + y = np.random.choice([0, 1], size=30) + return Dataset(X, list(y[:10]) + [None] * 20), y + + +class TestUpdateBatchWithQuire(unittest.TestCase): + + def test_end_state_matches_sequential(self): + ds_batch, y = _make_dataset() + ds_seq, _ = _make_dataset() + quire_batch = QUIRE(ds_batch) + quire_seq = QUIRE(ds_seq) + + ids = [12, 15, 20] + labels = [int(y[i]) for i in ids] + + ds_batch.update_batch(ids, labels) + for i, label in zip(ids, labels): + ds_seq.update(i, label) + + self.assertEqual(quire_batch.Uindex, quire_seq.Uindex) + self.assertEqual(quire_batch.Lindex, quire_seq.Lindex) + self.assertEqual(list(quire_batch.y), list(quire_seq.y)) + # The newly labeled ids moved from Uindex to Lindex. + for i in ids: + self.assertIn(i, quire_batch.Lindex) + self.assertNotIn(i, quire_batch.Uindex) + + +class TestUpdateBatchWithQueryByCommittee(unittest.TestCase): + + def test_committee_retrains_once_per_entry(self): + ds, y = _make_dataset() + np.random.seed(0) + qbc = QueryByCommittee( + ds, + models=[ + SklearnProbaAdapter( + LogisticRegression(C=c, max_iter=200, solver='liblinear') + ) + for c in [0.1, 1.0] + ], + random_state=0, + ) + + calls = [] + original = qbc.teach_students + + def spy(*args, **kwargs): + calls.append(1) + return original(*args, **kwargs) + + qbc.teach_students = spy + + ids = [11, 14, 17] + ds.update_batch(ids, [int(y[i]) for i in ids]) + # Exactly the same notification stream as 3 sequential updates. + self.assertEqual(len(calls), 3) + + +class TestUpdateBatchWithEpsilonUS(unittest.TestCase): + + def test_observer_retrains_without_error(self): + ds, y = _make_dataset() + qs = EpsilonUncertaintySampling( + ds, + model=SklearnProbaAdapter( + LogisticRegression(max_iter=200, solver='liblinear') + ), + epsilon=0.1, + random_state=0, + ) + ids = [13, 18] + ds.update_batch(ids, [int(y[i]) for i in ids]) + # The strategy stays usable after the bulk update. + self.assertIsInstance(qs.make_query(), (int, np.integer)) + + +class TestUpdateBatchWithALBL(unittest.TestCase): + + def _make_albl(self): + ds, y = _make_dataset() + albl = ActiveLearningByLearning( + ds, + query_strategies=[ + UncertaintySampling( + ds, + model=SklearnProbaAdapter( + LogisticRegression( + C=1., max_iter=200, solver='liblinear') + ) + ), + ], + T=10, + model=SklearnProbaAdapter( + LogisticRegression(max_iter=200, solver='liblinear') + ), + random_state=42, + ) + return ds, y, albl + + def test_queried_id_through_update_batch(self): + # The supported flow: ids obtained from ALBL's own make_query may + # be applied through update_batch, and ALBL's bookkeeping matches + # the sequential path. + ds_b, y_b, albl_b = self._make_albl() + ds_s, y_s, albl_s = self._make_albl() + + ask_b = albl_b.make_query() + ask_s = albl_s.make_query() + # Identical fixtures and seeds: both instances pick the same id. + self.assertEqual(ask_b, ask_s) + + ds_b.update_batch([ask_b], [int(y_b[ask_b])]) + ds_s.update(ask_s, int(y_s[ask_s])) + + self.assertEqual(albl_b.queried_hist_, albl_s.queried_hist_) + self.assertEqual(albl_b.W, albl_s.W) + + def test_update_before_make_query_fails_like_sequential(self): + # ALBL assumes each update corresponds to an entry it has itself + # queried via make_query() — before any query its query + # distribution is uninitialized and update() raises TypeError. + # ALBL does NOT otherwise guard against unqueried ids (see the + # parity test below); what update_batch must guarantee is that it + # fails exactly like the sequential update() path does. + ds_b, y_b, _albl_b = self._make_albl() + ds_s, y_s, _albl_s = self._make_albl() + with self.assertRaises(TypeError): + ds_b.update_batch([15], [int(y_b[15])]) + with self.assertRaises(TypeError): + ds_s.update(15, int(y_s[15])) + + def test_arbitrary_id_parity_with_sequential(self): + # Pre-existing ALBL behavior: after make_query(), updating an id + # ALBL did NOT query is accepted silently (the bandit bookkeeping + # uses whatever query distribution is current). That behavior is + # out of scope to change here — this test pins that update_batch + # reproduces the sequential path exactly, corrupt bookkeeping + # included. + ds_b, y_b, albl_b = self._make_albl() + ds_s, y_s, albl_s = self._make_albl() + + ask_b = albl_b.make_query() + ask_s = albl_s.make_query() + self.assertEqual(ask_b, ask_s) + + # Pick an unlabeled id that ALBL did not query. + unlabeled_ids = ds_b.get_unlabeled_entries()[0].tolist() + other = next(i for i in unlabeled_ids if i != ask_b) + + ds_b.update_batch([other], [int(y_b[other])]) + ds_s.update(other, int(y_s[other])) + + self.assertEqual(albl_b.queried_hist_, albl_s.queried_hist_) + self.assertEqual(albl_b.W, albl_s.W) + + +if __name__ == '__main__': + unittest.main() diff --git a/libact/query_strategies/tests/test_variance_reduction.py b/libact/query_strategies/tests/test_variance_reduction.py index a182136..a65dfc0 100644 --- a/libact/query_strategies/tests/test_variance_reduction.py +++ b/libact/query_strategies/tests/test_variance_reduction.py @@ -22,7 +22,7 @@ def test_variance_reduction(self): [None] * (len(self.y) - 2)])) qs = VarianceReduction( trn_ds, - model=LogisticRegression(solver='liblinear', multi_class="ovr"), + model=LogisticRegression(solver='liblinear'), sigma=0.1 ) qseq = run_qs(trn_ds, qs, self.y, self.quota) diff --git a/libact/query_strategies/variance_reduction.py b/libact/query_strategies/variance_reduction.py index e64d33a..6add852 100644 --- a/libact/query_strategies/variance_reduction.py +++ b/libact/query_strategies/variance_reduction.py @@ -74,6 +74,22 @@ def _get_scores(self): "for batch mode." ) + def make_query_batch(self, batch_size): + """VarianceReduction does not support batch queries. + + Raises + ------ + NotImplementedError + Always. The variance estimation is tightly coupled to the C + extension and provides no per-sample scoring; use + :py:meth:`make_query` instead. + """ + raise NotImplementedError( + "VarianceReduction does not support batch querying: its " + "computation is tightly coupled to the C extension and " + "provides no per-sample scoring. Use make_query() instead." + ) + @inherit_docstring_from(QueryStrategy) def make_query(self): Xlabeled, y = self.dataset.get_labeled_entries()