blueprint update: laws of sums of rewards

blueprint/lean_decls

@@ -100,6 +100,15 @@ ProbabilityTheory.HasSubgaussianMGF.add_of_indepFun
 ProbabilityTheory.HasSubgaussianMGF.measure_ge_le
 ProbabilityTheory.HasSubgaussianMGF.measure_sum_range_ge_le_of_iIndepFun
 ProbabilityTheory.HasSubgaussianMGF.measure_sum_le_sum_le'
+Bandits.ArrayModel.identDistrib_pullCount_prod_sumRewards
+Bandits.ArrayModel.identDistrib_sum_range_snd
+Bandits.ArrayModel.prob_pullCount_prod_sumRewards_mem_le
+Bandits.prob_pullCount_prod_sumRewards_mem_le
+Bandits.ArrayModel.prob_sumRewards_le_sumRewards_le
+Bandits.probReal_sumRewards_le_sumRewards_le
+Bandits.probReal_sum_le_sum_streamMeasure
+Bandits.prob_sum_le_sqrt_log
+Bandits.prob_sum_ge_sqrt_log
 Bandits.ETC.nextArm
 Bandits.etcAlgorithm
 Bandits.ETC.pullCount_of_ge

blueprint/src/appendix/conditional_independence.tex

@@ -7,7 +7,7 @@ \chapter{Conditional independence}
 If $X \ind Y \mid Z$, then $X \ind (Y, Z) \mid Z$.
 \end{lemma}
 
-\begin{proof}
+\begin{proof}\leanok
 
 \end{proof}
 

blueprint/src/chapters/bandit.tex

@@ -418,18 +418,14 @@ \subsection{Laws}
 \end{proof}
 
 
-\section{Alternative models: rewards indexed by time or pull count}\label{sec:alt_model}
 
-The description of the bandit model above considers that at time $t$, a reward $R_t$ is generated, depending on the arm $A_t$ pulled at that time.
-An alternative way to talk about that process is to imagine that there is a stream of rewards from each arm, and that the algorithm sees the first, then second, etc. reward from the arms at it pulls them.
-This uses the random variables $Y_{n, a}$ defined in Definition~\ref{def:rewardByCount}, which represent the $n^{th}$ reward obtained from arm $a$.
-We now describe the distribution of those rewards.
 
-The probability space $\Omega_{\mathcal{T}}$ has been augmented with $\Omega' = \mathbb{R}^{\mathbb{N} \times \mathcal{A}}$, on which we put the product measure $\bigotimes_{n \in \mathbb{N}, a \in \mathcal{A}} \nu(a)$.
-With that measure, the law of $Z_{n,a}$ is $\nu(a)$.
+\section{Law of the n\textsuperscript{th} pull}\label{sec:alt_model}
+
+This section describes the law of $Y_{n, a}$ the $n^{th}$ reward obtained from an arm $a \in \mathcal{A}$ (Definition~\ref{def:rewardByCount}).
 
-Our main goal in this section is to prove that $(Y_{n,a})_{n \in \mathbb{N}, a \in \mathcal{A}}$ and $(Z_{n,a})_{n \in \mathbb{N}, a \in \mathcal{A}}$ are identically distributed.
-This can be done by proving that for each $n$ and $a$, $Y_{n,a}$ and $Z_{n,a}$ are identically distributed (with law $\nu(a)$), and that the family $(Y_{n,a})_{n \in \mathbb{N}, a \in \mathcal{A}}$ is independent.
+We augment the probability space $\Omega$ on which we have an algorithm-environment sequence with $\Omega' = \mathbb{R}^{\mathbb{N} \times \mathcal{A}}$, on which we put the product measure $\bigotimes_{n \in \mathbb{N}, a \in \mathcal{A}} \nu(a)$.
+With that measure, the law of $Z_{n,a}$ is $\nu(a)$.
 
 
 \begin{lemma}\label{lem:measurable_comap_indicator_stepsUntil_eq}
@@ -525,116 +521,116 @@ \section{Alternative models: rewards indexed by time or pull count}\label{sec:al
 \end{proof}
 
 
-\begin{lemma}\label{lem:indepFun_rewardByCount_stepsUntil}
-  \uses{def:stepsUntil, def:rewardByCount, def:Bandit.measure}
-For $n > 0$, $Y_{n, a} \ind T_{n,a}$.
-\end{lemma}
+% \begin{lemma}\label{lem:indepFun_rewardByCount_stepsUntil}
+%   \uses{def:stepsUntil, def:rewardByCount, def:Bandit.measure}
+% For $n > 0$, $Y_{n, a} \ind T_{n,a}$.
+% \end{lemma}
 
-\begin{proof}
-  \uses{lem:condDistrib_rewardByCount_stepsUntil, lem:hasLaw_rewardByCount}
-It suffices to prove that $\mathcal{L}(Y_{n,a} \mid T_{n,a}) = \mathcal{L}(Y_{n,a})$.
+% \begin{proof}
+%   \uses{lem:condDistrib_rewardByCount_stepsUntil, lem:hasLaw_rewardByCount}
+% It suffices to prove that $\mathcal{L}(Y_{n,a} \mid T_{n,a}) = \mathcal{L}(Y_{n,a})$.
 
-By Lemma~\ref{lem:hasLaw_rewardByCount}, $\mathcal{L}(Y_{n,a}) = \nu(a)$.
-By Lemma~\ref{lem:condDistrib_rewardByCount_stepsUntil}, $\mathcal{L}(Y_{n,a} \mid T_{n,a}) = \nu(a)$.
-\end{proof}
+% By Lemma~\ref{lem:hasLaw_rewardByCount}, $\mathcal{L}(Y_{n,a}) = \nu(a)$.
+% By Lemma~\ref{lem:condDistrib_rewardByCount_stepsUntil}, $\mathcal{L}(Y_{n,a} \mid T_{n,a}) = \nu(a)$.
+% \end{proof}
 
 
-\begin{lemma}\label{lem:condIndepFun_rewardByCount_hist}
-  \uses{def:rewardByCount, def:stepsUntil, def:Bandit.measure}
-For $n > 0$, $Y_{n, a} \ind H_{T_{n,a}-1} \mid T_{n,a}$, in which $H_{\infty}$ is interpreted as the whole history.
-\end{lemma}
+% \begin{lemma}\label{lem:condIndepFun_rewardByCount_hist}
+%   \uses{def:rewardByCount, def:stepsUntil, def:Bandit.measure}
+% For $n > 0$, $Y_{n, a} \ind H_{T_{n,a}-1} \mid T_{n,a}$, in which $H_{\infty}$ is interpreted as the whole history.
+% \end{lemma}
 
-\begin{proof}
-  \uses{lem:condDistrib_rewardByCount_stepsUntil, lem:condIndepFun_reward_hist_action}
-By Lemma~\ref{lem:condDistrib_rewardByCount_stepsUntil}, $\mathcal{L}(Y_{n,a} \mid T_{n,a}) = \nu(a)$.
-We need to prove that $\mathcal{L}(Y_{n,a} \mid H_{T_{n,a}-1}, T_{n,a}) = \nu(a)$.
-By Lemma~\ref{lem:stepsUntil_basic}, if $T_{n,a} = t \in \mathbb{N}$, then $A_t = a$.
-We get
-\begin{align*}
-  \mathcal{L}(Y_{n,a} \mid H_{T_{n,a}-1}, T_{n,a} = t)
-  &= \mathcal{L}(R_t \mid H_{t-1}, T_{n,a} = t, A_t = a)
-  \: .
-\end{align*}
-Then, $\mathbb{I}\{T_{n,a} = t\}$ is a function of $(H_{t-1}, A_t)$ by Lemma~\ref{lem:measurable_comap_indicator_stepsUntil_eq}, such that
+% \begin{proof}
+%   \uses{lem:condDistrib_rewardByCount_stepsUntil, lem:condIndepFun_reward_hist_action}
+% By Lemma~\ref{lem:condDistrib_rewardByCount_stepsUntil}, $\mathcal{L}(Y_{n,a} \mid T_{n,a}) = \nu(a)$.
+% We need to prove that $\mathcal{L}(Y_{n,a} \mid H_{T_{n,a}-1}, T_{n,a}) = \nu(a)$.
+% By Lemma~\ref{lem:stepsUntil_basic}, if $T_{n,a} = t \in \mathbb{N}$, then $A_t = a$.
+% We get
+% \begin{align*}
+%   \mathcal{L}(Y_{n,a} \mid H_{T_{n,a}-1}, T_{n,a} = t)
+%   &= \mathcal{L}(R_t \mid H_{t-1}, T_{n,a} = t, A_t = a)
+%   \: .
+% \end{align*}
+% Then, $\mathbb{I}\{T_{n,a} = t\}$ is a function of $(H_{t-1}, A_t)$ by Lemma~\ref{lem:measurable_comap_indicator_stepsUntil_eq}, such that
 
-\begin{align*}
-  \mathcal{L}(R_t \mid H_{t-1}, T_{n,a} = t, A_t = a)
-  &= \mathcal{L}(R_t \mid H_{t-1}, A_t = a)
-  \: .
-\end{align*}
-Thus, using Lemma~\ref{lem:condIndepFun_reward_hist_action}, we have
-\begin{align*}
-  \mathcal{L}(R_t \mid H_{t-1}, A_t = a)
-  = \nu(a)
-  \: .
-\end{align*}
+% \begin{align*}
+%   \mathcal{L}(R_t \mid H_{t-1}, T_{n,a} = t, A_t = a)
+%   &= \mathcal{L}(R_t \mid H_{t-1}, A_t = a)
+%   \: .
+% \end{align*}
+% Thus, using Lemma~\ref{lem:condIndepFun_reward_hist_action}, we have
+% \begin{align*}
+%   \mathcal{L}(R_t \mid H_{t-1}, A_t = a)
+%   = \nu(a)
+%   \: .
+% \end{align*}
 
-If $T_{n,a} = \infty$, then $\mathcal{L}(Y_{n,a} \mid H_{T_{n,a}-1}, T_{n,a} = \infty) = \mathcal{L}(Z_{n,a} \mid H_{\infty}, T_{n,a} = \infty) = \nu(a)$ by independence of $Z_{n,a}$ from the history and $T_{n,a}$.
+% If $T_{n,a} = \infty$, then $\mathcal{L}(Y_{n,a} \mid H_{T_{n,a}-1}, T_{n,a} = \infty) = \mathcal{L}(Z_{n,a} \mid H_{\infty}, T_{n,a} = \infty) = \nu(a)$ by independence of $Z_{n,a}$ from the history and $T_{n,a}$.
 
-\end{proof}
+% \end{proof}
 
 
-\begin{lemma}\label{lem:indepFun_rewardByCount_hist_stepsUntil}
-  \uses{def:rewardByCount, def:stepsUntil, def:Bandit.measure}
-For $n > 0$, $Y_{n, a} \ind (H_{T_{n,a}-1}, T_{n,a})$.
-\end{lemma}
+% \begin{lemma}\label{lem:indepFun_rewardByCount_hist_stepsUntil}
+%   \uses{def:rewardByCount, def:stepsUntil, def:Bandit.measure}
+% For $n > 0$, $Y_{n, a} \ind (H_{T_{n,a}-1}, T_{n,a})$.
+% \end{lemma}
 
-\begin{proof}
-  \uses{lem:condIndepFun_rewardByCount_hist}
-By Lemma~\ref{lem:condIndepFun_rewardByCount_hist}, $Y_{n, a} \ind H_{T_{n,a}-1} \mid T_{n,a}$.
-By Lemma~\ref{lem:indepFun_rewardByCount_stepsUntil}, $Y_{n, a} \ind T_{n,a}$.
-By the contraction property of conditional independence (Lemma~\ref{lem:indepFun_contraction}), we have $Y_{n, a} \ind (H_{T_{n,a}-1}, T_{n,a})$.
-\end{proof}
+% \begin{proof}
+%   \uses{lem:condIndepFun_rewardByCount_hist}
+% By Lemma~\ref{lem:condIndepFun_rewardByCount_hist}, $Y_{n, a} \ind H_{T_{n,a}-1} \mid T_{n,a}$.
+% By Lemma~\ref{lem:indepFun_rewardByCount_stepsUntil}, $Y_{n, a} \ind T_{n,a}$.
+% By the contraction property of conditional independence (Lemma~\ref{lem:indepFun_contraction}), we have $Y_{n, a} \ind (H_{T_{n,a}-1}, T_{n,a})$.
+% \end{proof}
 
 
-\begin{lemma}\label{lem:iIndepFun_rewardByCount}
-  \uses{def:rewardByCount}
-The rewards $(Y_{n,a})_{n \in \mathbb{N}}$ are independent.
-\end{lemma}
+% \begin{lemma}\label{lem:iIndepFun_rewardByCount}
+%   \uses{def:rewardByCount}
+% The rewards $(Y_{n,a})_{n \in \mathbb{N}}$ are independent.
+% \end{lemma}
 
-\begin{proof}
-  \uses{lem:iIndepFun_nat_iff_forall_indepFun, lem:indepFun_contraction, lem:indepFun_rewardByCount_stepsUntil}
-By Lemma~\ref{lem:iIndepFun_nat_iff_forall_indepFun}, it suffices to show that for all $n \in \mathbb{N}$, $Y_{n+1, a}$ is independent of $(Y_{1,a}, \ldots, Y_{n,a})$.
+% \begin{proof}
+%   \uses{lem:iIndepFun_nat_iff_forall_indepFun, lem:indepFun_contraction, lem:indepFun_rewardByCount_stepsUntil}
+% By Lemma~\ref{lem:iIndepFun_nat_iff_forall_indepFun}, it suffices to show that for all $n \in \mathbb{N}$, $Y_{n+1, a}$ is independent of $(Y_{1,a}, \ldots, Y_{n,a})$.
 
-By the contraction property of conditional independence (Lemma~\ref{lem:indepFun_contraction}), it suffices to show that $Y_{n+1, a}$ is independent of $(Y_{1,a}, \ldots, Y_{n,a})$ conditionally on $T_{n+1, a}$ and that $Y_{n+1, a}$ is independent of $T_{n+1, a}$.
+% By the contraction property of conditional independence (Lemma~\ref{lem:indepFun_contraction}), it suffices to show that $Y_{n+1, a}$ is independent of $(Y_{1,a}, \ldots, Y_{n,a})$ conditionally on $T_{n+1, a}$ and that $Y_{n+1, a}$ is independent of $T_{n+1, a}$.
 
-The fact that $Y_{n+1, a}$ is independent of $T_{n+1, a}$ is Lemma~\ref{lem:indepFun_rewardByCount_stepsUntil}.
+% The fact that $Y_{n+1, a}$ is independent of $T_{n+1, a}$ is Lemma~\ref{lem:indepFun_rewardByCount_stepsUntil}.
 
-TODO
+% TODO
 
-\end{proof}
+% \end{proof}
 
 
-\begin{lemma}\label{lem:independent_rewardByCount}
-  \uses{def:rewardByCount}
-For $a \in \mathcal{A}$, let $Y^{(a)} = (Y_{n,a})_{n \in \mathbb{N}} \in \mathbb{R}^{\mathbb{N}}$ be the sequence of rewards obtained from pulling arm $a$. Then the sequences $(Y^{(a)})_{a \in \mathcal{A}}$ are independent.
-\end{lemma}
+% \begin{lemma}\label{lem:independent_rewardByCount}
+%   \uses{def:rewardByCount}
+% For $a \in \mathcal{A}$, let $Y^{(a)} = (Y_{n,a})_{n \in \mathbb{N}} \in \mathbb{R}^{\mathbb{N}}$ be the sequence of rewards obtained from pulling arm $a$. Then the sequences $(Y^{(a)})_{a \in \mathcal{A}}$ are independent.
+% \end{lemma}
 
-\begin{proof}
+% \begin{proof}
 
-\end{proof}
+% \end{proof}
 
 
-\begin{lemma}\label{lem:identDistrib_rewardByCount_stream}
-  \uses{def:rewardByCount}
-The random sequences $(Y_{n+1,a})_{n \in \mathbb{N}}$ and $(Z_{n,a})_{n \in \mathbb{N}}$ are identically distributed.
-\end{lemma}
+% \begin{lemma}\label{lem:identDistrib_rewardByCount_stream}
+%   \uses{def:rewardByCount}
+% The random sequences $(Y_{n+1,a})_{n \in \mathbb{N}}$ and $(Z_{n,a})_{n \in \mathbb{N}}$ are identically distributed.
+% \end{lemma}
 
-\begin{proof}
-  \uses{lem:hasLaw_rewardByCount, lem:iIndepFun_rewardByCount}
+% \begin{proof}
+%   \uses{lem:hasLaw_rewardByCount, lem:iIndepFun_rewardByCount}
 
-\end{proof}
+% \end{proof}
 
 
-\begin{lemma}\label{lem:identDistrib_sum_Icc_rewardByCount}
-  \uses{def:rewardByCount}
-The random variables $\sum_{i=1}^n Y_{i,a}$ and $\sum_{i=0}^{n-1} Z_{i,a}$ are identically distributed.
-\end{lemma}
+% \begin{lemma}\label{lem:identDistrib_sum_Icc_rewardByCount}
+%   \uses{def:rewardByCount}
+% The random variables $\sum_{i=1}^n Y_{i,a}$ and $\sum_{i=0}^{n-1} Z_{i,a}$ are identically distributed.
+% \end{lemma}
 
-\begin{proof}
-  \uses{lem:identDistrib_rewardByCount_stream}
-Immediate consequence of Lemma~\ref{lem:identDistrib_rewardByCount_stream}.
-\end{proof}
+% \begin{proof}
+%   \uses{lem:identDistrib_rewardByCount_stream}
+% Immediate consequence of Lemma~\ref{lem:identDistrib_rewardByCount_stream}.
+% \end{proof}
 
 
 \section{Regret and other bandit quantities}

blueprint/src/chapters/concentration.tex

@@ -1,5 +1,8 @@
 \chapter{Concentration inequalities}
 
+
+\section{Sub-Gaussian random variables}
+
 \begin{definition}[Sub-Gaussian]\label{def:subGaussian}
   \mathlibok
   \lean{ProbabilityTheory.HasSubgaussianMGF}
@@ -79,3 +82,116 @@ \chapter{Concentration inequalities}
   \uses{lem:subGaussian_add_of_indepFun, thm:hoeffding}
 
 \end{proof}
+
+
+
+
+\section{Laws of sums of rewards}
+
+
+\begin{lemma}\label{lem:AM.identDistrib_pullCount_prod_sumRewards}
+  \uses{def:rewardByCount, def:sumRewards, def:pullCount, def:AM.history}
+  \leanok
+  \lean{Bandits.ArrayModel.identDistrib_pullCount_prod_sumRewards}
+In the array model, for $t \in \mathbb{N}$, the random variable $(N_{t,a}, S_{t, a})_{a \in \mathcal{A}}$ has the same distribution as $(N_{t,a}, \sum_{s=0}^{N_{t,a}-1} \omega_{2, s, a})_{a \in \mathcal{A}}$.
+\end{lemma}
+
+\begin{proof}\leanok
+  \uses{def:AM.history, lem:AM.measurable_hist, lem:sum_rewardByCount}
+
+\end{proof}
+
+
+\begin{lemma}\label{lem:AM.identDistrib_sum_range_snd}
+  \uses{def:AM.history}
+  \leanok
+  \lean{Bandits.ArrayModel.identDistrib_sum_range_snd}
+In the array model, for $k \in \mathbb{N}$, the random variable $\sum_{s=0}^{k-1} \omega_{2, s, a}$ has the same distribution as a sum of $k$ i.i.d. random variables with law $\nu(a)$.
+\end{lemma}
+
+\begin{proof}\leanok
+By definition of $P_{\mathcal{A}}$ (Definition~\ref{def:arrayMeasure}).
+\end{proof}
+
+
+\begin{lemma}\label{lem:prob_pullCount_prod_sumRewards_mem_le}
+  \uses{def:sumRewards, def:pullCount, def:AM.history}
+  \leanok
+  \lean{Bandits.ArrayModel.prob_pullCount_prod_sumRewards_mem_le, Bandits.prob_pullCount_prod_sumRewards_mem_le}
+In the array model, for $t \in \mathbb{N}$, $a \in \mathcal{A}$, and a measurable set $B \subseteq \mathbb{N} \times \mathbb{R}$,
+\begin{align*}
+  P_{\mathcal{A}}\left((N_{t,a}, S_{t, a}) \in B\right)
+  &\le \sum_{k < t, \exists r, (k, r) \in B} \nu(a)^{\otimes \mathbb{N}} \left(\sum_{s=0}^{k-1} \omega_{s} \in \{x \mid \exists n, (n, x) \in B\}\right)
+  \: .
+\end{align*}
+As a consequence, this also holds for any algorithm-environment sequence.
+\end{lemma}
+
+\begin{proof}\leanok
+  \uses{lem:AM.identDistrib_pullCount_prod_sumRewards, lem:AM.identDistrib_sum_range_snd, thm:isAlgEnvSeq_unique, thm:isAlgEnvSeq_arrayMeasure}
+
+\end{proof}
+
+
+\begin{lemma}\label{lem:prob_sumRewards_le_sumRewards_le}
+  \uses{def:sumRewards, def:pullCount, def:AM.history}
+  \leanok
+  \lean{Bandits.ArrayModel.prob_sumRewards_le_sumRewards_le, Bandits.probReal_sumRewards_le_sumRewards_le}
+In the array model,
+\begin{align*}
+  P_{\mathcal{A}}\left( N_{t, a^*} = m_1 \wedge N_{t, a} = m_2 \wedge S_{t, a^*} \le S_{t, a}\right)
+  &\le (\otimes_a \nu(a))^{\otimes \mathbb{N}} \left( \sum_{s=0}^{m_1-1} \omega_{s, a^*} \le \sum_{s=0}^{m_2-1} \omega_{s, a} \right)
+  \: .
+\end{align*}
+As a consequence, this also holds for any algorithm-environment sequence.
+\end{lemma}
+
+\begin{proof}\leanok
+  \uses{lem:AM.identDistrib_pullCount_prod_sumRewards, lem:AM.identDistrib_sum_range_snd, thm:isAlgEnvSeq_unique,thm:isAlgEnvSeq_arrayMeasure}
+
+\end{proof}
+
+
+
+\subsection{Sub-Gaussian rewards}
+
+
+\begin{lemma}\label{lem:probReal_sum_le_sum_streamMeasure}
+  \leanok
+  \lean{Bandits.probReal_sum_le_sum_streamMeasure}
+Let $\nu(a)$ be a 1-sub-Gaussian distribution on $\mathbb{R}$ for each arm $a \in \mathcal{A}$.
+\begin{align*}
+  (\otimes_a \nu(a))^{\otimes \mathbb{N}} \left( \sum_{s=0}^{m-1} \omega_{s, a^*} \le \sum_{s=0}^{m-1} \omega_{s, a} \right)
+  &\le \exp\left( -m \frac{\Delta_a^2}{4 \sigma^2} \right)
+\end{align*}
+\end{lemma}
+
+\begin{proof}\leanok
+  \uses{lem:measure_sum_le_sum_le'}
+
+\end{proof}
+
+
+\begin{lemma}\label{lem:prob_sum_le_sqrt_log}
+  \leanok
+  \lean{Bandits.prob_sum_le_sqrt_log, Bandits.prob_sum_ge_sqrt_log}
+Let $\nu(a)$ be a 1-sub-Gaussian distribution on $\mathbb{R}$ for each arm $a \in \mathcal{A}$.
+Let $c \ge 0$ be a real number and $k$ a positive natural number.
+Then
+\begin{align*}
+  \nu(a)^{\otimes \mathbb{N}} \left( \sum_{s=0}^{k-1} (\omega_{s} - \mu_a) \le - \sqrt{c k \log(n + 1)} \right)
+  &\le \frac{1}{(n + 1)^{c / 2}}
+  \: .
+\end{align*}
+The same upper bound holds for the upper tail:
+\begin{align*}
+  \nu(a)^{\otimes \mathbb{N}} \left( \sum_{s=0}^{k-1} (\omega_{s} - \mu_a) \ge \sqrt{c k \log(n + 1)} \right)
+  &\le \frac{1}{(n + 1)^{c / 2}}
+  \: .
+\end{align*}
+\end{lemma}
+
+\begin{proof}\leanok
+  \uses{thm:hoeffding}
+
+\end{proof}

blueprint/src/chapters/etc.tex

@@ -1,5 +1,7 @@
 \chapter{Bandit algorithms}
 
+TODO: update this chapter to reflect the changes in the formalization.
+
 \section{Explore-Then-Commit}
 
 Note: times start at 0 to be consistent with Lean.

blueprint/src/chapters/ucb.tex

@@ -1,5 +1,7 @@
 \section{UCB}
 
+TODO: update this chapter to reflect the changes in the formalization.
+
 \begin{definition}[UCB algorithm]\label{def:ucbAlgorithm}
   \uses{def:IT.actionReward, def:pullCount, def:empMean}
   \leanok