diff --git a/.github/workflows/blueprint.yml b/.github/workflows/blueprint.yml index 2428c569..b13d4b0a 100644 --- a/.github/workflows/blueprint.yml +++ b/.github/workflows/blueprint.yml @@ -53,13 +53,13 @@ jobs: - name: Build Verso Documentation run: | - ./scripts/build_tutorial.sh + ./build_tutorial.sh - name: Compile blueprint and documentation uses: leanprover-community/docgen-action@7b5b9a1822650bd45aaeb4182d94bceba2030f89 # 2026-04-14 with: homepage: home_page - blueprint: true + blueprint: false build-page: false deploy: false diff --git a/LeanMachineLearning/Online/Bandit/Algorithms/ETC.lean b/LeanMachineLearning/Online/Bandit/Algorithms/ETC.lean index c59cb570..054ff2d1 100644 --- a/LeanMachineLearning/Online/Bandit/Algorithms/ETC.lean +++ b/LeanMachineLearning/Online/Bandit/Algorithms/ETC.lean @@ -66,16 +66,16 @@ variable {hK : 0 < K} {m : ℕ} {ν : Kernel (Fin K) ℝ} [IsMarkovKernel ν] lemma isAlgEnvSeqUntil_roundRobinAlgorithm [Nonempty (Fin K)] (h : IsAlgEnvSeq A R (etcAlgorithm hK m) (stationaryEnv ν) P) : IsAlgEnvSeqUntil A R (roundRobinAlgorithm hK) (stationaryEnv ν) P (K * m - 1) where - measurable_A := h.measurable_A - measurable_R := h.measurable_R + measurable_action := h.measurable_action + measurable_feedback := h.measurable_feedback hasLaw_action_zero := h.hasLaw_action_zero - hasCondDistrib_reward_zero := h.hasCondDistrib_reward_zero + hasCondDistrib_feedback_zero := h.hasCondDistrib_feedback_zero hasCondDistrib_action n hn := by convert h.hasCondDistrib_action n using 1 simp only [roundRobinAlgorithm, detAlgorithm_policy, etcAlgorithm] congr 1 with h simp [ETC.nextArm, hn] - hasCondDistrib_reward n _ := h.hasCondDistrib_reward n + hasCondDistrib_feedback n _ := h.hasCondDistrib_feedback n section AlgorithmBehavior @@ -229,7 +229,7 @@ lemma expectation_pullCount_le [Nonempty (Fin K)] (a : Fin K) (hm : m ≠ 0) {n : ℕ} (hn : K * m ≤ n) : P[fun ω ↦ (pullCount A a n ω : ℝ)] ≤ m + (n - K * m) * Real.exp (- (m : ℝ) * gap ν a ^ 2 / (4 * σ2)) := by - have hA := h.measurable_A + have hA := h.measurable_action have : (fun ω ↦ (pullCount A a n ω : ℝ)) =ᵐ[P] fun ω ↦ m + (n - K * m) * {ω' | A (K * m) ω' = a}.indicator (fun _ ↦ 1) ω := by filter_upwards [pullCount_of_ge h a hm hn] with ω h diff --git a/LeanMachineLearning/Online/Bandit/Algorithms/UCB.lean b/LeanMachineLearning/Online/Bandit/Algorithms/UCB.lean index 4a4553cc..f107347f 100644 --- a/LeanMachineLearning/Online/Bandit/Algorithms/UCB.lean +++ b/LeanMachineLearning/Online/Bandit/Algorithms/UCB.lean @@ -67,16 +67,16 @@ variable {hK : 0 < K} {c : ℝ} {ν : Kernel (Fin K) ℝ} [IsMarkovKernel ν] lemma isAlgEnvSeqUntil_roundRobinAlgorithm [Nonempty (Fin K)] (h : IsAlgEnvSeq A R (ucbAlgorithm hK c) (stationaryEnv ν) P) : IsAlgEnvSeqUntil A R (roundRobinAlgorithm hK) (stationaryEnv ν) P (K - 1) where - measurable_A := h.measurable_A - measurable_R := h.measurable_R + measurable_action := h.measurable_action + measurable_feedback := h.measurable_feedback hasLaw_action_zero := h.hasLaw_action_zero - hasCondDistrib_reward_zero := h.hasCondDistrib_reward_zero + hasCondDistrib_feedback_zero := h.hasCondDistrib_feedback_zero hasCondDistrib_action n hn := by convert h.hasCondDistrib_action n using 1 simp only [roundRobinAlgorithm, detAlgorithm_policy, ucbAlgorithm] congr 1 with h simp [UCB.nextArm, hn] - hasCondDistrib_reward n _ := h.hasCondDistrib_reward n + hasCondDistrib_feedback n _ := h.hasCondDistrib_feedback n section AlgorithmBehavior @@ -452,7 +452,6 @@ lemma constSum_lt_top (c : ℝ) (n : ℕ) : constSum c n < ∞ := by simp only [one_div, ENNReal.inv_lt_top] positivity -set_option backward.isDefEq.respectTransparency false in /-- Bound on the expectation of the number of pulls of each arm by the UCB algorithm. -/ lemma expectation_pullCount_le' [Nonempty (Fin K)] (h : IsAlgEnvSeq A R (ucbAlgorithm hK (c * σ2)) (stationaryEnv ν) P) @@ -460,8 +459,8 @@ lemma expectation_pullCount_le' [Nonempty (Fin K)] (hσ2 : σ2 ≠ 0) (hc : 0 < c) (a : Fin K) (h_gap : 0 < gap ν a) (n : ℕ) : ∫⁻ ω, pullCount A a n ω ∂P ≤ ENNReal.ofReal (8 * c * σ2 * log (n + 1) / gap ν a ^ 2 + 1) + 1 + 2 * constSum c n := by - have hA := h.measurable_A - have hR := h.measurable_R + have hA := h.measurable_action + have hR := h.measurable_feedback by_cases hn_zero : n = 0 · simp [hn_zero] let C a : ℕ := ⌈8 * c * σ2 * log (n + 1) / gap ν a ^ 2⌉₊ @@ -549,7 +548,7 @@ lemma expectation_pullCount_le [Nonempty (Fin K)] (hσ2 : σ2 ≠ 0) (hc : 0 < c) (a : Fin K) (h_gap : 0 < gap ν a) (n : ℕ) : P[fun ω ↦ (pullCount A a n ω : ℝ)] ≤ 8 * c * σ2 * log (n + 1) / gap ν a ^ 2 + 2 + 2 * (constSum c n).toReal := by - have hA := h.measurable_A + have hA := h.measurable_action have h := expectation_pullCount_le' h hν hσ2 hc a h_gap n (hK := hK) simp_rw [← ENNReal.ofReal_natCast] at h rw [← ofReal_integral_eq_lintegral_ofReal] at h diff --git a/LeanMachineLearning/Online/Bandit/ArrayProbSpace.lean b/LeanMachineLearning/Online/Bandit/ArrayProbSpace.lean index fabc7ca8..f9302a1f 100644 --- a/LeanMachineLearning/Online/Bandit/ArrayProbSpace.lean +++ b/LeanMachineLearning/Online/Bandit/ArrayProbSpace.lean @@ -30,9 +30,9 @@ arms and rewards specified in the bandit model determine their laws uniquely. * `streamMeasure ν`: probability measure on the space of infinite arrays of rewards, where the rewards in each row are i.i.d. according to `ν`. -* `probSpace α R`: probability space for the array model of stochastic bandits with action space `α` +* `probSpace 𝓐 R`: probability space for the array model of stochastic bandits with action space `𝓐` and reward space `R`. -* `arrayMeasure ν`: probability measure on `probSpace α R` for the array model of stochastic bandits +* `arrayMeasure ν`: probability measure on `probSpace 𝓐 R` for the array model of stochastic bandits with reward kernel `ν`. -/ @@ -45,16 +45,16 @@ open scoped ENNReal NNReal namespace Bandits -variable {α R : Type*} {mα : MeasurableSpace α} {mR : MeasurableSpace R} +variable {𝓐 R : Type*} {m𝓐 : MeasurableSpace 𝓐} {mR : MeasurableSpace R} section MeasureSpace /-- Measure of an infinite stream of rewards from each action. -/ noncomputable -def streamMeasure (ν : Kernel α R) : Measure (ℕ → α → R) := +def streamMeasure (ν : Kernel 𝓐 R) : Measure (ℕ → 𝓐 → R) := Measure.infinitePi fun _ ↦ Measure.infinitePi ν -instance (ν : Kernel α R) [IsMarkovKernel ν] : IsProbabilityMeasure (streamMeasure ν) := by +instance (ν : Kernel 𝓐 R) [IsMarkovKernel ν] : IsProbabilityMeasure (streamMeasure ν) := by unfold streamMeasure infer_instance @@ -66,16 +66,16 @@ lemma _root_.hasLaw_eval_infinitePi {ι : Type*} {X : ι → Type*} {mX : ∀ i, aemeasurable := Measurable.aemeasurable (by fun_prop) map_eq := by exact (measurePreserving_eval_infinitePi μ i).map_eq -lemma hasLaw_eval_streamMeasure (ν : Kernel α R) [IsMarkovKernel ν] (n : ℕ) : - HasLaw (fun h : ℕ → α → R ↦ h n) (Measure.infinitePi ν) (streamMeasure ν) := +lemma hasLaw_eval_streamMeasure (ν : Kernel 𝓐 R) [IsMarkovKernel ν] (n : ℕ) : + HasLaw (fun h : ℕ → 𝓐 → R ↦ h n) (Measure.infinitePi ν) (streamMeasure ν) := hasLaw_eval_infinitePi (fun _ ↦ Measure.infinitePi ν) n -lemma hasLaw_eval_eval_streamMeasure (ν : Kernel α R) [IsMarkovKernel ν] (n : ℕ) (a : α) : - HasLaw (fun h : ℕ → α → R ↦ h n a) (ν a) (streamMeasure ν) := +lemma hasLaw_eval_eval_streamMeasure (ν : Kernel 𝓐 R) [IsMarkovKernel ν] (n : ℕ) (a : 𝓐) : + HasLaw (fun h : ℕ → 𝓐 → R ↦ h n a) (ν a) (streamMeasure ν) := (hasLaw_eval_infinitePi ν a).comp (hasLaw_eval_streamMeasure ν n) -lemma identDistrib_eval_eval_id_streamMeasure (ν : Kernel α R) [IsMarkovKernel ν] (n : ℕ) (a : α) : - IdentDistrib (fun h : ℕ → α → R ↦ h n a) id (streamMeasure ν) (ν a) where +lemma identDistrib_eval_eval_id_streamMeasure (ν : Kernel 𝓐 R) [IsMarkovKernel ν] (n : ℕ) (a : 𝓐) : + IdentDistrib (fun h : ℕ → 𝓐 → R ↦ h n a) id (streamMeasure ν) (ν a) where aemeasurable_fst := Measurable.aemeasurable (by fun_prop) aemeasurable_snd := Measurable.aemeasurable (by fun_prop) map_eq := by @@ -83,39 +83,39 @@ lemma identDistrib_eval_eval_id_streamMeasure (ν : Kernel α R) [IsMarkovKernel Measure.map_map (by fun_prop) (by fun_prop)] simp -lemma integrable_eval_streamMeasure (ν : Kernel α ℝ) [IsMarkovKernel ν] (n : ℕ) (a : α) +lemma integrable_eval_streamMeasure (ν : Kernel 𝓐 ℝ) [IsMarkovKernel ν] (n : ℕ) (a : 𝓐) (h_int : Integrable id (ν a)) : - Integrable (fun h : ℕ → α → ℝ ↦ h n a) (streamMeasure ν) := + Integrable (fun h : ℕ → 𝓐 → ℝ ↦ h n a) (streamMeasure ν) := Integrable.congr_identDistrib h_int (identDistrib_eval_eval_id_streamMeasure ν n a).symm -lemma integral_eval_streamMeasure (ν : Kernel α ℝ) [IsMarkovKernel ν] (n : ℕ) (a : α) : +lemma integral_eval_streamMeasure (ν : Kernel 𝓐 ℝ) [IsMarkovKernel ν] (n : ℕ) (a : 𝓐) : ∫ h, h n a ∂(streamMeasure ν) = (ν a)[id] := by calc ∫ h, h n a ∂(streamMeasure ν) _ = ∫ x, x ∂((streamMeasure ν).map (fun h ↦ h n a)) := by rw [integral_map (Measurable.aemeasurable (by fun_prop)) (by fun_prop)] _ = (ν a)[id] := by simp [(hasLaw_eval_eval_streamMeasure ν n a).map_eq] -lemma iIndepFun_eval_streamMeasure' (ν : Kernel α R) [IsMarkovKernel ν] : +lemma iIndepFun_eval_streamMeasure' (ν : Kernel 𝓐 R) [IsMarkovKernel ν] : iIndepFun (fun n ω ↦ ω n) (streamMeasure ν) := - iIndepFun_infinitePi (P := fun (_ : ℕ) ↦ Measure.infinitePi ν) (Ω := fun _ ↦ α → R) + iIndepFun_infinitePi (P := fun (_ : ℕ) ↦ Measure.infinitePi ν) (Ω := fun _ ↦ 𝓐 → R) (X := fun i u ↦ u) (fun i ↦ by fun_prop) -lemma iIndepFun_eval_streamMeasure'' (ν : Kernel α R) [IsMarkovKernel ν] (a : α) : +lemma iIndepFun_eval_streamMeasure'' (ν : Kernel 𝓐 R) [IsMarkovKernel ν] (a : 𝓐) : iIndepFun (fun n ω ↦ ω n a) (streamMeasure ν) := (iIndepFun_eval_streamMeasure' ν).comp (g := fun i ω ↦ ω a) (by fun_prop) -lemma iIndepFun_eval_streamMeasure (ν : Kernel α R) [IsMarkovKernel ν] : - iIndepFun (fun (p : ℕ × α) ω ↦ ω p.1 p.2) (streamMeasure ν) := +lemma iIndepFun_eval_streamMeasure (ν : Kernel 𝓐 R) [IsMarkovKernel ν] : + iIndepFun (fun (p : ℕ × 𝓐) ω ↦ ω p.1 p.2) (streamMeasure ν) := iIndepFun_uncurry_infinitePi' (X := fun _ _ ↦ id) (fun _ ↦ ν) (by fun_prop) -lemma indepFun_eval_streamMeasure (ν : Kernel α R) [IsMarkovKernel ν] {n m : ℕ} {a b : α} +lemma indepFun_eval_streamMeasure (ν : Kernel 𝓐 R) [IsMarkovKernel ν] {n m : ℕ} {a b : 𝓐} (h : n ≠ m ∨ a ≠ b) : IndepFun (fun ω ↦ ω n a) (fun ω ↦ ω m b) (streamMeasure ν) := by change IndepFun (fun ω ↦ ω (n, a).1 (n, a).2) (fun ω ↦ ω (m, b).1 (m, b).2) (streamMeasure ν) exact (iIndepFun_eval_streamMeasure ν).indepFun (by grind) -lemma indepFun_eval_streamMeasure' (ν : Kernel α R) [IsMarkovKernel ν] {a b : α} (h : a ≠ b) : +lemma indepFun_eval_streamMeasure' (ν : Kernel 𝓐 R) [IsMarkovKernel ν] {a b : 𝓐} (h : a ≠ b) : IndepFun (fun ω n ↦ ω n a) (fun ω n ↦ ω n b) (streamMeasure ν) := indepFun_proj_infinitePi_infinitePi h @@ -127,80 +127,80 @@ open unitInterval section ProbabilitySpace -variable (α R) in +variable (𝓐 R) in /-- Probability space for the array model of stochastic bandits. -/ -def probSpace : Type _ := (ℕ → I) × (ℕ → α → R) +def probSpace : Type _ := (ℕ → I) × (ℕ → 𝓐 → R) -instance {α R : Type*} [MeasurableSpace R] : MeasurableSpace (probSpace α R) := - inferInstanceAs (MeasurableSpace ((ℕ → I) × (ℕ → α → R))) +instance {𝓐 R : Type*} [MeasurableSpace R] : MeasurableSpace (probSpace 𝓐 R) := + inferInstanceAs (MeasurableSpace ((ℕ → I) × (ℕ → 𝓐 → R))) -instance {α R : Type*} [Countable α] [MeasurableSpace R] [StandardBorelSpace R] : - StandardBorelSpace (probSpace α R) := - inferInstanceAs (StandardBorelSpace ((ℕ → I) × (ℕ → α → R))) +instance {𝓐 R : Type*} [Countable 𝓐] [MeasurableSpace R] [StandardBorelSpace R] : + StandardBorelSpace (probSpace 𝓐 R) := + inferInstanceAs (StandardBorelSpace ((ℕ → I) × (ℕ → 𝓐 → R))) /-- Probability measure for the array model of stochastic bandits. -/ noncomputable -def arrayMeasure (ν : Kernel α R) : Measure (probSpace α R) := +def arrayMeasure (ν : Kernel 𝓐 R) : Measure (probSpace 𝓐 R) := (Measure.infinitePi fun _ ↦ volume).prod (streamMeasure ν) -instance (ν : Kernel α R) [IsMarkovKernel ν] : IsProbabilityMeasure (arrayMeasure ν) := +instance (ν : Kernel 𝓐 R) [IsMarkovKernel ν] : IsProbabilityMeasure (arrayMeasure ν) := Measure.prod.instIsProbabilityMeasure _ _ -variable [Nonempty α] [StandardBorelSpace α] +variable [Nonempty 𝓐] [StandardBorelSpace 𝓐] /-- The initial action is the image of a uniform random variable by this function. -/ noncomputable -def initAlgFunction (alg : Algorithm α R) : I → α := +def initAlgFunction (alg : Algorithm 𝓐 R) : I → 𝓐 := (Measure.exists_measurable_map_eq alg.p0).choose -lemma initAlgFunction_map (alg : Algorithm α R) : volume.map (initAlgFunction alg) = alg.p0 := +lemma initAlgFunction_map (alg : Algorithm 𝓐 R) : volume.map (initAlgFunction alg) = alg.p0 := (Measure.exists_measurable_map_eq alg.p0).choose_spec.2 @[fun_prop] -lemma measurable_initAlgFunction (alg : Algorithm α R) : +lemma measurable_initAlgFunction (alg : Algorithm 𝓐 R) : Measurable (initAlgFunction alg) := (Measure.exists_measurable_map_eq alg.p0).choose_spec.1 /-- The next action is the image of the history and a uniform random variable by this function. -/ noncomputable -def algFunction (alg : Algorithm α R) (n : ℕ) : - (Iic n → α × R) → I → α := +def algFunction (alg : Algorithm 𝓐 R) (n : ℕ) : + (Iic n → 𝓐 × R) → I → 𝓐 := (Kernel.exists_measurable_map_eq_unitInterval (alg.policy n)).choose -lemma algFunction_map (alg : Algorithm α R) (n : ℕ) (h : Iic n → α × R) : +lemma algFunction_map (alg : Algorithm 𝓐 R) (n : ℕ) (h : Iic n → 𝓐 × R) : volume.map (algFunction alg n h) = alg.policy n h := (Kernel.exists_measurable_map_eq_unitInterval (alg.policy n)).choose_spec.2 h @[fun_prop] -lemma measurable_algFunction (alg : Algorithm α R) (n : ℕ) : +lemma measurable_algFunction (alg : Algorithm 𝓐 R) (n : ℕ) : Measurable (Function.uncurry (algFunction alg n)) := (Kernel.exists_measurable_map_eq_unitInterval (alg.policy n)).choose_spec.1 end ProbabilitySpace -variable [Nonempty α] [StandardBorelSpace α] +variable [Nonempty 𝓐] [StandardBorelSpace 𝓐] section HistoryActionReward /-- History of actions and rewards up to time `n` in the array model. -/ noncomputable -def hist [DecidableEq α] (alg : Algorithm α R) (ω : probSpace α R) : (n : ℕ) → Iic n → α × R +def hist [DecidableEq 𝓐] (alg : Algorithm 𝓐 R) (ω : probSpace 𝓐 R) : (n : ℕ) → Iic n → 𝓐 × R | 0 => fun _ ↦ (initAlgFunction alg (ω.1 0), ω.2 0 (initAlgFunction alg (ω.1 0))) | n + 1 => - let hn : Iic n → α × R := hist alg ω n - let a : α := algFunction alg n hn (ω.1 (n + 1)) + let hn : Iic n → 𝓐 × R := hist alg ω n + let a : 𝓐 := algFunction alg n hn (ω.1 (n + 1)) fun i ↦ if hin : i ≤ n then hn ⟨i, by simp [hin]⟩ else (a, ω.2 (pullCount' n hn a) a) @[simp] -lemma hist_zero [DecidableEq α] (alg : Algorithm α R) (ω : probSpace α R) : +lemma hist_zero [DecidableEq 𝓐] (alg : Algorithm 𝓐 R) (ω : probSpace 𝓐 R) : hist alg ω 0 = fun _ ↦ (initAlgFunction alg (ω.1 0), ω.2 0 (initAlgFunction alg (ω.1 0))) := rfl -lemma hist_add_one [DecidableEq α] (alg : Algorithm α R) (ω : probSpace α R) (n : ℕ) : - let a : α := algFunction alg n (hist alg ω n) (ω.1 (n + 1)) +lemma hist_add_one [DecidableEq 𝓐] (alg : Algorithm 𝓐 R) (ω : probSpace 𝓐 R) (n : ℕ) : + let a : 𝓐 := algFunction alg n (hist alg ω n) (ω.1 (n + 1)) hist alg ω (n + 1) = fun (i : Iic (n + 1)) ↦ if hin : i ≤ n then hist alg ω n ⟨i, by simp [hin]⟩ else (a, ω.2 (pullCount' n (hist alg ω n) a) a) := rfl -lemma hist_eq [DecidableEq α] (alg : Algorithm α R) (ω : probSpace α R) (n : ℕ) : +lemma hist_eq [DecidableEq 𝓐] (alg : Algorithm 𝓐 R) (ω : probSpace 𝓐 R) (n : ℕ) : hist alg ω n = fun i : Iic n ↦ hist alg ω i ⟨i.1, by simp⟩ := by induction n with | zero => @@ -217,11 +217,11 @@ lemma hist_eq [DecidableEq α] (alg : Algorithm α R) (ω : probSpace α R) (n : simp_rw [hn] · grind -lemma hist_add_one_eq_IicSuccProd' [DecidableEq α] (alg : Algorithm α R) (ω : probSpace α R) +lemma hist_add_one_eq_IicSuccProd' [DecidableEq 𝓐] (alg : Algorithm 𝓐 R) (ω : probSpace 𝓐 R) (n : ℕ) : - let a : α := algFunction alg n (hist alg ω n) (ω.1 (n + 1)) + let a : 𝓐 := algFunction alg n (hist alg ω n) (ω.1 (n + 1)) hist alg ω (n + 1) = - (MeasurableEquiv.IicSuccProd (fun _ ↦ α × R) n).symm + (MeasurableEquiv.IicSuccProd (fun _ ↦ 𝓐 × R) n).symm (hist alg ω n, (a, ω.2 (pullCount' n (hist alg ω n) a) a)) := by intro a rw [hist_add_one] @@ -234,15 +234,15 @@ lemma hist_add_one_eq_IicSuccProd' [DecidableEq α] (alg : Algorithm α R) (ω : /-- Action taken at time `n` in the array model. -/ noncomputable -def action [DecidableEq α] (alg : Algorithm α R) (n : ℕ) (ω : probSpace α R) : α := +def action [DecidableEq 𝓐] (alg : Algorithm 𝓐 R) (n : ℕ) (ω : probSpace 𝓐 R) : 𝓐 := (hist alg ω n ⟨n, by simp⟩).1 -lemma action_zero [DecidableEq α] (alg : Algorithm α R) : +lemma action_zero [DecidableEq 𝓐] (alg : Algorithm 𝓐 R) : action alg 0 = fun ω ↦ initAlgFunction alg (ω.1 0) := by ext simp [action, hist_zero] -lemma action_add_one_eq [DecidableEq α] (alg : Algorithm α R) (n : ℕ) : +lemma action_add_one_eq [DecidableEq 𝓐] (alg : Algorithm 𝓐 R) (n : ℕ) : action alg (n + 1) = fun ω ↦ algFunction alg n (hist alg ω n) (ω.1 (n + 1)) := by ext ω rw [action, hist_add_one] @@ -250,21 +250,21 @@ lemma action_add_one_eq [DecidableEq α] (alg : Algorithm α R) (n : ℕ) : /-- Reward received at time `n` in the array model. -/ noncomputable -def reward [DecidableEq α] (alg : Algorithm α R) (n : ℕ) (ω : probSpace α R) : R := +def reward [DecidableEq 𝓐] (alg : Algorithm 𝓐 R) (n : ℕ) (ω : probSpace 𝓐 R) : R := (hist alg ω n ⟨n, by simp⟩).2 -lemma reward_zero [DecidableEq α] (alg : Algorithm α R) : +lemma reward_zero [DecidableEq 𝓐] (alg : Algorithm 𝓐 R) : reward alg 0 = fun ω ↦ ω.2 0 (action alg 0 ω) := by ext simp [reward, hist_zero, action_zero] -lemma reward_add_one [DecidableEq α] (alg : Algorithm α R) (n : ℕ) : +lemma reward_add_one [DecidableEq 𝓐] (alg : Algorithm 𝓐 R) (n : ℕ) : reward alg (n + 1) = fun ω ↦ ω.2 (pullCount' n (hist alg ω n) (action alg (n + 1) ω)) (action alg (n + 1) ω) := by ext ω simp [reward, hist_add_one, action_add_one_eq] -lemma reward_eq [DecidableEq α] (alg : Algorithm α R) (n : ℕ) : +lemma reward_eq [DecidableEq 𝓐] (alg : Algorithm 𝓐 R) (n : ℕ) : reward alg n = fun ω ↦ ω.2 (pullCount (action alg) (action alg n ω) n ω) (action alg n ω) := by cases n with | zero => ext; simp [reward_zero, action_zero] @@ -279,28 +279,28 @@ lemma reward_eq [DecidableEq α] (alg : Algorithm α R) (n : ℕ) : section Measurability -lemma measurable_action_add_one' [DecidableEq α] {alg : Algorithm α R} +lemma measurable_action_add_one' [DecidableEq 𝓐] {alg : Algorithm 𝓐 R} (n : ℕ) (h : Measurable (hist alg · n)) : Measurable (fun x ↦ algFunction alg n (hist alg x n) (x.1 (n + 1))) := by fun_prop -lemma measurable_pullCount'_action_add_one [DecidableEq α] {alg : Algorithm α R} +lemma measurable_pullCount'_action_add_one [DecidableEq 𝓐] {alg : Algorithm 𝓐 R} (n : ℕ) (h_hist : Measurable (hist alg · n)) : Measurable (fun x ↦ pullCount' n (hist alg x n) (algFunction alg n (hist alg x n) (x.1 (n + 1)))) := by have h_alg_meas : Measurable (fun x ↦ algFunction alg n (hist alg x n) (x.1 (n + 1))) := measurable_action_add_one' n h_hist - exact (measurable_uncurry_pullCount' (α := α) n).comp (h_hist.prodMk h_alg_meas) + exact (measurable_uncurry_pullCount' (𝓐 := 𝓐) n).comp (h_hist.prodMk h_alg_meas) @[fun_prop] -lemma measurable_hist [DecidableEq α] [Countable α] (alg : Algorithm α R) (n : ℕ) : +lemma measurable_hist [DecidableEq 𝓐] [Countable 𝓐] (alg : Algorithm 𝓐 R) (n : ℕ) : Measurable (fun ω ↦ hist alg ω n) := by induction n with | zero => simp_rw [hist_zero, measurable_pi_iff] refine fun _ ↦ Measurable.prodMk (by fun_prop) ?_ - change Measurable ((fun x : α × ((ℕ → I) × (ℕ → α → R)) ↦ x.2.2 0 x.1) ∘ - (fun x : (ℕ → I) × (ℕ → α → R) ↦ (initAlgFunction alg (x.1 0), x))) - have : Measurable (fun x : α × ((ℕ → I) × (ℕ → α → R)) ↦ x.2.2 0 x.1) := + change Measurable ((fun x : 𝓐 × ((ℕ → I) × (ℕ → 𝓐 → R)) ↦ x.2.2 0 x.1) ∘ + (fun x : (ℕ → I) × (ℕ → 𝓐 → R) ↦ (initAlgFunction alg (x.1 0), x))) + have : Measurable (fun x : 𝓐 × ((ℕ → I) × (ℕ → 𝓐 → R)) ↦ x.2.2 0 x.1) := measurable_from_prod_countable_right fun p ↦ by simp only; fun_prop exact Measurable.comp (by fun_prop) (Measurable.prodMk (by fun_prop) (by fun_prop)) | succ n hn => @@ -311,48 +311,48 @@ lemma measurable_hist [DecidableEq α] [Countable α] (alg : Algorithm α R) (n exact hn ⟨i.1, by simp [hin]⟩ · simp only [hist, hin, ↓reduceDIte] refine Measurable.prodMk (by fun_prop) ?_ - change Measurable ((fun (x : (ℕ → α → R) × ℕ × α) ↦ x.1 x.2.1 x.2.2) ∘ + change Measurable ((fun (x : (ℕ → 𝓐 → R) × ℕ × 𝓐) ↦ x.1 x.2.1 x.2.2) ∘ (fun x ↦ (x.2, pullCount' n (hist alg x n) (algFunction alg n (hist alg x n) (x.1 (n + 1))), (algFunction alg n (hist alg x n) (x.1 (n + 1)))))) - have h1 : Measurable (fun (x : (ℕ → α → R) × ℕ × α) ↦ x.1 x.2.1 x.2.2) := - measurable_from_prod_countable_left fun p : ℕ × α ↦ (by simp only; fun_prop) + have h1 : Measurable (fun (x : (ℕ → 𝓐 → R) × ℕ × 𝓐) ↦ x.1 x.2.1 x.2.2) := + measurable_from_prod_countable_left fun p : ℕ × 𝓐 ↦ (by simp only; fun_prop) refine Measurable.comp (by fun_prop) (Measurable.prodMk (by fun_prop) ?_) refine Measurable.prodMk ?_ (by fun_prop) exact measurable_pullCount'_action_add_one n hn @[fun_prop] -lemma measurable_action [DecidableEq α] [Countable α] (alg : Algorithm α R) (n : ℕ) : +lemma measurable_action [DecidableEq 𝓐] [Countable 𝓐] (alg : Algorithm 𝓐 R) (n : ℕ) : Measurable (action alg n) := by unfold action; fun_prop @[fun_prop] -lemma measurable_reward [DecidableEq α] [Countable α] (alg : Algorithm α R) (n : ℕ) : +lemma measurable_reward [DecidableEq 𝓐] [Countable 𝓐] (alg : Algorithm 𝓐 R) (n : ℕ) : Measurable (reward alg n) := by unfold reward; fun_prop -lemma hist_add_one_eq_IicSuccProd [DecidableEq α] (alg : Algorithm α R) (ω : probSpace α R) +lemma hist_add_one_eq_IicSuccProd [DecidableEq 𝓐] (alg : Algorithm 𝓐 R) (ω : probSpace 𝓐 R) (n : ℕ) : hist alg ω (n + 1) = - (MeasurableEquiv.IicSuccProd (fun _ ↦ α × R) n).symm + (MeasurableEquiv.IicSuccProd (fun _ ↦ 𝓐 × R) n).symm (hist alg ω n, (action alg (n + 1) ω, reward alg (n + 1) ω)) := by rw [hist_add_one_eq_IicSuccProd', reward_add_one, action_add_one_eq] @[fun_prop] -lemma measurable_pullCount_action_add_one [DecidableEq α] [Countable α] (alg : Algorithm α R) +lemma measurable_pullCount_action_add_one [DecidableEq 𝓐] [Countable 𝓐] (alg : Algorithm 𝓐 R) (n : ℕ) : Measurable (fun ω ↦ pullCount (action alg) (action alg (n + 1) ω) (n + 1) ω) := by - change Measurable ((fun p : (probSpace α R) × α ↦ pullCount (action alg) p.2 (n + 1) p.1) ∘ - (fun ω : probSpace α R ↦ (ω, action alg (n + 1) ω))) + change Measurable ((fun p : (probSpace 𝓐 R) × 𝓐 ↦ pullCount (action alg) p.2 (n + 1) p.1) ∘ + (fun ω : probSpace 𝓐 R ↦ (ω, action alg (n + 1) ω))) exact (measurable_uncurry_pullCount (by fun_prop) _).comp (by fun_prop) end Measurability end HistoryActionReward -variable [DecidableEq α] +variable [DecidableEq 𝓐] section Congruence -- very useful to prove measurability -lemma hist_congr (alg : Algorithm α R) (n : ℕ) {ω ω' : probSpace α R} +lemma hist_congr (alg : Algorithm 𝓐 R) (n : ℕ) {ω ω' : probSpace 𝓐 R} (hω1 : ∀ i ≤ n, ω.1 i = ω'.1 i) (hω2 : ∀ i a, i < pullCount (action alg) a (n + 1) ω → ω.2 i a = ω'.2 i a) : hist alg ω n = hist alg ω' n := by @@ -385,8 +385,8 @@ lemma hist_congr (alg : Algorithm α R) (n : ℕ) {ω ω' : probSpace α R} pullCount' n (fun i ↦ (action alg i ω, reward alg i ω)) (action alg (n + 1) ω') + 1 grind -lemma stepsUntil_congr_aux (alg : Algorithm α R) - (a : α) (m n : ℕ) {ω ω' : probSpace α R} +lemma stepsUntil_congr_aux (alg : Algorithm 𝓐 R) + (a : 𝓐) (m n : ℕ) {ω ω' : probSpace 𝓐 R} (hω1 : ∀ i, ω.1 i = ω'.1 i) (hω2_ne : ∀ i b, b ≠ a → ω.2 i b = ω'.2 i b) (hω2_eq : ∀ i, i + 1 ≤ m → ω.2 i a = ω'.2 i a) (h_eq : action alg (n + 1) ω = a ∧ pullCount (action alg) a (n + 1) ω = m) : @@ -411,7 +411,7 @@ lemma stepsUntil_congr_aux (alg : Algorithm α R) simp only at h_hist rw [h_hist] -lemma stepsUntil_congr (alg : Algorithm α R) (a : α) (m n : ℕ) {ω ω' : probSpace α R} +lemma stepsUntil_congr (alg : Algorithm 𝓐 R) (a : 𝓐) (m n : ℕ) {ω ω' : probSpace 𝓐 R} (hω1 : ∀ i, ω.1 i = ω'.1 i) (hω2_ne : ∀ i b, b ≠ a → ω.2 i b = ω'.2 i b) (hω2_eq : ∀ i, i + 1 ≤ m → ω.2 i a = ω'.2 i a) : (action alg (n + 1) ω = a ∧ pullCount (action alg) a (n + 1) ω = m) ↔ @@ -419,7 +419,7 @@ lemma stepsUntil_congr (alg : Algorithm α R) (a : α) (m n : ℕ) {ω ω' : pro ⟨stepsUntil_congr_aux alg a m n hω1 hω2_ne hω2_eq, stepsUntil_congr_aux alg a m n (by grind) (by grind) (by grind)⟩ -lemma stepsUntil_indicator_congr (alg : Algorithm α R) (a : α) (m n : ℕ) {ω ω' : probSpace α R} +lemma stepsUntil_indicator_congr (alg : Algorithm 𝓐 R) (a : 𝓐) (m n : ℕ) {ω ω' : probSpace 𝓐 R} (hω1 : ∀ i, ω.1 i = ω'.1 i) (hω2_ne : ∀ i b, b ≠ a → ω.2 i b = ω'.2 i b) (hω2_eq : ∀ i, i + 1 ≤ m → ω.2 i a = ω'.2 i a) : {ω | action alg (n + 1) ω = a ∧ pullCount (action alg) a (n + 1) ω = m}.indicator (fun _ ↦ 1) @@ -433,7 +433,7 @@ end Congruence section MeasurabilityAdvanced -lemma measurable_hist_todo [Countable α] (alg : Algorithm α R) (n : ℕ) : +lemma measurable_hist_todo [Countable 𝓐] (alg : Algorithm 𝓐 R) (n : ℕ) : Measurable[MeasurableSpace.comap (fun ω ↦ (fun (i : Iic n) ↦ ω.1 i, ω.2)) inferInstance] (hist alg · n) := by have h_eq : (hist alg · n) = @@ -446,7 +446,7 @@ lemma measurable_hist_todo [Countable α] (alg : Algorithm α R) (n : ℕ) : refine Measurable.prodMk ?_ (by fun_prop) rw [measurable_pi_iff] intro i - change Measurable ((fun p ↦ p ⟨min i n, by simp⟩) ∘ (fun x : (Iic n → I) × (ℕ → α → R) ↦ x.1)) + change Measurable ((fun p ↦ p ⟨min i n, by simp⟩) ∘ (fun x : (Iic n → I) × (ℕ → 𝓐 → R) ↦ x.1)) exact Measurable.comp (by fun_prop) measurable_fst variable [Nonempty R] @@ -455,28 +455,28 @@ variable [Nonempty R] /-- All random variables in the space, except for the unseen rewards for action `a` after time `n`. -/ noncomputable -def truePast (alg : Algorithm α R) (a : α) (n : ℕ) (ω : probSpace α R) : - probSpace α R := +def truePast (alg : Algorithm 𝓐 R) (a : 𝓐) (n : ℕ) (ω : probSpace 𝓐 R) : + probSpace 𝓐 R := (ω.1, fun i b ↦ if b = a then if pullCount (action alg) a (n + 1) ω ≠ 0 then ω.2 (min i ((pullCount (action alg) a (n + 1) ω) - 1)) a else Nonempty.some inferInstance else ω.2 i b) -lemma truePast_eq_of_pullCount_eq (alg : Algorithm α R) - (a : α) (n m : ℕ) (ω : probSpace α R) +lemma truePast_eq_of_pullCount_eq (alg : Algorithm 𝓐 R) + (a : 𝓐) (n m : ℕ) (ω : probSpace 𝓐 R) (h_pc : pullCount (action alg) a (n + 1) ω = m) : truePast alg a n ω = (ω.1, fun i b ↦ if b = a then if m ≠ 0 then ω.2 (min i (m - 1)) a else Nonempty.some inferInstance else ω.2 i b) := by simp [truePast, h_pc] -lemma truePast_eq_of_pullCount_eq_of_ne_zero (alg : Algorithm α R) - (a : α) (n m : ℕ) (ω : probSpace α R) +lemma truePast_eq_of_pullCount_eq_of_ne_zero (alg : Algorithm 𝓐 R) + (a : 𝓐) (n m : ℕ) (ω : probSpace 𝓐 R) (h_pc : pullCount (action alg) a (n + 1) ω = m) (hm : m ≠ 0) : truePast alg a n ω = (ω.1, fun i b ↦ if b = a then ω.2 (min i (m - 1)) a else ω.2 i b) := by simp [truePast, h_pc, hm] -lemma measurable_hist_truePast [Countable α] (alg : Algorithm α R) - (a : α) (n : ℕ) : +lemma measurable_hist_truePast [Countable 𝓐] (alg : Algorithm 𝓐 R) + (a : 𝓐) (n : ℕ) : Measurable[MeasurableSpace.comap (truePast alg a n) inferInstance] (hist alg · n) := by have h_eq : (hist alg · n) = (hist alg · n) ∘ (truePast alg a n) := by ext ω : 1 @@ -491,8 +491,8 @@ lemma measurable_hist_truePast [Countable α] (alg : Algorithm α R) refine Measurable.comp ?_ (Measurable.of_comap_le le_rfl) fun_prop -lemma measurable_action_add_one_truePast [Countable α] (alg : Algorithm α R) - (a : α) (n : ℕ) : +lemma measurable_action_add_one_truePast [Countable 𝓐] (alg : Algorithm 𝓐 R) + (a : 𝓐) (n : ℕ) : Measurable[MeasurableSpace.comap (truePast alg a n) inferInstance] (action alg (n + 1)) := by rw [action_add_one_eq] @@ -501,11 +501,11 @@ lemma measurable_action_add_one_truePast [Countable α] (alg : Algorithm α R) refine (measurable_algFunction alg n).comp (Measurable.prodMk ?_ ?_) · exact measurable_hist_truePast alg a n · have : (fun ω ↦ ω.1 (n + 1)) = - (fun (p : probSpace α R) ↦ p.1 (n + 1)) ∘ (truePast alg a n) := rfl + (fun (p : probSpace 𝓐 R) ↦ p.1 (n + 1)) ∘ (truePast alg a n) := rfl rw [this] exact Measurable.comp (by fun_prop) (Measurable.of_comap_le le_rfl) -lemma measurable_pullCount_add_one_truePast [Countable α] (alg : Algorithm α R) (a : α) (n : ℕ) : +lemma measurable_pullCount_add_one_truePast [Countable 𝓐] (alg : Algorithm 𝓐 R) (a : 𝓐) (n : ℕ) : Measurable[MeasurableSpace.comap (truePast alg a n) inferInstance] (pullCount (action alg) a (n + 1)) := by change Measurable[MeasurableSpace.comap (truePast alg a n) inferInstance] @@ -517,7 +517,7 @@ lemma measurable_pullCount_add_one_truePast [Countable α] (alg : Algorithm α R simp_rw [hist_eq _ _ n, @measurable_pi_iff] at h_meas exact (h_meas ⟨i, by grind⟩).fst -lemma measurable_stepsUntil [Countable α] (alg : Algorithm α R) (a : α) (m n : ℕ) : +lemma measurable_stepsUntil [Countable 𝓐] (alg : Algorithm 𝓐 R) (a : 𝓐) (m n : ℕ) : Measurable[MeasurableSpace.comap (fun ω ↦ (ω.1, fun k b ↦ if b = a then if m ≠ 0 then ω.2 (min k (m - 1)) b else Nonempty.some inferInstance else ω.2 k b)) inferInstance] @@ -540,7 +540,7 @@ lemma measurable_stepsUntil [Countable α] (alg : Algorithm α R) (a : α) (m n ((measurableSet_singleton _).preimage (by fun_prop)) omit [Nonempty R] in -lemma measurable_pullCount_action_add_one_hist (alg : Algorithm α R) (n : ℕ) : +lemma measurable_pullCount_action_add_one_hist (alg : Algorithm 𝓐 R) (n : ℕ) : Measurable[MeasurableSpace.comap (fun ω ↦ (action alg (n + 1) ω, hist alg ω n)) inferInstance] (fun ω ↦ pullCount (action alg) (action alg (n + 1) ω) (n + 1) ω) := by simp_rw [pullCount_eq_sum] @@ -552,15 +552,15 @@ lemma measurable_pullCount_action_add_one_hist (alg : Algorithm α R) (n : ℕ) unfold action refine Measurable.fst (mγ := inferInstance) ?_ have : (hist alg · i ⟨i, by grind⟩) = - (fun ω : α × (Iic n → α × R) ↦ ω.2 ⟨i, by grind⟩) ∘ + (fun ω : 𝓐 × (Iic n → 𝓐 × R) ↦ ω.2 ⟨i, by grind⟩) ∘ (fun ω ↦ (action alg (n + 1) ω, fun i : Iic n ↦ hist alg ω i ⟨i, by grind⟩)) := rfl rw [this] exact measurable_comp_comap _ (Measurable.prodMk (by fun_prop) (by fun_prop)) end MeasurabilityAdvanced -omit [Nonempty α] [StandardBorelSpace α] [DecidableEq α] in -lemma map_snd_apply_arrayMeasure {ν : Kernel α R} [IsMarkovKernel ν] (n : ℕ) (a : α) : +omit [Nonempty 𝓐] [StandardBorelSpace 𝓐] [DecidableEq 𝓐] in +lemma map_snd_apply_arrayMeasure {ν : Kernel 𝓐 R} [IsMarkovKernel ν] (n : ℕ) (a : 𝓐) : (arrayMeasure ν).map (fun ω ↦ ω.2 n a) = ν a := by calc (arrayMeasure ν).map (fun ω ↦ ω.2 n a) _ = (arrayMeasure ν).snd.map (fun ω ↦ ω n a) := by @@ -568,29 +568,29 @@ lemma map_snd_apply_arrayMeasure {ν : Kernel α R} [IsMarkovKernel ν] (n : ℕ rfl _ = ν a := by rw [arrayMeasure, Measure.snd_prod, streamMeasure] - have : (fun ω ↦ ω n a) = (fun h : α → R ↦ h a) ∘ (fun ω : ℕ → α → R ↦ ω n) := rfl + have : (fun ω ↦ ω n a) = (fun h : 𝓐 → R ↦ h a) ∘ (fun ω : ℕ → 𝓐 → R ↦ ω n) := rfl rw [this, ← Measure.map_map (by fun_prop) (by fun_prop), Measure.infinitePi_map_eval, Measure.infinitePi_map_eval] section Independence -omit [Nonempty α] [StandardBorelSpace α] [DecidableEq α] in -lemma indepFun_fst_snd (ν : Kernel α R) [IsMarkovKernel ν] : +omit [Nonempty 𝓐] [StandardBorelSpace 𝓐] [DecidableEq 𝓐] in +lemma indepFun_fst_snd (ν : Kernel 𝓐 R) [IsMarkovKernel ν] : IndepFun Prod.fst Prod.snd (arrayMeasure ν) := indepFun_prod measurable_id measurable_id -omit [Nonempty α] [StandardBorelSpace α] [DecidableEq α] in -lemma indepFun_fst_zero_snd_zero_action (ν : Kernel α R) [IsMarkovKernel ν] (a : α) : +omit [Nonempty 𝓐] [StandardBorelSpace 𝓐] [DecidableEq 𝓐] in +lemma indepFun_fst_zero_snd_zero_action (ν : Kernel 𝓐 R) [IsMarkovKernel ν] (a : 𝓐) : IndepFun (fun ω ↦ ω.1 0) (fun ω ↦ ω.2 0 a) (arrayMeasure ν) := - indepFun_prod (X := fun ω : ℕ → I ↦ ω 0) (Y := fun ω : ℕ → α → R ↦ ω 0 a) + indepFun_prod (X := fun ω : ℕ → I ↦ ω 0) (Y := fun ω : ℕ → 𝓐 → R ↦ ω 0 a) (by fun_prop) (by fun_prop) -- proved by Claude, then slightly golfed -omit [DecidableEq α] [Nonempty α] [StandardBorelSpace α] in -lemma indepFun_fst_add_one_aux (ν : Kernel α R) [IsMarkovKernel ν] (n : ℕ) : +omit [DecidableEq 𝓐] [Nonempty 𝓐] [StandardBorelSpace 𝓐] in +lemma indepFun_fst_add_one_aux (ν : Kernel 𝓐 R) [IsMarkovKernel ν] (n : ℕ) : (fun ω ↦ ω.1 (n + 1)) ⟂ᵢ[arrayMeasure ν] (fun ω ↦ (fun (i : Iic n) ↦ ω.1 i, ω.2)) := by let μ₁ : Measure (ℕ → I) := Measure.infinitePi fun _ ↦ volume - let μ₂ : Measure (ℕ → α → R) := streamMeasure ν + let μ₂ : Measure (ℕ → 𝓐 → R) := streamMeasure ν -- Coordinates of μ₁ are independent have h_indep : iIndepFun (fun i (ω : ℕ → I) ↦ ω i) μ₁ := iIndepFun_infinitePi (fun _ ↦ measurable_id) @@ -600,8 +600,8 @@ lemma indepFun_fst_add_one_aux (ν : Kernel α R) [IsMarkovKernel ν] (n : ℕ) convert h.comp (measurable_pi_apply ⟨n + 1, by simp⟩) measurable_id using 1 rw [indepFun_iff_measure_inter_preimage_eq_mul] intro s t hs ht - let X : (ℕ → I) × (ℕ → α → R) → I := fun ω ↦ ω.1 (n + 1) - let Y : (ℕ → I) × (ℕ → α → R) → (Iic n → I) × (ℕ → α → R) := fun ω ↦ (fun i ↦ ω.1 i, ω.2) + let X : (ℕ → I) × (ℕ → 𝓐 → R) → I := fun ω ↦ ω.1 (n + 1) + let Y : (ℕ → I) × (ℕ → 𝓐 → R) → (Iic n → I) × (ℕ → 𝓐 → R) := fun ω ↦ (fun i ↦ ω.1 i, ω.2) change (μ₁.prod μ₂) (X ⁻¹' s ∩ Y ⁻¹' t) = (μ₁.prod μ₂) (X ⁻¹' s) * (μ₁.prod μ₂) (Y ⁻¹' t) -- Rewrite using Fubini rw [Measure.prod_apply (hs.preimage (by fun_prop : Measurable X)), @@ -635,36 +635,36 @@ lemma indepFun_fst_add_one_aux (ν : Kernel α R) [IsMarkovKernel ν] (n : ℕ) variable [StandardBorelSpace R] [Nonempty R] -lemma indepFun_fst_add_one_hist [Countable α] (alg : Algorithm α R) - (ν : Kernel α R) [IsMarkovKernel ν] (n : ℕ) : +lemma indepFun_fst_add_one_hist [Countable 𝓐] (alg : Algorithm 𝓐 R) + (ν : Kernel 𝓐 R) [IsMarkovKernel ν] (n : ℕ) : IndepFun (fun ω ↦ ω.1 (n + 1)) (hist alg · n) (arrayMeasure ν) := (indepFun_fst_add_one_aux ν n).of_measurable_right (measurable_hist_todo alg n) -- proved by Claude -omit [Nonempty α] [StandardBorelSpace α] [StandardBorelSpace R] in -lemma indepFun_snd_apply_aux (ν : Kernel α R) [IsMarkovKernel ν] (a : α) (m : ℕ) : +omit [Nonempty 𝓐] [StandardBorelSpace 𝓐] [StandardBorelSpace R] in +lemma indepFun_snd_apply_aux (ν : Kernel 𝓐 R) [IsMarkovKernel ν] (a : 𝓐) (m : ℕ) : (fun ω ↦ ω.2 m a) ⟂ᵢ[arrayMeasure ν] (fun ω ↦ (ω.1, fun k b ↦ if b = a then if m ≠ 0 then ω.2 (min k (m - 1)) b else Nonempty.some inferInstance else ω.2 k b)) := by unfold arrayMeasure let μ₁ : Measure (ℕ → I) := Measure.infinitePi fun _ ↦ volume - let μ₂ : Measure (ℕ → α → R) := Measure.infinitePi fun _ ↦ Measure.infinitePi ν + let μ₂ : Measure (ℕ → 𝓐 → R) := Measure.infinitePi fun _ ↦ Measure.infinitePi ν -- Independence within μ₂: coordinates ω i are independent - have h_indep₂ : iIndepFun (fun i (ω : ℕ → α → R) ↦ ω i) μ₂ := + have h_indep₂ : iIndepFun (fun i (ω : ℕ → 𝓐 → R) ↦ ω i) μ₂ := iIndepFun_infinitePi (fun _ ↦ measurable_id) -- Independence within each infinitePi ν: coordinates f b are independent - have h_indep_inner : iIndepFun (fun (b : α) (f : α → R) ↦ f b) (Measure.infinitePi ν) := + have h_indep_inner : iIndepFun (fun (b : 𝓐) (f : 𝓐 → R) ↦ f b) (Measure.infinitePi ν) := iIndepFun_infinitePi (fun _ ↦ measurable_id) rw [indepFun_iff_measure_inter_preimage_eq_mul] intro s t hs ht - let X : (ℕ → I) × (ℕ → α → R) → R := fun ω ↦ ω.2 m a - let Y : (ℕ → I) × (ℕ → α → R) → (ℕ → I) × (ℕ → α → R) := + let X : (ℕ → I) × (ℕ → 𝓐 → R) → R := fun ω ↦ ω.2 m a + let Y : (ℕ → I) × (ℕ → 𝓐 → R) → (ℕ → I) × (ℕ → 𝓐 → R) := fun ω ↦ (ω.1, fun k b ↦ if b = a then if m ≠ 0 then ω.2 (min k (m - 1)) b else Nonempty.some inferInstance else ω.2 k b) have hX_meas : Measurable X := (measurable_pi_apply a).comp ((measurable_pi_apply m).comp measurable_snd) have hY_meas : Measurable Y := by - change Measurable (fun ω : (ℕ → I) × (ℕ → α → R) ↦ + change Measurable (fun ω : (ℕ → I) × (ℕ → 𝓐 → R) ↦ (ω.1, fun k b ↦ if b = a then if m ≠ 0 then ω.2 (min k (m - 1)) b else Nonempty.some inferInstance else ω.2 k b)) refine Measurable.prod measurable_fst ?_ @@ -691,7 +691,7 @@ lemma indepFun_snd_apply_aux (ν : Kernel α R) [IsMarkovKernel ν] (a : α) (m -- The LHS integral: fiber of X ∩ Y at ω₁ -- Key: X depends only on ω₂ m a, while Y's dependence on ω₂ avoids (m, a) -- Define the "truncation" map on ω₂ - let trunc : (ℕ → α → R) → (ℕ → α → R) := + let trunc : (ℕ → 𝓐 → R) → (ℕ → 𝓐 → R) := fun ω₂ k b ↦ if b = a then if m ≠ 0 then ω₂ (min k (m - 1)) b else Nonempty.some inferInstance else ω₂ k b -- The fiber of Y at ω₁ only depends on trunc(ω₂) @@ -718,7 +718,7 @@ lemma indepFun_snd_apply_aux (ν : Kernel α R) [IsMarkovKernel ν] (a : α) (m -- Key independence: (ω₂ m a) ⟂ trunc because trunc only uses coordinates ≠ (m, a) have h_indep_trunc : IndepFun (fun ω₂ ↦ ω₂ m a) trunc μ₂ := by -- Factor trunc through proj which extracts the relevant coordinates - let proj : (ℕ → α → R) → ((ℕ → R) × (ℕ → {b : α // b ≠ a} → R)) := fun ω₂ ↦ + let proj : (ℕ → 𝓐 → R) → ((ℕ → R) × (ℕ → {b : 𝓐 // b ≠ a} → R)) := fun ω₂ ↦ (fun k ↦ if m ≠ 0 then ω₂ (min k (m - 1)) a else Nonempty.some inferInstance, fun k ⟨b, _⟩ ↦ ω₂ k b) have h_trunc_proj : ∀ ω₂, trunc ω₂ = (fun p k b ↦ @@ -734,7 +734,7 @@ lemma indepFun_snd_apply_aux (ν : Kernel α R) [IsMarkovKernel ν] (a : α) (m · simp [proj, hm] · exact measurable_pi_lambda _ (fun k ↦ measurable_pi_lambda _ (fun ⟨b, _⟩ ↦ (measurable_pi_apply b).comp (measurable_pi_apply k))) - have h_g_meas : Measurable (fun p : (ℕ → R) × (ℕ → {b : α // b ≠ a} → R) ↦ + have h_g_meas : Measurable (fun p : (ℕ → R) × (ℕ → {b : 𝓐 // b ≠ a} → R) ↦ (fun k b ↦ if h : b = a then if m ≠ 0 then p.1 k else Nonempty.some inferInstance else p.2 k ⟨b, h⟩)) := by refine measurable_pi_lambda _ (fun k ↦ measurable_pi_lambda _ (fun b ↦ ?_)) @@ -745,7 +745,7 @@ lemma indepFun_snd_apply_aux (ν : Kernel α R) [IsMarkovKernel ν] (a : α) (m exact (measurable_pi_apply k).comp measurable_fst · simp [hm] · simp only [hb, ↓reduceDIte] - exact (measurable_pi_apply (⟨b, hb⟩ : {b : α // b ≠ a})).comp + exact (measurable_pi_apply (⟨b, hb⟩ : {b : 𝓐 // b ≠ a})).comp ((measurable_pi_apply k).comp measurable_snd) -- Show (ω₂ m a) ⟂ proj: proj uses coordinates disjoint from (m, a) have h_indep_proj : IndepFun (fun ω₂ ↦ ω₂ m a) proj μ₂ := by @@ -756,8 +756,8 @@ lemma indepFun_snd_apply_aux (ν : Kernel α R) [IsMarkovKernel ν] (a : α) (m rw [indepFun_iff_measure_inter_preimage_eq_mul] intro s t' hs ht' -- rows_lt_m extracts column a at rows < m, other_cols extracts columns ≠ a - let rows_lt_m : (ℕ → α → R) → (Iio m → R) := fun ω₂ ⟨j, _⟩ ↦ ω₂ j a - let other_cols : (ℕ → α → R) → (ℕ → {b : α // b ≠ a} → R) := fun ω₂ k ⟨b, _⟩ ↦ ω₂ k b + let rows_lt_m : (ℕ → 𝓐 → R) → (Iio m → R) := fun ω₂ ⟨j, _⟩ ↦ ω₂ j a + let other_cols : (ℕ → 𝓐 → R) → (ℕ → {b : 𝓐 // b ≠ a} → R) := fun ω₂ k ⟨b, _⟩ ↦ ω₂ k b have h_proj_factor : ∀ ω₂, proj ω₂ = ((fun r k ↦ if hm : m ≠ 0 then r ⟨min k (m - 1), Finset.mem_Iio.mpr (h_row_bound hm k)⟩ else Nonempty.some inferInstance) (rows_lt_m ω₂), @@ -769,7 +769,7 @@ lemma indepFun_snd_apply_aux (ν : Kernel α R) [IsMarkovKernel ν] (a : α) (m · simp [proj, hm] · rfl -- Use iIndepFun structure of the doubly-indexed infinite product - have h_iindep : iIndepFun (fun (p : ℕ × α) ω ↦ ω p.1 p.2) μ₂ := + have h_iindep : iIndepFun (fun (p : ℕ × 𝓐) ω ↦ ω p.1 p.2) μ₂ := iIndepFun_uncurry_infinitePi' (X := fun _ _ ↦ id) (fun _ ↦ ν) (by fun_prop) have h_rows_meas : Measurable rows_lt_m := measurable_pi_lambda _ (fun ⟨j, _⟩ ↦ (measurable_pi_apply a).comp (measurable_pi_apply j)) @@ -782,7 +782,7 @@ lemma indepFun_snd_apply_aux (ν : Kernel α R) [IsMarkovKernel ν] (a : α) (m rw [IndepFun_iff_Indep] have h_comap_le : (MeasurableSpace.pi.prod MeasurableSpace.pi).comap (fun ω₂ ↦ (rows_lt_m ω₂, other_cols ω₂)) ≤ - ⨆ (p : {p : ℕ × α // p ≠ (m, a)}), mR.comap (fun ω ↦ ω p.val.1 p.val.2) := by + ⨆ (p : {p : ℕ × 𝓐 // p ≠ (m, a)}), mR.comap (fun ω ↦ ω p.val.1 p.val.2) := by rw [MeasurableSpace.comap_prodMk] refine sup_le ?_ ?_ · rw [MeasurableSpace.comap_pi] @@ -796,11 +796,11 @@ lemma indepFun_snd_apply_aux (ν : Kernel α R) [IsMarkovKernel ν] (a : α) (m have h_ne : (k, b) ≠ (m, a) := fun h ↦ hb (Prod.mk.inj h).2 exact le_iSup_of_le ⟨(k, b), h_ne⟩ le_rfl refine indep_of_indep_of_le_right ?_ h_comap_le - have h_disjoint : Disjoint ({(m, a)} : Set (ℕ × α)) {p | p ≠ (m, a)} := by simp - have h_le : ∀ p : ℕ × α, mR.comap (fun ω : ℕ → α → R ↦ ω p.1 p.2) ≤ + have h_disjoint : Disjoint ({(m, a)} : Set (ℕ × 𝓐)) {p | p ≠ (m, a)} := by simp + have h_le : ∀ p : ℕ × 𝓐, mR.comap (fun ω : ℕ → 𝓐 → R ↦ ω p.1 p.2) ≤ MeasurableSpace.pi (m := fun _ ↦ MeasurableSpace.pi) := fun p ↦ Measurable.comap_le ((measurable_pi_apply p.2).comp (measurable_pi_apply p.1)) - have h_iindep' : iIndep (fun p : ℕ × α ↦ mR.comap (fun ω : ℕ → α → R ↦ ω p.1 p.2)) μ₂ := + have h_iindep' : iIndep (fun p : ℕ × 𝓐 ↦ mR.comap (fun ω : ℕ → 𝓐 → R ↦ ω p.1 p.2)) μ₂ := h_iindep.iIndep have h_indep := indep_iSup_of_disjoint h_le h_iindep' h_disjoint convert h_indep using 2 @@ -813,7 +813,7 @@ lemma indepFun_snd_apply_aux (ν : Kernel α R) [IsMarkovKernel ν] (a : α) (m := by ext ω₂; simp only [Set.mem_preimage, Set.mem_setOf_eq, h_proj_factor] rw [indepFun_iff_measure_inter_preimage_eq_mul] at h_indep_combined rw [h_proj_preimage] - let T : Set ((Iio m → R) × (ℕ → {b : α // b ≠ a} → R)) := + let T : Set ((Iio m → R) × (ℕ → {b : 𝓐 // b ≠ a} → R)) := {p | ((fun r k ↦ if hm : m ≠ 0 then r ⟨min k (m - 1), Finset.mem_Iio.mpr (h_row_bound hm k)⟩ else Nonempty.some inferInstance) p.1, p.2) ∈ t'} @@ -851,24 +851,24 @@ lemma indepFun_snd_apply_aux (ν : Kernel α R) [IsMarkovKernel ν] (a : α) (m lintegral_const, measure_univ, mul_one] omit [StandardBorelSpace R] in -lemma indepFun_snd_apply_pullCount_action [Countable α] (alg : Algorithm α R) - (ν : Kernel α R) [IsMarkovKernel ν] (a : α) (m n : ℕ) : +lemma indepFun_snd_apply_pullCount_action [Countable 𝓐] (alg : Algorithm 𝓐 R) + (ν : Kernel 𝓐 R) [IsMarkovKernel ν] (a : 𝓐) (m n : ℕ) : (fun ω ↦ ω.2 m a) ⟂ᵢ[arrayMeasure ν] ({ω | action alg (n + 1) ω = a ∧ pullCount (action alg) a (n + 1) ω = m}).indicator (fun _ ↦ 1) := (indepFun_snd_apply_aux ν a m).of_measurable_right (measurable_stepsUntil alg a m n) -lemma indepFun_todo {α β γ δ : Type*} {mα : MeasurableSpace α} {mβ : MeasurableSpace β} - {mγ : MeasurableSpace γ} {mδ : MeasurableSpace δ} [MeasurableSingletonClass δ] {μ : Measure α} - {X : α → β} {Y : α → γ} (hXY : X ⟂ᵢ[μ] Y) (hY : Measurable Y) +lemma indepFun_todo {𝓐 β γ δ : Type*} {m𝓐 : MeasurableSpace 𝓐} {mβ : MeasurableSpace β} + {mγ : MeasurableSpace γ} {mδ : MeasurableSpace δ} [MeasurableSingletonClass δ] {μ : Measure 𝓐} + {X : 𝓐 → β} {Y : 𝓐 → γ} (hXY : X ⟂ᵢ[μ] Y) (hY : Measurable Y) {Z : γ → δ} (hZ : Measurable Z) (z : δ) : X ⟂ᵢ[μ[|(Z ∘ Y) ⁻¹' {z}]] Y := by have h_preim : (Z ∘ Y) ⁻¹' {z} = Y ⁻¹' (Z ⁻¹' {z}) := by grind simp_rw [h_preim] exact indepFun_cond_of_indepFun hXY hY (hZ (measurableSet_singleton z)) -lemma indepFun_snd_hist_cond [Countable α] (alg : Algorithm α R) - (ν : Kernel α R) [IsMarkovKernel ν] (a : α) (n m : ℕ) : +lemma indepFun_snd_hist_cond [Countable 𝓐] (alg : Algorithm 𝓐 R) + (ν : Kernel 𝓐 R) [IsMarkovKernel ν] (a : 𝓐) (n m : ℕ) : (fun ω ↦ ω.2 m a) ⟂ᵢ[(arrayMeasure ν)[|(fun ω ↦ (action alg (n + 1) ω, pullCount (action alg) (action alg (n + 1) ω) (n + 1) ω)) ⁻¹' {(a, m)}]] (hist alg · n) := by @@ -882,8 +882,8 @@ lemma indepFun_snd_hist_cond [Countable α] (alg : Algorithm α R) · refine (measurableSet_singleton _).preimage ?_ have h_meas_pc : Measurable fun ω ↦ pullCount (action alg) (action alg (n + 1) ω) (n + 1) ω := by - change Measurable ((fun p : (probSpace α R) × α ↦ pullCount (action alg) p.2 (n + 1) p.1) ∘ - (fun ω : probSpace α R ↦ (ω, action alg (n + 1) ω))) + change Measurable ((fun p : (probSpace 𝓐 R) × 𝓐 ↦ pullCount (action alg) p.2 (n + 1) p.1) ∘ + (fun ω : probSpace 𝓐 R ↦ (ω, action alg (n + 1) ω))) exact (measurable_uncurry_pullCount (by fun_prop) _).comp (by fun_prop) fun_prop simp only [Set.mem_preimage, Set.mem_singleton_iff, Prod.mk.injEq] at hx @@ -921,9 +921,9 @@ end Independence section Laws -variable [Countable α] +variable [Countable 𝓐] -lemma hasLaw_action_zero (alg : Algorithm α R) (ν : Kernel α R) [IsMarkovKernel ν] : +lemma hasLaw_action_zero (alg : Algorithm 𝓐 R) (ν : Kernel 𝓐 R) [IsMarkovKernel ν] : HasLaw (action alg 0) alg.p0 (arrayMeasure ν) where map_eq := by calc (arrayMeasure ν).map (fun ω ↦ initAlgFunction alg (ω.1 0)) @@ -938,7 +938,7 @@ lemma hasLaw_action_zero (alg : Algorithm α R) (ν : Kernel α R) [IsMarkovKern variable [StandardBorelSpace R] [Nonempty R] -lemma hasCondDistrib_reward_zero (alg : Algorithm α R) (ν : Kernel α R) [IsMarkovKernel ν] : +lemma hasCondDistrib_reward_zero (alg : Algorithm 𝓐 R) (ν : Kernel 𝓐 R) [IsMarkovKernel ν] : HasCondDistrib (reward alg 0) (action alg 0) ν (arrayMeasure ν) where condDistrib_eq := by refine (condDistrib_ae_eq_cond (by fun_prop) (by fun_prop)).trans ?_ @@ -964,7 +964,7 @@ lemma hasCondDistrib_reward_zero (alg : Algorithm α R) (ν : Kernel α R) [IsMa · simp · rwa [Measure.map_apply (by fun_prop) (by simp)] at ha -lemma hasCondDistrib_action' (alg : Algorithm α R) (ν : Kernel α R) [IsMarkovKernel ν] (n : ℕ) : +lemma hasCondDistrib_action' (alg : Algorithm 𝓐 R) (ν : Kernel 𝓐 R) [IsMarkovKernel ν] (n : ℕ) : HasCondDistrib (action alg (n + 1)) (hist alg · n) (alg.policy n) (arrayMeasure ν) := by rw [action_add_one_eq] have h_fun ω := algFunction_map alg n (hist alg ω n) @@ -1023,13 +1023,13 @@ lemma hasCondDistrib_action' (alg : Algorithm α R) (ν : Kernel α R) [IsMarkov and the number of times that action has been pulled before time `n + 1`, is equal to the kernel `ν`. -/ lemma hasCondDistrib_reward_pullCount_action - (alg : Algorithm α R) (ν : Kernel α R) [IsMarkovKernel ν] (n : ℕ) : + (alg : Algorithm 𝓐 R) (ν : Kernel 𝓐 R) [IsMarkovKernel ν] (n : ℕ) : HasCondDistrib (reward alg (n + 1)) (fun ω ↦ (action alg (n + 1) ω, pullCount (action alg) (action alg (n + 1) ω) (n + 1) ω)) (ν.prodMkRight _) (arrayMeasure ν) := by have h_meas : Measurable fun ω ↦ pullCount (action alg) (action alg (n + 1) ω) (n + 1) ω := by - change Measurable ((fun p : (probSpace α R) × α ↦ pullCount (action alg) p.2 (n + 1) p.1) ∘ - (fun ω : probSpace α R ↦ (ω, action alg (n + 1) ω))) + change Measurable ((fun p : (probSpace 𝓐 R) × 𝓐 ↦ pullCount (action alg) p.2 (n + 1) p.1) ∘ + (fun ω : probSpace 𝓐 R ↦ (ω, action alg (n + 1) ω))) exact (measurable_uncurry_pullCount (by fun_prop) _).comp (by fun_prop) refine ⟨by fun_prop, by fun_prop, ?_⟩ refine (condDistrib_ae_eq_cond @@ -1077,15 +1077,15 @@ lemma hasCondDistrib_reward_pullCount_action simp [ha] omit [StandardBorelSpace R] [Nonempty R] in -lemma reward_ae_eq_cond (alg : Algorithm α R) (ν : Kernel α R) (a : α) (n m : ℕ) : +lemma reward_ae_eq_cond (alg : Algorithm 𝓐 R) (ν : Kernel 𝓐 R) (a : 𝓐) (n m : ℕ) : reward alg (n + 1) =ᵐ[(arrayMeasure ν)[|(fun ω ↦ (action alg (n + 1) ω, pullCount (action alg) (action alg (n + 1) ω) (n + 1) ω)) ⁻¹' {(a, m)}]] (fun ω ↦ ω.2 m a) := by rw [reward_eq] refine ae_cond_of_forall_mem ?_ ?_ · have : Measurable fun ω ↦ pullCount (action alg) (action alg (n + 1) ω) (n + 1) ω := by - change Measurable ((fun p : (probSpace α R) × α ↦ pullCount (action alg) p.2 (n + 1) p.1) ∘ - (fun ω : probSpace α R ↦ (ω, action alg (n + 1) ω))) + change Measurable ((fun p : (probSpace 𝓐 R) × 𝓐 ↦ pullCount (action alg) p.2 (n + 1) p.1) ∘ + (fun ω : probSpace 𝓐 R ↦ (ω, action alg (n + 1) ω))) exact (measurable_uncurry_pullCount (by fun_prop) _).comp (by fun_prop) exact (measurableSet_singleton _).preimage (by fun_prop) intro ω hω @@ -1097,19 +1097,19 @@ lemma reward_ae_eq_cond (alg : Algorithm α R) (ν : Kernel α R) (a : α) (n m the action at time `n + 1`, and the number of times that action has been pulled before time `n + 1`, is equal to the kernel `ν`. -/ lemma hasCondDistrib_reward_hist_action_pullCount - (alg : Algorithm α R) (ν : Kernel α R) [IsMarkovKernel ν] (n : ℕ) : + (alg : Algorithm 𝓐 R) (ν : Kernel 𝓐 R) [IsMarkovKernel ν] (n : ℕ) : HasCondDistrib (reward alg (n + 1)) (fun ω ↦ (hist alg ω n, action alg (n + 1) ω, pullCount (action alg) (action alg (n + 1) ω) (n + 1) ω)) ((ν.prodMkRight _).prodMkLeft _) (arrayMeasure ν) := by have h_meas : Measurable fun ω ↦ pullCount (action alg) (action alg (n + 1) ω) (n + 1) ω := by - change Measurable ((fun p : (probSpace α R) × α ↦ pullCount (action alg) p.2 (n + 1) p.1) ∘ - (fun ω : probSpace α R ↦ (ω, action alg (n + 1) ω))) + change Measurable ((fun p : (probSpace 𝓐 R) × 𝓐 ↦ pullCount (action alg) p.2 (n + 1) p.1) ∘ + (fun ω : probSpace 𝓐 R ↦ (ω, action alg (n + 1) ω))) exact (measurable_uncurry_pullCount (by fun_prop) _).comp (by fun_prop) refine ⟨by fun_prop, by fun_prop, ?_⟩ refine condDistrib_prod_of_forall_condDistrib_cond (by fun_prop) (by fun_prop) (by fun_prop) _ ?_ intro (a, m) ham - have h_eq : ((ν.prodMkRight _).prodMkLeft _).comap (fun ω : (Iic n → α × R) ↦ (ω, a, m)) + have h_eq : ((ν.prodMkRight _).prodMkLeft _).comap (fun ω : (Iic n → 𝓐 × R) ↦ (ω, a, m)) (by fun_prop) = Kernel.const _ (ν a) := by ext; simp rw [h_eq, condDistrib_congr_left (reward_ae_eq_cond alg ν a n m)] @@ -1141,7 +1141,7 @@ lemma hasCondDistrib_reward_hist_action_pullCount /-- The reward at time `n + 1` is conditionally independent of the history up to time `n`, given the action at time `n + 1` and the number of times that action has been pulled before time `n + 1`. -/ -lemma condIndepFun_reward_hist (alg : Algorithm α R) (ν : Kernel α R) [IsMarkovKernel ν] (n : ℕ) : +lemma condIndepFun_reward_hist (alg : Algorithm 𝓐 R) (ν : Kernel 𝓐 R) [IsMarkovKernel ν] (n : ℕ) : (reward alg (n + 1)) ⟂ᵢ[(fun ω ↦ (action alg (n + 1) ω, pullCount (action alg) (action alg (n + 1) ω) (n + 1) ω)), Measurable.prodMk (by fun_prop) (measurable_pullCount_action_add_one alg n); @@ -1154,7 +1154,7 @@ lemma condIndepFun_reward_hist (alg : Algorithm α R) (ν : Kernel α R) [IsMark /-- The conditional distribution of the reward at time `n + 1`, given the history up to time `n` and the action at time `n + 1`, is equal to the kernel `ν`. -/ -lemma hasCondDistrib_reward' (alg : Algorithm α R) (ν : Kernel α R) [IsMarkovKernel ν] (n : ℕ) : +lemma hasCondDistrib_reward' (alg : Algorithm 𝓐 R) (ν : Kernel 𝓐 R) [IsMarkovKernel ν] (n : ℕ) : HasCondDistrib (reward alg (n + 1)) (fun ω ↦ (hist alg ω n, action alg (n + 1) ω)) (ν.prodMkLeft _) (arrayMeasure ν) := by let R' := reward alg (n + 1) @@ -1178,7 +1178,7 @@ lemma hasCondDistrib_reward' (alg : Algorithm α R) (ν : Kernel α R) [IsMarkov rwa [hasCondDistrib_prod_right_iff _ _ hf_meas] at this suffices HasCondDistrib R' (fun ω ↦ ((A ω, P ω), H ω)) ((ν.prodMkRight _).prodMkRight _) (arrayMeasure ν) by - let e : ((α × ℕ) × (Iic n → α × R)) ≃ᵐ ((α × (Iic n → α × R)) × ℕ) := + let e : ((𝓐 × ℕ) × (Iic n → 𝓐 × R)) ≃ᵐ ((𝓐 × (Iic n → 𝓐 × R)) × ℕ) := { toFun := fun x ↦ ((x.1.1, x.2), x.1.2) invFun := fun x ↦ ((x.1.1, x.2), x.1.2) measurable_toFun := by fun_prop @@ -1200,7 +1200,7 @@ lemma hasCondDistrib_reward' (alg : Algorithm α R) (ν : Kernel α R) [IsMarkov · exact Kernel.measurableSet_eq _ _ exact hasCondDistrib_reward_pullCount_action alg ν n -lemma hasCondDistrib_action (alg : Algorithm α R) (ν : Kernel α R) [IsMarkovKernel ν] (n : ℕ) : +lemma hasCondDistrib_action (alg : Algorithm 𝓐 R) (ν : Kernel 𝓐 R) [IsMarkovKernel ν] (n : ℕ) : HasCondDistrib (action alg (n + 1)) (fun ω (i : Iic n) ↦ (action alg i ω, reward alg i ω)) (alg.policy n) (arrayMeasure ν) := by @@ -1210,7 +1210,7 @@ lemma hasCondDistrib_action (alg : Algorithm α R) (ν : Kernel α R) [IsMarkovK · simp only [reward] rw [hist_eq _ _ n] -lemma hasCondDistrib_reward (alg : Algorithm α R) (ν : Kernel α R) [IsMarkovKernel ν] +lemma hasCondDistrib_reward (alg : Algorithm 𝓐 R) (ν : Kernel 𝓐 R) [IsMarkovKernel ν] (n : ℕ) : HasCondDistrib (reward alg (n + 1)) (fun ω ↦ (fun (i : Iic n) ↦ (action alg i ω, reward alg i ω), action alg (n + 1) ω)) @@ -1221,12 +1221,12 @@ lemma hasCondDistrib_reward (alg : Algorithm α R) (ν : Kernel α R) [IsMarkovK · simp only [reward] rw [hist_eq _ _ n] -lemma isAlgEnvSeq_arrayMeasure (alg : Algorithm α R) (ν : Kernel α R) [IsMarkovKernel ν] : +lemma isAlgEnvSeq_arrayMeasure (alg : Algorithm 𝓐 R) (ν : Kernel 𝓐 R) [IsMarkovKernel ν] : IsAlgEnvSeq (action alg) (reward alg) alg (stationaryEnv ν) (arrayMeasure ν) where hasLaw_action_zero := hasLaw_action_zero alg ν - hasCondDistrib_reward_zero := hasCondDistrib_reward_zero alg ν + hasCondDistrib_feedback_zero := hasCondDistrib_reward_zero alg ν hasCondDistrib_action := hasCondDistrib_action alg ν - hasCondDistrib_reward := hasCondDistrib_reward alg ν + hasCondDistrib_feedback := hasCondDistrib_reward alg ν end Laws diff --git a/LeanMachineLearning/Online/Bandit/Regret.lean b/LeanMachineLearning/Online/Bandit/Regret.lean index b11be810..4f3c20bb 100644 --- a/LeanMachineLearning/Online/Bandit/Regret.lean +++ b/LeanMachineLearning/Online/Bandit/Regret.lean @@ -14,8 +14,8 @@ public import LeanMachineLearning.SequentialLearning.FiniteActions * `gap ν a` is the gap of an action `a`, i.e., the difference between the highest mean of the actions and the mean of `a`. -* `regret ν A t ω` is the regret of a sequence of pulls `A : ℕ → Ω → α` at time `t` for the reward - kernel `ν : Kernel α ℝ` and the outcome `ω : Ω`. +* `regret ν A t ω` is the regret of a sequence of pulls `A : ℕ → Ω → 𝓐` at time `t` for the reward + kernel `ν : Kernel 𝓐 ℝ` and the outcome `ω : Ω`. * `bestArm ν` is an action with the highest mean. -/ @@ -28,50 +28,50 @@ open scoped ENNReal NNReal namespace Bandits -variable {α Ω : Type*} [DecidableEq α] {mα : MeasurableSpace α} {mΩ : MeasurableSpace Ω} - {ν : Kernel α ℝ} - {A : ℕ → Ω → α} {R : ℕ → Ω → ℝ} - {ω : Ω} {m n t : ℕ} {a : α} +variable {𝓐 Ω : Type*} [DecidableEq 𝓐] {m𝓐 : MeasurableSpace 𝓐} {mΩ : MeasurableSpace Ω} + {ν : Kernel 𝓐 ℝ} + {A : ℕ → Ω → 𝓐} {R : ℕ → Ω → ℝ} + {ω : Ω} {m n t : ℕ} {a : 𝓐} /-- Gap of an action `a`: difference between the highest mean of the actions and the mean of `a`. -/ noncomputable -- ANCHOR: gap -def gap (ν : Kernel α ℝ) (a : α) : ℝ := (⨆ i, (ν i)[id]) - (ν a)[id] +def gap (ν : Kernel 𝓐 ℝ) (a : 𝓐) : ℝ := (⨆ i, (ν i)[id]) - (ν a)[id] -- ANCHOR_END: gap -omit [DecidableEq α] in -lemma gap_nonneg [Finite α] : 0 ≤ gap ν a := by +omit [DecidableEq 𝓐] in +lemma gap_nonneg [Finite 𝓐] : 0 ≤ gap ν a := by rw [gap, sub_nonneg] exact le_ciSup (f := fun i ↦ (ν i)[id]) (by simp) a -/-- Regret of a sequence of pulls `k : ℕ → α` at time `t` for the reward kernel `ν ; Kernel α ℝ`. -/ +/-- Regret of a sequence of pulls `k : ℕ → 𝓐` at time `t` for the reward kernel `ν ; Kernel 𝓐 ℝ`. -/ noncomputable -- ANCHOR: regret -def regret (ν : Kernel α ℝ) (A : ℕ → Ω → α) (t : ℕ) (ω : Ω) : ℝ := +def regret (ν : Kernel 𝓐 ℝ) (A : ℕ → Ω → 𝓐) (t : ℕ) (ω : Ω) : ℝ := t * (⨆ a, (ν a)[id]) - ∑ s ∈ range t, (ν (A s ω))[id] -- ANCHOR_END: regret -omit [DecidableEq α] in +omit [DecidableEq 𝓐] in lemma regret_eq_sum_gap : regret ν A t ω = ∑ s ∈ range t, gap ν (A s ω) := by simp [regret, gap] -omit [DecidableEq α] in -lemma regret_nonneg [Finite α] : 0 ≤ regret ν A t ω := by +omit [DecidableEq 𝓐] in +lemma regret_nonneg [Finite 𝓐] : 0 ≤ regret ν A t ω := by rw [regret_eq_sum_gap] exact sum_nonneg (fun _ _ ↦ gap_nonneg) -omit [DecidableEq α] in -lemma gap_eq_zero_of_regret_eq_zero [Finite α] (hr : regret ν A t ω = 0) {s : ℕ} (hs : s < t) : +omit [DecidableEq 𝓐] in +lemma gap_eq_zero_of_regret_eq_zero [Finite 𝓐] (hr : regret ν A t ω = 0) {s : ℕ} (hs : s < t) : gap ν (A s ω) = 0 := by rw [regret_eq_sum_gap] at hr exact (sum_eq_zero_iff_of_nonneg fun _ _ ↦ gap_nonneg).1 hr s (mem_range.2 hs) -lemma regret_eq_sum_pullCount_mul_gap [Fintype α] : +lemma regret_eq_sum_pullCount_mul_gap [Fintype 𝓐] : regret ν A t ω = ∑ a, pullCount A a t ω * gap ν a := by simp_rw [regret_eq_sum_gap, sum_pullCount_mul] lemma integral_regret_eq_sum_gap_mul_integral_pullCount - [StandardBorelSpace α] [Fintype α] {P : Measure Ω} [IsProbabilityMeasure P] + [StandardBorelSpace 𝓐] [Fintype 𝓐] {P : Measure Ω} [IsProbabilityMeasure P] (hA : ∀ n, Measurable (A n)) : P[regret ν A n] = ∑ a, gap ν a * P[fun ω ↦ (pullCount A a n ω : ℝ)] := by simp_rw [regret_eq_sum_pullCount_mul_gap] @@ -83,12 +83,12 @@ lemma integral_regret_eq_sum_gap_mul_integral_pullCount /-- To bound the expected regret, it suffices to bound the expected number of pulls for each action with positive gap. -/ lemma integral_regret_le_of_forall_integral_pullCount_le - [Nonempty α] [StandardBorelSpace α] [Fintype α] {P : Measure Ω} [IsProbabilityMeasure P] - {alg : Algorithm α ℝ} {env : Environment α ℝ} {B : α → ℝ} + [Nonempty 𝓐] [StandardBorelSpace 𝓐] [Fintype 𝓐] {P : Measure Ω} [IsProbabilityMeasure P] + {alg : Algorithm 𝓐 ℝ} {env : Environment 𝓐 ℝ} {B : 𝓐 → ℝ} (h : IsAlgEnvSeq A R alg env P) (h_le : ∀ a, gap ν a ≠ 0 → ∫ ω, (pullCount A a n ω : ℝ) ∂P ≤ B a) : P[regret ν A n] ≤ ∑ a, gap ν a * B a := by - have hA := h.measurable_A + have hA := h.measurable_action rw [integral_regret_eq_sum_gap_mul_integral_pullCount hA] gcongr 1 with a by_cases h_gap : gap ν a = 0 @@ -99,30 +99,30 @@ lemma integral_regret_le_of_forall_integral_pullCount_le section bestArm -variable [Fintype α] [Nonempty α] +variable [Fintype 𝓐] [Nonempty 𝓐] /-- action with the highest mean. -/ -noncomputable def bestArm (ν : Kernel α ℝ) : α := +noncomputable def bestArm (ν : Kernel 𝓐 ℝ) : 𝓐 := (exists_max_image univ (fun a ↦ (ν a)[id]) (univ_nonempty_iff.mpr inferInstance)).choose -omit [DecidableEq α] in -lemma le_bestArm (a : α) : (ν a)[id] ≤ (ν (bestArm ν))[id] := +omit [DecidableEq 𝓐] in +lemma le_bestArm (a : 𝓐) : (ν a)[id] ≤ (ν (bestArm ν))[id] := (exists_max_image univ (fun a ↦ (ν a)[id]) (univ_nonempty_iff.mpr inferInstance)).choose_spec.2 _ (mem_univ a) -omit [DecidableEq α] in +omit [DecidableEq 𝓐] in lemma gap_eq_bestArm_sub : gap ν a = (ν (bestArm ν))[id] - (ν a)[id] := by rw [gap] congr refine le_antisymm ?_ (le_ciSup (f := fun a ↦ (ν a)[id]) (by simp) (bestArm ν)) exact ciSup_le le_bestArm -omit [DecidableEq α] in +omit [DecidableEq 𝓐] in @[simp] lemma gap_bestArm : gap ν (bestArm ν) = 0 := by rw [gap_eq_bestArm_sub, sub_self] -omit [DecidableEq α] in +omit [DecidableEq 𝓐] in lemma integral_eq_of_gap_eq_zero (hg : gap ν a = 0) : (ν (bestArm ν))[id] = (ν a)[id] := by rwa [← sub_eq_zero, ← gap_eq_bestArm_sub] @@ -130,7 +130,7 @@ end bestArm section Asymptotics -omit [DecidableEq α] in +omit [DecidableEq 𝓐] in /-- If the regret is sublinear, the average mean reward tends to the highest mean of the arms. -/ lemma avg_mean_reward_tendsto_of_sublinear_regret (hr : (regret ν A · ω) =o[atTop] fun t ↦ (t : ℝ)) : @@ -146,10 +146,10 @@ lemma avg_mean_reward_tendsto_of_sublinear_regret ring /-- If the regret is sublinear, the rate of suboptimal arm pulls tends to zero. -/ -lemma pullCount_rate_tendsto_of_sublinear_regret [Finite α] +lemma pullCount_rate_tendsto_of_sublinear_regret [Finite 𝓐] (hr : (regret ν A · ω) =o[atTop] fun t ↦ (t : ℝ)) (hg : 0 < gap ν a) : Tendsto (fun t ↦ (pullCount A a t ω : ℝ) / t) atTop (nhds 0) := by - have := Fintype.ofFinite α + have := Fintype.ofFinite 𝓐 have hb (t : ℕ) : (pullCount A a t ω : ℝ) * gap ν a ≤ regret ν A t ω := by rw [regret_eq_sum_pullCount_mul_gap] exact single_le_sum (f := fun a ↦ pullCount A a t ω * gap ν a) diff --git a/LeanMachineLearning/Online/Bandit/RewardByCountMeasure.lean b/LeanMachineLearning/Online/Bandit/RewardByCountMeasure.lean index 70c2c047..44e91fdc 100644 --- a/LeanMachineLearning/Online/Bandit/RewardByCountMeasure.lean +++ b/LeanMachineLearning/Online/Bandit/RewardByCountMeasure.lean @@ -17,16 +17,16 @@ open scoped ENNReal NNReal namespace Bandits -variable {α Ω : Type*} {mα : MeasurableSpace α} {mΩ : MeasurableSpace Ω} [DecidableEq α] - [StandardBorelSpace α] [Nonempty α] - {A : ℕ → Ω → α} {R : ℕ → Ω → ℝ} {P : Measure Ω} [IsProbabilityMeasure P] - {alg : Algorithm α ℝ} {ν : Kernel α ℝ} [IsMarkovKernel ν] +variable {𝓐 Ω : Type*} {m𝓐 : MeasurableSpace 𝓐} {mΩ : MeasurableSpace Ω} [DecidableEq 𝓐] + [StandardBorelSpace 𝓐] [Nonempty 𝓐] + {A : ℕ → Ω → 𝓐} {R : ℕ → Ω → ℝ} {P : Measure Ω} [IsProbabilityMeasure P] + {alg : Algorithm 𝓐 ℝ} {ν : Kernel 𝓐 ℝ} [IsMarkovKernel ν] {h_inter : IsAlgEnvSeq A R alg (stationaryEnv ν) P} local notation "𝔓" => P.prod (streamMeasure ν) -omit [DecidableEq α] [StandardBorelSpace α] [Nonempty α] in -lemma hasLaw_Z (a : α) (m : ℕ) : +omit [DecidableEq 𝓐] [StandardBorelSpace 𝓐] [Nonempty 𝓐] in +lemma hasLaw_Z (a : 𝓐) (m : ℕ) : HasLaw (fun ω ↦ ω.2 m a) (ν a) 𝔓 where map_eq := by calc (𝔓).map (fun ω ↦ ω.2 m a) @@ -47,13 +47,13 @@ notation "𝓛[" Y " | " X " in " s "; " μ "]" => Measure.map Y (μ[|X ⁻¹' s /-- Law of `Y` conditioned on the event that `X` equals `x`. -/ notation "𝓛[" Y " | " X " ← " x "; " μ "]" => Measure.map Y (μ[|X ⁻¹' {x}]) -omit [DecidableEq α] in -lemma condDistrib_reward'' [Countable α] +omit [DecidableEq 𝓐] in +lemma condDistrib_reward'' [Countable 𝓐] (h : IsAlgEnvSeq A R alg (stationaryEnv ν) P) (n : ℕ) : 𝓛[fun ω ↦ R n ω.1 | fun ω ↦ A n ω.1; 𝔓] =ᵐ[(𝔓).map (fun ω ↦ A n ω.1)] ν := by - have hA := h.measurable_A - have hR := h.measurable_R - have h_ra' : 𝓛[R n | A n; P] =ᵐ[P.map (A n)] ν := h.condDistrib_reward_stationaryEnv n + have hA := h.measurable_action + have hR := h.measurable_feedback + have h_ra' : 𝓛[R n | A n; P] =ᵐ[P.map (A n)] ν := h.condDistrib_feedback_stationaryEnv n have h_law : (𝔓).map (fun ω ↦ A n ω.1) = P.map (A n) := by change ((𝔓).map (A n ∘ Prod.fst)) = _ rw [← Measure.map_map (by fun_prop) (by fun_prop), ← Measure.fst, Measure.fst_prod] @@ -64,13 +64,13 @@ lemma condDistrib_reward'' [Countable α] filter_upwards [h_ra', h_prod] with ω h_eq h_prod rw [h_prod, h_eq] -omit [DecidableEq α] in -lemma reward_cond_action [Countable α] - (h : IsAlgEnvSeq A R alg (stationaryEnv ν) P) (a : α) (n : ℕ) +omit [DecidableEq 𝓐] in +lemma reward_cond_action [Countable 𝓐] + (h : IsAlgEnvSeq A R alg (stationaryEnv ν) P) (a : 𝓐) (n : ℕ) (hμa : (𝔓).map (fun ω ↦ A n ω.1) {a} ≠ 0) : 𝓛[fun ω ↦ R n ω.1 | fun ω ↦ A n ω.1 ← a; 𝔓] = ν a := by - have hA := h.measurable_A - have hR := h.measurable_R + have hA := h.measurable_action + have hR := h.measurable_feedback have h_ra : 𝓛[fun ω ↦ R n ω.1 | fun ω ↦ A n ω.1; 𝔓] =ᵐ[(𝔓).map (fun ω ↦ A n ω.1)] ν := condDistrib_reward'' h n have h_eq := condDistrib_ae_eq_cond (μ := 𝔓) @@ -82,13 +82,13 @@ lemma reward_cond_action [Countable α] exact h_eq.symm lemma condIndepFun_reward_stepsUntil_action' [StandardBorelSpace Ω] - (h : IsAlgEnvSeq A R alg (stationaryEnv ν) P) (a : α) (m n : ℕ) : - R n ⟂ᵢ[A n, h.measurable_A n; P] {ω | stepsUntil A a m ω = ↑n}.indicator (fun _ ↦ 1) := by + (h : IsAlgEnvSeq A R alg (stationaryEnv ν) P) (a : 𝓐) (m n : ℕ) : + R n ⟂ᵢ[A n, h.measurable_action n; P] {ω | stepsUntil A a m ω = ↑n}.indicator (fun _ ↦ 1) := by -- the indicator of `stepsUntil ... = n` is a function of `hist (n-1)` and `action n`. -- It thus suffices to use the independence of `reward n` and `hist (n-1)` conditionally -- on `action n`. - have hA := h.measurable_A - have hR := h.measurable_R + have hA := h.measurable_action + have hR := h.measurable_feedback by_cases hn : n = 0 · have h_indep : R 0 ⟂ᵢ[A 0, hA 0; P] A 0 := condIndepFun_self_right (by fun_prop) (by fun_prop) @@ -96,30 +96,30 @@ lemma condIndepFun_reward_stepsUntil_action' [StandardBorelSpace Ω] refine h_indep.of_measurable_right (hX := hA 0) ?_ exact measurable_comap_indicator_stepsUntil_eq_zero a m · have h_indep : R n ⟂ᵢ[A n, hA n; P] fun ω ↦ (IsAlgEnvSeq.hist A R (n - 1) ω, A n ω) := - IsAlgEnvSeq.condIndepFun_reward_hist_action_action' h n (by grind) + IsAlgEnvSeq.condIndepFun_feedback_hist_action_action' h n (by grind) refine h_indep.of_measurable_right (hX := hA n) ?_ exact measurable_comap_indicator_stepsUntil_eq hA hR a m n -lemma condIndepFun_reward_stepsUntil_action [StandardBorelSpace Ω] [Countable α] +lemma condIndepFun_reward_stepsUntil_action [StandardBorelSpace Ω] [Countable 𝓐] (h : IsAlgEnvSeq A R alg (stationaryEnv ν) P) - (a : α) (m n : ℕ) : - CondIndepFun (mα.comap (fun ω ↦ A n ω.1)) ((h.measurable_A n).comp measurable_fst).comap_le + (a : 𝓐) (m n : ℕ) : + CondIndepFun (m𝓐.comap (fun ω ↦ A n ω.1)) ((h.measurable_action n).comp measurable_fst).comap_le (fun ω ↦ R n ω.1) ({ω | stepsUntil A a m ω.1 = ↑n}.indicator (fun _ ↦ 1)) 𝔓 := by - have hA := h.measurable_A - have hR := h.measurable_R + have hA := h.measurable_action + have hR := h.measurable_feedback exact condIndepFun_fst_prod (ν := streamMeasure ν) (measurable_indicator_stepsUntil_eq hA hR a m n) (by fun_prop) (by fun_prop) (condIndepFun_reward_stepsUntil_action' h a m n) -lemma reward_cond_stepsUntil [StandardBorelSpace Ω] [Countable α] - (h : IsAlgEnvSeq A R alg (stationaryEnv ν) P) (a : α) (m n : ℕ) +lemma reward_cond_stepsUntil [StandardBorelSpace Ω] [Countable 𝓐] + (h : IsAlgEnvSeq A R alg (stationaryEnv ν) P) (a : 𝓐) (m n : ℕ) (hm : m ≠ 0) (hμn : 𝔓 ((fun ω ↦ stepsUntil A a m ω.1) ⁻¹' {↑n}) ≠ 0) : 𝓛[fun ω ↦ R n ω.1 | fun ω ↦ stepsUntil A a m ω.1 ← ↑n; 𝔓] = ν a := by - have hA := h.measurable_A - have hR := h.measurable_R + have hA := h.measurable_action + have hR := h.measurable_feedback have hμna : 𝔓 ((fun ω ↦ stepsUntil A a m ω.1) ⁻¹' {↑n} ∩ (fun ω ↦ A n ω.1) ⁻¹' {a}) ≠ 0 := by - suffices ((fun ω : Ω × (ℕ → α → ℝ) ↦ + suffices ((fun ω : Ω × (ℕ → 𝓐 → ℝ) ↦ stepsUntil A a m ω.1) ⁻¹' {↑n} ∩ (fun ω ↦ A n ω.1) ⁻¹' {a}) = (fun ω ↦ stepsUntil A a m ω.1) ⁻¹' {↑n} by simpa [this] using hμn ext ω @@ -137,7 +137,7 @@ lemma reward_cond_stepsUntil [StandardBorelSpace Ω] [Countable α] simp only [Set.mem_preimage, Set.mem_singleton_iff, Set.mem_inter_iff, iff_self_and] exact action_eq_of_stepsUntil_eq_coe hm _ = (𝔓[|(fun ω ↦ A n ω.1) ⁻¹' {a} - ∩ {ω : Ω × (ℕ → α → ℝ) | stepsUntil A a m ω.1 = ↑n}.indicator 1 ⁻¹' {1} ]).map + ∩ {ω : Ω × (ℕ → 𝓐 → ℝ) | stepsUntil A a m ω.1 = ↑n}.indicator 1 ⁻¹' {1} ]).map (fun ω ↦ R n ω.1) := by congr 2 with ω simp only [Set.mem_inter_iff, Set.mem_preimage, Set.mem_singleton_iff, Set.indicator_apply, @@ -157,12 +157,12 @@ lemma reward_cond_stepsUntil [StandardBorelSpace Ω] [Countable α] /-- The conditional distribution of the reward received at the `m`-th pull of action `a` given the time at which number of pulls is `m` is the constant kernel with value `ν a`. -/ -theorem condDistrib_rewardByCount_stepsUntil [StandardBorelSpace Ω] [Countable α] - (h : IsAlgEnvSeq A R alg (stationaryEnv ν) P) (a : α) (m : ℕ) (hm : m ≠ 0) : +theorem condDistrib_rewardByCount_stepsUntil [StandardBorelSpace Ω] [Countable 𝓐] + (h : IsAlgEnvSeq A R alg (stationaryEnv ν) P) (a : 𝓐) (m : ℕ) (hm : m ≠ 0) : condDistrib (rewardByCount A R a m) (fun ω ↦ stepsUntil A a m ω.1) 𝔓 =ᵐ[(𝔓).map (fun ω ↦ stepsUntil A a m ω.1)] Kernel.const _ (ν a) := by - have hA := h.measurable_A - have hR := h.measurable_R + have hA := h.measurable_action + have hR := h.measurable_feedback refine (condDistrib_ae_eq_cond (μ := 𝔓) (X := fun ω ↦ stepsUntil A a m ω.1) (by fun_prop) (by fun_prop)).trans ?_ rw [Filter.EventuallyEq, ae_iff_of_countable] @@ -179,7 +179,7 @@ theorem condDistrib_rewardByCount_stepsUntil [StandardBorelSpace Ω] [Countable · exact (hasLaw_Z a m).map_eq · rwa [Measure.map_apply (by fun_prop) (measurableSet_singleton _)] at hn · exact indepFun_prod (X := fun ω : Ω ↦ stepsUntil A a m ω) - (Y := fun ω : ℕ → α → ℝ ↦ ω m a) (by fun_prop) (by fun_prop) + (Y := fun ω : ℕ → 𝓐 → ℝ ↦ ω m a) (by fun_prop) (by fun_prop) | coe n => rw [Measure.map_congr (g := fun ω ↦ R n ω.1)] swap @@ -190,13 +190,14 @@ theorem condDistrib_rewardByCount_stepsUntil [StandardBorelSpace Ω] [Countable rwa [Measure.map_apply (by fun_prop) (measurableSet_singleton _)] at hn /-- The reward received at the `m`-th pull of action `a` has law `ν a`. -/ -lemma hasLaw_rewardByCount [StandardBorelSpace Ω] [Countable α] - (h : IsAlgEnvSeq A R alg (stationaryEnv ν) P) (a : α) (m : ℕ) (hm : m ≠ 0) : +lemma hasLaw_rewardByCount [StandardBorelSpace Ω] [Countable 𝓐] + (h : IsAlgEnvSeq A R alg (stationaryEnv ν) P) (a : 𝓐) (m : ℕ) (hm : m ≠ 0) : HasLaw (rewardByCount A R a m) (ν a) 𝔓 where - aemeasurable := (measurable_rewardByCount h.measurable_A h.measurable_R a m).aemeasurable + aemeasurable := + (measurable_rewardByCount h.measurable_action h.measurable_feedback a m).aemeasurable map_eq := by - have hA := h.measurable_A - have hR := h.measurable_R + have hA := h.measurable_action + have hR := h.measurable_feedback have h_condDistrib : condDistrib (rewardByCount A R a m) (fun ω ↦ stepsUntil A a m ω.1) 𝔓 =ᵐ[(𝔓).map (fun ω ↦ stepsUntil A a m ω.1)] @@ -212,23 +213,26 @@ lemma hasLaw_rewardByCount [StandardBorelSpace Ω] [Countable α] Measure.isProbabilityMeasure_map (by fun_prop) simp -lemma identDistrib_rewardByCount [StandardBorelSpace Ω] [Countable α] - (h : IsAlgEnvSeq A R alg (stationaryEnv ν) P) (a : α) (n m : ℕ) +lemma identDistrib_rewardByCount [StandardBorelSpace Ω] [Countable 𝓐] + (h : IsAlgEnvSeq A R alg (stationaryEnv ν) P) (a : 𝓐) (n m : ℕ) (hn : n ≠ 0) (hm : m ≠ 0) : IdentDistrib (rewardByCount A R a n) (rewardByCount A R a m) 𝔓 𝔓 where - aemeasurable_fst := (measurable_rewardByCount h.measurable_A h.measurable_R a n).aemeasurable - aemeasurable_snd := (measurable_rewardByCount h.measurable_A h.measurable_R a m).aemeasurable + aemeasurable_fst := + (measurable_rewardByCount h.measurable_action h.measurable_feedback a n).aemeasurable + aemeasurable_snd := + (measurable_rewardByCount h.measurable_action h.measurable_feedback a m).aemeasurable map_eq := by rw [(hasLaw_rewardByCount h a n hn).map_eq, (hasLaw_rewardByCount h a m hm).map_eq] -lemma identDistrib_rewardByCount_id [StandardBorelSpace Ω] [Countable α] - (h : IsAlgEnvSeq A R alg (stationaryEnv ν) P) (a : α) (n : ℕ) (hn : n ≠ 0) : +lemma identDistrib_rewardByCount_id [StandardBorelSpace Ω] [Countable 𝓐] + (h : IsAlgEnvSeq A R alg (stationaryEnv ν) P) (a : 𝓐) (n : ℕ) (hn : n ≠ 0) : IdentDistrib (rewardByCount A R a n) id 𝔓 (ν a) where - aemeasurable_fst := (measurable_rewardByCount h.measurable_A h.measurable_R a n).aemeasurable + aemeasurable_fst := + (measurable_rewardByCount h.measurable_action h.measurable_feedback a n).aemeasurable aemeasurable_snd := Measurable.aemeasurable <| by fun_prop map_eq := by rw [(hasLaw_rewardByCount h a n hn).map_eq, Measure.map_id] -lemma identDistrib_rewardByCount_eval [StandardBorelSpace Ω] [Countable α] - (h : IsAlgEnvSeq A R alg (stationaryEnv ν) P) (a : α) (n m : ℕ) (hn : n ≠ 0) : +lemma identDistrib_rewardByCount_eval [StandardBorelSpace Ω] [Countable 𝓐] + (h : IsAlgEnvSeq A R alg (stationaryEnv ν) P) (a : 𝓐) (n m : ℕ) (hn : n ≠ 0) : IdentDistrib (rewardByCount A R a n) (fun ω ↦ ω m a) 𝔓 (streamMeasure ν) := (identDistrib_rewardByCount_id h a n hn).trans (identDistrib_eval_eval_id_streamMeasure ν m a).symm diff --git a/LeanMachineLearning/Online/Bandit/SumRewards.lean b/LeanMachineLearning/Online/Bandit/SumRewards.lean index e9ef8537..5a49fc32 100644 --- a/LeanMachineLearning/Online/Bandit/SumRewards.lean +++ b/LeanMachineLearning/Online/Bandit/SumRewards.lean @@ -31,9 +31,9 @@ lemma sum_Icc_one_eq_sum_range {m : ℕ} {f : ℕ → ℝ} : rw [h, Finset.sum_image (fun _ _ _ _ h => by omega)] simp -variable {α : Type*} {mα : MeasurableSpace α} [DecidableEq α] [Countable α] - [StandardBorelSpace α] [Nonempty α] - {alg : Algorithm α ℝ} {ν : Kernel α ℝ} [IsMarkovKernel ν] +variable {𝓐 : Type*} {m𝓐 : MeasurableSpace 𝓐} [DecidableEq 𝓐] [Countable 𝓐] + [StandardBorelSpace 𝓐] [Nonempty 𝓐] + {alg : Algorithm 𝓐 ℝ} {ν : Kernel 𝓐 ℝ} [IsMarkovKernel ν] local notation "A" => action alg local notation "R" => reward alg @@ -57,7 +57,7 @@ lemma identDistrib_pullCount_prod_sum_Icc_rewardByCount' (n : ℕ) : map_eq := by by_cases hn : n = 0 · simp [hn] - have h_eq (a : α) (i : ℕ) (ω : probSpace α ℝ × (ℕ → α → ℝ)) + have h_eq (a : 𝓐) (i : ℕ) (ω : probSpace 𝓐 ℝ × (ℕ → 𝓐 → ℝ)) (hi : i ∈ Icc 1 (pullCount A a n ω.1)) : rewardByCount A R a i ω = ω.1.2 (i - 1) a := by rw [rewardByCount_of_stepsUntil_ne_top] @@ -76,7 +76,7 @@ lemma identDistrib_pullCount_prod_sum_Icc_rewardByCount' (n : ℕ) : simp only [mem_Icc] at hi refine hi.2.trans ?_ exact pullCount_mono _ (by grind) _ - have h_sum_eq (a : α) (ω : probSpace α ℝ × (ℕ → α → ℝ)) : + have h_sum_eq (a : 𝓐) (ω : probSpace 𝓐 ℝ × (ℕ → 𝓐 → ℝ)) : ∑ i ∈ Icc 1 (pullCount A a n ω.1), rewardByCount A R a i ω = ∑ i ∈ Icc 1 (pullCount A a n ω.1), ω.1.2 (i - 1) a := Finset.sum_congr rfl fun i hi ↦ h_eq a i ω hi @@ -125,7 +125,7 @@ lemma identDistrib_pullCount_prod_sumRewards (n : ℕ) : simp_rw [← sum_rewardByCount_eq_sumRewards] exact identDistrib_pullCount_prod_sum_Icc_rewardByCount n -lemma identDistrib_pullCount_prod_sumRewards_arm (a : α) (n : ℕ) : +lemma identDistrib_pullCount_prod_sumRewards_arm (a : 𝓐) (n : ℕ) : IdentDistrib (fun ω ↦ (pullCount A a n ω, sumRewards A R a n ω)) (fun ω ↦ (pullCount A a n ω, ∑ i ∈ range (pullCount A a n ω), ω.2 i a)) 𝔓 𝔓 := by have h1 : (fun ω ↦ (pullCount A a n ω, sumRewards A R a n ω)) = @@ -137,7 +137,7 @@ lemma identDistrib_pullCount_prod_sumRewards_arm (a : α) (n : ℕ) : refine (identDistrib_pullCount_prod_sumRewards n).comp ?_ fun_prop -lemma identDistrib_pullCount_prod_sumRewards_two_arms (a b : α) (n : ℕ) : +lemma identDistrib_pullCount_prod_sumRewards_two_arms (a b : 𝓐) (n : ℕ) : IdentDistrib (fun ω ↦ (pullCount A a n ω, pullCount A b n ω, sumRewards A R a n ω, sumRewards A R b n ω)) (fun ω ↦ (pullCount A a n ω, pullCount A b n ω, @@ -152,7 +152,7 @@ lemma identDistrib_sumRewards (n : ℕ) : have h_ident := identDistrib_pullCount_prod_sumRewards (ν := ν) (alg := alg) n exact h_ident.comp (u := fun p a ↦ (p a).2) (by fun_prop) -lemma identDistrib_sumRewards_arm (a : α) (n : ℕ) : +lemma identDistrib_sumRewards_arm (a : 𝓐) (n : ℕ) : IdentDistrib (sumRewards A R a n) (fun ω ↦ ∑ i ∈ range (pullCount A a n ω), ω.2 i a) 𝔓 𝔓 := by have h1 : sumRewards A R a n = (fun p ↦ p a) ∘ (fun ω a ↦ sumRewards A R a n ω) := rfl @@ -162,8 +162,8 @@ lemma identDistrib_sumRewards_arm (a : α) (n : ℕ) : refine (identDistrib_sumRewards n).comp ?_ fun_prop -omit [DecidableEq α] [StandardBorelSpace α] [Nonempty α] in -lemma identDistrib_sum_range_snd (a : α) (k : ℕ) : +omit [DecidableEq 𝓐] [StandardBorelSpace 𝓐] [Nonempty 𝓐] in +lemma identDistrib_sum_range_snd (a : 𝓐) (k : ℕ) : IdentDistrib (fun ω ↦ ∑ i ∈ range k, ω.2 i a) (fun ω ↦ ∑ i ∈ range k, ω i a) 𝔓 (streamMeasure ν) where aemeasurable_fst := by fun_prop @@ -173,7 +173,7 @@ lemma identDistrib_sum_range_snd (a : α) (k : ℕ) : (ν := streamMeasure ν), Measure.snd, Measure.map_map (by fun_prop) (by fun_prop)] rfl -lemma prob_pullCount_prod_sumRewards_mem_le (a : α) (n : ℕ) +lemma prob_pullCount_prod_sumRewards_mem_le (a : 𝓐) (n : ℕ) {s : Set (ℕ × ℝ)} [DecidablePred (· ∈ Prod.fst '' s)] (hs : MeasurableSet s) : 𝔓 {ω | (pullCount A a n ω, sumRewards A R a n ω) ∈ s} ≤ ∑ k ∈ (range (n + 1)).filter (· ∈ Prod.fst '' s), @@ -207,7 +207,7 @@ lemma prob_pullCount_prod_sumRewards_mem_le (a : α) (n : ℕ) rwa [Measure.map_apply (by fun_prop) (hs.preimage (by fun_prop)), Measure.map_apply (by fun_prop) (hs.preimage (by fun_prop))] at this -lemma prob_pullCount_mem_and_sumRewards_mem_le (a : α) (n : ℕ) +lemma prob_pullCount_mem_and_sumRewards_mem_le (a : 𝓐) (n : ℕ) {s : Set ℕ} [DecidablePred (· ∈ s)] (hs : MeasurableSet s) {B : Set ℝ} (hB : MeasurableSet B) : 𝔓 {ω | pullCount A a n ω ∈ s ∧ sumRewards A R a n ω ∈ B} ≤ ∑ k ∈ (range (n + 1)).filter (· ∈ s), @@ -224,7 +224,7 @@ lemma prob_pullCount_mem_and_sumRewards_mem_le (a : α) (n : ℕ) exists_eq_right, mem_filter, mem_range] at hk simp [hk.2.1] -lemma prob_sumRewards_le_sumRewards_le [Fintype α] (a : α) (n m₁ m₂ : ℕ) : +lemma prob_sumRewards_le_sumRewards_le [Fintype 𝓐] (a : 𝓐) (n m₁ m₂ : ℕ) : (𝔓) {ω | pullCount A (bestArm ν) n ω = m₁ ∧ pullCount A a n ω = m₂ ∧ sumRewards A R (bestArm ν) n ω ≤ sumRewards A R a n ω} ≤ streamMeasure ν @@ -261,7 +261,7 @@ lemma prob_sumRewards_le_sumRewards_le [Fintype α] (a : α) (n m₁ m₂ : ℕ) simp only [measurableSet_setOf] fun_prop -lemma probReal_sumRewards_le_sumRewards_le [Fintype α] (a : α) (n m₁ m₂ : ℕ) : +lemma probReal_sumRewards_le_sumRewards_le [Fintype 𝓐] (a : 𝓐) (n m₁ m₂ : ℕ) : (𝔓).real {ω | pullCount A (bestArm ν) n ω = m₁ ∧ pullCount A a n ω = m₂ ∧ sumRewards A R (bestArm ν) n ω ≤ sumRewards A R a n ω} ≤ (streamMeasure ν).real @@ -273,12 +273,12 @@ lemma probReal_sumRewards_le_sumRewards_le [Fintype α] (a : α) (n m₁ m₂ : end ArrayModel -variable {α Ω Ω' : Type*} [DecidableEq α] {mα : MeasurableSpace α} {mΩ : MeasurableSpace Ω} +variable {𝓐 Ω Ω' : Type*} [DecidableEq 𝓐] {m𝓐 : MeasurableSpace 𝓐} {mΩ : MeasurableSpace Ω} {mΩ' : MeasurableSpace Ω'} {P : Measure Ω} [IsProbabilityMeasure P] {P' : Measure Ω'} [IsProbabilityMeasure P'] - {alg : Algorithm α ℝ} {ν : Kernel α ℝ} [IsMarkovKernel ν] - {A : ℕ → Ω → α} {R : ℕ → Ω → ℝ} {A₂ : ℕ → Ω' → α} {R₂ : ℕ → Ω' → ℝ} - {ω : Ω} {m n t : ℕ} {a : α} + {alg : Algorithm 𝓐 ℝ} {ν : Kernel 𝓐 ℝ} [IsMarkovKernel ν] + {A : ℕ → Ω → 𝓐} {R : ℕ → Ω → ℝ} {A₂ : ℕ → Ω' → 𝓐} {R₂ : ℕ → Ω' → ℝ} + {ω : Ω} {m n t : ℕ} {a : 𝓐} lemma sumRewards_eq_comp : sumRewards A R a n = @@ -292,17 +292,17 @@ lemma pullCount_eq_comp : ext simp [pullCount] -variable [StandardBorelSpace α] [Nonempty α] +variable [StandardBorelSpace 𝓐] [Nonempty 𝓐] -- todo: write those lemmas with IdentDistrib instead of equality of maps lemma _root_.Learning.IsAlgEnvSeq.law_sumRewards_unique (h1 : IsAlgEnvSeq A R alg (stationaryEnv ν) P) (h2 : IsAlgEnvSeq A₂ R₂ alg (stationaryEnv ν) P') : P.map (sumRewards A R a n) = P'.map (sumRewards A₂ R₂ a n) := by - have hA := h1.measurable_A - have hR := h1.measurable_R - have hA2 := h2.measurable_A - have hR2 := h2.measurable_R + have hA := h1.measurable_action + have hR := h1.measurable_feedback + have hA2 := h2.measurable_action + have hR2 := h2.measurable_feedback have h_unique := isAlgEnvSeq_unique h1 h2 rw [sumRewards_eq_comp, sumRewards_eq_comp, ← Measure.map_map, h_unique, Measure.map_map, ← sumRewards_eq_comp] @@ -320,10 +320,10 @@ lemma _root_.Learning.IsAlgEnvSeq.law_pullCount_sumRewards_unique' (h2 : IsAlgEnvSeq A₂ R₂ alg (stationaryEnv ν) P') : IdentDistrib (fun ω a ↦ (pullCount A a n ω, sumRewards A R a n ω)) (fun ω a ↦ (pullCount A₂ a n ω, sumRewards A₂ R₂ a n ω)) P P' := by - have hA := h1.measurable_A - have hR := h1.measurable_R - have hA2 := h2.measurable_A - have hR2 := h2.measurable_R + have hA := h1.measurable_action + have hR := h1.measurable_feedback + have hA2 := h2.measurable_action + have hR2 := h2.measurable_feedback constructor · refine Measurable.aemeasurable ?_ rw [measurable_pi_iff] @@ -332,7 +332,7 @@ lemma _root_.Learning.IsAlgEnvSeq.law_pullCount_sumRewards_unique' rw [measurable_pi_iff] exact fun a ↦ Measurable.prod (by fun_prop) (measurable_sumRewards hA2 hR2 _ _) have h_unique := isAlgEnvSeq_unique h1 h2 - let f := fun (p : ℕ → α × ℝ ) (a : α) ↦ (∑ i ∈ range n, if (p i).1 = a then 1 else 0, + let f := fun (p : ℕ → 𝓐 × ℝ ) (a : 𝓐) ↦ (∑ i ∈ range n, if (p i).1 = a then 1 else 0, ∑ i ∈ range n, if (p i).1 = a then (p i).2 else 0) have hf : Measurable f := by rw [measurable_pi_iff] @@ -369,14 +369,14 @@ lemma _root_.Learning.IsAlgEnvSeq.law_pullCount_sumRewards_unique ((h1.law_pullCount_sumRewards_unique' h2 (n := n)).comp (u := fun f ↦ f a) (by fun_prop)).map_eq -- this is what we will use for UCB -lemma prob_pullCount_prod_sumRewards_mem_le [Countable α] +lemma prob_pullCount_prod_sumRewards_mem_le [Countable 𝓐] (h : IsAlgEnvSeq A R alg (stationaryEnv ν) P) {s : Set (ℕ × ℝ)} [DecidablePred (· ∈ Prod.fst '' s)] (hs : MeasurableSet s) : P {ω | (pullCount A a n ω, sumRewards A R a n ω) ∈ s} ≤ ∑ k ∈ (range (n + 1)).filter (· ∈ Prod.fst '' s), streamMeasure ν {ω | ∑ i ∈ range k, ω i a ∈ Prod.mk k ⁻¹' s} := by - have hA := h.measurable_A - have hR := h.measurable_R + have hA := h.measurable_action + have hR := h.measurable_feedback calc P {ω | (pullCount A a n ω, sumRewards A R a n ω) ∈ s} _ = (P.map (fun ω ↦ (pullCount A a n ω, sumRewards A R a n ω))) s := by rw [Measure.map_apply (by fun_prop) hs]; rfl @@ -391,7 +391,7 @@ lemma prob_pullCount_prod_sumRewards_mem_le [Countable α] streamMeasure ν {ω | ∑ i ∈ range k, ω i a ∈ Prod.mk k ⁻¹' s} := ArrayModel.prob_pullCount_prod_sumRewards_mem_le a n hs -lemma prob_pullCount_mem_and_sumRewards_mem_le [Countable α] +lemma prob_pullCount_mem_and_sumRewards_mem_le [Countable 𝓐] (h : IsAlgEnvSeq A R alg (stationaryEnv ν) P) {s : Set ℕ} [DecidablePred (· ∈ s)] (hs : MeasurableSet s) {B : Set ℝ} (hB : MeasurableSet B) : P {ω | pullCount A a n ω ∈ s ∧ sumRewards A R a n ω ∈ B} ≤ @@ -409,7 +409,7 @@ lemma prob_pullCount_mem_and_sumRewards_mem_le [Countable α] exists_eq_right, mem_filter, mem_range] at hk simp [hk.2.1] -lemma prob_sumRewards_mem_le [Countable α] (h : IsAlgEnvSeq A R alg (stationaryEnv ν) P) +lemma prob_sumRewards_mem_le [Countable 𝓐] (h : IsAlgEnvSeq A R alg (stationaryEnv ν) P) {B : Set ℝ} (hB : MeasurableSet B) : P (sumRewards A R a n ⁻¹' B) ≤ ∑ k ∈ range (n + 1), streamMeasure ν {ω | ∑ i ∈ range k, ω i a ∈ B} := by @@ -417,7 +417,7 @@ lemma prob_sumRewards_mem_le [Countable α] (h : IsAlgEnvSeq A R alg (stationary have h_le := prob_pullCount_mem_and_sumRewards_mem_le h .univ hB (a := a) (n := n) simpa using h_le -lemma prob_pullCount_eq_and_sumRewards_mem_le [Countable α] +lemma prob_pullCount_eq_and_sumRewards_mem_le [Countable 𝓐] (h : IsAlgEnvSeq A R alg (stationaryEnv ν) P) {m : ℕ} (hm : m ≤ n) {B : Set ℝ} (hB : MeasurableSet B) : P {ω | pullCount A a n ω = m ∧ sumRewards A R a n ω ∈ B} ≤ @@ -426,14 +426,14 @@ lemma prob_pullCount_eq_and_sumRewards_mem_le [Countable α] have hm' : m < n + 1 := by lia simpa [hm'] using h_le -lemma probReal_sumRewards_le_sumRewards_le [Fintype α] (h : IsAlgEnvSeq A R alg (stationaryEnv ν) P) - (a : α) (n m₁ m₂ : ℕ) : +lemma probReal_sumRewards_le_sumRewards_le [Fintype 𝓐] (h : IsAlgEnvSeq A R alg (stationaryEnv ν) P) + (a : 𝓐) (n m₁ m₂ : ℕ) : P.real {ω | pullCount A (bestArm ν) n ω = m₁ ∧ pullCount A a n ω = m₂ ∧ sumRewards A R (bestArm ν) n ω ≤ sumRewards A R a n ω} ≤ (streamMeasure ν).real {ω | ∑ i ∈ range m₁, ω i (bestArm ν) ≤ ∑ i ∈ range m₂, ω i a} := by - have hA := h.measurable_A - have hR := h.measurable_R + have hA := h.measurable_action + have hR := h.measurable_feedback refine le_trans (le_of_eq ?_) (ArrayModel.probReal_sumRewards_le_sumRewards_le (alg := alg) a n m₁ m₂) let s := {p : ℕ × ℕ × ℝ × ℝ | p.1 = m₁ ∧ p.2.1 = m₂ ∧ p.2.2.1 ≤ p.2.2.2} @@ -460,9 +460,9 @@ lemma probReal_sumRewards_le_sumRewards_le [Fintype α] (h : IsAlgEnvSeq A R alg section Subgaussian -omit [DecidableEq α] [StandardBorelSpace α] in -lemma probReal_sum_le_sum_streamMeasure [Fintype α] {c : ℝ≥0} - (hν : ∀ a, HasSubgaussianMGF (fun x ↦ x - (ν a)[id]) c (ν a)) (a : α) (m : ℕ) : +omit [DecidableEq 𝓐] [StandardBorelSpace 𝓐] in +lemma probReal_sum_le_sum_streamMeasure [Fintype 𝓐] {c : ℝ≥0} + (hν : ∀ a, HasSubgaussianMGF (fun x ↦ x - (ν a)[id]) c (ν a)) (a : 𝓐) (m : ℕ) : (streamMeasure ν).real {ω | ∑ s ∈ range m, ω s (bestArm ν) ≤ ∑ s ∈ range m, ω s a} ≤ Real.exp (-↑m * gap ν a ^ 2 / (4 * c)) := by @@ -491,10 +491,10 @@ lemma probReal_sum_le_sum_streamMeasure [Fintype α] {c : ℝ≥0} field_simp ring -omit [DecidableEq α] [StandardBorelSpace α] [Nonempty α] in +omit [DecidableEq 𝓐] [StandardBorelSpace 𝓐] [Nonempty 𝓐] in lemma prob_sum_le_sqrt_log {σ2 : ℝ≥0} (hν : ∀ a, HasSubgaussianMGF (fun x ↦ x - (ν a)[id]) σ2 (ν a)) - (hσ2 : σ2 ≠ 0) {c : ℝ} (hc : 0 ≤ c) (a : α) (k : ℕ) (hk : k ≠ 0) : + (hσ2 : σ2 ≠ 0) {c : ℝ} (hc : 0 ≤ c) (a : 𝓐) (k : ℕ) (hk : k ≠ 0) : streamMeasure ν {ω | (∑ s ∈ range k, (ω s a - (ν a)[id])) ≤ - √(2 * c * k * σ2 * Real.log (n + 1))} ≤ 1 / (n + 1) ^ c := by @@ -519,10 +519,10 @@ lemma prob_sum_le_sqrt_log {σ2 : ℝ≥0} ← ENNReal.ofReal_rpow_of_nonneg (by positivity) (by positivity)] norm_cast -omit [DecidableEq α] [StandardBorelSpace α] [Nonempty α] in +omit [DecidableEq 𝓐] [StandardBorelSpace 𝓐] [Nonempty 𝓐] in lemma prob_sum_ge_sqrt_log {σ2 : ℝ≥0} (hν : ∀ a, HasSubgaussianMGF (fun x ↦ x - (ν a)[id]) σ2 (ν a)) - (hσ2 : σ2 ≠ 0) {c : ℝ} (hc : 0 ≤ c) (a : α) (k : ℕ) (hk : k ≠ 0) : + (hσ2 : σ2 ≠ 0) {c : ℝ} (hc : 0 ≤ c) (a : 𝓐) (k : ℕ) (hk : k ≠ 0) : streamMeasure ν {ω | √(2 * c * k * σ2 * Real.log (n + 1)) ≤ (∑ s ∈ range k, (ω s a - (ν a)[id]))} ≤ 1 / (n + 1) ^ c := by @@ -549,10 +549,10 @@ lemma prob_sum_ge_sqrt_log {σ2 : ℝ≥0} open Real -omit [DecidableEq α] [StandardBorelSpace α] [Nonempty α] in +omit [DecidableEq 𝓐] [StandardBorelSpace 𝓐] [Nonempty 𝓐] in lemma todo {σ2 : ℝ≥0} {c : ℝ} (hν : ∀ a, HasSubgaussianMGF (fun x ↦ x - (ν a)[id]) σ2 (ν a)) (hσ2 : σ2 ≠ 0) - (hc : 0 ≤ c) (a : α) (n k : ℕ) (hk : k ≠ 0) : + (hc : 0 ≤ c) (a : 𝓐) (n k : ℕ) (hk : k ≠ 0) : streamMeasure ν {ω | (∑ m ∈ range k, ω m a) / k + √(2 * c * σ2 * log (n + 1) / k) ≤ (ν a)[id]} ≤ 1 / (n + 1) ^ c := by have h_log_nonneg : 0 ≤ log (n + 1) := log_nonneg (by simp) @@ -575,10 +575,10 @@ lemma todo {σ2 : ℝ≥0} {c : ℝ} sqrt_mul (x := (k : ℝ)) (by positivity), mul_comm] _ ≤ 1 / (n + 1) ^ c := prob_sum_le_sqrt_log hν hσ2 hc a k hk -omit [DecidableEq α] [StandardBorelSpace α] [Nonempty α] in +omit [DecidableEq 𝓐] [StandardBorelSpace 𝓐] [Nonempty 𝓐] in lemma todo' {σ2 : ℝ≥0} {c : ℝ} (hν : ∀ a, HasSubgaussianMGF (fun x ↦ x - (ν a)[id]) σ2 (ν a)) (hσ2 : σ2 ≠ 0) - (hc : 0 ≤ c) (a : α) (n k : ℕ) (hk : k ≠ 0) : + (hc : 0 ≤ c) (a : 𝓐) (n k : ℕ) (hk : k ≠ 0) : streamMeasure ν {ω | (ν a)[id] ≤ (∑ m ∈ range k, ω m a) / k - √(2 * c * σ2 *log (n + 1) / k)} ≤ 1 / (n + 1) ^ c := by diff --git a/LeanMachineLearning/SequentialLearning/Algorithm.lean b/LeanMachineLearning/SequentialLearning/Algorithm.lean index 809e7e53..2662ecf6 100644 --- a/LeanMachineLearning/SequentialLearning/Algorithm.lean +++ b/LeanMachineLearning/SequentialLearning/Algorithm.lean @@ -12,17 +12,17 @@ public import LeanMachineLearning.Probability.Kernel.IonescuTulcea.Traj # Algorithms and environments We define structures for stochastic, sequential algorithms and environments, and the notion of an -algorithm-environment sequence, which is a sequence of actions and rewards generated by an algorithm -interacting with an environment. +algorithm-environment sequence, which is a sequence of actions and feedbacks generated by +an algorithm interacting with an environment. ## Main definitions -* `Algorithm α R`: a stochastic, sequential algorithm. -* `Environment α R`: a stochastic environment. -* `IsAlgEnvSeq A R' alg env P`: an algorithm-environment sequence. That is, a sequence of - actions `A` and feedback `R'` that have the correct conditional distributions to be generated by +* `Algorithm 𝓐 𝓨`: a stochastic, sequential algorithm. +* `Environment 𝓐 𝓨`: a stochastic environment. +* `IsAlgEnvSeq A 𝓨' alg env P`: an algorithm-environment sequence. That is, a sequence of + actions `A` and feedback `Y` that have the correct conditional distributions to be generated by an algorithm `alg` interacting with an environment `env`, defined on a probability space `(Ω, P)`. -* `IsAlgEnvSeqUntil A R' alg env P N`: `A` and `R'` form an algorithm-environment sequence until +* `IsAlgEnvSeqUntil A Y alg env P N`: `A` and `Y` form an algorithm-environment sequence until time `N`. ## Notes @@ -39,209 +39,203 @@ open scoped ENNReal NNReal namespace Learning -variable {α R Ω : Type*} {mα : MeasurableSpace α} {mR : MeasurableSpace R} {mΩ : MeasurableSpace Ω} +variable {𝓐 𝓨 Ω : Type*} {m𝓐 : MeasurableSpace 𝓐} {m𝓨 : MeasurableSpace 𝓨} {mΩ : MeasurableSpace Ω} /-- A stochastic, sequential algorithm. -/ -- ANCHOR: Algorithm -structure Algorithm (α R : Type*) [MeasurableSpace α] [MeasurableSpace R] where +structure Algorithm (𝓐 𝓨 : Type*) [MeasurableSpace 𝓐] [MeasurableSpace 𝓨] where /-- Policy or sampling rule: distribution of the next action. -/ - policy : (n : ℕ) → Kernel (Iic n → α × R) α + policy : (n : ℕ) → Kernel (Iic n → 𝓐 × 𝓨) 𝓐 [h_policy : ∀ n, IsMarkovKernel (policy n)] /-- Distribution of the first action. -/ - p0 : Measure α + p0 : Measure 𝓐 [hp0 : IsProbabilityMeasure p0] -- ANCHOR_END: Algorithm -instance (alg : Algorithm α R) (n : ℕ) : IsMarkovKernel (alg.policy n) := alg.h_policy n -instance (alg : Algorithm α R) : IsProbabilityMeasure alg.p0 := alg.hp0 +instance (alg : Algorithm 𝓐 𝓨) (n : ℕ) : IsMarkovKernel (alg.policy n) := alg.h_policy n +instance (alg : Algorithm 𝓐 𝓨) : IsProbabilityMeasure alg.p0 := alg.hp0 /-- A stochastic environment. -/ -- ANCHOR: Environment -structure Environment (α R : Type*) [MeasurableSpace α] [MeasurableSpace R] where +structure Environment (𝓐 𝓨 : Type*) [MeasurableSpace 𝓐] [MeasurableSpace 𝓨] where /-- Distribution of the next observation as function of the past history. -/ - feedback : (n : ℕ) → Kernel ((Iic n → α × R) × α) R + feedback : (n : ℕ) → Kernel ((Iic n → 𝓐 × 𝓨) × 𝓐) 𝓨 [h_feedback : ∀ n, IsMarkovKernel (feedback n)] /-- Distribution of the first observation given the first action. -/ - ν0 : Kernel α R + ν0 : Kernel 𝓐 𝓨 [hp0 : IsMarkovKernel ν0] -- ANCHOR_END: Environment -instance (env : Environment α R) (n : ℕ) : IsMarkovKernel (env.feedback n) := env.h_feedback n -instance (env : Environment α R) : IsMarkovKernel env.ν0 := env.hp0 +instance (env : Environment 𝓐 𝓨) (n : ℕ) : IsMarkovKernel (env.feedback n) := env.h_feedback n +instance (env : Environment 𝓐 𝓨) : IsMarkovKernel env.ν0 := env.hp0 -/-- Kernel describing the distribution of the next action-reward pair given the history +/-- Kernel describing the distribution of the next action-feedback pair given the history up to `n`. -/ noncomputable -def stepKernel (alg : Algorithm α R) (env : Environment α R) (n : ℕ) : - Kernel (Iic n → α × R) (α × R) := +def stepKernel (alg : Algorithm 𝓐 𝓨) (env : Environment 𝓐 𝓨) (n : ℕ) : + Kernel (Iic n → 𝓐 × 𝓨) (𝓐 × 𝓨) := alg.policy n ⊗ₖ env.feedback n deriving IsMarkovKernel @[simp] -lemma fst_stepKernel (alg : Algorithm α R) (env : Environment α R) (n : ℕ) : +lemma fst_stepKernel (alg : Algorithm 𝓐 𝓨) (env : Environment 𝓐 𝓨) (n : ℕ) : (stepKernel alg env n).fst = alg.policy n := by rw [stepKernel, Kernel.fst_compProd] section IsAlgEnvSeq -variable {A : ℕ → Ω → α} {R' : ℕ → Ω → R} {alg : Algorithm α R} {env : Environment α R} +variable {A : ℕ → Ω → 𝓐} {Y : ℕ → Ω → 𝓨} {alg : Algorithm 𝓐 𝓨} {env : Environment 𝓐 𝓨} {P : Measure Ω} [IsFiniteMeasure P] {N : ℕ} -/-- Step of the algorithm-environment sequence: the action-reward pair at time `n`. -/ -def IsAlgEnvSeq.step (A : ℕ → Ω → α) (R' : ℕ → Ω → R) (n : ℕ) (ω : Ω) : α × R := - (A n ω, R' n ω) +/-- Step of the algorithm-environment sequence: the action-feedback pair at time `n`. -/ +def IsAlgEnvSeq.step (A : ℕ → Ω → 𝓐) (Y : ℕ → Ω → 𝓨) (n : ℕ) (ω : Ω) : 𝓐 × 𝓨 := + (A n ω, Y n ω) @[fun_prop] lemma IsAlgEnvSeq.measurable_step (n : ℕ) (hA : Measurable (A n)) - (hR' : Measurable (R' n)) : - Measurable (IsAlgEnvSeq.step A R' n) := by + (hY : Measurable (Y n)) : + Measurable (IsAlgEnvSeq.step A Y n) := by unfold IsAlgEnvSeq.step fun_prop /-- History of the algorithm-environment sequence up to time `n`. -/ -def IsAlgEnvSeq.hist (A : ℕ → Ω → α) (R' : ℕ → Ω → R) (n : ℕ) (ω : Ω) : Iic n → α × R := - fun i ↦ (A i ω, R' i ω) +def IsAlgEnvSeq.hist (A : ℕ → Ω → 𝓐) (Y : ℕ → Ω → 𝓨) (n : ℕ) (ω : Ω) : Iic n → 𝓐 × 𝓨 := + fun i ↦ (A i ω, Y i ω) @[fun_prop] lemma IsAlgEnvSeq.measurable_hist (hA : ∀ n, Measurable (A n)) - (hR' : ∀ n, Measurable (R' n)) (n : ℕ) : - Measurable (IsAlgEnvSeq.hist A R' n) := by + (hY : ∀ n, Measurable (Y n)) (n : ℕ) : + Measurable (IsAlgEnvSeq.hist A Y n) := by unfold IsAlgEnvSeq.hist fun_prop lemma IsAlgEnvSeq.eval_comp_hist (n : ℕ) : - (fun x ↦ x ⟨n, by simp⟩) ∘ (hist A R' n) = step A R' n := rfl + (fun x ↦ x ⟨n, by simp⟩) ∘ (hist A Y n) = step A Y n := rfl lemma IsAlgEnvSeq.fst_eval_comp_hist (n : ℕ) : - (fun x ↦ (x ⟨n, by simp⟩).1) ∘ (hist A R' n) = A n := rfl + (fun x ↦ (x ⟨n, by simp⟩).1) ∘ (hist A Y n) = A n := rfl lemma IsAlgEnvSeq.snd_eval_comp_hist (n : ℕ) : - (fun x ↦ (x ⟨n, by simp⟩).2) ∘ (hist A R' n) = R' n := rfl + (fun x ↦ (x ⟨n, by simp⟩).2) ∘ (hist A Y n) = Y n := rfl -/-- An algorithm-environment sequence: a sequence of actions and rewards generated +section IsAlgEnvSeq + +variable [StandardBorelSpace 𝓐] [Nonempty 𝓐] [StandardBorelSpace 𝓨] [Nonempty 𝓨] + +/-- An algorithm-environment sequence: a sequence of actions and feedbacks generated by an algorithm interacting with an environment. -/ -- ANCHOR: IsAlgEnvSeq structure IsAlgEnvSeq - [StandardBorelSpace α] [Nonempty α] [StandardBorelSpace R] [Nonempty R] - (A : ℕ → Ω → α) (R' : ℕ → Ω → R) (alg : Algorithm α R) (env : Environment α R) + (A : ℕ → Ω → 𝓐) (Y : ℕ → Ω → 𝓨) (alg : Algorithm 𝓐 𝓨) (env : Environment 𝓐 𝓨) (P : Measure Ω) [IsFiniteMeasure P] : Prop where - measurable_A n : Measurable (A n) := by fun_prop - measurable_R n : Measurable (R' n) := by fun_prop + measurable_action n : Measurable (A n) := by fun_prop + measurable_feedback n : Measurable (Y n) := by fun_prop hasLaw_action_zero : HasLaw (fun ω ↦ (A 0 ω)) alg.p0 P - hasCondDistrib_reward_zero : HasCondDistrib (R' 0) (A 0) env.ν0 P + hasCondDistrib_feedback_zero : HasCondDistrib (Y 0) (A 0) env.ν0 P hasCondDistrib_action n : - HasCondDistrib (A (n + 1)) (IsAlgEnvSeq.hist A R' n) (alg.policy n) P - hasCondDistrib_reward n : - HasCondDistrib (R' (n + 1)) (fun ω ↦ (IsAlgEnvSeq.hist A R' n ω, A (n + 1) ω)) + HasCondDistrib (A (n + 1)) (IsAlgEnvSeq.hist A Y n) (alg.policy n) P + hasCondDistrib_feedback n : + HasCondDistrib (Y (n + 1)) (fun ω ↦ (IsAlgEnvSeq.hist A Y n ω, A (n + 1) ω)) (env.feedback n) P -- ANCHOR_END: IsAlgEnvSeq -/-- An algorithm-environment sequence: a sequence of actions and rewards generated +/-- An algorithm-environment sequence: a sequence of actions and feedbacks generated by an algorithm interacting with an environment. -/ structure IsAlgEnvSeqUntil - [StandardBorelSpace α] [Nonempty α] [StandardBorelSpace R] [Nonempty R] - (A : ℕ → Ω → α) (R' : ℕ → Ω → R) (alg : Algorithm α R) (env : Environment α R) + (A : ℕ → Ω → 𝓐) (Y : ℕ → Ω → 𝓨) (alg : Algorithm 𝓐 𝓨) (env : Environment 𝓐 𝓨) (P : Measure Ω) [IsFiniteMeasure P] (N : ℕ) : Prop where - measurable_A n : Measurable (A n) := by fun_prop - measurable_R n : Measurable (R' n) := by fun_prop + measurable_action n : Measurable (A n) := by fun_prop + measurable_feedback n : Measurable (Y n) := by fun_prop hasLaw_action_zero : HasLaw (fun ω ↦ (A 0 ω)) alg.p0 P - hasCondDistrib_reward_zero : HasCondDistrib (R' 0) (A 0) env.ν0 P + hasCondDistrib_feedback_zero : HasCondDistrib (Y 0) (A 0) env.ν0 P hasCondDistrib_action n (hn : n < N) : - HasCondDistrib (A (n + 1)) (IsAlgEnvSeq.hist A R' n) (alg.policy n) P - hasCondDistrib_reward n (hn : n < N) : - HasCondDistrib (R' (n + 1)) (fun ω ↦ (IsAlgEnvSeq.hist A R' n ω, A (n + 1) ω)) + HasCondDistrib (A (n + 1)) (IsAlgEnvSeq.hist A Y n) (alg.policy n) P + hasCondDistrib_feedback n (hn : n < N) : + HasCondDistrib (Y (n + 1)) (fun ω ↦ (IsAlgEnvSeq.hist A Y n ω, A (n + 1) ω)) (env.feedback n) P -lemma IsAlgEnvSeqUntil.mono [StandardBorelSpace α] [Nonempty α] [StandardBorelSpace R] [Nonempty R] - (h : IsAlgEnvSeqUntil A R' alg env P N) {N' : ℕ} (hN : N' ≤ N) : - IsAlgEnvSeqUntil A R' alg env P N' where - measurable_A := h.measurable_A - measurable_R := h.measurable_R +lemma IsAlgEnvSeqUntil.mono (h : IsAlgEnvSeqUntil A Y alg env P N) {N' : ℕ} (hN : N' ≤ N) : + IsAlgEnvSeqUntil A Y alg env P N' where + measurable_action := h.measurable_action + measurable_feedback := h.measurable_feedback hasLaw_action_zero := h.hasLaw_action_zero - hasCondDistrib_reward_zero := h.hasCondDistrib_reward_zero + hasCondDistrib_feedback_zero := h.hasCondDistrib_feedback_zero hasCondDistrib_action n hn := h.hasCondDistrib_action n (hn.trans_le hN) - hasCondDistrib_reward n hn := h.hasCondDistrib_reward n (hn.trans_le hN) - -lemma IsAlgEnvSeq.isAlgEnvSeqUntil - [StandardBorelSpace α] [Nonempty α] [StandardBorelSpace R] [Nonempty R] - (h : IsAlgEnvSeq A R' alg env P) (N : ℕ) : - IsAlgEnvSeqUntil A R' alg env P N where - measurable_A := h.measurable_A - measurable_R := h.measurable_R + hasCondDistrib_feedback n hn := h.hasCondDistrib_feedback n (hn.trans_le hN) + +lemma IsAlgEnvSeq.isAlgEnvSeqUntil (h : IsAlgEnvSeq A Y alg env P) (N : ℕ) : + IsAlgEnvSeqUntil A Y alg env P N where + measurable_action := h.measurable_action + measurable_feedback := h.measurable_feedback hasLaw_action_zero := h.hasLaw_action_zero - hasCondDistrib_reward_zero := h.hasCondDistrib_reward_zero + hasCondDistrib_feedback_zero := h.hasCondDistrib_feedback_zero hasCondDistrib_action n _ := h.hasCondDistrib_action n - hasCondDistrib_reward n _ := h.hasCondDistrib_reward n - -lemma IsAlgEnvSeq.hasLaw_step_zero - [StandardBorelSpace α] [Nonempty α] [StandardBorelSpace R] [Nonempty R] - (h : IsAlgEnvSeq A R' alg env P) : - HasLaw (step A R' 0) (alg.p0 ⊗ₘ env.ν0) P := - HasLaw.prod_of_hasCondDistrib h.hasLaw_action_zero h.hasCondDistrib_reward_zero - -lemma IsAlgEnvSeqUntil.hasLaw_step_zero - [StandardBorelSpace α] [Nonempty α] [StandardBorelSpace R] [Nonempty R] - (h : IsAlgEnvSeqUntil A R' alg env P N) : - HasLaw (IsAlgEnvSeq.step A R' 0) (alg.p0 ⊗ₘ env.ν0) P := - HasLaw.prod_of_hasCondDistrib h.hasLaw_action_zero h.hasCondDistrib_reward_zero - -lemma IsAlgEnvSeq.hasCondDistrib_step - [StandardBorelSpace α] [Nonempty α] [StandardBorelSpace R] [Nonempty R] - (h : IsAlgEnvSeq A R' alg env P) (n : ℕ) : - HasCondDistrib (step A R' (n + 1)) (hist A R' n) (stepKernel alg env n) P := - HasCondDistrib.prod (h.hasCondDistrib_action n) (h.hasCondDistrib_reward n) - -lemma IsAlgEnvSeqUntil.hasCondDistrib_step - [StandardBorelSpace α] [Nonempty α] [StandardBorelSpace R] [Nonempty R] - (h : IsAlgEnvSeqUntil A R' alg env P N) (n : ℕ) (hn : n < N) : - HasCondDistrib (IsAlgEnvSeq.step A R' (n + 1)) (IsAlgEnvSeq.hist A R' n) + hasCondDistrib_feedback n _ := h.hasCondDistrib_feedback n + +lemma IsAlgEnvSeq.hasLaw_step_zero (h : IsAlgEnvSeq A Y alg env P) : + HasLaw (step A Y 0) (alg.p0 ⊗ₘ env.ν0) P := + HasLaw.prod_of_hasCondDistrib h.hasLaw_action_zero h.hasCondDistrib_feedback_zero + +lemma IsAlgEnvSeqUntil.hasLaw_step_zero (h : IsAlgEnvSeqUntil A Y alg env P N) : + HasLaw (IsAlgEnvSeq.step A Y 0) (alg.p0 ⊗ₘ env.ν0) P := + HasLaw.prod_of_hasCondDistrib h.hasLaw_action_zero h.hasCondDistrib_feedback_zero + +lemma IsAlgEnvSeq.hasCondDistrib_step (h : IsAlgEnvSeq A Y alg env P) (n : ℕ) : + HasCondDistrib (step A Y (n + 1)) (hist A Y n) (stepKernel alg env n) P := + HasCondDistrib.prod (h.hasCondDistrib_action n) (h.hasCondDistrib_feedback n) + +lemma IsAlgEnvSeqUntil.hasCondDistrib_step (h : IsAlgEnvSeqUntil A Y alg env P N) + (n : ℕ) (hn : n < N) : + HasCondDistrib (IsAlgEnvSeq.step A Y (n + 1)) (IsAlgEnvSeq.hist A Y n) (stepKernel alg env n) P := - HasCondDistrib.prod (h.hasCondDistrib_action n hn) (h.hasCondDistrib_reward n hn) + HasCondDistrib.prod (h.hasCondDistrib_action n hn) (h.hasCondDistrib_feedback n hn) + +end IsAlgEnvSeq /-- Filtration generated by the history up to time `n`. -/ -def IsAlgEnvSeq.filtration (hA : ∀ n, Measurable (A n)) (hR' : ∀ n, Measurable (R' n)) : +def IsAlgEnvSeq.filtration (hA : ∀ n, Measurable (A n)) (hY : ∀ n, Measurable (Y n)) : Filtration ℕ mΩ where - seq i := MeasurableSpace.comap (hist A R' i) inferInstance + seq i := MeasurableSpace.comap (hist A Y i) inferInstance mono' i j hij := by simp only rw [← measurable_iff_comap_le] - have : hist A R' i = (fun h k ↦ h ⟨k.1, by grind⟩) ∘ hist A R' j := rfl + have : hist A Y i = (fun h k ↦ h ⟨k.1, by grind⟩) ∘ hist A Y j := rfl rw [this] exact measurable_comp_comap _ (by fun_prop) le' i := by rw [← measurable_iff_comap_le] - exact measurable_hist hA hR' i + exact measurable_hist hA hY i lemma IsAlgEnvSeq.adapted_hist - (hA : ∀ n, Measurable (A n)) (hR' : ∀ n, Measurable (R' n)) : - Adapted (filtration hA hR') (IsAlgEnvSeq.hist A R') := + (hA : ∀ n, Measurable (A n)) (hY : ∀ n, Measurable (Y n)) : + Adapted (filtration hA hY) (IsAlgEnvSeq.hist A Y) := fun _ ↦ measurable_iff_comap_le.mpr le_rfl lemma IsAlgEnvSeq.adapted_step - (hA : ∀ n, Measurable (A n)) (hR' : ∀ n, Measurable (R' n)) : - Adapted (filtration hA hR') (step A R') := by + (hA : ∀ n, Measurable (A n)) (hY : ∀ n, Measurable (Y n)) : + Adapted (filtration hA hY) (step A Y) := by intro n - have : step A R' n = (fun h ↦ (h ⟨n, by simp⟩)) ∘ (hist A R' n) := by + have : step A Y n = (fun h ↦ (h ⟨n, by simp⟩)) ∘ (hist A Y n) := by ext ω : 1 simp [hist, step] rw [this] exact measurable_comp_comap _ (by fun_prop) lemma IsAlgEnvSeq.adapted_action - (hA : ∀ n, Measurable (A n)) (hR' : ∀ n, Measurable (R' n)) : - Adapted (filtration hA hR') A := by + (hA : ∀ n, Measurable (A n)) (hY : ∀ n, Measurable (Y n)) : + Adapted (filtration hA hY) A := by intro n - have : A n = (fun h ↦ (h ⟨n, by simp⟩).1) ∘ (hist A R' n) := by + have : A n = (fun h ↦ (h ⟨n, by simp⟩).1) ∘ (hist A Y n) := by ext ω : 1 simp [IsAlgEnvSeq.hist] rw [this] exact measurable_comp_comap _ (by fun_prop) -lemma IsAlgEnvSeq.adapted_reward - (hA : ∀ n, Measurable (A n)) (hR' : ∀ n, Measurable (R' n)) : - Adapted (filtration hA hR') R' := by +lemma IsAlgEnvSeq.adapted_feedback + (hA : ∀ n, Measurable (A n)) (hY : ∀ n, Measurable (Y n)) : + Adapted (filtration hA hY) Y := by intro n - have : R' n = (fun h ↦ (h ⟨n, by simp⟩).2) ∘ (hist A R' n) := by + have : Y n = (fun h ↦ (h ⟨n, by simp⟩).2) ∘ (hist A Y n) := by ext ω : 1 simp [IsAlgEnvSeq.hist] rw [this] @@ -249,10 +243,10 @@ lemma IsAlgEnvSeq.adapted_reward /-- Filtration generated by the history at time `n-1` together with the action at time `n`. -/ def IsAlgEnvSeq.filtrationAction - (hA : ∀ n, Measurable (A n)) (hR' : ∀ n, Measurable (R' n)) : + (hA : ∀ n, Measurable (A n)) (hY : ∀ n, Measurable (Y n)) : Filtration ℕ mΩ where seq n := if n = 0 then MeasurableSpace.comap (A 0) inferInstance - else IsAlgEnvSeq.filtration hA hR' (n - 1) ⊔ MeasurableSpace.comap (A n) inferInstance + else IsAlgEnvSeq.filtration hA hY (n - 1) ⊔ MeasurableSpace.comap (A n) inferInstance mono' n m hnm := by simp only by_cases hn : n = 0 @@ -261,24 +255,24 @@ def IsAlgEnvSeq.filtrationAction · simp only [hn, ↓reduceIte, hm] refine le_sup_of_le_left ?_ rw [← measurable_iff_comap_le] - suffices Measurable[IsAlgEnvSeq.filtration hA hR' 0] (A 0) from - this.mono ((IsAlgEnvSeq.filtration hA hR').mono zero_le') le_rfl - exact adapted_action hA hR' 0 + suffices Measurable[IsAlgEnvSeq.filtration hA hY 0] (A 0) from + this.mono ((IsAlgEnvSeq.filtration hA hY).mono zero_le') le_rfl + exact adapted_action hA hY 0 have hm : m ≠ 0 := by grind simp only [hn, hm, ↓reduceIte] have hnm' : n - 1 ≤ m - 1 := by grind simp only [sup_le_iff] constructor · refine le_sup_of_le_left ?_ - exact (IsAlgEnvSeq.filtration hA hR').mono hnm' + exact (IsAlgEnvSeq.filtration hA hY).mono hnm' · rcases eq_or_lt_of_le hnm with rfl | hlt · exact le_sup_of_le_right le_rfl refine le_sup_of_le_left ?_ rw [← measurable_iff_comap_le] have h_le : n ≤ m - 1 := by grind - suffices Measurable[IsAlgEnvSeq.filtration hA hR' n] (A n) from - this.mono ((IsAlgEnvSeq.filtration hA hR').mono h_le) le_rfl - exact adapted_action hA hR' n + suffices Measurable[IsAlgEnvSeq.filtration hA hY n] (A n) from + this.mono ((IsAlgEnvSeq.filtration hA hY).mono h_le) le_rfl + exact adapted_action hA hY n le' n := by by_cases hn : n = 0 · simp only [hn, ↓reduceIte] @@ -286,19 +280,19 @@ def IsAlgEnvSeq.filtrationAction fun_prop simp only [hn, ↓reduceIte, sup_le_iff] constructor - · exact (IsAlgEnvSeq.filtration hA hR').le _ + · exact (IsAlgEnvSeq.filtration hA hY).le _ · rw [← measurable_iff_comap_le] fun_prop lemma IsAlgEnvSeq.filtrationAction_zero_eq_comap - {hA : ∀ n, Measurable (A n)} {hR' : ∀ n, Measurable (R' n)} : - filtrationAction hA hR' 0 = MeasurableSpace.comap (A 0) inferInstance := by + {hA : ∀ n, Measurable (A n)} {hY : ∀ n, Measurable (Y n)} : + filtrationAction hA hY 0 = MeasurableSpace.comap (A 0) inferInstance := by simp [filtrationAction] lemma IsAlgEnvSeq.filtrationAction_eq_comap - {hA : ∀ n, Measurable (A n)} {hR' : ∀ n, Measurable (R' n)} (n : ℕ) (hn : n ≠ 0) : - filtrationAction hA hR' n = - MeasurableSpace.comap (fun ω ↦ (hist A R' (n - 1) ω, A n ω)) inferInstance := by + {hA : ∀ n, Measurable (A n)} {hY : ∀ n, Measurable (Y n)} (n : ℕ) (hn : n ≠ 0) : + filtrationAction hA hY n = + MeasurableSpace.comap (fun ω ↦ (hist A Y (n - 1) ω, A n ω)) inferInstance := by simp only [filtrationAction, filtration, ← MeasurableSpace.comap_prodMk, hn, ↓reduceIte] rfl diff --git a/LeanMachineLearning/SequentialLearning/Algorithms/RandomSampling.lean b/LeanMachineLearning/SequentialLearning/Algorithms/RandomSampling.lean index f6a35718..7e6e2410 100644 --- a/LeanMachineLearning/SequentialLearning/Algorithms/RandomSampling.lean +++ b/LeanMachineLearning/SequentialLearning/Algorithms/RandomSampling.lean @@ -33,24 +33,24 @@ open scoped Topology namespace Learning -variable {α β Ω : Type*} [MeasurableSpace α] [MeasurableSpace β] [StandardBorelSpace α] [Nonempty α] - [StandardBorelSpace β] [Nonempty β] {μ : Measure α} [IsProbabilityMeasure μ] [MeasurableSpace Ω] +variable {𝓐 𝓨 Ω : Type*} [MeasurableSpace 𝓐] [MeasurableSpace 𝓨] [StandardBorelSpace 𝓐] [Nonempty 𝓐] + [StandardBorelSpace 𝓨] [Nonempty 𝓨] {μ : Measure 𝓐} [IsProbabilityMeasure μ] [MeasurableSpace Ω] {P : Measure Ω} [IsProbabilityMeasure P] open Set in /-- The _Random Sampling_ algorithm, which samples from a fixed probability measure at each iteration. -/ @[simps] -noncomputable def randomSampling (μ : Measure α) [IsProbabilityMeasure μ] : Algorithm α β where +noncomputable def randomSampling (μ : Measure 𝓐) [IsProbabilityMeasure μ] : Algorithm 𝓐 𝓨 where policy _ := Kernel.const _ μ p0 := μ namespace randomSampling -variable {A : ℕ → Ω → α} {R : ℕ → Ω → β} {env : Environment α β} +variable {A : ℕ → Ω → 𝓐} {Y : ℕ → Ω → 𝓨} {env : Environment 𝓐 𝓨} /-- Each action follows the distribution μ. -/ -lemma hasLaw_action (h : IsAlgEnvSeq A R (randomSampling μ) env P) (n : ℕ) : +lemma hasLaw_action (h : IsAlgEnvSeq A Y (randomSampling μ) env P) (n : ℕ) : HasLaw (A n) μ P := by by_cases hn : n = 0 · rw [hn] @@ -60,19 +60,19 @@ lemma hasLaw_action (h : IsAlgEnvSeq A R (randomSampling μ) env P) (n : ℕ) : exact hasLaw_of_hasCondDistrib_const <| h.hasCondDistrib_action k /-- Actions are mutually independent. -/ -lemma iIndep_action (h : IsAlgEnvSeq A R (randomSampling μ) env P) : +lemma iIndep_action (h : IsAlgEnvSeq A Y (randomSampling μ) env P) : iIndepFun A P := by - have hA := h.measurable_A + have hA := h.measurable_action rw [iIndepFun_nat_iff_forall_indepFun (by fun_prop)] intro n have condDistrib_eq := (h.hasCondDistrib_action n).condDistrib_eq simp only [randomSampling_policy] at condDistrib_eq have law_eq := (hasLaw_action h (n + 1)).map_eq rw [← law_eq, ← indepFun_iff_condDistrib_eq_const ?_ (by fun_prop)] at condDistrib_eq - · have meas_fst : Measurable (fun (f : Iic n → α × β) ↦ (fun i ↦ (f i).1)) := by + · have meas_fst : Measurable (fun (f : Iic n → 𝓐 × 𝓨) ↦ (fun i ↦ (f i).1)) := by fun_prop exact (condDistrib_eq.comp meas_fst measurable_id).symm - · exact (IsAlgEnvSeq.measurable_hist (h.measurable_A) (h.measurable_R) n).aemeasurable + · exact (IsAlgEnvSeq.measurable_hist (h.measurable_action) (h.measurable_feedback) n).aemeasurable end randomSampling diff --git a/LeanMachineLearning/SequentialLearning/Algorithms/RoundRobin.lean b/LeanMachineLearning/SequentialLearning/Algorithms/RoundRobin.lean index bfac2e07..4bebce42 100644 --- a/LeanMachineLearning/SequentialLearning/Algorithms/RoundRobin.lean +++ b/LeanMachineLearning/SequentialLearning/Algorithms/RoundRobin.lean @@ -69,7 +69,7 @@ end Aux namespace Learning -variable {K : ℕ} +variable {𝓨 : Type*} {m𝓨 : MeasurableSpace 𝓨} {K : ℕ} section AlgorithmDefinition @@ -79,32 +79,33 @@ def RoundRobin.nextAction (hK : 0 < K) (n : ℕ) : Fin K := ⟨(n + 1) % K, Nat. /-- The Round-Robin algorithm: deterministic algorithm that chooses action `n % K` at time `n`. -/ noncomputable -def roundRobinAlgorithm (hK : 0 < K) : Algorithm (Fin K) ℝ := +def roundRobinAlgorithm (hK : 0 < K) : Algorithm (Fin K) 𝓨 := detAlgorithm (fun n _ ↦ RoundRobin.nextAction hK n) (by fun_prop) ⟨0, hK⟩ end AlgorithmDefinition namespace RoundRobin -variable {hK : 0 < K} {ν : Kernel (Fin K) ℝ} [IsMarkovKernel ν] +variable [StandardBorelSpace 𝓨] [Nonempty 𝓨] + {hK : 0 < K} {ν : Kernel (Fin K) 𝓨} [IsMarkovKernel ν] {Ω : Type*} {mΩ : MeasurableSpace Ω} {P : Measure Ω} [IsProbabilityMeasure P] - {A : ℕ → Ω → Fin K} {R : ℕ → Ω → ℝ} + {A : ℕ → Ω → Fin K} {Y : ℕ → Ω → 𝓨} lemma action_zero [Nonempty (Fin K)] - (h : IsAlgEnvSeqUntil A R (roundRobinAlgorithm hK) (stationaryEnv ν) P 0) : + (h : IsAlgEnvSeqUntil A Y (roundRobinAlgorithm hK) (stationaryEnv ν) P 0) : A 0 =ᵐ[P] fun _ ↦ ⟨0, hK⟩ := by have : Nonempty (Fin K) := Fin.pos_iff_nonempty.mp hK exact h.action_zero_detAlgorithm lemma action_ae_eq_roundRobinNextAction [Nonempty (Fin K)] (n : ℕ) - (h : IsAlgEnvSeqUntil A R (roundRobinAlgorithm hK) (stationaryEnv ν) P (n + 1)) : + (h : IsAlgEnvSeqUntil A Y (roundRobinAlgorithm hK) (stationaryEnv ν) P (n + 1)) : A (n + 1) =ᵐ[P] fun _ ↦ nextAction hK n := h.action_detAlgorithm_ae_eq (by grind) /-- The action chosen at time `n` is the action `n % K`. -/ lemma action_ae_eq [Nonempty (Fin K)] (n : ℕ) - (h : IsAlgEnvSeqUntil A R (roundRobinAlgorithm hK) (stationaryEnv ν) P n) : + (h : IsAlgEnvSeqUntil A Y (roundRobinAlgorithm hK) (stationaryEnv ν) P n) : A n =ᵐ[P] fun _ ↦ ⟨n % K, Nat.mod_lt _ hK⟩ := by cases n with | zero => exact action_zero h @@ -114,7 +115,7 @@ lemma action_ae_eq [Nonempty (Fin K)] (n : ℕ) /-- At time `K * m`, the number of times each action is chosen is equal to `m`. -/ lemma pullCount_mul [Nonempty (Fin K)] (m : ℕ) - (h : IsAlgEnvSeqUntil A R (roundRobinAlgorithm hK) (stationaryEnv ν) P (K * m - 1)) + (h : IsAlgEnvSeqUntil A Y (roundRobinAlgorithm hK) (stationaryEnv ν) P (K * m - 1)) (a : Fin K) : pullCount A a (K * m) =ᵐ[P] fun _ ↦ m := by rw [Filter.EventuallyEq] @@ -130,15 +131,15 @@ lemma pullCount_mul [Nonempty (Fin K)] (m : ℕ) _ = m := sum_mod_range_mul hK m a lemma pullCount_eq_one [Nonempty (Fin K)] - (h : IsAlgEnvSeqUntil A R (roundRobinAlgorithm hK) (stationaryEnv ν) P (K - 1)) + (h : IsAlgEnvSeqUntil A Y (roundRobinAlgorithm hK) (stationaryEnv ν) P (K - 1)) (a : Fin K) : pullCount A a K =ᵐ[P] fun _ ↦ 1 := by suffices pullCount A a (K * 1) =ᵐ[P] fun _ ↦ 1 by simpa using this - refine pullCount_mul 1 (P := P) (ν := ν) (R := R) (hK := hK) ?_ a + refine pullCount_mul 1 (P := P) (ν := ν) (Y := Y) (hK := hK) ?_ a simpa lemma time_gt_of_pullCount_gt_one [Nonempty (Fin K)] - (h : IsAlgEnvSeqUntil A R (roundRobinAlgorithm hK) (stationaryEnv ν) P (K - 1)) (a : Fin K) : + (h : IsAlgEnvSeqUntil A Y (roundRobinAlgorithm hK) (stationaryEnv ν) P (K - 1)) (a : Fin K) : ∀ᵐ ω ∂P, ∀ n, 1 < pullCount A a n ω → K < n := by filter_upwards [pullCount_eq_one h a] with h h_eq n hn rw [← h_eq] at hn @@ -146,7 +147,7 @@ lemma time_gt_of_pullCount_gt_one [Nonempty (Fin K)] exact hn.not_ge (pullCount_mono _ h_lt _) lemma pullCount_pos_of_time_ge [Nonempty (Fin K)] - (h : IsAlgEnvSeqUntil A R (roundRobinAlgorithm hK) (stationaryEnv ν) P (K - 1)) : + (h : IsAlgEnvSeqUntil A Y (roundRobinAlgorithm hK) (stationaryEnv ν) P (K - 1)) : ∀ᵐ ω ∂P, ∀ n, K ≤ n → ∀ b : Fin K, 0 < pullCount A b n ω := by have h_ae a := pullCount_eq_one h a simp_rw [Filter.EventuallyEq, ← ae_all_iff] at h_ae @@ -156,7 +157,7 @@ lemma pullCount_pos_of_time_ge [Nonempty (Fin K)] exact pullCount_mono _ hn _ lemma pullCount_pos_of_pullCount_gt_one [Nonempty (Fin K)] - (h : IsAlgEnvSeqUntil A R (roundRobinAlgorithm hK) (stationaryEnv ν) P (K - 1)) (a : Fin K) : + (h : IsAlgEnvSeqUntil A Y (roundRobinAlgorithm hK) (stationaryEnv ν) P (K - 1)) (a : Fin K) : ∀ᵐ ω ∂P, ∀ n, 1 < pullCount A a n ω → ∀ b : Fin K, 0 < pullCount A b n ω := by filter_upwards [time_gt_of_pullCount_gt_one h a, pullCount_pos_of_time_ge h] with ω h1 h2 n h_gt a exact h2 n (h1 n h_gt).le a diff --git a/LeanMachineLearning/SequentialLearning/Deterministic.lean b/LeanMachineLearning/SequentialLearning/Deterministic.lean index 0b1b1e1a..4411541d 100644 --- a/LeanMachineLearning/SequentialLearning/Deterministic.lean +++ b/LeanMachineLearning/SequentialLearning/Deterministic.lean @@ -51,86 +51,86 @@ open scoped ENNReal NNReal namespace Learning -variable {α R : Type*} {mα : MeasurableSpace α} {mR : MeasurableSpace R} +variable {𝓐 𝓨 : Type*} {m𝓐 : MeasurableSpace 𝓐} {m𝓨 : MeasurableSpace 𝓨} /-- An algorithm is deterministic if its initial action and subsequent actions are determined by measurable functions (and not possibly random kernels). -/ -class IsDeterministicAlg (alg : Algorithm α R) : Prop where +class IsDeterministicAlg (alg : Algorithm 𝓐 𝓨) : Prop where exists_action0 : ∃ action0, alg.p0 = Measure.dirac action0 - exists_nextAction n : ∃ (nextAction : (Iic n → α × R) → α) (h_meas : Measurable nextAction), + exists_nextAction n : ∃ (nextAction : (Iic n → 𝓐 × 𝓨) → 𝓐) (h_meas : Measurable nextAction), alg.policy n = Kernel.deterministic nextAction h_meas /-- The initial action of a deterministic algorithm. -/ noncomputable -def actionZero (alg : Algorithm α R) [h_det : IsDeterministicAlg alg] : α := +def actionZero (alg : Algorithm 𝓐 𝓨) [h_det : IsDeterministicAlg alg] : 𝓐 := h_det.exists_action0.choose /-- The next action of a deterministic algorithm after step `n`. -/ noncomputable -def nextAction (alg : Algorithm α R) [h_det : IsDeterministicAlg alg] (n : ℕ) : - (Iic n → α × R) → α := +def nextAction (alg : Algorithm 𝓐 𝓨) [h_det : IsDeterministicAlg alg] (n : ℕ) : + (Iic n → 𝓐 × 𝓨) → 𝓐 := (h_det.exists_nextAction n).choose @[fun_prop] -lemma measurable_nextAction (alg : Algorithm α R) [IsDeterministicAlg alg] (n : ℕ) : +lemma measurable_nextAction (alg : Algorithm 𝓐 𝓨) [IsDeterministicAlg alg] (n : ℕ) : Measurable (nextAction alg n) := (IsDeterministicAlg.exists_nextAction n).choose_spec.choose -lemma p0_eq_dirac (alg : Algorithm α R) [h_det : IsDeterministicAlg alg] : +lemma p0_eq_dirac (alg : Algorithm 𝓐 𝓨) [h_det : IsDeterministicAlg alg] : alg.p0 = Measure.dirac (actionZero alg) := h_det.exists_action0.choose_spec -lemma policy_eq_deterministic (alg : Algorithm α R) [h_det : IsDeterministicAlg alg] (n : ℕ) : +lemma policy_eq_deterministic (alg : Algorithm 𝓐 𝓨) [h_det : IsDeterministicAlg alg] (n : ℕ) : alg.policy n = Kernel.deterministic (nextAction alg n) (measurable_nextAction alg n) := (IsDeterministicAlg.exists_nextAction n).choose_spec.choose_spec namespace IsDeterministicAlg variable {Ω : Type*} {mΩ : MeasurableSpace Ω} - [StandardBorelSpace α] [Nonempty α] [StandardBorelSpace R] [Nonempty R] - {alg : Algorithm α R} {env : Environment α R} {P : Measure Ω} [IsFiniteMeasure P] - {A : ℕ → Ω → α} {R' : ℕ → Ω → R} {n N : ℕ} + [StandardBorelSpace 𝓐] [Nonempty 𝓐] [StandardBorelSpace 𝓨] [Nonempty 𝓨] + {alg : Algorithm 𝓐 𝓨} {env : Environment 𝓐 𝓨} {P : Measure Ω} [IsFiniteMeasure P] + {A : ℕ → Ω → 𝓐} {Y : ℕ → Ω → 𝓨} {n N : ℕ} lemma hasLaw_action_zero_of_IsAlgEnvSeqUntil [h_det : IsDeterministicAlg alg] - (h : IsAlgEnvSeqUntil A R' alg env P N) : + (h : IsAlgEnvSeqUntil A Y alg env P N) : HasLaw (A 0) (Measure.dirac (actionZero alg)) P where - aemeasurable := have hA := h.measurable_A; by fun_prop + aemeasurable := have hA := h.measurable_action; by fun_prop map_eq := (h.hasLaw_action_zero).map_eq.trans (p0_eq_dirac alg) lemma action_zero_of_IsAlgEnvSeqUntil [h_det : IsDeterministicAlg alg] - (h : IsAlgEnvSeqUntil A R' alg env P N) : + (h : IsAlgEnvSeqUntil A Y alg env P N) : A 0 =ᵐ[P] fun _ ↦ actionZero alg := by have h_eq : ∀ᵐ x ∂(P.map (A 0)), x = actionZero alg := by simp [(hasLaw_action_zero_of_IsAlgEnvSeqUntil h).map_eq] - have hA := h.measurable_A + have hA := h.measurable_action exact ae_of_ae_map (by fun_prop) h_eq lemma action_ae_eq_of_IsAlgEnvSeqUntil [h_det : IsDeterministicAlg alg] - (h : IsAlgEnvSeqUntil A R' alg env P N) (hn : n < N) : - A (n + 1) =ᵐ[P] fun ω ↦ nextAction alg n (IsAlgEnvSeq.hist A R' n ω) := by - have hA := h.measurable_A - have hR' := h.measurable_R + (h : IsAlgEnvSeqUntil A Y alg env P N) (hn : n < N) : + A (n + 1) =ᵐ[P] fun ω ↦ nextAction alg n (IsAlgEnvSeq.hist A Y n ω) := by + have hA := h.measurable_action + have hY := h.measurable_feedback have h_eq := (h.hasCondDistrib_action n hn).condDistrib_eq rw [policy_eq_deterministic alg n] at h_eq refine ae_eq_of_condDistrib_eq_deterministic (by fun_prop : Measurable (nextAction alg n)) (by fun_prop) (by fun_prop) h_eq -lemma hasLaw_action_zero [h_det : IsDeterministicAlg alg] (h : IsAlgEnvSeq A R' alg env P) : +lemma hasLaw_action_zero [h_det : IsDeterministicAlg alg] (h : IsAlgEnvSeq A Y alg env P) : HasLaw (A 0) (Measure.dirac (actionZero alg)) P where - aemeasurable := have hA := h.measurable_A; by fun_prop + aemeasurable := have hA := h.measurable_action; by fun_prop map_eq := (h.hasLaw_action_zero).map_eq.trans (p0_eq_dirac alg) -lemma action_zero_ae_eq [h_det : IsDeterministicAlg alg] (h : IsAlgEnvSeq A R' alg env P) : +lemma action_zero_ae_eq [h_det : IsDeterministicAlg alg] (h : IsAlgEnvSeq A Y alg env P) : A 0 =ᵐ[P] fun _ ↦ actionZero alg := action_zero_of_IsAlgEnvSeqUntil (h.isAlgEnvSeqUntil 0) -lemma action_ae_eq [h_det : IsDeterministicAlg alg] (h : IsAlgEnvSeq A R' alg env P) (n : ℕ) : - A (n + 1) =ᵐ[P] fun ω ↦ nextAction alg n (IsAlgEnvSeq.hist A R' n ω) := +lemma action_ae_eq [h_det : IsDeterministicAlg alg] (h : IsAlgEnvSeq A Y alg env P) (n : ℕ) : + A (n + 1) =ᵐ[P] fun ω ↦ nextAction alg n (IsAlgEnvSeq.hist A Y n ω) := action_ae_eq_of_IsAlgEnvSeqUntil (h.isAlgEnvSeqUntil (n + 1)) (by simp) -lemma action_ae_all_eq [h_det : IsDeterministicAlg alg] (h : IsAlgEnvSeq A R' alg env P) : +lemma action_ae_all_eq [h_det : IsDeterministicAlg alg] (h : IsAlgEnvSeq A Y alg env P) : ∀ᵐ ω ∂P, A 0 ω = actionZero alg ∧ - ∀ n, A (n + 1) ω = nextAction alg n (IsAlgEnvSeq.hist A R' n ω) := by + ∀ n, A (n + 1) ω = nextAction alg n (IsAlgEnvSeq.hist A Y n ω) := by rw [eventually_and, ae_all_iff] exact ⟨action_zero_ae_eq h, action_ae_eq h⟩ @@ -138,77 +138,77 @@ end IsDeterministicAlg /-- An environment is deterministic if its initial feedbacks are determined by measurable functions (and not possibly random kernels). -/ -class IsDeterministicEnv (env : Environment α R) : Prop where - exists_f0 : ∃ (f0 : α → R) (hf0 : Measurable f0), env.ν0 = Kernel.deterministic f0 hf0 - exists_f : ∀ n, ∃ (f : ((Iic n → α × R) × α) → R) (hf : Measurable f), +class IsDeterministicEnv (env : Environment 𝓐 𝓨) : Prop where + exists_f0 : ∃ (f0 : 𝓐 → 𝓨) (hf0 : Measurable f0), env.ν0 = Kernel.deterministic f0 hf0 + exists_f : ∀ n, ∃ (f : ((Iic n → 𝓐 × 𝓨) × 𝓐) → 𝓨) (hf : Measurable f), env.feedback n = Kernel.deterministic f hf /-- The initial feedback function of a deterministic environment. -/ noncomputable -def feedbackFunZero (env : Environment α R) [h_det : IsDeterministicEnv env] : α → R := +def feedbackFunZero (env : Environment 𝓐 𝓨) [h_det : IsDeterministicEnv env] : 𝓐 → 𝓨 := h_det.exists_f0.choose @[fun_prop] -lemma measurable_feedbackFunZero (env : Environment α R) [IsDeterministicEnv env] : +lemma measurable_feedbackFunZero (env : Environment 𝓐 𝓨) [IsDeterministicEnv env] : Measurable (feedbackFunZero env) := (IsDeterministicEnv.exists_f0).choose_spec.choose -lemma ν0_eq_deterministic (env : Environment α R) [IsDeterministicEnv env] : +lemma ν0_eq_deterministic (env : Environment 𝓐 𝓨) [IsDeterministicEnv env] : env.ν0 = Kernel.deterministic (feedbackFunZero env) (measurable_feedbackFunZero env) := (IsDeterministicEnv.exists_f0).choose_spec.choose_spec /-- The feedback function of a deterministic environment at step `n`. -/ noncomputable -def feedbackFun (env : Environment α R) [h_det : IsDeterministicEnv env] (n : ℕ) : - ((Iic n → α × R) × α) → R := +def feedbackFun (env : Environment 𝓐 𝓨) [h_det : IsDeterministicEnv env] (n : ℕ) : + ((Iic n → 𝓐 × 𝓨) × 𝓐) → 𝓨 := (h_det.exists_f n).choose @[fun_prop] -lemma measurable_feedbackFun (env : Environment α R) [IsDeterministicEnv env] (n : ℕ) : +lemma measurable_feedbackFun (env : Environment 𝓐 𝓨) [IsDeterministicEnv env] (n : ℕ) : Measurable (feedbackFun env n) := (IsDeterministicEnv.exists_f n).choose_spec.choose -lemma feedback_eq_deterministic (env : Environment α R) [IsDeterministicEnv env] (n : ℕ) : +lemma feedback_eq_deterministic (env : Environment 𝓐 𝓨) [IsDeterministicEnv env] (n : ℕ) : env.feedback n = Kernel.deterministic (feedbackFun env n) (measurable_feedbackFun env n) := (IsDeterministicEnv.exists_f n).choose_spec.choose_spec namespace IsDeterministicEnv variable {Ω : Type*} {mΩ : MeasurableSpace Ω} - [StandardBorelSpace α] [Nonempty α] [StandardBorelSpace R] [Nonempty R] - {alg : Algorithm α R} {env : Environment α R} {P : Measure Ω} [IsFiniteMeasure P] - {A : ℕ → Ω → α} {R' : ℕ → Ω → R} - {f : (n : ℕ) → ((Iic n → α × R) × α) → R} {hf : ∀ n, Measurable (f n)} - {f0 : α → R} {hf0 : Measurable f0} - -lemma hasCondDistrib_reward_zero [h_det : IsDeterministicEnv env] - (h : IsAlgEnvSeq A R' alg env P) : - HasCondDistrib (R' 0) (A 0) + [StandardBorelSpace 𝓐] [Nonempty 𝓐] [StandardBorelSpace 𝓨] [Nonempty 𝓨] + {alg : Algorithm 𝓐 𝓨} {env : Environment 𝓐 𝓨} {P : Measure Ω} [IsFiniteMeasure P] + {A : ℕ → Ω → 𝓐} {Y : ℕ → Ω → 𝓨} + {f : (n : ℕ) → ((Iic n → 𝓐 × 𝓨) × 𝓐) → 𝓨} {hf : ∀ n, Measurable (f n)} + {f0 : 𝓐 → 𝓨} {hf0 : Measurable f0} + +lemma hasCondDistrib_feedback_zero [h_det : IsDeterministicEnv env] + (h : IsAlgEnvSeq A Y alg env P) : + HasCondDistrib (Y 0) (A 0) (Kernel.deterministic (feedbackFunZero env) (measurable_feedbackFunZero env)) P := by rw [← ν0_eq_deterministic] - exact h.hasCondDistrib_reward_zero + exact h.hasCondDistrib_feedback_zero -lemma hasCondDistrib_reward [h_det : IsDeterministicEnv env] - (h : IsAlgEnvSeq A R' alg env P) (n : ℕ) : - HasCondDistrib (R' (n + 1)) (fun ω ↦ (IsAlgEnvSeq.hist A R' n ω, A (n + 1) ω)) +lemma hasCondDistrib_feedback [h_det : IsDeterministicEnv env] + (h : IsAlgEnvSeq A Y alg env P) (n : ℕ) : + HasCondDistrib (Y (n + 1)) (fun ω ↦ (IsAlgEnvSeq.hist A Y n ω, A (n + 1) ω)) (Kernel.deterministic (feedbackFun env n) (measurable_feedbackFun env n)) P := by rw [← feedback_eq_deterministic] - exact h.hasCondDistrib_reward n + exact h.hasCondDistrib_feedback n end IsDeterministicEnv -variable {nextA : (n : ℕ) → (Iic n → α × R) → α} {h_next : ∀ n, Measurable (nextA n)} - {action0 : α} {env : Environment α R} - {f0 : α → R} {hf0 : Measurable f0} - {f : (n : ℕ) → ((Iic n → α × R) × α) → R} {hf : ∀ n, Measurable (f n)} +variable {nextA : (n : ℕ) → (Iic n → 𝓐 × 𝓨) → 𝓐} {h_next : ∀ n, Measurable (nextA n)} + {action0 : 𝓐} {env : Environment 𝓐 𝓨} + {f0 : 𝓐 → 𝓨} {hf0 : Measurable f0} + {f : (n : ℕ) → ((Iic n → 𝓐 × 𝓨) × 𝓐) → 𝓨} {hf : ∀ n, Measurable (f n)} /-- A deterministic algorithm, which chooses the action given by the function `nextAction`. -/ @[simps] noncomputable -- ANCHOR: detAlgorithm -def detAlgorithm (nextA : (n : ℕ) → (Iic n → α × R) → α) - (h_next : ∀ n, Measurable (nextA n)) (action0 : α) : - Algorithm α R where +def detAlgorithm (nextA : (n : ℕ) → (Iic n → 𝓐 × 𝓨) → 𝓐) + (h_next : ∀ n, Measurable (nextA n)) (action0 : 𝓐) : + Algorithm 𝓐 𝓨 where policy n := Kernel.deterministic (nextA n) (h_next n) p0 := Measure.dirac action0 -- ANCHOR_END: detAlgorithm @@ -218,7 +218,7 @@ instance : IsDeterministicAlg (detAlgorithm nextA h_next action0) where exists_nextAction n := ⟨nextA n, h_next n, rfl⟩ @[simp] -lemma actionZero_detAlgorithm [MeasurableSpace.SeparatesPoints α] : +lemma actionZero_detAlgorithm [MeasurableSpace.SeparatesPoints 𝓐] : actionZero (detAlgorithm nextA h_next action0) = action0 := by have h_eq := p0_eq_dirac (detAlgorithm nextA h_next action0) simp only [detAlgorithm] at h_eq @@ -226,7 +226,7 @@ lemma actionZero_detAlgorithm [MeasurableSpace.SeparatesPoints α] : exact h_eq.symm @[simp] -lemma nextAction_detAlgorithm [MeasurableSpace.SeparatesPoints α] (n : ℕ) : +lemma nextAction_detAlgorithm [MeasurableSpace.SeparatesPoints 𝓐] (n : ℕ) : nextAction (detAlgorithm nextA h_next action0) n = nextA n := by have h_eq := policy_eq_deterministic (detAlgorithm nextA h_next action0) n simpa [detAlgorithm] using h_eq.symm @@ -234,9 +234,9 @@ lemma nextAction_detAlgorithm [MeasurableSpace.SeparatesPoints α] (n : ℕ) : /-- A deterministic environment, where the feedback is given by evaluating fixed measurable functions. -/ noncomputable def detEnvironment - (f0 : α → R) (hf0 : Measurable f0) - (f : (n : ℕ) → ((Iic n → α × R) × α) → R) (hf : ∀ n, Measurable (f n)) : - Environment α R where + (f0 : 𝓐 → 𝓨) (hf0 : Measurable f0) + (f : (n : ℕ) → ((Iic n → 𝓐 × 𝓨) × 𝓐) → 𝓨) (hf : ∀ n, Measurable (f n)) : + Environment 𝓐 𝓨 where feedback n := (Kernel.deterministic (f n) (hf n)) ν0 := Kernel.deterministic f0 hf0 @@ -245,65 +245,64 @@ instance : IsDeterministicEnv (detEnvironment f0 hf0 f hf) where exists_f n := ⟨f n, hf n, rfl⟩ @[simp] -lemma feedbackFunZero_detEnvironment [MeasurableSpace.SeparatesPoints R] : +lemma feedbackFunZero_detEnvironment [MeasurableSpace.SeparatesPoints 𝓨] : feedbackFunZero (detEnvironment f0 hf0 f hf) = f0 := by simpa [detEnvironment] using (ν0_eq_deterministic (detEnvironment f0 hf0 f hf)).symm @[simp] -lemma feedbackFun_detEnvironment [MeasurableSpace.SeparatesPoints R] (n : ℕ) : +lemma feedbackFun_detEnvironment [MeasurableSpace.SeparatesPoints 𝓨] (n : ℕ) : feedbackFun (detEnvironment f0 hf0 f hf) n = f n := by simpa [detEnvironment] using (feedback_eq_deterministic (detEnvironment f0 hf0 f hf) n).symm namespace IsAlgEnvSeq variable {Ω : Type*} {mΩ : MeasurableSpace Ω} - [StandardBorelSpace α] [Nonempty α] [StandardBorelSpace R] [Nonempty R] - {alg : Algorithm α R} {ν : Kernel α R} [IsMarkovKernel ν] - {P : Measure Ω} [IsProbabilityMeasure P] {A : ℕ → Ω → α} {R' : ℕ → Ω → R} + [StandardBorelSpace 𝓐] [Nonempty 𝓐] [StandardBorelSpace 𝓨] [Nonempty 𝓨] + {alg : Algorithm 𝓐 𝓨} {ν : Kernel 𝓐 𝓨} [IsMarkovKernel ν] + {P : Measure Ω} [IsProbabilityMeasure P] {A : ℕ → Ω → 𝓐} {Y : ℕ → Ω → 𝓨} lemma hasLaw_action_zero_detAlgorithm - (h : IsAlgEnvSeq A R' (detAlgorithm nextA h_next action0) env P) : + (h : IsAlgEnvSeq A Y (detAlgorithm nextA h_next action0) env P) : HasLaw (A 0) (Measure.dirac action0) P := by simpa using IsDeterministicAlg.hasLaw_action_zero h lemma action_zero_detAlgorithm - (h : IsAlgEnvSeq A R' (detAlgorithm nextA h_next action0) env P) : + (h : IsAlgEnvSeq A Y (detAlgorithm nextA h_next action0) env P) : A 0 =ᵐ[P] fun _ ↦ action0 := (IsDeterministicAlg.action_zero_ae_eq h).trans (by simp) lemma action_detAlgorithm_ae_eq - (h : IsAlgEnvSeq A R' (detAlgorithm nextA h_next action0) env P) (n : ℕ) : - A (n + 1) =ᵐ[P] fun ω ↦ nextA n (hist A R' n ω) := + (h : IsAlgEnvSeq A Y (detAlgorithm nextA h_next action0) env P) (n : ℕ) : + A (n + 1) =ᵐ[P] fun ω ↦ nextA n (hist A Y n ω) := (IsDeterministicAlg.action_ae_eq h n).trans (by simp) lemma action_detAlgorithm_ae_all_eq - (h : IsAlgEnvSeq A R' (detAlgorithm nextA h_next action0) env P) : - ∀ᵐ ω ∂P, A 0 ω = action0 ∧ ∀ n, A (n + 1) ω = nextA n (hist A R' n ω) := by + (h : IsAlgEnvSeq A Y (detAlgorithm nextA h_next action0) env P) : + ∀ᵐ ω ∂P, A 0 ω = action0 ∧ ∀ n, A (n + 1) ω = nextA n (hist A Y n ω) := by filter_upwards [IsDeterministicAlg.action_ae_all_eq h] with ω hω using by simp [hω] - end IsAlgEnvSeq namespace IsAlgEnvSeqUntil variable {Ω : Type*} {mΩ : MeasurableSpace Ω} - [StandardBorelSpace α] [Nonempty α] [StandardBorelSpace R] [Nonempty R] - {alg : Algorithm α R} {ν : Kernel α R} [IsMarkovKernel ν] - {P : Measure Ω} [IsProbabilityMeasure P] {A : ℕ → Ω → α} {R' : ℕ → Ω → R} {N n : ℕ} + [StandardBorelSpace 𝓐] [Nonempty 𝓐] [StandardBorelSpace 𝓨] [Nonempty 𝓨] + {alg : Algorithm 𝓐 𝓨} {ν : Kernel 𝓐 𝓨} [IsMarkovKernel ν] + {P : Measure Ω} [IsProbabilityMeasure P] {A : ℕ → Ω → 𝓐} {Y : ℕ → Ω → 𝓨} {N n : ℕ} lemma hasLaw_action_zero_detAlgorithm - (h : IsAlgEnvSeqUntil A R' (detAlgorithm nextA h_next action0) env P N) : + (h : IsAlgEnvSeqUntil A Y (detAlgorithm nextA h_next action0) env P N) : HasLaw (A 0) (Measure.dirac action0) P := by simpa using IsDeterministicAlg.hasLaw_action_zero_of_IsAlgEnvSeqUntil h lemma action_zero_detAlgorithm - (h : IsAlgEnvSeqUntil A R' (detAlgorithm nextA h_next action0) env P N) : + (h : IsAlgEnvSeqUntil A Y (detAlgorithm nextA h_next action0) env P N) : A 0 =ᵐ[P] fun _ ↦ action0 := (IsDeterministicAlg.action_zero_of_IsAlgEnvSeqUntil h).trans (by simp) lemma action_detAlgorithm_ae_eq - (h : IsAlgEnvSeqUntil A R' (detAlgorithm nextA h_next action0) env P N) (hn : n < N) : - A (n + 1) =ᵐ[P] fun ω ↦ nextA n (IsAlgEnvSeq.hist A R' n ω) := + (h : IsAlgEnvSeqUntil A Y (detAlgorithm nextA h_next action0) env P N) (hn : n < N) : + A (n + 1) =ᵐ[P] fun ω ↦ nextA n (IsAlgEnvSeq.hist A Y n ω) := (IsDeterministicAlg.action_ae_eq_of_IsAlgEnvSeqUntil h hn).trans (by simp) end IsAlgEnvSeqUntil diff --git a/LeanMachineLearning/SequentialLearning/EvaluationEnv.lean b/LeanMachineLearning/SequentialLearning/EvaluationEnv.lean index 684633a6..2a339656 100644 --- a/LeanMachineLearning/SequentialLearning/EvaluationEnv.lean +++ b/LeanMachineLearning/SequentialLearning/EvaluationEnv.lean @@ -12,25 +12,25 @@ public import LeanMachineLearning.Probability.Independence.CondDistrib /-! # Function evaluation environments -We define two environments, `onlineEvalEnv` and `evalEnv`, where the reward is given by evaluating +We define two environments, `onlineEvalEnv` and `evalEnv`, where the feedback is given by evaluating a measurable function at the chosen action. The first one allows the function to change at every time step, while the second one uses a fixed function at every time step. ## Main definitions -* `onlineEvalEnv g hg`: A stationary environment where the reward at time `n` is given by a +* `onlineEvalEnv g hg`: A stationary environment where the feedback at time `n` is given by a deterministic kernel that evaluates the measurable function `g n` at the chosen action. -* `evalEnv f hf`: A stationary environment where the reward is given by a deterministic kernel that - evaluates a fixed measurable function `f` at the chosen action. +* `evalEnv f hf`: A stationary environment where the feedback is given by a deterministic kernel + that evaluates a fixed measurable function `f` at the chosen action. They both satisfy the typeclasses `IsObliviousEnv` and `IsDeterministicEnv`. ## Main statements -* `forall_reward_onlineEvalEnv_ae_eq_eval_action`: For almost all `ω`, the reward at time `n` is +* `forall_feedback_onlineEvalEnv_ae_eq_eval_action`: For almost all `ω`, the feedback at time `n` is equal to `g n` evaluated at the action taken at time `n`. -* `forall_reward_evalEnv_ae_eq_eval_action`: For almost all `ω`, the reward at time `n` is equal to - `f` evaluated at the action taken at time `n`. +* `forall_feedback_evalEnv_ae_eq_eval_action`: For almost all `ω`, the feedback at time `n` is equal + to `f` evaluated at the action taken at time `n`. -/ @@ -40,13 +40,13 @@ open MeasureTheory ProbabilityTheory namespace Learning -variable {α R : Type*} {mα : MeasurableSpace α} {mR : MeasurableSpace R} - {g : ℕ → α → R} {hg : ∀ n, Measurable (g n)} - {f : α → R} {hf : Measurable f} +variable {𝓐 𝓨 : Type*} {m𝓐 : MeasurableSpace 𝓐} {m𝓨 : MeasurableSpace 𝓨} + {g : ℕ → 𝓐 → 𝓨} {hg : ∀ n, Measurable (g n)} + {f : 𝓐 → 𝓨} {hf : Measurable f} -/-- The evaluation environment where the reward is given by evaluating a fixed measurable function +/-- The evaluation environment where the feedback is given by evaluating a fixed measurable function `f` at the chosen action. -/ -noncomputable def onlineEvalEnv (g : ℕ → α → R) (hg : ∀ n, Measurable (g n)) := +noncomputable def onlineEvalEnv (g : ℕ → 𝓐 → 𝓨) (hg : ∀ n, Measurable (g n)) := obliviousEnv (fun n ↦ Kernel.deterministic (g n) (hg n)) instance : IsObliviousEnv (onlineEvalEnv g hg) := @@ -62,14 +62,14 @@ lemma feedbackCondAction_onlineEvalEnv (n : ℕ) : simp [onlineEvalEnv] @[simp] -lemma feedbackFunZero_onlineEvalEnv [MeasurableSpace.SeparatesPoints R] : +lemma feedbackFunZero_onlineEvalEnv [MeasurableSpace.SeparatesPoints 𝓨] : feedbackFunZero (onlineEvalEnv g hg) = g 0 := by have h_eq := ν0_eq_deterministic (onlineEvalEnv g hg) simpa only [onlineEvalEnv, ν0_obliviousEnv, Kernel.prodMkLeft_deterministic, Kernel.deterministic_inj] using h_eq.symm @[simp] -lemma feedbackFun_onlineEvalEnv [MeasurableSpace.SeparatesPoints R] (n : ℕ) : +lemma feedbackFun_onlineEvalEnv [MeasurableSpace.SeparatesPoints 𝓨] (n : ℕ) : feedbackFun (onlineEvalEnv g hg) n = fun p ↦ g (n + 1) p.2 := by have h_eq := feedback_eq_deterministic (onlineEvalEnv g hg) n simpa only [onlineEvalEnv, feedback_obliviousEnv, Kernel.prodMkLeft_deterministic, @@ -77,34 +77,35 @@ lemma feedbackFun_onlineEvalEnv [MeasurableSpace.SeparatesPoints R] (n : ℕ) : section OnlineEvalEnv -variable [StandardBorelSpace α] [Nonempty α] [StandardBorelSpace R] [Nonempty R] - {Ω : Type*} {mΩ : MeasurableSpace Ω} {alg : Algorithm α R} - {g : ℕ → α → R} {hg : ∀ n, Measurable (g n)} - {P : Measure Ω} [IsProbabilityMeasure P] {A : ℕ → Ω → α} {R' : ℕ → Ω → R} - -lemma hascondDistrib_reward_onlineEvalEnv - (h : IsAlgEnvSeq A R' alg (onlineEvalEnv g hg) P) (n : ℕ) : - HasCondDistrib (R' n) (A n) (Kernel.deterministic (g n) (hg n)) P := by - simpa using IsObliviousEnv.hasCondDistrib_reward h n - -lemma reward_onlineEvalEnv_ae_eq_eval_action - (h : IsAlgEnvSeq A R' alg (onlineEvalEnv g hg) P) (n : ℕ) : - R' n =ᵐ[P] g n ∘ A n := - ae_eq_of_condDistrib_eq_deterministic (hg n) (h.measurable_A n).aemeasurable - (h.measurable_R n).aemeasurable (hascondDistrib_reward_onlineEvalEnv h n).condDistrib_eq - -lemma forall_reward_onlineEvalEnv_ae_eq_eval_action - (h : IsAlgEnvSeq A R' alg (onlineEvalEnv g hg) P) : - ∀ᵐ ω ∂P, ∀ n, R' n ω = g n (A n ω) := by +variable [StandardBorelSpace 𝓐] [Nonempty 𝓐] [StandardBorelSpace 𝓨] [Nonempty 𝓨] + {Ω : Type*} {mΩ : MeasurableSpace Ω} {alg : Algorithm 𝓐 𝓨} + {g : ℕ → 𝓐 → 𝓨} {hg : ∀ n, Measurable (g n)} + {P : Measure Ω} [IsProbabilityMeasure P] {A : ℕ → Ω → 𝓐} {Y : ℕ → Ω → 𝓨} + +lemma hascondDistrib_feedback_onlineEvalEnv + (h : IsAlgEnvSeq A Y alg (onlineEvalEnv g hg) P) (n : ℕ) : + HasCondDistrib (Y n) (A n) (Kernel.deterministic (g n) (hg n)) P := by + simpa using IsObliviousEnv.hasCondDistrib_feedback h n + +lemma feedback_onlineEvalEnv_ae_eq_eval_action + (h : IsAlgEnvSeq A Y alg (onlineEvalEnv g hg) P) (n : ℕ) : + Y n =ᵐ[P] g n ∘ A n := + ae_eq_of_condDistrib_eq_deterministic (hg n) (h.measurable_action n).aemeasurable + (h.measurable_feedback n).aemeasurable + (hascondDistrib_feedback_onlineEvalEnv h n).condDistrib_eq + +lemma forall_feedback_onlineEvalEnv_ae_eq_eval_action + (h : IsAlgEnvSeq A Y alg (onlineEvalEnv g hg) P) : + ∀ᵐ ω ∂P, ∀ n, Y n ω = g n (A n ω) := by rw [ae_all_iff] intro n - exact reward_onlineEvalEnv_ae_eq_eval_action h n + exact feedback_onlineEvalEnv_ae_eq_eval_action h n end OnlineEvalEnv -/-- The evaluation environment where the reward is given by evaluating a fixed measurable function +/-- The evaluation environment where the feedback is given by evaluating a fixed measurable function `f` at the chosen action. -/ -noncomputable def evalEnv (f : α → R) (hf : Measurable f) := onlineEvalEnv (fun _ ↦ f) (fun _ ↦ hf) +noncomputable def evalEnv (f : 𝓐 → 𝓨) (hf : Measurable f) := onlineEvalEnv (fun _ ↦ f) (fun _ ↦ hf) instance : IsObliviousEnv (evalEnv f hf) := by unfold evalEnv; infer_instance @@ -115,34 +116,34 @@ lemma feedbackCondAction_evalEnv (n : ℕ) : feedbackCondAction (evalEnv f hf) n = Kernel.deterministic f hf := by simp [evalEnv] @[simp] -lemma feedbackFunZero_evalEnv [MeasurableSpace.SeparatesPoints R] : +lemma feedbackFunZero_evalEnv [MeasurableSpace.SeparatesPoints 𝓨] : feedbackFunZero (evalEnv f hf) = f := by simp [evalEnv] @[simp] -lemma feedbackFun_evalEnv [MeasurableSpace.SeparatesPoints R] (n : ℕ) : +lemma feedbackFun_evalEnv [MeasurableSpace.SeparatesPoints 𝓨] (n : ℕ) : feedbackFun (evalEnv f hf) n = fun p ↦ f p.2 := by simp [evalEnv] section EvalEnv -variable [StandardBorelSpace α] [Nonempty α] [StandardBorelSpace R] [Nonempty R] - {Ω : Type*} {mΩ : MeasurableSpace Ω} {alg : Algorithm α R} {f : α → R} {hf : Measurable f} - {P : Measure Ω} [IsProbabilityMeasure P] {A : ℕ → Ω → α} {R' : ℕ → Ω → R} +variable [StandardBorelSpace 𝓐] [Nonempty 𝓐] [StandardBorelSpace 𝓨] [Nonempty 𝓨] + {Ω : Type*} {mΩ : MeasurableSpace Ω} {alg : Algorithm 𝓐 𝓨} {f : 𝓐 → 𝓨} {hf : Measurable f} + {P : Measure Ω} [IsProbabilityMeasure P] {A : ℕ → Ω → 𝓐} {Y : ℕ → Ω → 𝓨} -lemma hascondDistrib_reward_evalEnv (h : IsAlgEnvSeq A R' alg (evalEnv f hf) P) (n : ℕ) : - HasCondDistrib (R' n) (A n) (Kernel.deterministic f hf) P := by - simpa using IsObliviousEnv.hasCondDistrib_reward h n +lemma hascondDistrib_feedback_evalEnv (h : IsAlgEnvSeq A Y alg (evalEnv f hf) P) (n : ℕ) : + HasCondDistrib (Y n) (A n) (Kernel.deterministic f hf) P := by + simpa using IsObliviousEnv.hasCondDistrib_feedback h n -lemma reward_evalEnv_ae_eq_eval_action (h : IsAlgEnvSeq A R' alg (evalEnv f hf) P) (n : ℕ) : - R' n =ᵐ[P] f ∘ A n := reward_onlineEvalEnv_ae_eq_eval_action h n +lemma feedback_evalEnv_ae_eq_eval_action (h : IsAlgEnvSeq A Y alg (evalEnv f hf) P) (n : ℕ) : + Y n =ᵐ[P] f ∘ A n := feedback_onlineEvalEnv_ae_eq_eval_action h n -lemma forall_reward_evalEnv_ae_eq_eval_action (h : IsAlgEnvSeq A R' alg (evalEnv f hf) P) : - ∀ᵐ ω ∂P, ∀ n, R' n ω = f (A n ω) := forall_reward_onlineEvalEnv_ae_eq_eval_action h +lemma forall_feedback_evalEnv_ae_eq_eval_action (h : IsAlgEnvSeq A Y alg (evalEnv f hf) P) : + ∀ᵐ ω ∂P, ∀ n, Y n ω = f (A n ω) := forall_feedback_onlineEvalEnv_ae_eq_eval_action h open Finset in -lemma reward_evalEnv_ae_eq_eval_action_comp {β : Type*} - (h : IsAlgEnvSeq A R' alg (evalEnv f hf) P) {n : ℕ} (g : (Iic n → R) → β) : - ∀ᵐ ω ∂P, g (fun i ↦ R' i ω) = g (fun i ↦ f (A i ω)) := by - filter_upwards [forall_reward_evalEnv_ae_eq_eval_action h] with ω hω +lemma feedback_evalEnv_ae_eq_eval_action_comp {β : Type*} + (h : IsAlgEnvSeq A Y alg (evalEnv f hf) P) {n : ℕ} (g : (Iic n → 𝓨) → β) : + ∀ᵐ ω ∂P, g (fun i ↦ Y i ω) = g (fun i ↦ f (A i ω)) := by + filter_upwards [forall_feedback_evalEnv_ae_eq_eval_action h] with ω hω simp_rw [hω] end EvalEnv diff --git a/LeanMachineLearning/SequentialLearning/FiniteActions.lean b/LeanMachineLearning/SequentialLearning/FiniteActions.lean index bea8c5d2..56037e5c 100644 --- a/LeanMachineLearning/SequentialLearning/FiniteActions.lean +++ b/LeanMachineLearning/SequentialLearning/FiniteActions.lean @@ -16,9 +16,9 @@ If the number of actions is finite, it makes sense to define the number of times chosen, the time at which an action was chosen for the nth time, the value of the reward at that time, the sum of rewards obtained for each action, the empirical mean reward for each action, etc. -For each definition that take as arguments a time `t : ℕ`, a history `h : ℕ → α × R`, and possibly +For each definition that take as arguments a time `t : ℕ`, a history `h : ℕ → 𝓐 × R`, and possibly other parameters, we put the time and history at the end in this order, so that the definition can -be seen as a stochastic process indexed by time `t` on the measurable space `ℕ → α × R`. +be seen as a stochastic process indexed by time `t` on the measurable space `ℕ → 𝓐 × R`. -/ @@ -28,29 +28,29 @@ open MeasureTheory Finset Learning namespace Learning -variable {α R Ω : Type*} {mα : MeasurableSpace α} {mR : MeasurableSpace R} {mΩ : MeasurableSpace Ω} - [DecidableEq α] - {alg : Algorithm α R} {env : Environment α R} +variable {𝓐 R Ω : Type*} {m𝓐 : MeasurableSpace 𝓐} {mR : MeasurableSpace R} {mΩ : MeasurableSpace Ω} + [DecidableEq 𝓐] + {alg : Algorithm 𝓐 R} {env : Environment 𝓐 R} {P : Measure Ω} [IsProbabilityMeasure P] - {A : ℕ → Ω → α} {R' : ℕ → Ω → R} - {a : α} {m n t : ℕ} {ω : Ω} + {A : ℕ → Ω → 𝓐} {R' : ℕ → Ω → R} + {a : 𝓐} {m n t : ℕ} {ω : Ω} section PullCount /-- Number of times action `a` was chosen up to time `t` (excluding `t`). -/ noncomputable -def pullCount (A : ℕ → Ω → α) (a : α) (t : ℕ) (ω : Ω) : ℕ := +def pullCount (A : ℕ → Ω → 𝓐) (a : 𝓐) (t : ℕ) (ω : Ω) : ℕ := #(filter (fun s ↦ A s ω = a) (range t)) /-- Number of pulls of arm `a` up to (and including) time `n`. This is the number of entries in `h` in which the arm is `a`. -/ noncomputable -def pullCount' (n : ℕ) (h : Iic n → α × R) (a : α) := #{s | (h s).1 = a} +def pullCount' (n : ℕ) (h : Iic n → 𝓐 × R) (a : 𝓐) := #{s | (h s).1 = a} @[simp] -lemma pullCount_zero (a : α) : pullCount A a 0 = 0 := by ext; simp [pullCount] +lemma pullCount_zero (a : 𝓐) : pullCount A a 0 = 0 := by ext; simp [pullCount] -lemma pullCount_zero_apply (a : α) (ω : Ω) : pullCount A a 0 ω = 0 := by simp +lemma pullCount_zero_apply (a : 𝓐) (ω : Ω) : pullCount A a 0 ω = 0 := by simp lemma pullCount_one : pullCount A a 1 ω = if A 0 ω = a then 1 else 0 := by simp only [pullCount, range_one] @@ -59,11 +59,11 @@ lemma pullCount_one : pullCount A a 1 ω = if A 0 ω = a then 1 else 0 := by refine ⟨0, by simp [h]⟩ · simp [h] -lemma monotone_pullCount (a : α) (ω : Ω) : Monotone (pullCount A a · ω) := +lemma monotone_pullCount (a : 𝓐) (ω : Ω) : Monotone (pullCount A a · ω) := fun _ _ _ ↦ card_le_card (filter_subset_filter _ (by simpa)) @[mono, gcongr] -lemma pullCount_mono (a : α) {n m : ℕ} (hnm : n ≤ m) (ω : Ω) : +lemma pullCount_mono (a : 𝓐) {n m : ℕ} (hnm : n ≤ m) (ω : Ω) : pullCount A a n ω ≤ pullCount A a m ω := monotone_pullCount a ω hnm @@ -81,10 +81,10 @@ lemma pullCount_add_one : · rw [← h, pullCount_action_eq_pullCount_add_one] · rw [pullCount_eq_pullCount_of_action_ne h, add_zero] -lemma pullCount_eq_sum (a : α) (t : ℕ) (ω : Ω) : +lemma pullCount_eq_sum (a : 𝓐) (t : ℕ) (ω : Ω) : pullCount A a t ω = ∑ s ∈ range t, if A s ω = a then 1 else 0 := by simp [pullCount] -lemma pullCount'_eq_sum (n : ℕ) (h : Iic n → α × R) (a : α) : +lemma pullCount'_eq_sum (n : ℕ) (h : Iic n → 𝓐 × R) (a : 𝓐) : pullCount' n h a = ∑ s : Iic n, if (h s).1 = a then 1 else 0 := by simp [pullCount'] lemma pullCount_add_one_eq_pullCount' {n : ℕ} {ω : Ω} : @@ -106,7 +106,7 @@ lemma pullCount'_mono {n m : ℕ} (hnm : n ≤ m) : rw [← pullCount_add_one_eq_pullCount', ← pullCount_add_one_eq_pullCount'] exact pullCount_mono a (by lia) _ -lemma pullCount_le (a : α) (t : ℕ) (ω : Ω) : pullCount A a t ω ≤ t := +lemma pullCount_le (a : 𝓐) (t : ℕ) (ω : Ω) : pullCount A a t ω ≤ t := (card_filter_le _ _).trans_eq (by simp) lemma pullCount_congr {ω' : Ω} (h_eq : ∀ i ≤ n, A i ω = A i ω') : @@ -134,7 +134,7 @@ lemma exists_pullCount_eq_of_le (hnm : t ≤ pullCount A a (n + 1) ω) (ht : t refine lt_of_lt_of_le ?_ hnm exact pullCount_lt_of_forall_ne h_contra ht -lemma pullCount_le_add (a : α) (n C : ℕ) (ω : Ω) : +lemma pullCount_le_add (a : 𝓐) (n C : ℕ) (ω : Ω) : pullCount A a n ω ≤ C + 1 + ∑ s ∈ range n, {s | A s ω = a ∧ C < pullCount A a s ω}.indicator 1 s := by rw [pullCount_eq_sum] @@ -172,8 +172,8 @@ lemma pullCount_le_add (a : α) (n C : ℕ) (ω : Ω) : section Measurability @[fun_prop] -lemma measurable_pullCount [MeasurableSingletonClass α] (hA : ∀ n, Measurable (A n)) - (a : α) (t : ℕ) : +lemma measurable_pullCount [MeasurableSingletonClass 𝓐] (hA : ∀ n, Measurable (A n)) + (a : 𝓐) (t : ℕ) : Measurable (fun ω : Ω ↦ pullCount A a t ω) := by simp_rw [pullCount_eq_sum] have h_meas s : Measurable (fun ω : Ω ↦ if A s ω = a then 1 else 0) := by @@ -182,37 +182,37 @@ lemma measurable_pullCount [MeasurableSingletonClass α] (hA : ∀ n, Measurable fun_prop @[fun_prop] -lemma measurable_uncurry_pullCount [MeasurableEq α] +lemma measurable_uncurry_pullCount [MeasurableEq 𝓐] (hA : ∀ n, Measurable (A n)) (t : ℕ) : - Measurable (fun p : Ω × α ↦ pullCount A p.2 t p.1) := by + Measurable (fun p : Ω × 𝓐 ↦ pullCount A p.2 t p.1) := by simp_rw [pullCount_eq_sum] - have h_meas s : Measurable (fun h : Ω × α ↦ if A s h.1 = h.2 then 1 else 0) := by + have h_meas s : Measurable (fun h : Ω × 𝓐 ↦ if A s h.1 = h.2 then 1 else 0) := by refine Measurable.ite ?_ (by fun_prop) (by fun_prop) exact measurableSet_eq_fun (by fun_prop) (by fun_prop) fun_prop @[fun_prop] -lemma measurable_pullCount' [MeasurableSingletonClass α] (n : ℕ) (a : α) : - Measurable (fun h : Iic n → α × R ↦ pullCount' n h a) := by +lemma measurable_pullCount' [MeasurableSingletonClass 𝓐] (n : ℕ) (a : 𝓐) : + Measurable (fun h : Iic n → 𝓐 × R ↦ pullCount' n h a) := by simp_rw [pullCount'_eq_sum] - have h_meas s : Measurable (fun (h : Iic n → α × R) ↦ if (h s).1 = a then 1 else 0) := by + have h_meas s : Measurable (fun (h : Iic n → 𝓐 × R) ↦ if (h s).1 = a then 1 else 0) := by refine Measurable.ite ?_ (by fun_prop) (by fun_prop) exact (measurableSet_singleton _).preimage (by fun_prop) fun_prop -lemma measurable_uncurry_pullCount' [MeasurableEq α] (n : ℕ) : - Measurable (fun p : (Iic n → α × R) × α ↦ pullCount' n p.1 p.2) := by +lemma measurable_uncurry_pullCount' [MeasurableEq 𝓐] (n : ℕ) : + Measurable (fun p : (Iic n → 𝓐 × R) × 𝓐 ↦ pullCount' n p.1 p.2) := by simp_rw [pullCount'_eq_sum] - have h_meas s : Measurable (fun h : (Iic n → α × R) × α ↦ if (h.1 s).1 = h.2 then 1 else 0) := by + have h_meas s : Measurable (fun h : (Iic n → 𝓐 × R) × 𝓐 ↦ if (h.1 s).1 = h.2 then 1 else 0) := by refine Measurable.ite ?_ (by fun_prop) (by fun_prop) exact measurableSet_eq_fun (by fun_prop) (by fun_prop) fun_prop -lemma adapted_pullCount_add_one [MeasurableSingletonClass α] - (hA : ∀ n, Measurable (A n)) (hR' : ∀ n, Measurable (R' n)) (a : α) : +lemma adapted_pullCount_add_one [MeasurableSingletonClass 𝓐] + (hA : ∀ n, Measurable (A n)) (hR' : ∀ n, Measurable (R' n)) (a : 𝓐) : Adapted (IsAlgEnvSeq.filtration hA hR') (fun n ↦ pullCount A a (n + 1)) := by intro n - have : pullCount A a (n + 1) = (fun h : Iic n → α × R ↦ pullCount' n h a) ∘ + have : pullCount A a (n + 1) = (fun h : Iic n → 𝓐 × R ↦ pullCount' n h a) ∘ (IsAlgEnvSeq.hist A R' n) := by ext exact pullCount_add_one_eq_pullCount' @@ -221,16 +221,16 @@ lemma adapted_pullCount_add_one [MeasurableSingletonClass α] rw [← measurable_iff_comap_le] exact measurable_comp_comap _ (measurable_pullCount' n a) -lemma isPredictable_pullCount [MeasurableSingletonClass α] - (hA : ∀ n, Measurable (A n)) (hR' : ∀ n, Measurable (R' n)) (a : α) : +lemma isPredictable_pullCount [MeasurableSingletonClass 𝓐] + (hA : ∀ n, Measurable (A n)) (hR' : ∀ n, Measurable (R' n)) (a : 𝓐) : IsPredictable (IsAlgEnvSeq.filtration hA hR') (pullCount A a) := by rw [isPredictable_iff_measurable_add_one] refine ⟨?_, adapted_pullCount_add_one hA hR' a⟩ simp only [pullCount_zero] fun_prop -lemma integrable_pullCount [MeasurableSingletonClass α] - (hA : ∀ n, Measurable (A n)) (a : α) (n : ℕ) : +lemma integrable_pullCount [MeasurableSingletonClass 𝓐] + (hA : ∀ n, Measurable (A n)) (a : 𝓐) (n : ℕ) : Integrable (fun ω ↦ (pullCount A a n ω : ℝ)) P := by refine integrable_of_le_of_le (g₁ := 0) (g₂ := fun _ ↦ n) (by fun_prop) (ae_of_all _ fun ω ↦ by simp) (ae_of_all _ fun ω ↦ ?_) (integrable_const _) (integrable_const _) @@ -246,7 +246,7 @@ section StepsUntil -- TODO: replace this by leastGE, once leastGE is generalized /-- Number of steps until action `a` was pulled exactly `m` times. -/ noncomputable -def stepsUntil (A : ℕ → Ω → α) (a : α) (m : ℕ) (ω : Ω) : ℕ∞ := +def stepsUntil (A : ℕ → Ω → 𝓐) (a : 𝓐) (m : ℕ) (ω : Ω) : ℕ∞ := sInf ((↑) '' {s | pullCount A a (s + 1) ω = m}) lemma stepsUntil_eq_top_iff : stepsUntil A a m ω = ⊤ ↔ ∀ s, pullCount A a (s + 1) ω ≠ m := by @@ -276,7 +276,7 @@ lemma stepsUntil_zero_of_eq (hka : A 0 ω = a) : stepsUntil A a 0 ω = ⊤ := by rw [← hka, ← zero_add 1, pullCount_action_eq_pullCount_add_one] simp -lemma stepsUntil_eq_dite (a : α) (m : ℕ) (ω : Ω) +lemma stepsUntil_eq_dite (a : 𝓐) (m : ℕ) (ω : Ω) [Decidable (∃ s, pullCount A a (s + 1) ω = m)] : stepsUntil A a m ω = if h : ∃ s, pullCount A a (s + 1) ω = m then (Nat.find h : ℕ∞) else ⊤ := by @@ -295,7 +295,7 @@ lemma stepsUntil_eq_dite (a : α) (m : ℕ) (ω : Ω) set_option backward.isDefEq.respectTransparency false in -- todo: this is in ℝ because of the limited def of leastGE -lemma stepsUntil_eq_leastGE (a : α) (hm : m ≠ 0) : +lemma stepsUntil_eq_leastGE (a : 𝓐) (hm : m ≠ 0) : stepsUntil A a m = leastGE (fun n (ω : Ω) ↦ pullCount A a (n + 1) ω) m := by classical ext ω @@ -324,13 +324,13 @@ lemma stepsUntil_eq_leastGE (a : α) (hm : m ≠ 0) : refine hn.not_ge ?_ exact csInf_le (by simp) (by simp [h_contra]) -lemma stepsUntil_mono (a : α) (ω : Ω) {n m : ℕ} (hn : n ≠ 0) (hnm : n ≤ m) : +lemma stepsUntil_mono (a : 𝓐) (ω : Ω) {n m : ℕ} (hn : n ≠ 0) (hnm : n ≤ m) : stepsUntil A a n ω ≤ stepsUntil A a m ω := by rw [stepsUntil_eq_leastGE a hn, stepsUntil_eq_leastGE a (by lia)] simp_rw [leastGE] exact hittingAfter_anti (fun n ω ↦ (pullCount A a (n + 1) ω)) 0 (fun x ↦ by grind) ω -lemma stepsUntil_pullCount_le (ω : Ω) (a : α) (t : ℕ) : +lemma stepsUntil_pullCount_le (ω : Ω) (a : 𝓐) (t : ℕ) : stepsUntil A a (pullCount A a (t + 1) ω) ω ≤ t := by rw [stepsUntil] exact csInf_le (OrderBot.bddBelow _) ⟨t, rfl, rfl⟩ @@ -416,7 +416,7 @@ lemma pullCount_stepsUntil (hm : m ≠ 0) (h_exists : ∃ s, pullCount A a (s + swap; · simpa [stepsUntil_eq_top_iff] grind -lemma pullCount_lt_of_le_stepsUntil (a : α) {n m : ℕ} (ω : Ω) +lemma pullCount_lt_of_le_stepsUntil (a : 𝓐) {n m : ℕ} (ω : Ω) (h_exists : ∃ s, pullCount A a (s + 1) ω = m) (hn : n < stepsUntil A a m ω) : pullCount A a (n + 1) ω < m := by classical @@ -495,8 +495,8 @@ lemma stepsUntil_eq_congr {ω' : Ω} (h_eq : ∀ i ≤ n, A i ω = A i ω') : section Measurability -lemma isStoppingTime_stepsUntil [MeasurableSingletonClass α] - (hA : ∀ n, Measurable (A n)) (hR' : ∀ n, Measurable (R' n)) (a : α) (hm : m ≠ 0) : +lemma isStoppingTime_stepsUntil [MeasurableSingletonClass 𝓐] + (hA : ∀ n, Measurable (A n)) (hR' : ∀ n, Measurable (R' n)) (a : 𝓐) (hm : m ≠ 0) : IsStoppingTime (IsAlgEnvSeq.filtration hA hR') (stepsUntil A a m) := by rw [stepsUntil_eq_leastGE _ hm] refine StronglyAdapted.isStoppingTime_leastGE _ fun n ↦ ?_ @@ -507,8 +507,8 @@ lemma isStoppingTime_stepsUntil [MeasurableSingletonClass α] -- todo: get this from the stopping time property? @[fun_prop] -lemma measurable_stepsUntil [MeasurableSingletonClass α] - (hA : ∀ n, Measurable (A n)) (a : α) (m : ℕ) : +lemma measurable_stepsUntil [MeasurableSingletonClass 𝓐] + (hA : ∀ n, Measurable (A n)) (a : 𝓐) (m : ℕ) : Measurable (stepsUntil A a m) := by classical have h_union : {h' : Ω | ∃ s, pullCount A a (s + 1) h' = m} @@ -539,13 +539,13 @@ lemma measurable_stepsUntil [MeasurableSingletonClass α] exact (measurableSet_singleton _).preimage (by fun_prop) exact (measurableSet_singleton _).preimage (by fun_prop) -lemma measurable_stepsUntil' [MeasurableSingletonClass α] - (hA : ∀ n, Measurable (A n)) (a : α) (m : ℕ) : - Measurable (fun ω : Ω × (ℕ → α → R) ↦ stepsUntil A a m ω.1) := +lemma measurable_stepsUntil' [MeasurableSingletonClass 𝓐] + (hA : ∀ n, Measurable (A n)) (a : 𝓐) (m : ℕ) : + Measurable (fun ω : Ω × (ℕ → 𝓐 → R) ↦ stepsUntil A a m ω.1) := (measurable_stepsUntil hA a m).comp measurable_fst -lemma measurable_comap_indicator_stepsUntil_eq [MeasurableSingletonClass α] - (hA : ∀ n, Measurable (A n)) (hR' : ∀ n, Measurable (R' n)) (a : α) (m n : ℕ) : +lemma measurable_comap_indicator_stepsUntil_eq [MeasurableSingletonClass 𝓐] + (hA : ∀ n, Measurable (A n)) (hR' : ∀ n, Measurable (R' n)) (a : 𝓐) (m n : ℕ) : Measurable[MeasurableSpace.comap (fun ω : Ω ↦ (IsAlgEnvSeq.hist A R' (n-1) ω, A n ω)) inferInstance] ({ω | stepsUntil A a m ω = ↑n}.indicator fun _ ↦ 1) := by @@ -582,14 +582,14 @@ lemma measurable_comap_indicator_stepsUntil_eq [MeasurableSingletonClass α] have h_meas := adapted_pullCount_add_one hA hR' a (n - 1) grind -lemma measurable_indicator_stepsUntil_eq [MeasurableSingletonClass α] - (hA : ∀ n, Measurable (A n)) (hR' : ∀ n, Measurable (R' n)) (a : α) (m n : ℕ) : +lemma measurable_indicator_stepsUntil_eq [MeasurableSingletonClass 𝓐] + (hA : ∀ n, Measurable (A n)) (hR' : ∀ n, Measurable (R' n)) (a : 𝓐) (m n : ℕ) : Measurable ({ω : Ω | stepsUntil A a m ω = ↑n}.indicator fun _ ↦ 1) := by refine (measurable_comap_indicator_stepsUntil_eq hA hR' a m n).mono ?_ le_rfl refine Measurable.comap_le ?_ fun_prop -lemma measurableSet_stepsUntil_eq_zero [MeasurableSingletonClass α] (a : α) (m : ℕ) : +lemma measurableSet_stepsUntil_eq_zero [MeasurableSingletonClass 𝓐] (a : 𝓐) (m : ℕ) : MeasurableSet[MeasurableSpace.comap (A 0) inferInstance] {ω : Ω | stepsUntil A a m ω = 0} := by simp only [stepsUntil_eq_zero_iff (a := a) (m := m), ne_eq] @@ -603,14 +603,14 @@ lemma measurableSet_stepsUntil_eq_zero [MeasurableSingletonClass α] (a : α) (m refine (measurableSet_singleton _).preimage ?_ rw [measurable_iff_comap_le] -lemma measurable_comap_indicator_stepsUntil_eq_zero [MeasurableSingletonClass α] (a : α) (m : ℕ) : +lemma measurable_comap_indicator_stepsUntil_eq_zero [MeasurableSingletonClass 𝓐] (a : 𝓐) (m : ℕ) : Measurable[MeasurableSpace.comap (A 0) inferInstance] ({ω | stepsUntil A a m ω = 0}.indicator fun _ ↦ 1) := by rw [measurable_indicator_const_iff] exact measurableSet_stepsUntil_eq_zero a m -lemma measurableSet_stepsUntil_eq [MeasurableSingletonClass α] - (hA : ∀ n, Measurable (A n)) (hR' : ∀ n, Measurable (R' n)) (a : α) (m n : ℕ) : +lemma measurableSet_stepsUntil_eq [MeasurableSingletonClass 𝓐] + (hA : ∀ n, Measurable (A n)) (hR' : ∀ n, Measurable (R' n)) (a : 𝓐) (m n : ℕ) : MeasurableSet[MeasurableSpace.comap (fun ω : Ω ↦ (IsAlgEnvSeq.hist A R' (n-1) ω, A n ω)) inferInstance] {ω : Ω | stepsUntil A a m ω = ↑n} := by @@ -621,8 +621,8 @@ lemma measurableSet_stepsUntil_eq [MeasurableSingletonClass α] exact measurable_comap_indicator_stepsUntil_eq hA hR' a m n /-- `stepsUntil a m` is a stopping time with respect to the filtration `filtrationAction`. -/ -theorem isStoppingTime_stepsUntil_filtrationAction [MeasurableSingletonClass α] - (hA : ∀ n, Measurable (A n)) (hR' : ∀ n, Measurable (R' n)) (a : α) (m : ℕ) : +theorem isStoppingTime_stepsUntil_filtrationAction [MeasurableSingletonClass 𝓐] + (hA : ∀ n, Measurable (A n)) (hR' : ∀ n, Measurable (R' n)) (a : 𝓐) (m : ℕ) : IsStoppingTime (IsAlgEnvSeq.filtrationAction hA hR') (stepsUntil A a m) := by refine isStoppingTime_of_measurableSet_eq fun n ↦ ?_ by_cases hn : n = 0 @@ -642,14 +642,14 @@ section RewardByCount If it is never pulled `m` times, the reward is given by the second component of `ω`, which in applications will be indepedent with same law. -/ noncomputable -def rewardByCount (A : ℕ → Ω → α) (R' : ℕ → Ω → R) (a : α) (m : ℕ) (ω : Ω × (ℕ → α → R)) : R := +def rewardByCount (A : ℕ → Ω → 𝓐) (R' : ℕ → Ω → R) (a : 𝓐) (m : ℕ) (ω : Ω × (ℕ → 𝓐 → R)) : R := match (stepsUntil A a m ω.1) with | ⊤ => ω.2 m a | (n : ℕ) => R' n ω.1 -variable {ω : Ω × (ℕ → α → R)} +variable {ω : Ω × (ℕ → 𝓐 → R)} -lemma rewardByCount_eq_ite (a : α) (m : ℕ) (ω : Ω × (ℕ → α → R)) : +lemma rewardByCount_eq_ite (a : 𝓐) (m : ℕ) (ω : Ω × (ℕ → 𝓐 → R)) : rewardByCount A R' a m ω = if (stepsUntil A a m ω.1) = ⊤ then ω.2 m a else R' (stepsUntil A a m ω.1).toNat ω.1 := by unfold rewardByCount @@ -657,9 +657,9 @@ lemma rewardByCount_eq_ite (a : α) (m : ℕ) (ω : Ω × (ℕ → α → R)) : · simp; rfl · simp -lemma rewardByCount_eq_add [AddMonoid R] (a : α) (m : ℕ) : +lemma rewardByCount_eq_add [AddMonoid R] (a : 𝓐) (m : ℕ) : rewardByCount A R' a m = - {ω : Ω × (ℕ → α → R) | stepsUntil A a m ω.1 ≠ ⊤}.indicator + {ω : Ω × (ℕ → 𝓐 → R) | stepsUntil A a m ω.1 ≠ ⊤}.indicator (fun ω ↦ R' (stepsUntil A a m ω.1).toNat ω.1) + {ω | stepsUntil A a m ω.1 = ⊤}.indicator (fun ω ↦ ω.2 m a) := by ext ω @@ -686,28 +686,28 @@ lemma rewardByCount_of_stepsUntil_eq_coe (h : stepsUntil A a m ω.1 = n) : /-- The value at 0 does not matter (it would be the "zeroth" reward). It should be considered a junk value. -/ @[simp] -lemma rewardByCount_zero (a : α) (ω : Ω × (ℕ → α → R)) : +lemma rewardByCount_zero (a : 𝓐) (ω : Ω × (ℕ → 𝓐 → R)) : rewardByCount A R' a 0 ω = if A 0 ω.1 = a then ω.2 0 a else R' 0 ω.1 := by rw [rewardByCount_eq_ite] by_cases ha : A 0 ω.1 = a · simp [ha, stepsUntil_zero_of_eq] · simp [stepsUntil_zero_of_ne, ha] -lemma rewardByCount_pullCount_add_one_eq_reward (t : ℕ) (ω : Ω × (ℕ → α → R)) : +lemma rewardByCount_pullCount_add_one_eq_reward (t : ℕ) (ω : Ω × (ℕ → 𝓐 → R)) : rewardByCount A R' (A t ω.1) (pullCount A (A t ω.1) t ω.1 + 1) ω = R' t ω.1 := by rw [rewardByCount, ← pullCount_action_eq_pullCount_add_one, stepsUntil_pullCount_eq] @[fun_prop] -lemma measurable_rewardByCount [MeasurableSingletonClass α] - (hA : ∀ n, Measurable (A n)) (hR' : ∀ n, Measurable (R' n)) (a : α) (m : ℕ) : - Measurable (fun ω : Ω × (ℕ → α → R) ↦ rewardByCount A R' a m ω) := by +lemma measurable_rewardByCount [MeasurableSingletonClass 𝓐] + (hA : ∀ n, Measurable (A n)) (hR' : ∀ n, Measurable (R' n)) (a : 𝓐) (m : ℕ) : + Measurable (fun ω : Ω × (ℕ → 𝓐 → R) ↦ rewardByCount A R' a m ω) := by simp_rw [rewardByCount_eq_ite] refine Measurable.ite ?_ ?_ ?_ · exact (measurableSet_singleton _).preimage <| measurable_stepsUntil' hA a m · fun_prop · change Measurable ((fun p : ℕ × Ω ↦ R' p.1 p.2) - ∘ (fun ω : Ω × (ℕ → α → R) ↦ ((stepsUntil A a m ω.1).toNat, ω.1))) - have : Measurable fun ω : Ω × (ℕ → α → R) ↦ ((stepsUntil A a m ω.1).toNat, ω.1) := + ∘ (fun ω : Ω × (ℕ → 𝓐 → R) ↦ ((stepsUntil A a m ω.1).toNat, ω.1))) + have : Measurable fun ω : Ω × (ℕ → 𝓐 → R) ↦ ((stepsUntil A a m ω.1).toNat, ω.1) := (measurable_stepsUntil' hA a m).toNat.prodMk (by fun_prop) refine Measurable.comp ?_ this refine measurable_from_prod_countable_right fun n ↦ ?_ @@ -716,7 +716,7 @@ lemma measurable_rewardByCount [MeasurableSingletonClass α] end RewardByCount -lemma sum_pullCount_mul [Fintype α] [Semiring R] (ω : Ω) (f : α → R) (t : ℕ) : +lemma sum_pullCount_mul [Fintype 𝓐] [Semiring R] (ω : Ω) (f : 𝓐 → R) (t : ℕ) : ∑ a, pullCount A a t ω * f a = ∑ s ∈ range t, f (A s ω) := by unfold pullCount classical @@ -726,7 +726,7 @@ lemma sum_pullCount_mul [Fintype α] [Semiring R] (ω : Ω) (f : α → R) (t : exact sum_fiberwise' (range t) (A · ω) f -- todo: only in ℝ for now -lemma sum_pullCount [Fintype α] {ω : Ω} : ∑ a, pullCount A a t ω = t := by +lemma sum_pullCount [Fintype 𝓐] {ω : Ω} : ∑ a, pullCount A a t ω = t := by suffices ∑ a, pullCount A a t ω * (1 : ℝ) = t by norm_cast at this; simpa rw [sum_pullCount_mul] simp @@ -734,25 +734,25 @@ lemma sum_pullCount [Fintype α] {ω : Ω} : ∑ a, pullCount A a t ω = t := by section SumRewards /-- Sum of rewards obtained when pulling action `a` up to time `t` (exclusive). -/ -def sumRewards (A : ℕ → Ω → α) (R' : ℕ → Ω → ℝ) (a : α) (t : ℕ) (ω : Ω) : ℝ := +def sumRewards (A : ℕ → Ω → 𝓐) (R' : ℕ → Ω → ℝ) (a : 𝓐) (t : ℕ) (ω : Ω) : ℝ := ∑ s ∈ range t, if A s ω = a then R' s ω else 0 /-- Sum of rewards of arm `a` up to (and including) time `n`. -/ noncomputable -def sumRewards' (n : ℕ) (h : Iic n → α × ℝ) (a : α) := +def sumRewards' (n : ℕ) (h : Iic n → 𝓐 × ℝ) (a : 𝓐) := ∑ s, if (h s).1 = a then (h s).2 else 0 /-- Empirical mean reward obtained when pulling action `a` up to time `t` (exclusive). -/ noncomputable -- ANCHOR: empMean -def empMean (A : ℕ → Ω → α) (R' : ℕ → Ω → ℝ) (a : α) (t : ℕ) (ω : Ω) : ℝ := +def empMean (A : ℕ → Ω → 𝓐) (R' : ℕ → Ω → ℝ) (a : 𝓐) (t : ℕ) (ω : Ω) : ℝ := sumRewards A R' a t ω / pullCount A a t ω -- ANCHOR_END: empMean /-- Empirical mean of arm `a` at time `n`. -/ noncomputable -- ANCHOR: empMean' -def empMean' (n : ℕ) (h : Iic n → α × ℝ) (a : α) := +def empMean' (n : ℕ) (h : Iic n → 𝓐 × ℝ) (a : 𝓐) := (sumRewards' n h a) / (pullCount' n h a) -- ANCHOR_END: empMean' @@ -786,7 +786,7 @@ lemma sumRewards_eq_pullCount_mul_empMean {R' : ℕ → Ω → ℝ} {ω : Ω} (h_pull : pullCount A a t ω ≠ 0) : sumRewards A R' a t ω = pullCount A a t ω * empMean A R' a t ω := by unfold empMean; field_simp -lemma sum_rewardByCount_eq_sumRewards {R' : ℕ → Ω → ℝ} (a : α) (t : ℕ) (ω : Ω × (ℕ → α → ℝ)) : +lemma sum_rewardByCount_eq_sumRewards {R' : ℕ → Ω → ℝ} (a : 𝓐) (t : ℕ) (ω : Ω × (ℕ → 𝓐 → ℝ)) : ∑ m ∈ Icc 1 (pullCount A a t ω.1), rewardByCount A R' a m ω = sumRewards A R' a t ω.1 := by induction t with | zero => simp [pullCount, sumRewards] @@ -823,7 +823,7 @@ lemma empMean_eq_empMean' {R' : ℕ → Ω → ℝ} {n : ℕ} {ω : Ω} (hn : n unfold empMean empMean' rw [sumRewards_eq_sumRewards' hn, pullCount_eq_pullCount' hn] -lemma sumRewards_sub_pullCount_mul_eq_sum {R' : ℕ → Ω → ℝ} (c : α → ℝ) : +lemma sumRewards_sub_pullCount_mul_eq_sum {R' : ℕ → Ω → ℝ} (c : 𝓐 → ℝ) : sumRewards A R' a (n + 1) ω - pullCount A a (n + 1) ω * c a = ∑ i ∈ range (n + 1), (if A i ω = a then R' i ω - c a else 0) := by induction n with @@ -843,8 +843,8 @@ lemma sumRewards_sub_pullCount_mul_eq_sum {R' : ℕ → Ω → ℝ} (c : α → simp [ha] @[fun_prop] -lemma measurable_sumRewards [MeasurableSingletonClass α] {R' : ℕ → Ω → ℝ} - (hA : ∀ n, Measurable (A n)) (hR' : ∀ n, Measurable (R' n)) (a : α) (t : ℕ) : +lemma measurable_sumRewards [MeasurableSingletonClass 𝓐] {R' : ℕ → Ω → ℝ} + (hA : ∀ n, Measurable (A n)) (hR' : ∀ n, Measurable (R' n)) (a : 𝓐) (t : ℕ) : Measurable (sumRewards A R' a t) := by unfold sumRewards have h_meas s : Measurable (fun h : Ω ↦ if A s h = a then R' s h else 0) := by @@ -853,50 +853,51 @@ lemma measurable_sumRewards [MeasurableSingletonClass α] {R' : ℕ → Ω → fun_prop @[fun_prop] -lemma measurable_empMean [MeasurableSingletonClass α] {R' : ℕ → Ω → ℝ} (hA : ∀ n, Measurable (A n)) - (hR' : ∀ n, Measurable (R' n)) (a : α) (n : ℕ) : +lemma measurable_empMean [MeasurableSingletonClass 𝓐] {R' : ℕ → Ω → ℝ} (hA : ∀ n, Measurable (A n)) + (hR' : ∀ n, Measurable (R' n)) (a : 𝓐) (n : ℕ) : Measurable (empMean A R' a n) := by unfold empMean fun_prop @[fun_prop] -lemma measurable_sumRewards' [MeasurableSingletonClass α] (n : ℕ) (a : α) : +lemma measurable_sumRewards' [MeasurableSingletonClass 𝓐] (n : ℕ) (a : 𝓐) : Measurable (fun h ↦ sumRewards' n h a) := by simp_rw [sumRewards'] - have h_meas s : Measurable (fun (h : Iic n → α × ℝ) ↦ if (h s).1 = a then (h s).2 else 0) := by + have h_meas s : Measurable (fun (h : Iic n → 𝓐 × ℝ) ↦ if (h s).1 = a then (h s).2 else 0) := by refine Measurable.ite ?_ (by fun_prop) (by fun_prop) exact (measurableSet_singleton _).preimage (by fun_prop) fun_prop @[fun_prop] -lemma measurable_empMean' [MeasurableSingletonClass α] (n : ℕ) (a : α) : +lemma measurable_empMean' [MeasurableSingletonClass 𝓐] (n : ℕ) (a : 𝓐) : Measurable (fun h ↦ empMean' n h a) := by unfold empMean' fun_prop -lemma IsAlgEnvSeq.isPredictable_sumRewards [StandardBorelSpace α] [Nonempty α] {R' : ℕ → Ω → ℝ} - {alg : Algorithm α ℝ} {env : Environment α ℝ} - (h : IsAlgEnvSeq A R' alg env P) (a : α) : - IsPredictable (IsAlgEnvSeq.filtration h.measurable_A h.measurable_R) (sumRewards A R' a) := by +lemma IsAlgEnvSeq.isPredictable_sumRewards [StandardBorelSpace 𝓐] [Nonempty 𝓐] {R' : ℕ → Ω → ℝ} + {alg : Algorithm 𝓐 ℝ} {env : Environment 𝓐 ℝ} + (h : IsAlgEnvSeq A R' alg env P) (a : 𝓐) : + IsPredictable (IsAlgEnvSeq.filtration h.measurable_action h.measurable_feedback) + (sumRewards A R' a) := by rw [isPredictable_iff_measurable_add_one] constructor · simp only [sumRewards_zero] fun_prop refine fun n ↦ measurable_fun_sum _ fun i hi ↦ Measurable.ite ?_ ?_ (by fun_prop) · refine (measurableSet_singleton a).preimage ?_ - have h_meas_i := IsAlgEnvSeq.adapted_action h.measurable_A h.measurable_R i + have h_meas_i := IsAlgEnvSeq.adapted_action h.measurable_action h.measurable_feedback i simp only [mem_range] at hi - exact h_meas_i.mono ((IsAlgEnvSeq.filtration h.measurable_A h.measurable_R).mono (by lia)) - le_rfl - · have h_meas_i := IsAlgEnvSeq.adapted_reward h.measurable_A h.measurable_R i + exact h_meas_i.mono ((IsAlgEnvSeq.filtration h.measurable_action h.measurable_feedback).mono + (by lia)) le_rfl + · have h_meas_i := IsAlgEnvSeq.adapted_feedback h.measurable_action h.measurable_feedback i simp only [mem_range] at hi - exact h_meas_i.mono ((IsAlgEnvSeq.filtration h.measurable_A h.measurable_R).mono (by lia)) - le_rfl + exact h_meas_i.mono ((IsAlgEnvSeq.filtration h.measurable_action h.measurable_feedback).mono + (by lia)) le_rfl -lemma IsAlgEnvSeq.adapted_sumRewards_add_one [StandardBorelSpace α] [Nonempty α] {R' : ℕ → Ω → ℝ} - {alg : Algorithm α ℝ} {env : Environment α ℝ} - (h : IsAlgEnvSeq A R' alg env P) (a : α) : - Adapted (IsAlgEnvSeq.filtration h.measurable_A h.measurable_R) +lemma IsAlgEnvSeq.adapted_sumRewards_add_one [StandardBorelSpace 𝓐] [Nonempty 𝓐] {R' : ℕ → Ω → ℝ} + {alg : Algorithm 𝓐 ℝ} {env : Environment 𝓐 ℝ} + (h : IsAlgEnvSeq A R' alg env P) (a : 𝓐) : + Adapted (IsAlgEnvSeq.filtration h.measurable_action h.measurable_feedback) (fun n ↦ sumRewards A R' a (n + 1)) := by have h_predictable := h.isPredictable_sumRewards a rw [isPredictable_iff_measurable_add_one] at h_predictable @@ -906,12 +907,12 @@ section CopiedFromPR open Set -theorem _root_.MeasureTheory.StronglyMeasurable.div₀' {α β : Type*} - {mα : MeasurableSpace α} [TopologicalSpace β] +theorem _root_.MeasureTheory.StronglyMeasurable.div₀' {𝓐 β : Type*} + {m𝓐 : MeasurableSpace 𝓐} [TopologicalSpace β] [GroupWithZero β] [ContinuousMul β] [ContinuousInv₀ β] [TopologicalSpace.PseudoMetrizableSpace β] [MeasurableSpace β] [BorelSpace β] [MeasurableSingletonClass β] - {f g : α → β} (hf : StronglyMeasurable f) (hg : StronglyMeasurable g) : + {f g : 𝓐 → β} (hf : StronglyMeasurable f) (hg : StronglyMeasurable g) : StronglyMeasurable (f / g) := by refine ⟨fun n => hf.approx n / (hg.approx n).restrict {x | g x ≠ 0}, fun x => ?_⟩ have : MeasurableSet {x | g x ≠ 0} := ((MeasurableSet.singleton 0).preimage hg.measurable).compl @@ -925,20 +926,21 @@ theorem _root_.MeasureTheory.StronglyMeasurable.div₀' {α β : Type*} end CopiedFromPR -lemma IsAlgEnvSeq.isPredictable_empMean [StandardBorelSpace α] [Nonempty α] {R' : ℕ → Ω → ℝ} - {alg : Algorithm α ℝ} {env : Environment α ℝ} - (h : IsAlgEnvSeq A R' alg env P) (a : α) : - IsPredictable (IsAlgEnvSeq.filtration h.measurable_A h.measurable_R) (empMean A R' a) := by +lemma IsAlgEnvSeq.isPredictable_empMean [StandardBorelSpace 𝓐] [Nonempty 𝓐] {R' : ℕ → Ω → ℝ} + {alg : Algorithm 𝓐 ℝ} {env : Environment 𝓐 ℝ} + (h : IsAlgEnvSeq A R' alg env P) (a : 𝓐) : + IsPredictable (IsAlgEnvSeq.filtration h.measurable_action h.measurable_feedback) + (empMean A R' a) := by unfold empMean refine StronglyMeasurable.div₀' ?_ ?_ · exact h.isPredictable_sumRewards a - · have h_meas := (isPredictable_pullCount h.measurable_A h.measurable_R a).measurable + · have h_meas := (isPredictable_pullCount h.measurable_action h.measurable_feedback a).measurable fun_prop -lemma IsAlgEnvSeq.adapted_empMean_add_one [StandardBorelSpace α] [Nonempty α] {R' : ℕ → Ω → ℝ} - {alg : Algorithm α ℝ} {env : Environment α ℝ} - (h : IsAlgEnvSeq A R' alg env P) (a : α) : - Adapted (IsAlgEnvSeq.filtration h.measurable_A h.measurable_R) +lemma IsAlgEnvSeq.adapted_empMean_add_one [StandardBorelSpace 𝓐] [Nonempty 𝓐] {R' : ℕ → Ω → ℝ} + {alg : Algorithm 𝓐 ℝ} {env : Environment 𝓐 ℝ} + (h : IsAlgEnvSeq A R' alg env P) (a : 𝓐) : + Adapted (IsAlgEnvSeq.filtration h.measurable_action h.measurable_feedback) (fun n ↦ empMean A R' a (n + 1)) := by have h_predictable := h.isPredictable_empMean a rw [isPredictable_iff_measurable_add_one] at h_predictable diff --git a/LeanMachineLearning/SequentialLearning/IonescuTulceaSpace.lean b/LeanMachineLearning/SequentialLearning/IonescuTulceaSpace.lean index e59547b1..fc1e9d69 100644 --- a/LeanMachineLearning/SequentialLearning/IonescuTulceaSpace.lean +++ b/LeanMachineLearning/SequentialLearning/IonescuTulceaSpace.lean @@ -33,22 +33,22 @@ open scoped ENNReal NNReal namespace Learning -variable {α R Ω : Type*} {mα : MeasurableSpace α} {mR : MeasurableSpace R} {mΩ : MeasurableSpace Ω} +variable {𝓐 𝓨 Ω : Type*} {m𝓐 : MeasurableSpace 𝓐} {m𝓨 : MeasurableSpace 𝓨} {mΩ : MeasurableSpace Ω} /-- Measure on the sequence of actions and observations generated by the algorithm/environment. -/ noncomputable -def trajMeasure (alg : Algorithm α R) (env : Environment α R) : - Measure (ℕ → α × R) := +def trajMeasure (alg : Algorithm 𝓐 𝓨) (env : Environment 𝓐 𝓨) : + Measure (ℕ → 𝓐 × 𝓨) := Kernel.trajMeasure (alg.p0 ⊗ₘ env.ν0) (stepKernel alg env) deriving IsProbabilityMeasure section ModelEquivalence variable {Ω Ω' : Type*} {mΩ : MeasurableSpace Ω} {mΩ' : MeasurableSpace Ω'} - [StandardBorelSpace α] [Nonempty α] [StandardBorelSpace R] [Nonempty R] - {alg : Algorithm α R} {env : Environment α R} + [StandardBorelSpace 𝓐] [Nonempty 𝓐] [StandardBorelSpace 𝓨] [Nonempty 𝓨] + {alg : Algorithm 𝓐 𝓨} {env : Environment 𝓐 𝓨} {P : Measure Ω} [IsProbabilityMeasure P] {P' : Measure Ω'} [IsProbabilityMeasure P'] - {A₁ : ℕ → Ω → α} {R₁ : ℕ → Ω → R} {A₂ : ℕ → Ω' → α} {R₂ : ℕ → Ω' → R} {N : ℕ} + {A₁ : ℕ → Ω → 𝓐} {R₁ : ℕ → Ω → 𝓨} {A₂ : ℕ → Ω' → 𝓐} {R₂ : ℕ → Ω' → 𝓨} {N : ℕ} theorem eq_trajMeasure_of_isAlgEnvSeq (h : IsAlgEnvSeq A₁ R₁ alg env P) : P.map (fun ω n ↦ (A₁ n ω, R₁ n ω)) = trajMeasure alg env := by @@ -56,8 +56,8 @@ theorem eq_trajMeasure_of_isAlgEnvSeq (h : IsAlgEnvSeq A₁ R₁ alg env P) : have h := Kernel.eq_trajMeasure (Y := fun n ω ↦ (A₁ n ω, R₁ n ω)) (P := P) (μ₀ := alg.p0 ⊗ₘ env.ν0) (κ := stepKernel alg env) (fun n ↦ ?_) ?_ (fun n ↦ ?_) · exact h - · have hA := h.measurable_A n - have hR := h.measurable_R n + · have hA := h.measurable_action n + have hR := h.measurable_feedback n fun_prop · simp only exact h.hasLaw_step_zero @@ -94,99 +94,99 @@ end ModelEquivalence namespace IT -/-- Action and reward at step `n`. -/ -def step (n : ℕ) (h : ℕ → α × R) : α × R := h n +/-- Action and feedback at step `n`. -/ +def step (n : ℕ) (h : ℕ → 𝓐 × 𝓨) : 𝓐 × 𝓨 := h n /-- `action n` is the action pulled at time `n`. This is a random variable on the measurable space -`ℕ → α × ℝ`. -/ -def action (n : ℕ) (h : ℕ → α × R) : α := (h n).1 +`ℕ → 𝓐 × 𝓨`. -/ +def action (n : ℕ) (h : ℕ → 𝓐 × 𝓨) : 𝓐 := (h n).1 -/-- `reward n` is the reward at time `n`. This is a random variable on the measurable space -`ℕ → α × R`. -/ -def reward (n : ℕ) (h : ℕ → α × R) : R := (h n).2 +/-- `feedback n` is the feedback at time `n`. This is a random variable on the measurable space +`ℕ → 𝓐 × 𝓨`. -/ +def feedback (n : ℕ) (h : ℕ → 𝓐 × 𝓨) : 𝓨 := (h n).2 /-- `hist n` is the history up to time `n`. This is a random variable on the measurable space -`ℕ → α × R`. -/ -def hist (n : ℕ) (h : ℕ → α × R) : Iic n → α × R := fun i ↦ h i +`ℕ → 𝓐 × 𝓨`. -/ +def hist (n : ℕ) (h : ℕ → 𝓐 × 𝓨) : Iic n → 𝓐 × 𝓨 := fun i ↦ h i -lemma fst_comp_step (n : ℕ) : Prod.fst ∘ step (α := α) (R := R) n = action n := rfl +lemma fst_comp_step (n : ℕ) : Prod.fst ∘ step (𝓐 := 𝓐) (𝓨 := 𝓨) n = action n := rfl @[fun_prop] -lemma measurable_step (n : ℕ) : Measurable (step n (α := α) (R := R)) := by +lemma measurable_step (n : ℕ) : Measurable (step n (𝓐 := 𝓐) (𝓨 := 𝓨)) := by unfold step; fun_prop @[fun_prop] -lemma measurable_step_prod : Measurable (fun p : ℕ × (ℕ → α × R) ↦ step p.1 p.2) := +lemma measurable_step_prod : Measurable (fun p : ℕ × (ℕ → 𝓐 × 𝓨) ↦ step p.1 p.2) := measurable_from_prod_countable_right fun n ↦ (by simp only; fun_prop) @[fun_prop] -lemma measurable_action (n : ℕ) : Measurable (action n (α := α) (R := R)) := by +lemma measurable_action (n : ℕ) : Measurable (action n (𝓐 := 𝓐) (𝓨 := 𝓨)) := by unfold action; fun_prop @[fun_prop] -lemma measurable_action_prod : Measurable (fun p : ℕ × (ℕ → α × R) ↦ action p.1 p.2) := +lemma measurable_action_prod : Measurable (fun p : ℕ × (ℕ → 𝓐 × 𝓨) ↦ action p.1 p.2) := measurable_from_prod_countable_right fun n ↦ (by simp only; fun_prop) @[fun_prop] -lemma measurable_reward (n : ℕ) : Measurable (reward n (α := α) (R := R)) := by - unfold reward; fun_prop +lemma measurable_feedback (n : ℕ) : Measurable (feedback n (𝓐 := 𝓐) (𝓨 := 𝓨)) := by + unfold feedback; fun_prop @[fun_prop] -lemma measurable_reward_prod : Measurable (fun p : ℕ × (ℕ → α × R) ↦ reward p.1 p.2) := +lemma measurable_feedback_prod : Measurable (fun p : ℕ × (ℕ → 𝓐 × 𝓨) ↦ feedback p.1 p.2) := measurable_from_prod_countable_right fun n ↦ (by simp only; fun_prop) @[fun_prop] -lemma measurable_hist (n : ℕ) : Measurable (hist n (α := α) (R := R)) := by unfold hist; fun_prop +lemma measurable_hist (n : ℕ) : Measurable (hist n (𝓐 := 𝓐) (𝓨 := 𝓨)) := by unfold hist; fun_prop lemma hist_eq_frestrictLe : - hist = Preorder.frestrictLe («π» := fun _ ↦ α × R) := by + hist = Preorder.frestrictLe («π» := fun _ ↦ 𝓐 × 𝓨) := by ext n h i : 3 simp [hist, Preorder.frestrictLe] /-- Filtration of the algorithm Seq. -/ -protected def filtration (α R : Type*) [MeasurableSpace α] [MeasurableSpace R] : - Filtration ℕ (inferInstance : MeasurableSpace (ℕ → α × R)) := - MeasureTheory.Filtration.piLE (X := fun _ ↦ α × R) +protected def filtration (𝓐 𝓨 : Type*) [MeasurableSpace 𝓐] [MeasurableSpace 𝓨] : + Filtration ℕ (inferInstance : MeasurableSpace (ℕ → 𝓐 × 𝓨)) := + MeasureTheory.Filtration.piLE (X := fun _ ↦ 𝓐 × 𝓨) lemma filtration_eq_comap (n : ℕ) : - IT.filtration α R n = MeasurableSpace.comap (hist n) inferInstance := by + IT.filtration 𝓐 𝓨 n = MeasurableSpace.comap (hist n) inferInstance := by simp [IT.filtration, Filtration.piLE_eq_comap_frestrictLe, ← hist_eq_frestrictLe] lemma step_eq_eval_comp_hist (n : ℕ) : - step (α := α) (R := R) n = (fun x ↦ x ⟨n, by simp⟩) ∘ (hist n) := rfl + step (𝓐 := 𝓐) (𝓨 := 𝓨) n = (fun x ↦ x ⟨n, by simp⟩) ∘ (hist n) := rfl lemma action_eq_eval_comp_hist (n : ℕ) : - action (α := α) (R := R) n = (fun x ↦ (x ⟨n, by simp⟩).1) ∘ (hist n) := rfl + action (𝓐 := 𝓐) (𝓨 := 𝓨) n = (fun x ↦ (x ⟨n, by simp⟩).1) ∘ (hist n) := rfl -lemma reward_eq_eval_comp_hist (n : ℕ) : - reward (α := α) (R := R) n = (fun x ↦ (x ⟨n, by simp⟩).2) ∘ (hist n) := rfl +lemma feedback_eq_eval_comp_hist (n : ℕ) : + feedback (𝓐 := 𝓐) (𝓨 := 𝓨) n = (fun x ↦ (x ⟨n, by simp⟩).2) ∘ (hist n) := rfl -lemma adapted_step : Adapted (IT.filtration α R) (step (α := α) (R := R)) := by +lemma adapted_step : Adapted (IT.filtration 𝓐 𝓨) (step (𝓐 := 𝓐) (𝓨 := 𝓨)) := by intro n rw [filtration_eq_comap, step_eq_eval_comp_hist] exact measurable_comp_comap _ (by fun_prop) -lemma adapted_hist : Adapted (IT.filtration α R) hist := by +lemma adapted_hist : Adapted (IT.filtration 𝓐 𝓨) hist := by intro n simp [filtration_eq_comap, measurable_iff_comap_le] -lemma adapted_action : Adapted (IT.filtration α R) action := by +lemma adapted_action : Adapted (IT.filtration 𝓐 𝓨) action := by intro n rw [filtration_eq_comap, action_eq_eval_comp_hist] exact measurable_comp_comap _ (by fun_prop) -lemma adapted_reward : Adapted (IT.filtration α R) reward := by +lemma adapted_feedback : Adapted (IT.filtration 𝓐 𝓨) feedback := by intro n - rw [filtration_eq_comap, reward_eq_eval_comp_hist] + rw [filtration_eq_comap, feedback_eq_eval_comp_hist] exact measurable_comp_comap _ (by fun_prop) section FiltrationAction /-- Filtration generated by the history at time `n-1` together with the action at time `n`. -/ -def filtrationAction (α R : Type*) [MeasurableSpace α] [MeasurableSpace R] : - Filtration ℕ (inferInstance : MeasurableSpace (ℕ → α × R)) where +def filtrationAction (𝓐 𝓨 : Type*) [MeasurableSpace 𝓐] [MeasurableSpace 𝓨] : + Filtration ℕ (inferInstance : MeasurableSpace (ℕ → 𝓐 × 𝓨)) where seq n := if n = 0 then MeasurableSpace.comap (action 0) inferInstance - else IT.filtration α R (n - 1) ⊔ MeasurableSpace.comap (action n) inferInstance + else IT.filtration 𝓐 𝓨 (n - 1) ⊔ MeasurableSpace.comap (action n) inferInstance mono' n m hnm := by simp only by_cases hn : n = 0 @@ -195,8 +195,8 @@ def filtrationAction (α R : Type*) [MeasurableSpace α] [MeasurableSpace R] : · simp only [hn, ↓reduceIte, hm] refine le_sup_of_le_left ?_ rw [← measurable_iff_comap_le] - suffices Measurable[IT.filtration α R 0] (action 0) from - this.mono ((IT.filtration α R).mono zero_le') le_rfl + suffices Measurable[IT.filtration 𝓐 𝓨 0] (action 0) from + this.mono ((IT.filtration 𝓐 𝓨).mono zero_le') le_rfl exact adapted_action 0 have hm : m ≠ 0 := by grind simp only [hn, hm, ↓reduceIte] @@ -204,14 +204,14 @@ def filtrationAction (α R : Type*) [MeasurableSpace α] [MeasurableSpace R] : simp only [sup_le_iff] constructor · refine le_sup_of_le_left ?_ - exact (IT.filtration α R).mono hnm' + exact (IT.filtration 𝓐 𝓨).mono hnm' · rcases eq_or_lt_of_le hnm with rfl | hlt · exact le_sup_of_le_right le_rfl refine le_sup_of_le_left ?_ rw [← measurable_iff_comap_le] have h_le : n ≤ m - 1 := by grind - suffices Measurable[IT.filtration α R n] (action n) from - this.mono ((IT.filtration α R).mono h_le) le_rfl + suffices Measurable[IT.filtration 𝓐 𝓨 n] (action n) from + this.mono ((IT.filtration 𝓐 𝓨).mono h_le) le_rfl exact adapted_action n le' n := by by_cases hn : n = 0 @@ -220,46 +220,46 @@ def filtrationAction (α R : Type*) [MeasurableSpace α] [MeasurableSpace R] : fun_prop simp only [hn, ↓reduceIte, sup_le_iff] constructor - · exact (IT.filtration α R).le _ + · exact (IT.filtration 𝓐 𝓨).le _ · rw [← measurable_iff_comap_le] fun_prop lemma filtrationAction_zero_eq_comap : - filtrationAction α R 0 = MeasurableSpace.comap (action 0) inferInstance := by + filtrationAction 𝓐 𝓨 0 = MeasurableSpace.comap (action 0) inferInstance := by simp [filtrationAction] lemma filtrationAction_eq_comap (n : ℕ) (hn : n ≠ 0) : - filtrationAction α R n = + filtrationAction 𝓐 𝓨 n = MeasurableSpace.comap (fun ω ↦ (hist (n - 1) ω, action n ω)) inferInstance := by simp only [filtrationAction, filtration_eq_comap, ← MeasurableSpace.comap_prodMk, hn, ↓reduceIte] rfl lemma filtration_le_filtrationAction_add_one (n : ℕ) : - IT.filtration α R n ≤ filtrationAction α R (n + 1) := le_sup_of_le_left le_rfl + IT.filtration 𝓐 𝓨 n ≤ filtrationAction 𝓐 𝓨 (n + 1) := le_sup_of_le_left le_rfl lemma filtration_le_filtrationAction {m n : ℕ} (h : n < m) : - IT.filtration α R n ≤ filtrationAction α R m := by + IT.filtration 𝓐 𝓨 n ≤ filtrationAction 𝓐 𝓨 m := by have h' : n + 1 ≤ m := by grind - exact (filtration_le_filtrationAction_add_one n).trans ((filtrationAction α R).mono h') + exact (filtration_le_filtrationAction_add_one n).trans ((filtrationAction 𝓐 𝓨).mono h') lemma filtrationAction_le_filtration_self (n : ℕ) : - filtrationAction α R n ≤ IT.filtration α R n := by + filtrationAction 𝓐 𝓨 n ≤ IT.filtration 𝓐 𝓨 n := by by_cases hn : n = 0 · simp only [hn, filtrationAction_zero_eq_comap] rw [← measurable_iff_comap_le] exact adapted_action 0 simp only [filtrationAction, hn, ↓reduceIte, sup_le_iff] constructor - · exact (IT.filtration α R).mono (by grind) + · exact (IT.filtration 𝓐 𝓨).mono (by grind) · rw [← measurable_iff_comap_le] exact adapted_action _ lemma filtrationAction_le_filtration {m n : ℕ} (h : m ≤ n) : - filtrationAction α R m ≤ IT.filtration α R n := - (filtrationAction_le_filtration_self m).trans ((IT.filtration α R).mono h) + filtrationAction 𝓐 𝓨 m ≤ IT.filtration 𝓐 𝓨 n := + (filtrationAction_le_filtration_self m).trans ((IT.filtration 𝓐 𝓨).mono h) lemma measurable_action_filtrationAction (n : ℕ) : - Measurable[filtrationAction α R n] (action n) := by + Measurable[filtrationAction 𝓐 𝓨 n] (action n) := by rw [measurable_iff_comap_le] simp only [filtrationAction] split_ifs with hn @@ -270,7 +270,7 @@ end FiltrationAction section Laws -lemma hasLaw_step_zero (alg : Algorithm α R) (env : Environment α R) : +lemma hasLaw_step_zero (alg : Algorithm 𝓐 𝓨) (env : Environment 𝓐 𝓨) : HasLaw (step 0) (alg.p0 ⊗ₘ env.ν0) (trajMeasure alg env) where aemeasurable := Measurable.aemeasurable (by fun_prop) map_eq := by @@ -282,29 +282,29 @@ lemma hasLaw_step_zero (alg : Algorithm α R) (env : Environment α R) : Measure.deterministic_comp_eq_map, Measure.map_map (by fun_prop) (by fun_prop)] exact Measure.map_id -lemma hasLaw_action_zero (alg : Algorithm α R) (env : Environment α R) : +lemma hasLaw_action_zero (alg : Algorithm 𝓐 𝓨) (env : Environment 𝓐 𝓨) : HasLaw (action 0) alg.p0 (trajMeasure alg env) where map_eq := by rw [← fst_comp_step, ← Measure.map_map (by fun_prop) (by fun_prop), (hasLaw_step_zero alg env).map_eq, ← Measure.fst, Measure.fst_compProd] -variable [StandardBorelSpace R] [Nonempty R] +variable [StandardBorelSpace 𝓨] [Nonempty 𝓨] -lemma condDistrib_reward_zero (alg : Algorithm α R) (env : Environment α R) : - condDistrib (reward 0) (action 0) (trajMeasure alg env) +lemma condDistrib_feedback_zero (alg : Algorithm 𝓐 𝓨) (env : Environment 𝓐 𝓨) : + condDistrib (feedback 0) (action 0) (trajMeasure alg env) =ᵐ[(trajMeasure alg env).map (action 0)] env.ν0 := by have h_step := (hasLaw_step_zero alg env).map_eq have h_action := (hasLaw_action_zero alg env).map_eq rwa [condDistrib_ae_eq_iff_measure_eq_compProd _ (by fun_prop), h_action] -variable [StandardBorelSpace α] [Nonempty α] +variable [StandardBorelSpace 𝓐] [Nonempty 𝓐] -lemma condDistrib_step (alg : Algorithm α R) (env : Environment α R) (n : ℕ) : +lemma condDistrib_step (alg : Algorithm 𝓐 𝓨) (env : Environment 𝓐 𝓨) (n : ℕ) : condDistrib (step (n + 1)) (hist n) (trajMeasure alg env) =ᵐ[(trajMeasure alg env).map (hist n)] stepKernel alg env n := Kernel.condDistrib_trajMeasure -lemma condDistrib_action (alg : Algorithm α R) (env : Environment α R) (n : ℕ) : +lemma condDistrib_action (alg : Algorithm 𝓐 𝓨) (env : Environment 𝓐 𝓨) (n : ℕ) : condDistrib (action (n + 1)) (hist n) (trajMeasure alg env) =ᵐ[(trajMeasure alg env).map (hist n)] alg.policy n := by rw [← fst_comp_step] @@ -313,8 +313,8 @@ lemma condDistrib_action (alg : Algorithm α R) (env : Environment α R) (n : rw [Kernel.map_apply _ (by fun_prop), h_eq, ← Kernel.map_apply _ (by fun_prop), ← Kernel.fst_eq, fst_stepKernel] -lemma condDistrib_reward (alg : Algorithm α R) (env : Environment α R) (n : ℕ) : - condDistrib (reward (n + 1)) (fun ω ↦ (hist n ω, action (n + 1) ω)) (trajMeasure alg env) +lemma condDistrib_feedback (alg : Algorithm 𝓐 𝓨) (env : Environment 𝓐 𝓨) (n : ℕ) : + condDistrib (feedback (n + 1)) (fun ω ↦ (hist n ω, action (n + 1) ω)) (trajMeasure alg env) =ᵐ[(trajMeasure alg env).map (fun ω ↦ (hist n ω, action (n + 1) ω))] env.feedback n := by have h_step := condDistrib_step alg env n have h_action := condDistrib_action alg env n @@ -323,12 +323,12 @@ lemma condDistrib_reward (alg : Algorithm α R) (env : Environment α R) (n : Measure.map_map (by fun_prop) (by fun_prop)] rfl -lemma isAlgEnvSeq_trajMeasure (alg : Algorithm α R) (env : Environment α R) : - IsAlgEnvSeq action reward alg env (trajMeasure alg env) where +lemma isAlgEnvSeq_trajMeasure (alg : Algorithm 𝓐 𝓨) (env : Environment 𝓐 𝓨) : + IsAlgEnvSeq action feedback alg env (trajMeasure alg env) where hasLaw_action_zero := hasLaw_action_zero alg env - hasCondDistrib_reward_zero := ⟨by fun_prop, by fun_prop, condDistrib_reward_zero alg env⟩ + hasCondDistrib_feedback_zero := ⟨by fun_prop, by fun_prop, condDistrib_feedback_zero alg env⟩ hasCondDistrib_action n := ⟨by fun_prop, by fun_prop, condDistrib_action alg env n⟩ - hasCondDistrib_reward n := ⟨by fun_prop, by fun_prop, condDistrib_reward alg env n⟩ + hasCondDistrib_feedback n := ⟨by fun_prop, by fun_prop, condDistrib_feedback alg env n⟩ end Laws diff --git a/LeanMachineLearning/SequentialLearning/StationaryEnv.lean b/LeanMachineLearning/SequentialLearning/StationaryEnv.lean index 4d0736c2..6a658c93 100644 --- a/LeanMachineLearning/SequentialLearning/StationaryEnv.lean +++ b/LeanMachineLearning/SequentialLearning/StationaryEnv.lean @@ -11,9 +11,9 @@ public import LeanMachineLearning.SequentialLearning.Algorithm /-! # Oblivious and stationary environments -An oblivious environment is an environment in which the distribution of the next reward depends only -on the last action (and not on the past history). -If the kernel that gives the distribution of the next reward given the last action is the same at +An oblivious environment is an environment in which the distribution of the next feedback depends +only on the last action (and not on the past history). +If the kernel that gives the distribution of the next feedback given the last action is the same at every time step, then we say that the environment is stationary. ## Main definitions @@ -27,12 +27,12 @@ Typeclass and related definitions: given the action at time `n` in an oblivious environment `env`. Constructors for oblivious environments: -* `obliviousEnv ν`: an oblivious environment, in which the distribution of the next reward depends +* `obliviousEnv ν`: an oblivious environment, in which the distribution of the next feedback depends only on the last action, but in a possibly time-dependent manner, and is given by a sequence of - Markov kernels `ν : ℕ → Kernel α R`. -* `stationaryEnv ν`: a stationary environment, in which the distribution of the next reward depends - only on the last action (and not on the past history), and is given by a Markov kernel - `ν : Kernel α R`. + Markov kernels `ν : ℕ → Kernel 𝓐 𝓨`. +* `stationaryEnv ν`: a stationary environment, in which the distribution of the next feedback + depends only on the last action (and not on the past history), and is given by a Markov kernel + `ν : Kernel 𝓐 𝓨`. -/ @@ -44,189 +44,189 @@ open scoped ENNReal NNReal namespace Learning -variable {α R : Type*} {mα : MeasurableSpace α} {mR : MeasurableSpace R} +variable {𝓐 𝓨 : Type*} {m𝓐 : MeasurableSpace 𝓐} {m𝓨 : MeasurableSpace 𝓨} /-- An environment is oblivious if the distribution of the next feedback depends only on the last action and not on the past history. -/ -class IsObliviousEnv (env : Environment α R) : Prop where - exists_eq_prodMkLeft : ∃ ν : ℕ → Kernel α R, (∀ n, IsMarkovKernel (ν n)) ∧ +class IsObliviousEnv (env : Environment 𝓐 𝓨) : Prop where + exists_eq_prodMkLeft : ∃ ν : ℕ → Kernel 𝓐 𝓨, (∀ n, IsMarkovKernel (ν n)) ∧ (env.ν0 = ν 0) ∧ (∀ n, env.feedback n = (ν (n + 1)).prodMkLeft _) /-- The kernel representing the conditional distribution of the feedback given the action at time `n` in an oblivious environment. -/ noncomputable -def feedbackCondAction (env : Environment α R) [h_obl : IsObliviousEnv env] (n : ℕ) : Kernel α R := +def feedbackCondAction (env : Environment 𝓐 𝓨) [h_obl : IsObliviousEnv env] (n : ℕ) : Kernel 𝓐 𝓨 := h_obl.exists_eq_prodMkLeft.choose n -instance (env : Environment α R) [IsObliviousEnv env] (n : ℕ) : +instance (env : Environment 𝓐 𝓨) [IsObliviousEnv env] (n : ℕ) : IsMarkovKernel (feedbackCondAction env n) := IsObliviousEnv.exists_eq_prodMkLeft.choose_spec.1 n -lemma ν0_eq_feedbackCondAction (env : Environment α R) [IsObliviousEnv env] : +lemma ν0_eq_feedbackCondAction (env : Environment 𝓐 𝓨) [IsObliviousEnv env] : env.ν0 = feedbackCondAction env 0 := IsObliviousEnv.exists_eq_prodMkLeft.choose_spec.2.1 -lemma feedback_eq_feedbackCondAction (env : Environment α R) [IsObliviousEnv env] (n : ℕ) : +lemma feedback_eq_feedbackCondAction (env : Environment 𝓐 𝓨) [IsObliviousEnv env] (n : ℕ) : env.feedback n = (feedbackCondAction env (n + 1)).prodMkLeft _ := IsObliviousEnv.exists_eq_prodMkLeft.choose_spec.2.2 n namespace IsObliviousEnv variable {Ω : Type*} {mΩ : MeasurableSpace Ω} - [StandardBorelSpace α] [Nonempty α] [StandardBorelSpace R] [Nonempty R] - {alg : Algorithm α R} {env : Environment α R} {P : Measure Ω} [IsFiniteMeasure P] - {A : ℕ → Ω → α} {R' : ℕ → Ω → R} {n N : ℕ} - {ν : ℕ → Kernel α R} [∀ n, IsMarkovKernel (ν n)] - -lemma hasCondDistrib_reward [IsObliviousEnv env] (h : IsAlgEnvSeq A R' alg env P) (n : ℕ) : - HasCondDistrib (R' n) (A n) (feedbackCondAction env n) P := by - have hA := h.measurable_A - have hR' := h.measurable_R + [StandardBorelSpace 𝓐] [Nonempty 𝓐] [StandardBorelSpace 𝓨] [Nonempty 𝓨] + {alg : Algorithm 𝓐 𝓨} {env : Environment 𝓐 𝓨} {P : Measure Ω} [IsFiniteMeasure P] + {A : ℕ → Ω → 𝓐} {Y : ℕ → Ω → 𝓨} {n N : ℕ} + {ν : ℕ → Kernel 𝓐 𝓨} [∀ n, IsMarkovKernel (ν n)] + +lemma hasCondDistrib_feedback [IsObliviousEnv env] (h : IsAlgEnvSeq A Y alg env P) (n : ℕ) : + HasCondDistrib (Y n) (A n) (feedbackCondAction env n) P := by + have hA := h.measurable_action + have hY := h.measurable_feedback cases n with - | zero => rw [← ν0_eq_feedbackCondAction]; exact h.hasCondDistrib_reward_zero + | zero => rw [← ν0_eq_feedbackCondAction]; exact h.hasCondDistrib_feedback_zero | succ n => refine ⟨by fun_prop, by fun_prop, ?_⟩ - have h_eq := (h.hasCondDistrib_reward n).condDistrib_eq + have h_eq := (h.hasCondDistrib_feedback n).condDistrib_eq rw [condDistrib_ae_eq_iff_measure_eq_compProd _ (by fun_prop)] at h_eq ⊢ have : P.map (A (n + 1)) = - (P.map (fun x ↦ (IsAlgEnvSeq.hist A R' n x, A (n + 1) x))).snd := by + (P.map (fun x ↦ (IsAlgEnvSeq.hist A Y n x, A (n + 1) x))).snd := by rw [Measure.snd_map_prodMk (by fun_prop)] simp only [feedback_eq_feedbackCondAction] at h_eq rw [this, ← Measure.snd_prodAssoc_compProd_prodMkLeft, ← h_eq, Measure.snd_map_prodMk (by fun_prop), Measure.map_map (by fun_prop) (by fun_prop)] congr -/-- The reward at time `n + 1` is conditionally independent of the history up to time `n` +/-- The feedback at time `n + 1` is conditionally independent of the history up to time `n` given the action at time `n + 1`. -/ -lemma condIndepFun_reward_hist_action [StandardBorelSpace Ω] - [IsObliviousEnv env] (h : IsAlgEnvSeq A R' alg env P) (n : ℕ) : - R' (n + 1) ⟂ᵢ[A (n + 1), h.measurable_A _ ; P] IsAlgEnvSeq.hist A R' n := by - have hA := h.measurable_A - have hR' := h.measurable_R +lemma condIndepFun_feedback_hist_action [StandardBorelSpace Ω] + [IsObliviousEnv env] (h : IsAlgEnvSeq A Y alg env P) (n : ℕ) : + Y (n + 1) ⟂ᵢ[A (n + 1), h.measurable_action _ ; P] IsAlgEnvSeq.hist A Y n := by + have hA := h.measurable_action + have hY := h.measurable_feedback refine condIndepFun_of_exists_condDistrib_prod_ae_eq_prodMkLeft (η := feedbackCondAction env (n + 1)) (by fun_prop) (by fun_prop) (by fun_prop) ?_ refine HasCondDistrib.condDistrib_eq ?_ rw [← feedback_eq_feedbackCondAction] - exact h.hasCondDistrib_reward n - -lemma condIndepFun_reward_hist_action_action [StandardBorelSpace Ω] - [IsObliviousEnv env] (h : IsAlgEnvSeq A R' alg env P) (n : ℕ) : - R' (n + 1) ⟂ᵢ[A (n + 1), h.measurable_A (n + 1); P] - (fun ω ↦ (IsAlgEnvSeq.hist A R' n ω, A (n + 1) ω)) := by - have h_indep : R' (n + 1) ⟂ᵢ[A (n + 1), h.measurable_A (n + 1); P] IsAlgEnvSeq.hist A R' n := - condIndepFun_reward_hist_action h n - have hA := h.measurable_A - have hR' := h.measurable_R + exact h.hasCondDistrib_feedback n + +lemma condIndepFun_feedback_hist_action_action [StandardBorelSpace Ω] + [IsObliviousEnv env] (h : IsAlgEnvSeq A Y alg env P) (n : ℕ) : + Y (n + 1) ⟂ᵢ[A (n + 1), h.measurable_action (n + 1); P] + (fun ω ↦ (IsAlgEnvSeq.hist A Y n ω, A (n + 1) ω)) := by + have h_indep : Y (n + 1) ⟂ᵢ[A (n + 1), h.measurable_action (n + 1); P] IsAlgEnvSeq.hist A Y n := + condIndepFun_feedback_hist_action h n + have hA := h.measurable_action + have hY := h.measurable_feedback exact h_indep.prod_right (by fun_prop) (by fun_prop) (by fun_prop) -lemma condIndepFun_reward_hist_action_action' [StandardBorelSpace Ω] - [IsObliviousEnv env] (h : IsAlgEnvSeq A R' alg env P) (n : ℕ) (hn : n ≠ 0) : - R' n ⟂ᵢ[A n, h.measurable_A n; P] (fun ω ↦ (IsAlgEnvSeq.hist A R' (n - 1) ω, A n ω)) := by - have := condIndepFun_reward_hist_action_action h (n - 1) +lemma condIndepFun_feedback_hist_action_action' [StandardBorelSpace Ω] + [IsObliviousEnv env] (h : IsAlgEnvSeq A Y alg env P) (n : ℕ) (hn : n ≠ 0) : + Y n ⟂ᵢ[A n, h.measurable_action n; P] (fun ω ↦ (IsAlgEnvSeq.hist A Y (n - 1) ω, A n ω)) := by + have := condIndepFun_feedback_hist_action_action h (n - 1) grind end IsObliviousEnv -/-- An oblivious environment, in which the distribution of the next reward depends only on the last -action, but in a possibly time-dependent manner. -/ +/-- An oblivious environment, in which the distribution of the next feedback depends only on +the last action, but in a possibly time-dependent manner. -/ @[simps] -- ANCHOR: obliviousEnv -def obliviousEnv (ν : ℕ → Kernel α R) [∀ n, IsMarkovKernel (ν n)] : Environment α R where +def obliviousEnv (ν : ℕ → Kernel 𝓐 𝓨) [∀ n, IsMarkovKernel (ν n)] : Environment 𝓐 𝓨 where feedback n := (ν (n + 1)).prodMkLeft _ ν0 := ν 0 -- ANCHOR_END: obliviousEnv @[simp] -lemma feedback_obliviousEnv (ν : ℕ → Kernel α R) [∀ n, IsMarkovKernel (ν n)] (n : ℕ) : +lemma feedback_obliviousEnv (ν : ℕ → Kernel 𝓐 𝓨) [∀ n, IsMarkovKernel (ν n)] (n : ℕ) : (obliviousEnv ν).feedback n = (ν (n + 1)).prodMkLeft _ := by simp [obliviousEnv] @[simp] -lemma ν0_obliviousEnv (ν : ℕ → Kernel α R) [∀ n, IsMarkovKernel (ν n)] : +lemma ν0_obliviousEnv (ν : ℕ → Kernel 𝓐 𝓨) [∀ n, IsMarkovKernel (ν n)] : (obliviousEnv ν).ν0 = ν 0 := by simp [obliviousEnv] -instance (ν : ℕ → Kernel α R) [∀ n, IsMarkovKernel (ν n)] : +instance (ν : ℕ → Kernel 𝓐 𝓨) [∀ n, IsMarkovKernel (ν n)] : IsObliviousEnv (obliviousEnv ν) where exists_eq_prodMkLeft := ⟨fun n ↦ ν n, inferInstance,rfl, fun _ ↦ rfl⟩ @[simp] -lemma feedbackCondAction_obliviousEnv (ν : ℕ → Kernel α R) [hν : ∀ n, IsMarkovKernel (ν n)] +lemma feedbackCondAction_obliviousEnv (ν : ℕ → Kernel 𝓐 𝓨) [hν : ∀ n, IsMarkovKernel (ν n)] (n : ℕ) : feedbackCondAction (obliviousEnv ν) n = ν n := by - rcases isEmpty_or_nonempty α with hα | hα + rcases isEmpty_or_nonempty 𝓐 with h𝓐 | h𝓐 · ext a : 1 - exact hα.elim a - rcases isEmpty_or_nonempty R with hR | hR + exact h𝓐.elim a + rcases isEmpty_or_nonempty 𝓨 with hR | hR · refine absurd (hν 0) ?_ simp only [Subsingleton.eq_zero ν, Pi.zero_apply] exact Kernel.not_isMarkovKernel_zero - have : Nonempty (Iic n → α × R) := ⟨fun _ ↦ (hα.some, hR.some)⟩ + have : Nonempty (Iic n → 𝓐 × 𝓨) := ⟨fun _ ↦ (h𝓐.some, hR.some)⟩ have h_eq_zero := ν0_eq_feedbackCondAction (obliviousEnv ν) have h_eq := feedback_eq_feedbackCondAction (obliviousEnv ν) (n - 1) cases n with | zero => exact h_eq_zero.symm | succ n => simp only [Nat.add_one_sub_one, obliviousEnv_feedback, add_tsub_cancel_right] at h_eq - rw [← Kernel.prodMkLeft_inj (γ := Iic n → α × R)] + rw [← Kernel.prodMkLeft_inj (γ := Iic n → 𝓐 × 𝓨)] exact h_eq.symm -/-- A stationary environment, in which the distribution of the next reward depends only on the last -action. -/ +/-- A stationary environment, in which the distribution of the next feedback depends only on the +last action. -/ -- ANCHOR: stationaryEnv -def stationaryEnv (ν : Kernel α R) [IsMarkovKernel ν] : Environment α R := obliviousEnv fun _ ↦ ν +def stationaryEnv (ν : Kernel 𝓐 𝓨) [IsMarkovKernel ν] : Environment 𝓐 𝓨 := obliviousEnv fun _ ↦ ν -- ANCHOR_END: stationaryEnv @[simp] -lemma feedback_stationaryEnv (ν : Kernel α R) [IsMarkovKernel ν] (n : ℕ) : +lemma feedback_stationaryEnv (ν : Kernel 𝓐 𝓨) [IsMarkovKernel ν] (n : ℕ) : (stationaryEnv ν).feedback n = ν.prodMkLeft _ := by simp [stationaryEnv] @[simp] -lemma ν0_stationaryEnv (ν : Kernel α R) [IsMarkovKernel ν] : (stationaryEnv ν).ν0 = ν := by +lemma ν0_stationaryEnv (ν : Kernel 𝓐 𝓨) [IsMarkovKernel ν] : (stationaryEnv ν).ν0 = ν := by simp [stationaryEnv] -instance (ν : Kernel α R) [IsMarkovKernel ν] : IsObliviousEnv (stationaryEnv ν) where +instance (ν : Kernel 𝓐 𝓨) [IsMarkovKernel ν] : IsObliviousEnv (stationaryEnv ν) where exists_eq_prodMkLeft := ⟨fun _ ↦ ν, inferInstance, rfl, fun _ ↦ rfl⟩ @[simp] -lemma feedbackCondAction_stationaryEnv (ν : Kernel α R) [hν : IsMarkovKernel ν] (n : ℕ) : +lemma feedbackCondAction_stationaryEnv (ν : Kernel 𝓐 𝓨) [hν : IsMarkovKernel ν] (n : ℕ) : feedbackCondAction (stationaryEnv ν) n = ν := feedbackCondAction_obliviousEnv _ _ variable {Ω : Type*} {mΩ : MeasurableSpace Ω} - [StandardBorelSpace α] [Nonempty α] [StandardBorelSpace R] [Nonempty R] - {alg : Algorithm α R} {ν : Kernel α R} [IsMarkovKernel ν] - {P : Measure Ω} [IsProbabilityMeasure P] {A : ℕ → Ω → α} {R' : ℕ → Ω → R} + [StandardBorelSpace 𝓐] [Nonempty 𝓐] [StandardBorelSpace 𝓨] [Nonempty 𝓨] + {alg : Algorithm 𝓐 𝓨} {ν : Kernel 𝓐 𝓨} [IsMarkovKernel ν] + {P : Measure Ω} [IsProbabilityMeasure P] {A : ℕ → Ω → 𝓐} {Y : ℕ → Ω → 𝓨} namespace IsAlgEnvSeq -/-- The conditional distribution of the reward at time `n` given the action at time `n` is `ν`. -/ -lemma hasCondDistrib_reward_stationaryEnv - (h : IsAlgEnvSeq A R' alg (stationaryEnv ν) P) (n : ℕ) : - HasCondDistrib (R' n) (A n) ν P := by - simpa using IsObliviousEnv.hasCondDistrib_reward h n +/-- The conditional distribution of the feedback at time `n` given the action at time `n` is `ν`. -/ +lemma hasCondDistrib_feedback_stationaryEnv + (h : IsAlgEnvSeq A Y alg (stationaryEnv ν) P) (n : ℕ) : + HasCondDistrib (Y n) (A n) ν P := by + simpa using IsObliviousEnv.hasCondDistrib_feedback h n -/-- The conditional distribution of the reward at time `n` given the action at time `n` is `ν`. -/ -lemma condDistrib_reward_stationaryEnv - (h : IsAlgEnvSeq A R' alg (stationaryEnv ν) P) (n : ℕ) : - condDistrib (R' n) (A n) P =ᵐ[P.map (A n)] ν := - (hasCondDistrib_reward_stationaryEnv h n).condDistrib_eq +/-- The conditional distribution of the feedback at time `n` given the action at time `n` is `ν`. -/ +lemma condDistrib_feedback_stationaryEnv + (h : IsAlgEnvSeq A Y alg (stationaryEnv ν) P) (n : ℕ) : + condDistrib (Y n) (A n) P =ᵐ[P.map (A n)] ν := + (hasCondDistrib_feedback_stationaryEnv h n).condDistrib_eq -/-- The reward at time `n + 1` is conditionally independent of the history up to time `n` +/-- The feedback at time `n + 1` is conditionally independent of the history up to time `n` given the action at time `n + 1`. -/ -lemma condIndepFun_reward_hist_action [StandardBorelSpace Ω] - (h : IsAlgEnvSeq A R' alg (stationaryEnv ν) P) (n : ℕ) : - R' (n + 1) ⟂ᵢ[A (n + 1), h.measurable_A _ ; P] hist A R' n := - IsObliviousEnv.condIndepFun_reward_hist_action h n - -lemma condIndepFun_reward_hist_action_action [StandardBorelSpace Ω] - (h : IsAlgEnvSeq A R' alg (stationaryEnv ν) P) (n : ℕ) : - R' (n + 1) ⟂ᵢ[A (n + 1), h.measurable_A (n + 1); P] - (fun ω ↦ (hist A R' n ω, A (n + 1) ω)) := - IsObliviousEnv.condIndepFun_reward_hist_action_action h n - -lemma condIndepFun_reward_hist_action_action' [StandardBorelSpace Ω] - (h : IsAlgEnvSeq A R' alg (stationaryEnv ν) P) (n : ℕ) (hn : n ≠ 0) : - R' n ⟂ᵢ[A n, h.measurable_A n; P] (fun ω ↦ (hist A R' (n - 1) ω, A n ω)) := - IsObliviousEnv.condIndepFun_reward_hist_action_action' h n hn +lemma condIndepFun_feedback_hist_action [StandardBorelSpace Ω] + (h : IsAlgEnvSeq A Y alg (stationaryEnv ν) P) (n : ℕ) : + Y (n + 1) ⟂ᵢ[A (n + 1), h.measurable_action _ ; P] hist A Y n := + IsObliviousEnv.condIndepFun_feedback_hist_action h n + +lemma condIndepFun_feedback_hist_action_action [StandardBorelSpace Ω] + (h : IsAlgEnvSeq A Y alg (stationaryEnv ν) P) (n : ℕ) : + Y (n + 1) ⟂ᵢ[A (n + 1), h.measurable_action (n + 1); P] + (fun ω ↦ (hist A Y n ω, A (n + 1) ω)) := + IsObliviousEnv.condIndepFun_feedback_hist_action_action h n + +lemma condIndepFun_feedback_hist_action_action' [StandardBorelSpace Ω] + (h : IsAlgEnvSeq A Y alg (stationaryEnv ν) P) (n : ℕ) (hn : n ≠ 0) : + Y n ⟂ᵢ[A n, h.measurable_action n; P] (fun ω ↦ (hist A Y (n - 1) ω, A n ω)) := + IsObliviousEnv.condIndepFun_feedback_hist_action_action' h n hn end IsAlgEnvSeq diff --git a/blueprint/lean_decls b/blueprint/lean_decls index 4a30eb78..e6b21f0d 100644 --- a/blueprint/lean_decls +++ b/blueprint/lean_decls @@ -12,10 +12,10 @@ Learning.IsAlgEnvSeq.filtrationAction Learning.IsAlgEnvSeq.adapted_step Learning.IsAlgEnvSeq.adapted_hist Learning.IsAlgEnvSeq.adapted_action -Learning.IsAlgEnvSeq.adapted_reward +Learning.IsAlgEnvSeq.adapted_feedback Learning.isAlgEnvSeq_unique -Learning.IsAlgEnvSeq.condDistrib_reward_stationaryEnv -Learning.IsAlgEnvSeq.condIndepFun_reward_hist_action +Learning.IsAlgEnvSeq.condDistrib_feedback_stationaryEnv +Learning.IsAlgEnvSeq.condIndepFun_feedback_hist_action ProbabilityTheory.Kernel.traj ProbabilityTheory.Kernel.trajMeasure Learning.IT.step @@ -26,13 +26,13 @@ Learning.IT.adapted_hist ProbabilityTheory.Kernel.condDistrib_trajMeasure Learning.IsAlgEnvSeq.hasLaw_step_zero Learning.IT.action -Learning.IT.reward +Learning.IT.feedback Learning.IT.adapted_action -Learning.IT.adapted_reward +Learning.IT.adapted_feedback Learning.IT.condDistrib_action -Learning.IT.condDistrib_reward +Learning.IT.condDistrib_feedback Learning.IT.hasLaw_action_zero -Learning.IT.condDistrib_reward_zero +Learning.IT.condDistrib_feedback_zero Learning.IT.isAlgEnvSeq_trajMeasure Learning.pullCount Learning.pullCount_zero diff --git a/blueprint/src/chapters/algorithm.tex b/blueprint/src/chapters/algorithm.tex index 802b4b4a..ccae536c 100644 --- a/blueprint/src/chapters/algorithm.tex +++ b/blueprint/src/chapters/algorithm.tex @@ -133,7 +133,7 @@ \chapter{Iterative stochastic algorithms} \begin{lemma}\label{lem:IsAlgEnvSeq.adapted} \uses{def:IsAlgEnvSeq.filtration,def:history} \leanok - \lean{Learning.IsAlgEnvSeq.adapted_step, Learning.IsAlgEnvSeq.adapted_hist, Learning.IsAlgEnvSeq.adapted_action, Learning.IsAlgEnvSeq.adapted_reward} + \lean{Learning.IsAlgEnvSeq.adapted_step, Learning.IsAlgEnvSeq.adapted_hist, Learning.IsAlgEnvSeq.adapted_action, Learning.IsAlgEnvSeq.adapted_feedback} The history, step, action and observation processes are adapted to the filtration $(\mathcal{F}_t)_{t \in \mathbb{N}}$. \end{lemma} @@ -166,7 +166,7 @@ \section{Stationary environment} \begin{lemma}\label{lem:condDistrib_reward_stationaryEnv} \uses{def:stationaryEnv,def:IsAlgEnvSeq,def:algorithm} \leanok - \lean{Learning.IsAlgEnvSeq.condDistrib_reward_stationaryEnv} + \lean{Learning.IsAlgEnvSeq.condDistrib_feedback_stationaryEnv} In a stationary environment, for any $t \in \mathbb{N}$, $P\left(R_t \mid A_t\right) = \nu$. \end{lemma} @@ -179,7 +179,7 @@ \section{Stationary environment} \begin{lemma}\label{lem:condIndepFun_reward_hist_action} \uses{def:stationaryEnv,def:environment,def:IsAlgEnvSeq,def:algorithm,def:history} \leanok - \lean{Learning.IsAlgEnvSeq.condIndepFun_reward_hist_action} + \lean{Learning.IsAlgEnvSeq.condIndepFun_feedback_hist_action} In a stationary environment, for any $t \in \mathbb{N}$, the reward $R_{t+1}$ is conditionally independent of the history $H_t$ given the action $A_{t+1}$ (more succinctly, $R_{t+1} \ind H_t \mid A_{t+1}$). \end{lemma} @@ -300,7 +300,7 @@ \subsection{Case of an algorithm-environment interaction} \begin{definition}\label{def:IT.actionReward} \uses{def:IT.history} \leanok - \lean{Learning.IT.action, Learning.IT.reward} + \lean{Learning.IT.action, Learning.IT.feedback} We write $A_t$ and $R_t$ for the projections of $X_t$ on $\mathcal{A}$ and $\mathcal{R}$ respectively. $A_t$ is the action taken at time $t$ and $R_t$ is the reward received at time $t$. Formally, $A_t(\omega) = \omega_{t,1}$ and $R_t(\omega) = \omega_{t,2}$ for $\omega = \prod_{t=0}^{+\infty}(\omega_{t,1}, \omega_{t,2}) \in \Omega_{\mathcal{T}} = \prod_{t=0}^{+\infty} \mathcal{A} \times \mathcal{R}$. @@ -310,7 +310,7 @@ \subsection{Case of an algorithm-environment interaction} \begin{lemma}\label{lem:IT.adapted_action_reward} \uses{def:IT.actionReward, def:IT.filtration} \leanok - \lean{Learning.IT.adapted_action, Learning.IT.adapted_reward} + \lean{Learning.IT.adapted_action, Learning.IT.adapted_feedback} The random variables $A_t$ and $R_t$ are $\mathcal{F}_t$-measurable. Said differently, the processes $(A_t)_{t \in \mathbb{N}}$ and $(R_t)_{t \in \mathbb{N}}$ are adapted to the filtration $(\mathcal{F}_t)_{t \in \mathbb{N}}$. \end{lemma} @@ -340,7 +340,7 @@ \subsection{Case of an algorithm-environment interaction} \begin{lemma}\label{lem:IT.condDistrib_R_add_one} \uses{def:environment,def:algorithm,thm:ionescu-tulcea,def:IT.actionReward,def:trajMeasure,def:IT.history} \leanok - \lean{Learning.IT.condDistrib_reward} + \lean{Learning.IT.condDistrib_feedback} For any $t \in \mathbb{N}$, $P_{\mathcal{T}}\left(R_{t+1} \mid H_t, A_{t+1}\right) = \nu_t$. \end{lemma} @@ -372,7 +372,7 @@ \subsection{Case of an algorithm-environment interaction} \begin{lemma}\label{lem:IT.condDistrib_R_zero} \uses{def:environment,def:algorithm,thm:ionescu-tulcea,def:IT.actionReward,def:trajMeasure} \leanok - \lean{Learning.IT.condDistrib_reward_zero} + \lean{Learning.IT.condDistrib_feedback_zero} $P_{\mathcal{T}}\left(R_0 \mid A_0\right) = \nu'_0$. \end{lemma} diff --git a/scripts/build_blueprint.sh b/build_blueprint.sh similarity index 100% rename from scripts/build_blueprint.sh rename to build_blueprint.sh diff --git a/scripts/build_tutorial.sh b/build_tutorial.sh similarity index 100% rename from scripts/build_tutorial.sh rename to build_tutorial.sh diff --git a/tutorial/Manual/Pages/DefiningAlgorithm.lean b/tutorial/Manual/Pages/DefiningAlgorithm.lean index 2e1c752c..45b657c8 100644 --- a/tutorial/Manual/Pages/DefiningAlgorithm.lean +++ b/tutorial/Manual/Pages/DefiningAlgorithm.lean @@ -24,20 +24,20 @@ In general, both action and feedback can depend on the entire history up to the The `Algorithm` structure is defined as follows: ```anchor Algorithm (module := LeanMachineLearning.SequentialLearning.Algorithm) -structure Algorithm (α R : Type*) [MeasurableSpace α] [MeasurableSpace R] where +structure Algorithm (𝓐 𝓨 : Type*) [MeasurableSpace 𝓐] [MeasurableSpace 𝓨] where /-- Policy or sampling rule: distribution of the next action. -/ - policy : (n : ℕ) → Kernel (Iic n → α × R) α + policy : (n : ℕ) → Kernel (Iic n → 𝓐 × 𝓨) 𝓐 [h_policy : ∀ n, IsMarkovKernel (policy n)] /-- Distribution of the first action. -/ - p0 : Measure α + p0 : Measure 𝓐 [hp0 : IsProbabilityMeasure p0] ``` -This structure refers to two types, the type of actions `α` and the type of feedback `R`. +This structure refers to two types, the type of actions `𝓐` and the type of feedback `𝓨`. Both are measurable spaces, since we consider stochastic algorithms and environments. The interaction will start with the algorithm playing a first action, which is in general random with distribution `p0`. The field `hp0` registers that `p0` is a probability measure (and it is in square brackets to tell Lean to infer it automatically whenever possible). -After time `n`, there is a history of actions and feedbacks `Iic n → α × R` (`n+1` pairs action and feedback). +After time `n`, there is a history of actions and feedbacks `Iic n → 𝓐 × 𝓨` (`n+1` pairs action and feedback). So after time 0 (the processes are 0-indexed) the history contains the action at time 0 and the feedback that followed. The `policy` field contain for each time `n` a kernel from that history to the action space. That is, it maps every possible history to a random next action (and that map is measurable). @@ -46,9 +46,9 @@ The `h_policy` field records that the measure describing the next action is a pr If the algorithms actions are not random, we can use the `detAlgorithm` definition to build an algorithm from the data of a measurable function for the next action and a choice for the first action. ```anchor detAlgorithm (module := LeanMachineLearning.SequentialLearning.Deterministic) -def detAlgorithm (nextA : (n : ℕ) → (Iic n → α × R) → α) - (h_next : ∀ n, Measurable (nextA n)) (action0 : α) : - Algorithm α R where +def detAlgorithm (nextA : (n : ℕ) → (Iic n → 𝓐 × 𝓨) → 𝓐) + (h_next : ∀ n, Measurable (nextA n)) (action0 : 𝓐) : + Algorithm 𝓐 𝓨 where policy n := Kernel.deterministic (nextA n) (h_next n) p0 := Measure.dirac action0 ``` @@ -58,12 +58,12 @@ Lean knows that deterministic kernels are Markov. The `Environment` structure is the mirror of the `Algorithm` structure, with a kernel for the feedback instead of the actions and a kernel for the first feedback instead of the first action. ```anchor Environment (module := LeanMachineLearning.SequentialLearning.Algorithm) -structure Environment (α R : Type*) [MeasurableSpace α] [MeasurableSpace R] where +structure Environment (𝓐 𝓨 : Type*) [MeasurableSpace 𝓐] [MeasurableSpace 𝓨] where /-- Distribution of the next observation as function of the past history. -/ - feedback : (n : ℕ) → Kernel ((Iic n → α × R) × α) R + feedback : (n : ℕ) → Kernel ((Iic n → 𝓐 × 𝓨) × 𝓐) 𝓨 [h_feedback : ∀ n, IsMarkovKernel (feedback n)] /-- Distribution of the first observation given the first action. -/ - ν0 : Kernel α R + ν0 : Kernel 𝓐 𝓨 [hp0 : IsMarkovKernel ν0] ``` @@ -73,15 +73,15 @@ In many applications the feedback depends only on the last action and not on the We provide an `obliviousEnv` definition that builds an environment for those cases. ```anchor obliviousEnv (module := LeanMachineLearning.SequentialLearning.StationaryEnv) -def obliviousEnv (ν : ℕ → Kernel α R) [∀ n, IsMarkovKernel (ν n)] : Environment α R where +def obliviousEnv (ν : ℕ → Kernel 𝓐 𝓨) [∀ n, IsMarkovKernel (ν n)] : Environment 𝓐 𝓨 where feedback n := (ν (n + 1)).prodMkLeft _ ν0 := ν 0 ``` -`(ν (n + 1)).prodMkLeft _` is the kernel `ν (n + 1)` seen as a `Kernel ((Iic n → α × R) × α) R` by ignoring the history. +`(ν (n + 1)).prodMkLeft _` is the kernel `ν (n + 1)` seen as a `Kernel ((Iic n → 𝓐 × 𝓨) × 𝓐) 𝓨` by ignoring the history. If furthermore the feedback kernel does not change with time, we can use the `stationaryEnv` definition to build the environment. ```anchor stationaryEnv (module := LeanMachineLearning.SequentialLearning.StationaryEnv) -def stationaryEnv (ν : Kernel α R) [IsMarkovKernel ν] : Environment α R := obliviousEnv fun _ ↦ ν +def stationaryEnv (ν : Kernel 𝓐 𝓨) [IsMarkovKernel ν] : Environment 𝓐 𝓨 := obliviousEnv fun _ ↦ ν ``` @@ -92,24 +92,23 @@ This is done by the `IsAlgEnvSeq` structure. ```anchor IsAlgEnvSeq (module := LeanMachineLearning.SequentialLearning.Algorithm) structure IsAlgEnvSeq - [StandardBorelSpace α] [Nonempty α] [StandardBorelSpace R] [Nonempty R] - (A : ℕ → Ω → α) (R' : ℕ → Ω → R) (alg : Algorithm α R) (env : Environment α R) + (A : ℕ → Ω → 𝓐) (Y : ℕ → Ω → 𝓨) (alg : Algorithm 𝓐 𝓨) (env : Environment 𝓐 𝓨) (P : Measure Ω) [IsFiniteMeasure P] : Prop where - measurable_A n : Measurable (A n) := by fun_prop - measurable_R n : Measurable (R' n) := by fun_prop + measurable_action n : Measurable (A n) := by fun_prop + measurable_feedback n : Measurable (Y n) := by fun_prop hasLaw_action_zero : HasLaw (fun ω ↦ (A 0 ω)) alg.p0 P - hasCondDistrib_reward_zero : HasCondDistrib (R' 0) (A 0) env.ν0 P + hasCondDistrib_feedback_zero : HasCondDistrib (Y 0) (A 0) env.ν0 P hasCondDistrib_action n : - HasCondDistrib (A (n + 1)) (IsAlgEnvSeq.hist A R' n) (alg.policy n) P - hasCondDistrib_reward n : - HasCondDistrib (R' (n + 1)) (fun ω ↦ (IsAlgEnvSeq.hist A R' n ω, A (n + 1) ω)) + HasCondDistrib (A (n + 1)) (IsAlgEnvSeq.hist A Y n) (alg.policy n) P + hasCondDistrib_feedback n : + HasCondDistrib (Y (n + 1)) (fun ω ↦ (IsAlgEnvSeq.hist A Y n ω, A (n + 1) ω)) (env.feedback n) P ``` -This structure takes as input two sequences of random variables (two stochastic processes), `A` and `R'`, which represent the actions and feedback generated by the interaction of the algorithm with the environment. +This structure takes as input two sequences of random variables (two stochastic processes), `A` and `Y`, which represent the actions and feedback generated by the interaction of the algorithm with the environment. It states that those sequences are measurable and that they have the correct conditional distributions given by the algorithm and environment. The measurable space `Ω` and the measure `P` are not imposed: they can be chosen as we want, as long as the conditions of `IsAlgEnvSeq` are satisfied. -This definition requires `α` and `R` to be nonempty standard Borel spaces, because Mathlib's theory about conditional distributions requires those assumptions. +This definition requires `𝓐` and `𝓨` to be nonempty standard Borel spaces, because Mathlib's theory about conditional distributions requires those assumptions. All spaces of interest in machine learning are standard Borel, so this is not a restriction. Given any algorithm and environment, there always exists a sequence of actions and feedback that satisfies `IsAlgEnvSeq` by the Ionescu-Tulcea theorem. @@ -176,7 +175,7 @@ We can now state a theorem about the regret of UCB in a stochastic bandit enviro Let's first define the regret, which for stochastic bandits is the difference between the mean rewar that the algorithm would have obtained if it played always the best action, and the sum of mean rewards of the actions played. ```anchor regret (module := LeanMachineLearning.Online.Bandit.Regret) -def regret (ν : Kernel α ℝ) (A : ℕ → Ω → α) (t : ℕ) (ω : Ω) : ℝ := +def regret (ν : Kernel 𝓐 ℝ) (A : ℕ → Ω → 𝓐) (t : ℕ) (ω : Ω) : ℝ := t * (⨆ a, (ν a)[id]) - ∑ s ∈ range t, (ν (A s ω))[id] ``` The quantity `(ν a)[id]` is the mean reward of action `a` in the environment defined by `ν`. @@ -203,17 +202,17 @@ The theorem gives an upper bound on the expected regret of UCB at time `n`. # Building vs analyzing algorithms -When building an algorithm, we describe it with functions from the history `(Iic n → α × R)` to the action space `α`. +When building an algorithm, we describe it with functions from the history `(Iic n → 𝓐 × R)` to the action space `𝓐`. Thus, to construct UCB, we used the following empirical mean function. ```anchor empMean' (module := LeanMachineLearning.SequentialLearning.FiniteActions) -def empMean' (n : ℕ) (h : Iic n → α × ℝ) (a : α) := +def empMean' (n : ℕ) (h : Iic n → 𝓐 × ℝ) (a : 𝓐) := (sumRewards' n h a) / (pullCount' n h a) ``` -When analyzing an algorithm, we work with sequences of actions and rewards `A : ℕ → Ω → α` and `R' : ℕ → Ω → R` that satisfy `IsAlgEnvSeq`. +When analyzing an algorithm, we work with sequences of actions and rewards `A : ℕ → Ω → 𝓐` and `R' : ℕ → Ω → R` that satisfy `IsAlgEnvSeq`. For the analysis, the empirical mean is defined as a stochastic process on the same probability space `Ω`. ```anchor empMean (module := LeanMachineLearning.SequentialLearning.FiniteActions) -def empMean (A : ℕ → Ω → α) (R' : ℕ → Ω → ℝ) (a : α) (t : ℕ) (ω : Ω) : ℝ := +def empMean (A : ℕ → Ω → 𝓐) (R' : ℕ → Ω → ℝ) (a : 𝓐) (t : ℕ) (ω : Ω) : ℝ := sumRewards A R' a t ω / pullCount A a t ω ``` `empMean A R' a` is a stochastic process with type `ℕ → Ω → ℝ` that gives the empirical mean of action `a` at each time.