diff --git a/LMLTutorial/Front.lean b/LMLTutorial/Front.lean index 3bdb0d89..a0ccbe92 100644 --- a/LMLTutorial/Front.lean +++ b/LMLTutorial/Front.lean @@ -3,12 +3,14 @@ Copyright (c) 2026 Rémy Degenne. All rights reserved. Released under Apache 2.0 license as described in the file LICENSE. Authors: Rémy Degenne -/ -import LMLTutorial.Pages.BasicProbability -import LMLTutorial.Pages.DefiningAlgorithm -import LMLTutorial.Pages.Installation -import LMLTutorial.Pages.MarkovKernels -import LMLTutorial.Pages.Martingales -import VersoManual +module + +public import LMLTutorial.Pages.BasicProbability +public import LMLTutorial.Pages.DefiningAlgorithm +public import LMLTutorial.Pages.Installation +public import LMLTutorial.Pages.MarkovKernels +public import LMLTutorial.Pages.Martingales +public import VersoManual set_option linter.style.header false set_option linter.style.setOption false @@ -17,6 +19,8 @@ set_option linter.style.longLine false set_option pp.rawOnError true set_option verso.code.warnLineLength 100 +@[expose] public section + open Verso.Genre Manual Verso.Genre.Manual.InlineLean Verso.Code.External #doc (Manual) "Lean Machine Learning" => diff --git a/LMLTutorial/Pages/BasicProbability.lean b/LMLTutorial/Pages/BasicProbability.lean index 3940b022..dc9d62b4 100644 --- a/LMLTutorial/Pages/BasicProbability.lean +++ b/LMLTutorial/Pages/BasicProbability.lean @@ -3,10 +3,12 @@ Copyright (c) 2025 Rémy Degenne. All rights reserved. Released under Apache 2.0 license as described in the file LICENSE. Authors: Rémy Degenne -/ -import VersoManual -import Mathlib.Probability.Distributions.Gaussian.Real -import Mathlib.Probability.Independence.Basic -import Mathlib.Probability.Moments.Basic +module + +public import VersoManual +public import Mathlib.Probability.Distributions.Gaussian.Real +public import Mathlib.Probability.Independence.Basic +public import Mathlib.Probability.Moments.Basic set_option linter.style.header false set_option linter.style.setOption false @@ -15,6 +17,8 @@ set_option linter.style.longLine false set_option pp.rawOnError true set_option verso.code.warnLineLength 100 +@[expose] public section + open Verso.Genre Manual Verso.Genre.Manual.InlineLean Verso.Code.External #doc (Manual) "Probability Spaces and Measures" => diff --git a/LMLTutorial/Pages/DefiningAlgorithm.lean b/LMLTutorial/Pages/DefiningAlgorithm.lean index bcf9b60c..2286647b 100644 --- a/LMLTutorial/Pages/DefiningAlgorithm.lean +++ b/LMLTutorial/Pages/DefiningAlgorithm.lean @@ -3,8 +3,10 @@ Copyright (c) 2025 Rémy Degenne. All rights reserved. Released under Apache 2.0 license as described in the file LICENSE. Authors: Rémy Degenne -/ -import VersoManual -import LeanMachineLearning +module + +public import VersoManual +public import LeanMachineLearning set_option linter.style.header false set_option linter.style.setOption false @@ -15,6 +17,8 @@ set_option verso.code.warnLineLength 100 set_option verso.docstring.allowMissing true +@[expose] public section + open Verso.Genre Manual Verso.Genre.Manual.InlineLean Verso.Code.External Learning @@ -37,33 +41,34 @@ The `Algorithm` structure is defined as follows: This structure refers to two types, the type of actions `𝓐` and the type of feedback `𝓨`. Both are measurable spaces, since we consider stochastic algorithms and environments. -The interaction will start with the algorithm playing a first action, which is in general random with distribution `p0`. -The field `hp0` registers that `p0` is a probability measure (and it is in square brackets to tell Lean to infer it automatically whenever possible). -After time `n`, there is a history of actions and feedbacks `Iic n → 𝓐 × 𝓨` (`n+1` pairs action and feedback). -So after time 0 (the processes are 0-indexed) the history contains the action at time 0 and the feedback that followed. -The `policy` field contain for each time `n` a kernel from that history to the action space. -That is, it maps every possible history to a random next action (and that map is measurable). -The `h_policy` field records that the measure describing the next action is a probability measure. +Before time `n`, there is a history of actions and feedbacks `Fin n → 𝓐 × 𝓨` (the `n` pairs of action and feedback at times `0, ..., n - 1`; the processes are 0-indexed). +The `policy` field contains for each time `n` a kernel from that history to the action space. +That is, it maps every possible history to a random action at time `n` (and that map is measurable). +The `h_policy` field records that the measure describing the action is a probability measure (and it is in square brackets to tell Lean to infer it automatically whenever possible). +At time `0` the history is empty: `Fin 0 → 𝓐 × 𝓨` has a unique element, and the distribution of the first action is `policy 0` applied to that element. +That distribution is called `Algorithm.p0`. -If the algorithms actions are not random, we can use the `detAlgorithm` definition to build an algorithm from the data of a measurable function for the next action and a choice for the first action. +If the algorithms actions are not random, we can use the `detAlgorithm` definition to build an algorithm from the data of a measurable function for the action at each time, as a function of the history before that time. +The first action is the value of that function at time `0` on the empty history. {docstring detAlgorithm} -We can see here that we did not need to prove that the kernels are `IsMarkovKernel` and that the distribution of the first action is a probability measure. +We can see here that we did not need to prove that the kernels are `IsMarkovKernel`. Lean knows that deterministic kernels are Markov. -The `Environment` structure is the mirror of the `Algorithm` structure, with a kernel for the feedback instead of the actions and a kernel for the first feedback instead of the first action. +The `Environment` structure is the mirror of the `Algorithm` structure, with a kernel for the feedback instead of the actions. {docstring Environment} -`ν0` gives the distribution of the first feedback given the first action, and `feedback` gives the distribution of the next feedback given the history and the next action. +`feedback n` gives the distribution of the feedback at time `n` given the history before `n` and the action at time `n`. +The distribution of the first feedback given the first action is `feedback 0` applied to the empty history; it is called `Environment.ν0`. In many applications the feedback depends only on the last action and not on the prior history. We provide an `obliviousEnv` definition that builds an environment for those cases. {docstring obliviousEnv} -`(ν (n + 1)).prodMkLeft _` is the kernel `ν (n + 1)` seen as a `Kernel ((Iic n → 𝓐 × 𝓨) × 𝓐) 𝓨` by ignoring the history. +`(ν n).prodMkLeft _` is the kernel `ν n` seen as a `Kernel ((Fin n → 𝓐 × 𝓨) × 𝓐) 𝓨` by ignoring the history. If furthermore the feedback kernel does not change with time, we can use the `stationaryEnv` definition to build the environment. @@ -100,8 +105,8 @@ The environment is thus simply `stationaryEnv ν` for some kernel `ν : Kernel ( ## Algorithm -The UCB algorithm chooses at time `n + 1` the action that maximizes the sum of the empirical mean reward and an exploration bonus. -It starts by choosing each action once and then chooses $`\arg\max_a (\hat{\mu}_{n,a} + \sqrt{\frac{2c \log (n + 2)}{N_{n,a}}})`, in which $`\hat{\mu}_{n,a}` is the empirical mean reward of action `a` at time `n` (`empMean'` in the code), $`N_{n,a}` is the number of times action `a` has been chosen up to time `n` (`pullCount'` in the code), and `c` is a parameter of the algorithm. +The UCB algorithm chooses at time `n` the action that maximizes the sum of the empirical mean reward and an exploration bonus. +It starts by choosing each action once and then chooses $`\arg\max_a (\hat{\mu}_{n,a} + \sqrt{\frac{2c \log (n + 1)}{N_{n,a}}})`, in which $`\hat{\mu}_{n,a}` is the empirical mean reward of action `a` before time `n` (`empMean'` in the code), $`N_{n,a}` is the number of times action `a` has been chosen before time `n` (`pullCount'` in the code), and `c` is a parameter of the algorithm. To define the algorithm, we first define the exploration bonus and the next action function, and then we use `detAlgorithm` to build the algorithm. We also need to prove that the next action function is measurable, which is done by the `measurable_nextArm` lemma. @@ -117,7 +122,7 @@ Note that we are careful to use a measurable version of the argmax function, `ar The last line builds the algorithm using `detAlgorithm` and the function `UCB.nextArm`. Its measurability is proved by the `fun_prop` tactic, which proves measurability of functions by using lemmas tagged with `@[fun_prop]`. -The last argument `⟨0, hK⟩` is the first action of the algorithm, which is 0 as an element of `Fin K`. +The first action of the algorithm is `UCB.nextArm hK c 0` applied to the empty history, which is 0 as an element of `Fin K`. ## A theorem about UCB @@ -144,7 +149,7 @@ The theorem gives an upper bound on the expected regret of UCB at time `n`. # Building vs analyzing algorithms -When building an algorithm, we describe it with functions from the history `(Iic n → 𝓐 × R)` to the action space `𝓐`. +When building an algorithm, we describe it with functions from the history `(Fin n → 𝓐 × R)` to the action space `𝓐`. Thus, to construct UCB, we used the following empirical mean function. {docstring empMean'} diff --git a/LMLTutorial/Pages/Installation.lean b/LMLTutorial/Pages/Installation.lean index d4c32409..2b5a0787 100644 --- a/LMLTutorial/Pages/Installation.lean +++ b/LMLTutorial/Pages/Installation.lean @@ -3,7 +3,11 @@ Copyright (c) 2025 Rémy Degenne. All rights reserved. Released under Apache 2.0 license as described in the file LICENSE. Authors: Rémy Degenne -/ -import VersoManual +module + +public import VersoManual + +@[expose] public section open Verso.Genre Manual diff --git a/LMLTutorial/Pages/MarkovKernels.lean b/LMLTutorial/Pages/MarkovKernels.lean index 81ce4031..e11d267f 100644 --- a/LMLTutorial/Pages/MarkovKernels.lean +++ b/LMLTutorial/Pages/MarkovKernels.lean @@ -3,9 +3,11 @@ Copyright (c) 2025 Rémy Degenne. All rights reserved. Released under Apache 2.0 license as described in the file LICENSE. Authors: Rémy Degenne -/ -import LMLTutorial.References -import VersoManual -import Mathlib.Probability.Kernel.Composition.Lemmas +module + +public import LMLTutorial.References +public import VersoManual +public import Mathlib.Probability.Kernel.Composition.Lemmas set_option linter.style.header false set_option linter.style.setOption false @@ -14,6 +16,8 @@ set_option linter.style.longLine false set_option pp.rawOnError true set_option verso.code.warnLineLength 100 +@[expose] public section + open Verso.Genre Manual Verso.Genre.Manual.InlineLean Verso.Code.External #doc (Manual) "Markov Kernels" => diff --git a/LMLTutorial/Pages/Martingales.lean b/LMLTutorial/Pages/Martingales.lean index 5ad7701c..ae0011c2 100644 --- a/LMLTutorial/Pages/Martingales.lean +++ b/LMLTutorial/Pages/Martingales.lean @@ -3,10 +3,11 @@ Copyright (c) 2025 Rémy Degenne. All rights reserved. Released under Apache 2.0 license as described in the file LICENSE. Authors: Rémy Degenne -/ -import VersoManual -import Mathlib.Probability.Martingale.Convergence -import Mathlib.Probability.Martingale.OptionalStopping -import Mathlib.Probability.Martingale.OptionalSampling +module +public import VersoManual +public import Mathlib.Probability.Martingale.Convergence +public import Mathlib.Probability.Martingale.OptionalStopping +public import Mathlib.Probability.Martingale.OptionalSampling set_option linter.style.header false set_option linter.style.setOption false @@ -15,6 +16,8 @@ set_option linter.style.longLine false set_option pp.rawOnError true set_option verso.code.warnLineLength 100 +@[expose] public section + open Verso.Genre Manual Verso.Genre.Manual.InlineLean Verso.Code.External #doc (Manual) "Stochastic Processes and Martingales" => diff --git a/LMLTutorial/References.lean b/LMLTutorial/References.lean index 78b8ba8a..5fc695d0 100644 --- a/LMLTutorial/References.lean +++ b/LMLTutorial/References.lean @@ -3,7 +3,13 @@ Copyright (c) 2026 Rémy Degenne. All rights reserved. Released under Apache 2.0 license as described in the file LICENSE. Authors: Rémy Degenne -/ -import VersoManual +module + +public import Verso.Doc.Concrete.InlineString +public import VersoManual + +@[expose] public section + open Verso.Genre.Manual namespace Docs diff --git a/LMLTutorial/Tutorial.lean b/LMLTutorial/Tutorial.lean index ae5cf8ee..d779e365 100644 --- a/LMLTutorial/Tutorial.lean +++ b/LMLTutorial/Tutorial.lean @@ -3,12 +3,15 @@ Copyright (c) 2026 Rémy Degenne. All rights reserved. Released under Apache 2.0 license as described in the file LICENSE. Authors: Rémy Degenne -/ -import VersoManual -import LMLTutorial.Front +module +public import VersoManual +public import LMLTutorial.Front set_option linter.style.header false set_option linter.style.setOption false +@[expose] public section + open Verso.Genre.Manual Verso.Output.Html def extraHead : Array Verso.Output.Html := #[ diff --git a/LeanMachineLearning.lean b/LeanMachineLearning.lean index af26795d..d9225776 100644 --- a/LeanMachineLearning.lean +++ b/LeanMachineLearning.lean @@ -2,6 +2,7 @@ module -- shake: keep-all --deprecated_module: ignore public import LeanMachineLearning.ForMathlib.InformationTheory.KullbackLeibler.ChainRule public import LeanMachineLearning.ForMathlib.MeasureTheory.Measurable +public import LeanMachineLearning.ForMathlib.MeasureTheory.MeasurableSpace.Embedding public import LeanMachineLearning.ForMathlib.MeasureTheory.Measure.AbsolutelyContinuous public import LeanMachineLearning.ForMathlib.MeasureTheory.Order.Lattice public import LeanMachineLearning.ForMathlib.MeasureTheory.Order.MeasurableArg diff --git a/LeanMachineLearning/ForMathlib/MeasureTheory/MeasurableSpace/Embedding.lean b/LeanMachineLearning/ForMathlib/MeasureTheory/MeasurableSpace/Embedding.lean new file mode 100644 index 00000000..47e61a4b --- /dev/null +++ b/LeanMachineLearning/ForMathlib/MeasureTheory/MeasurableSpace/Embedding.lean @@ -0,0 +1,156 @@ +/- +Copyright (c) 2025 Rémy Degenne. All rights reserved. +Released under Apache 2.0 license as described in the file LICENSE. +Authors: Rémy Degenne, Paulo Rauber +-/ +module + +public import Mathlib.MeasureTheory.MeasurableSpace.Embedding +public import Mathlib.Order.Restriction +public import Mathlib.Probability.Kernel.IonescuTulcea.Maps + +/-! +# Measurable equivalences + +Measurable equivalences between product and pi types, used to manipulate histories of sequential +learning algorithms (elements of `Fin n → 𝓐 × 𝓨` or `Iic n → 𝓐 × 𝓨`). + +* `MeasurableEquiv.uniqueProd`, `MeasurableEquiv.prodUnique`: drop a component that lives in a type + with a unique element. +* `MeasurableEquiv.IicSuccProd`: `(Π i : Iic (n + 1), X i) ≃ᵐ (Π i : Iic n, X i) × X (n + 1)`. +* `MeasurableEquiv.finSuccPiIic`: `(Π i : Fin (n + 1), X i) ≃ᵐ (Π i : Iic n, X i)`. +* `MeasurableEquiv.finSuccProd`: `(Fin (n + 1) → X) ≃ᵐ (Fin n → X) × X`. +-/ + +@[expose] public section + +open Finset Preorder + +lemma coe_default_Iic_zero : ((default : Iic 0) : ℕ) = 0 := rfl + +namespace MeasurableEquiv + +section Unique + +/-- The measurable equivalence `α × β ≃ᵐ β` when `α` has a unique element. -/ +def uniqueProd (α β : Type*) [MeasurableSpace α] [MeasurableSpace β] [Unique α] : + α × β ≃ᵐ β where + toFun := Prod.snd + invFun b := (default, b) + left_inv _ := Prod.ext (Unique.eq_default _).symm rfl + right_inv _ := rfl + measurable_toFun := measurable_snd + measurable_invFun := measurable_const.prodMk measurable_id + +@[simp] +lemma uniqueProd_apply {α β : Type*} [MeasurableSpace α] [MeasurableSpace β] [Unique α] + (p : α × β) : + uniqueProd α β p = p.2 := rfl + +@[simp] +lemma uniqueProd_symm_apply {α β : Type*} [MeasurableSpace α] [MeasurableSpace β] [Unique α] + (b : β) : + (uniqueProd α β).symm b = (default, b) := rfl + +/-- The measurable equivalence `α × β ≃ᵐ α` when `β` has a unique element. -/ +def prodUnique (α β : Type*) [MeasurableSpace α] [MeasurableSpace β] [Unique β] : + α × β ≃ᵐ α where + toFun := Prod.fst + invFun a := (a, default) + left_inv _ := Prod.ext rfl (Unique.eq_default _).symm + right_inv _ := rfl + measurable_toFun := measurable_fst + measurable_invFun := measurable_id.prodMk measurable_const + +@[simp] +lemma prodUnique_apply {α β : Type*} [MeasurableSpace α] [MeasurableSpace β] [Unique β] + (p : α × β) : + prodUnique α β p = p.1 := rfl + +@[simp] +lemma prodUnique_symm_apply {α β : Type*} [MeasurableSpace α] [MeasurableSpace β] [Unique β] + (a : α) : + (prodUnique α β).symm a = (a, default) := rfl + +end Unique + +section Iic + +variable {X : ℕ → Type*} [∀ n, MeasurableSpace (X n)] + +/-- Measurable equivalence between a product up to `n + 1` and the pair of the product up to `n` and +the space at `n + 1`. -/ +def IicSuccProd (X : ℕ → Type*) [∀ n, MeasurableSpace (X n)] (n : ℕ) : + MeasurableEquiv (Π i : Iic (n + 1), X i) ((Π i : Iic n, X i) × X (n + 1)) := + (IicProdIoc (Nat.le_succ n)).symm.trans + (prodCongr (refl _) (piSingleton n).symm) + +lemma symm_IicSuccProd (n : ℕ) : + (IicSuccProd X n).symm = + (prodCongr (refl _) (piSingleton n)).trans + (IicProdIoc (Nat.le_succ n)) := rfl + +@[simp] +lemma IicSuccProd_apply (n : ℕ) (h : Π i : Iic (n + 1), X i) : + IicSuccProd X n h = (fun i : Iic n ↦ h ⟨i.1, by grind⟩, h ⟨n + 1, by simp⟩) := + rfl + +lemma coe_prodCongr {α β γ δ : Type*} + {mα : MeasurableSpace α} {mβ : MeasurableSpace β} + {mγ : MeasurableSpace γ} {mδ : MeasurableSpace δ} + (e₁ : MeasurableEquiv α β) (e₂ : MeasurableEquiv γ δ) : + (prodCongr e₁ e₂ : (α × γ) → (β × δ)) = Prod.map e₁ e₂ := rfl + +lemma coe_refl {α : Type*} {mα : MeasurableSpace α} : + (refl α : α → α) = id := rfl + +end Iic + +section Fin + +variable {X : ℕ → Type*} [∀ n, MeasurableSpace (X n)] + +/-- Measurable equivalence between `Π i : Fin (n + 1), X i` and `Π i : Iic n, X i`. -/ +def finSuccPiIic (X : ℕ → Type*) [∀ n, MeasurableSpace (X n)] (n : ℕ) : + (Π i : Fin (n + 1), X i) ≃ᵐ (Π i : Iic n, X i) where + toFun h i := h ⟨i.1, Nat.lt_succ_of_le (mem_Iic.mp i.2)⟩ + invFun h i := h ⟨i.1, mem_Iic.mpr (Nat.le_of_lt_succ i.2)⟩ + left_inv _ := rfl + right_inv _ := rfl + measurable_toFun := measurable_pi_lambda _ fun _ ↦ measurable_pi_apply _ + measurable_invFun := measurable_pi_lambda _ fun _ ↦ measurable_pi_apply _ + +@[simp] +lemma finSuccPiIic_apply (n : ℕ) (h : Π i : Fin (n + 1), X i) (i : Iic n) : + finSuccPiIic X n h i = h ⟨i.1, Nat.lt_succ_of_le (mem_Iic.mp i.2)⟩ := rfl + +@[simp] +lemma finSuccPiIic_symm_apply (n : ℕ) (h : Π i : Iic n, X i) (i : Fin (n + 1)) : + (finSuccPiIic X n).symm h i = h ⟨i.1, mem_Iic.mpr (Nat.le_of_lt_succ i.2)⟩ := + rfl + +lemma finSuccPiIic_symm_comp_frestrictLe (n : ℕ) : + (finSuccPiIic X n).symm ∘ frestrictLe n = fun x (i : Fin (n + 1)) ↦ x i := rfl + +/-- Measurable equivalence between `Fin (n + 1) → X` and `(Fin n → X) × X`. -/ +def finSuccProd (X : Type*) [MeasurableSpace X] (n : ℕ) : + (Fin (n + 1) → X) ≃ᵐ (Fin n → X) × X := + (piFinSuccAbove (fun _ ↦ X) (Fin.last n)).trans prodComm + +@[simp] +lemma finSuccProd_apply {X : Type*} [MeasurableSpace X] (n : ℕ) + (h : Fin (n + 1) → X) : + finSuccProd X n h = (fun i ↦ h i.castSucc, h (Fin.last n)) := by + simp [finSuccProd] + rfl + +@[simp] +lemma finSuccProd_symm_apply {X : Type*} [MeasurableSpace X] (n : ℕ) + (p : (Fin n → X) × X) : + (finSuccProd X n).symm p = Fin.snoc p.1 p.2 := by + simp [finSuccProd] + rfl + +end Fin + +end MeasurableEquiv diff --git a/LeanMachineLearning/ForMathlib/Probability/HasCondDistrib.lean b/LeanMachineLearning/ForMathlib/Probability/HasCondDistrib.lean index 65528559..b6e91c89 100644 --- a/LeanMachineLearning/ForMathlib/Probability/HasCondDistrib.lean +++ b/LeanMachineLearning/ForMathlib/Probability/HasCondDistrib.lean @@ -5,6 +5,7 @@ Authors: Rémy Degenne, Paulo Rauber -/ module +public import LeanMachineLearning.ForMathlib.MeasureTheory.MeasurableSpace.Embedding public import LeanMachineLearning.ForMathlib.Probability.Independence.CondDistrib public import Mathlib.Probability.HasCondDistrib @@ -168,6 +169,95 @@ lemma ae_eq_of_hasCondDistrib_deterministic [MeasurableEq Ω] [SFinite μ] {f : AEMeasurable.map_map_of_aemeasurable (by fun_prop) (by fun_prop)] rfl +section Const + +section CompRight + +variable [SFinite μ] + +/-- Converse of `HasCondDistrib.comp_right` for a measurable embedding. -/ +lemma HasCondDistrib.of_measurableEmbedding_comp_right {f : β → γ} (hf : MeasurableEmbedding f) + {κ : Kernel γ Ω} [IsSFiniteKernel κ] (h : HasCondDistrib Y (f ∘ X) κ μ) : + HasCondDistrib Y X (κ.comap f hf.measurable) μ := by + have hX : AEMeasurable X μ := hf.aemeasurable_comp_iff.mp h.aemeasurable_fst + have hY : AEMeasurable Y μ := h.aemeasurable_snd + have hfm : Measurable (Prod.map f (id : Ω → Ω)) := hf.measurable.prodMap measurable_id + refine ⟨hX.prodMk hY, (hf.prodMap MeasurableEmbedding.id).map_injective ?_⟩ + rw [AEMeasurable.map_map_of_aemeasurable hfm.aemeasurable (by fun_prop)] + calc μ.map (Prod.map f id ∘ fun ω ↦ (X ω, Y ω)) + _ = μ.map (f ∘ X) ⊗ₘ κ := h.map_eq + _ = (μ.map X).map f ⊗ₘ κ := by + rw [AEMeasurable.map_map_of_aemeasurable hf.measurable.aemeasurable hX] + _ = (μ.map X ⊗ₘ κ.comap f hf.measurable).map (Prod.map f id) := by + symm + ext s hs + rw [Measure.map_apply hfm hs, Measure.compProd_apply (hs.preimage hfm), + Measure.compProd_apply hs, + lintegral_map (Kernel.measurable_kernel_prodMk_left hs) hf.measurable] + rfl + +/-- `HasCondDistrib.comp_right` is an equivalence for measurable embeddings. -/ +lemma hasCondDistrib_measurableEmbedding_comp_right_iff {f : β → γ} (hf : MeasurableEmbedding f) + {κ : Kernel γ Ω} [IsSFiniteKernel κ] : + HasCondDistrib Y (f ∘ X) κ μ ↔ HasCondDistrib Y X (κ.comap f hf.measurable) μ := + ⟨fun h ↦ h.of_measurableEmbedding_comp_right hf, fun h ↦ h.comp_right⟩ + +/-- `HasCondDistrib.comp_right` is an equivalence for measurable equivalences. -/ +lemma hasCondDistrib_measurableEquiv_comp_right_iff (e : β ≃ᵐ γ) {κ : Kernel γ Ω} + [IsSFiniteKernel κ] : + HasCondDistrib Y (e ∘ X) κ μ ↔ HasCondDistrib Y X (κ.comap e e.measurable) μ := + hasCondDistrib_measurableEmbedding_comp_right_iff e.measurableEmbedding + +end CompRight + +section UniqueComponent + +variable {δ : Type*} {mδ : MeasurableSpace δ} [Unique δ] [SFinite μ] + +/-- Conditioning on a pair whose first component takes values in a type with a unique element +is the same as conditioning on the second component. -/ +lemma hasCondDistrib_prodMk_left_unique_iff {U : α → δ} {η : Kernel (δ × β) Ω} + [IsSFiniteKernel η] : + HasCondDistrib Y (fun ω ↦ (U ω, X ω)) η μ ↔ HasCondDistrib Y X (η.sectR default) μ := by + have hU : U = fun _ ↦ default := funext fun _ ↦ Unique.eq_default _ + subst hU + exact hasCondDistrib_measurableEmbedding_comp_right_iff (measurableEmbedding_prodMk_left default) + +/-- Conditioning on a pair whose second component takes values in a type with a unique element +is the same as conditioning on the first component. -/ +lemma hasCondDistrib_prodMk_right_unique_iff {U : α → δ} {η : Kernel (β × δ) Ω} + [IsSFiniteKernel η] : + HasCondDistrib Y (fun ω ↦ (X ω, U ω)) η μ ↔ HasCondDistrib Y X (η.sectL default) μ := by + have hU : U = fun _ ↦ default := funext fun _ ↦ Unique.eq_default _ + subst hU + exact hasCondDistrib_measurableEmbedding_comp_right_iff + (measurableEmbedding_prod_mk_right default) + +end UniqueComponent + + +lemma _root_.MeasureTheory.Measure.dirac_compProd {κ : Kernel β Ω} [IsSFiniteKernel κ] (b : β) : + Measure.dirac b ⊗ₘ κ = (κ b).map (Prod.mk b) := by + ext s hs + rw [Measure.compProd_apply hs, lintegral_dirac' _ (Kernel.measurable_kernel_prodMk_left hs), + Measure.map_apply measurable_prodMk_left hs] + +/-- Conditioning on a constant is the same as having law `κ b`. -/ +lemma hasCondDistrib_const_iff [IsProbabilityMeasure μ] [IsSFiniteKernel κ] {b : β} : + HasCondDistrib Y (fun _ ↦ b) κ μ ↔ HasLaw Y (κ b) μ := by + refine ⟨fun h ↦ ⟨h.aemeasurable_snd, ?_⟩, fun h ↦ ⟨aemeasurable_const.prodMk h.aemeasurable, ?_⟩⟩ + · rw [← Measure.snd_map_prodMk₀ (X := fun _ ↦ b) (Y := Y) aemeasurable_const, h.map_eq, + Measure.map_const, measure_univ, one_smul, Measure.dirac_compProd, Measure.snd, + Measure.map_map measurable_snd measurable_prodMk_left] + exact Measure.map_id + · rw [Measure.map_const, measure_univ, one_smul, Measure.dirac_compProd, ← h.map_eq, + AEMeasurable.map_map_of_aemeasurable measurable_prodMk_left.aemeasurable h.aemeasurable] + rfl + +alias ⟨HasCondDistrib.hasLaw_of_const', HasLaw.hasCondDistrib_const⟩ := hasCondDistrib_const_iff + +end Const + section Cond variable [IsSFiniteKernel κ] @@ -260,4 +350,5 @@ lemma HasCondDistrib.hasCondDistrib_sectR [IsFiniteMeasure μ] [StandardBorelSpa rw [Kernel.map_apply _ hf] at ha filter_upwards [hc, ha] with b hcb hab using hcb.trans hab + end ProbabilityTheory diff --git a/LeanMachineLearning/ForMathlib/Probability/Kernel/IonescuTulcea/Traj.lean b/LeanMachineLearning/ForMathlib/Probability/Kernel/IonescuTulcea/Traj.lean index d4787afe..53df791c 100644 --- a/LeanMachineLearning/ForMathlib/Probability/Kernel/IonescuTulcea/Traj.lean +++ b/LeanMachineLearning/ForMathlib/Probability/Kernel/IonescuTulcea/Traj.lean @@ -5,6 +5,7 @@ Authors: Rémy Degenne, Paulo Rauber -/ module +public import LeanMachineLearning.ForMathlib.MeasureTheory.MeasurableSpace.Embedding public import LeanMachineLearning.ForMathlib.Probability.HasCondDistrib public import Mathlib.Probability.Kernel.IonescuTulcea.Traj public import Mathlib.Probability.Process.FiniteDimensionalLaws @@ -21,12 +22,6 @@ variable {Ω : Type*} {mΩ : MeasurableSpace Ω} {P : Measure Ω} {κ : (n : ℕ) → Kernel (Π i : Iic n, X i) (X (n + 1))} [∀ n, IsMarkovKernel (κ n)] {μ₀ : Measure (X 0)} [IsProbabilityMeasure μ₀] -section MeasurableEquiv - -lemma coe_default_Iic_zero : ((default : Iic 0) : ℕ) = 0 := rfl - -end MeasurableEquiv - namespace ProbabilityTheory.Kernel lemma traj_zero_map_eval_zero : @@ -40,32 +35,6 @@ lemma traj_zero_map_eval_zero : rw [← Kernel.traj_map_frestrictLe, ← Kernel.map_comp_right _ (by fun_prop) (by fun_prop)] rfl -/-- Measurable equivalence between a product up to `n + 1` and the pair of the product up to `n` and -the space at `n + 1`. -/ -def _root_.MeasurableEquiv.IicSuccProd (X : ℕ → Type*) [∀ n, MeasurableSpace (X n)] (n : ℕ) : - MeasurableEquiv (Π i : Iic (n + 1), X i) ((Π i : Iic n, X i) × X (n + 1)) := - (MeasurableEquiv.IicProdIoc (Nat.le_succ n)).symm.trans - (MeasurableEquiv.prodCongr (MeasurableEquiv.refl _) (MeasurableEquiv.piSingleton n).symm) - -lemma symm_IicSuccProd (n : ℕ) : - (MeasurableEquiv.IicSuccProd X n).symm = - (MeasurableEquiv.prodCongr (MeasurableEquiv.refl _) (MeasurableEquiv.piSingleton n)).trans - (MeasurableEquiv.IicProdIoc (Nat.le_succ n)) := rfl - -@[simp] -lemma MeasurableEquiv.IicSuccProd_apply (n : ℕ) (h : Π i : Iic (n + 1), X i) : - MeasurableEquiv.IicSuccProd X n h = (fun i : Iic n ↦ h ⟨i.1, by grind⟩, h ⟨n + 1, by simp⟩) := - rfl - -lemma MeasurableEquiv.coe_prodCongr {α β γ δ : Type*} - {mα : MeasurableSpace α} {mβ : MeasurableSpace β} - {mγ : MeasurableSpace γ} {mδ : MeasurableSpace δ} - (e₁ : MeasurableEquiv α β) (e₂ : MeasurableEquiv γ δ) : - (MeasurableEquiv.prodCongr e₁ e₂ : (α × γ) → (β × δ)) = Prod.map e₁ e₂ := rfl - -lemma MeasurableEquiv.coe_refl {α : Type*} {mα : MeasurableSpace α} : - (MeasurableEquiv.refl α : α → α) = id := rfl - set_option backward.isDefEq.respectTransparency false in lemma hasLaw_Iic_of_forall_hasCondDistrib' {Y : (n : ℕ) → Ω → X n} (h0 : HasLaw (Y 0) μ₀ P) {N n : ℕ} @@ -157,4 +126,98 @@ lemma hasLaw_trajMeasure [IsFiniteMeasure P] rw [(hasLaw_Iic_of_forall_hasCondDistrib h0 h_condDistrib n).map_eq, trajMeasure_map_frestrictLe] +section FinTraj + +variable {κ' : (n : ℕ) → Kernel (Π i : Fin n, X i) (X n)} [∀ n, IsMarkovKernel (κ' n)] + +/-- Kernels indexed by `Iic n` (as needed for `Kernel.traj`), obtained from kernels indexed by +`Fin n`: the kernel `κ' (n + 1)` on `Π i : Fin (n + 1), X i` is seen as a kernel on +`Π i : Iic n, X i`. -/ +noncomputable +def iicOfFin (κ' : (n : ℕ) → Kernel (Π i : Fin n, X i) (X n)) (n : ℕ) : + Kernel (Π i : Iic n, X i) (X (n + 1)) := + (κ' (n + 1)).comap (MeasurableEquiv.finSuccPiIic X n).symm (by fun_prop) + +instance (n : ℕ) : IsMarkovKernel (iicOfFin κ' n) := by unfold iicOfFin; infer_instance + +/-- Measure on trajectories `Π n, X n` built from kernels `κ' n : Kernel (Π i : Fin n, X i) (X n)` +describing the law of the coordinate `n` given the `n` previous coordinates. +The initial measure is `κ' 0 default`. -/ +noncomputable +def trajMeasureFin (κ' : (n : ℕ) → Kernel (Π i : Fin n, X i) (X n)) [∀ n, IsMarkovKernel (κ' n)] : + Measure (Π n, X n) := + trajMeasure (κ' 0 default) (iicOfFin κ') +deriving IsProbabilityMeasure + +lemma trajMeasureFin_def : + trajMeasureFin κ' = trajMeasure (κ' 0 default) (iicOfFin κ') := rfl + +omit [IsProbabilityMeasure μ₀] in +lemma hasLaw_eval_zero_trajMeasure : HasLaw (fun x ↦ x 0) μ₀ (trajMeasure μ₀ κ) where + aemeasurable := (measurable_pi_apply 0).aemeasurable + map_eq := by + have h := trajMeasure_map_frestrictLe (κ := κ) (μ₀ := μ₀) 0 + rw [partialTraj_self, Measure.id_comp] at h + have h2 := congrArg (Measure.map (MeasurableEquiv.piUnique (fun i : Iic 0 ↦ X i))) h + rw [Measure.map_map (MeasurableEquiv.measurable _) (by fun_prop)] at h2 + exact h2.trans (MeasurableEquiv.map_map_symm _) + +lemma hasLaw_eval_zero_trajMeasureFin : + HasLaw (fun x ↦ x 0) (κ' 0 default) (trajMeasureFin κ') := + hasLaw_eval_zero_trajMeasure + +lemma hasCondDistrib_trajMeasureFin (n : ℕ) : + HasCondDistrib (fun x ↦ x n) (fun x (i : Fin n) ↦ x i) (κ' n) (trajMeasureFin κ') := by + cases n with + | zero => + rw [show (fun (x : Π n, X n) (i : Fin 0) ↦ x i) = fun _ ↦ default from + funext fun _ ↦ Unique.eq_default _] + exact hasLaw_eval_zero_trajMeasureFin.hasCondDistrib_const + | succ n => + have h : HasCondDistrib (fun x ↦ x (n + 1)) (frestrictLe n) (iicOfFin κ' n) + (trajMeasureFin κ') := + ⟨by fun_prop, map_frestrictLe_trajMeasure_compProd_eq_map_trajMeasure.symm⟩ + exact h.comp_right + +/-- Uniqueness of `trajMeasureFin`. -/ +lemma hasLaw_trajMeasureFin [IsProbabilityMeasure P] + {Y : (n : ℕ) → Ω → X n} (hY_meas : ∀ n, Measurable (Y n)) + (h_condDistrib : ∀ n, HasCondDistrib (Y n) (fun ω (i : Fin n) ↦ Y i ω) (κ' n) P) : + HasLaw (fun ω n ↦ Y n ω) (trajMeasureFin κ') P := by + unfold trajMeasureFin + refine hasLaw_trajMeasure hY_meas ?_ fun n ↦ ?_ + · have h := h_condDistrib 0 + rw [show (fun ω (i : Fin 0) ↦ Y i ω) = fun _ ↦ default from + funext fun _ ↦ Unique.eq_default _] at h + exact h.hasLaw_of_const' + · exact (h_condDistrib (n + 1)).measurableEquiv_comp_right (MeasurableEquiv.finSuccPiIic X n) + +lemma eq_trajMeasureFin_map [IsProbabilityMeasure P] + {Y : (n : ℕ) → Ω → X n} (hY_meas : ∀ n, Measurable (Y n)) {N : ℕ} + (h_condDistrib : ∀ n < N, HasCondDistrib (Y n) (fun ω (i : Fin n) ↦ Y i ω) (κ' n) P) : + P.map (fun ω (i : Fin N) ↦ Y i ω) = (trajMeasureFin κ').map (fun x (i : Fin N) ↦ x i) := by + cases N with + | zero => + rw [show (fun ω (i : Fin 0) ↦ Y i ω) = fun _ ↦ default from + funext fun _ ↦ Unique.eq_default _, + show (fun (x : Π n, X n) (i : Fin 0) ↦ x i) = fun _ ↦ default from + funext fun _ ↦ Unique.eq_default _, + Measure.map_const, Measure.map_const, measure_univ, measure_univ] + | succ N => + have h0 : HasLaw (Y 0) (κ' 0 default) P := by + have h := h_condDistrib 0 (by omega) + rw [show (fun ω (i : Fin 0) ↦ Y i ω) = fun _ ↦ default from + funext fun _ ↦ Unique.eq_default _] at h + exact h.hasLaw_of_const' + have h := eq_trajMeasure_map_frestrictLe (κ := iicOfFin κ') h0 (N := N) fun n hn ↦ + (h_condDistrib (n + 1) (by omega)).measurableEquiv_comp_right + (MeasurableEquiv.finSuccPiIic X n) + have h1 : (fun ω (i : Fin (N + 1)) ↦ Y i ω) = + (MeasurableEquiv.finSuccPiIic X N).symm ∘ (fun ω (n : Iic N) ↦ Y n ω) := rfl + rw [h1, ← Measure.map_map (MeasurableEquiv.measurable _) (by fun_prop), h, trajMeasureFin_def, + Measure.map_map (MeasurableEquiv.measurable _) (by fun_prop), + MeasurableEquiv.finSuccPiIic_symm_comp_frestrictLe] + +end FinTraj + end ProbabilityTheory.Kernel diff --git a/LeanMachineLearning/Online/Bandit/Algorithms/ETC.lean b/LeanMachineLearning/Online/Bandit/Algorithms/ETC.lean index 97a1ba05..7b87e5f8 100644 --- a/LeanMachineLearning/Online/Bandit/Algorithms/ETC.lean +++ b/LeanMachineLearning/Online/Bandit/Algorithms/ETC.lean @@ -24,33 +24,38 @@ variable {K : ℕ} section AlgorithmDefinition -/-- Arm pulled by the ETC algorithm at time `n + 1`. -For `n < K * m - 1`, this is arm `(n + 1) % K`. -For `n = K * m - 1`, this is the arm with the highest empirical mean after the exploration phase. -For `n ≥ K * m`, this is the same arm as at time `n`. -/ +/-- Arm pulled by the ETC algorithm at time `n`, as a function of the history before `n`. +For `n < K * m`, this is arm `n % K`. +For `n = K * m`, this is the arm with the highest empirical mean after the exploration phase. +For `n > K * m`, this is the same arm as at time `n - 1`. -/ noncomputable -def ETC.nextArm (hK : 0 < K) (m n : ℕ) (h : Iic n → Fin K × ℝ) : Fin K := +def ETC.nextArm (hK : 0 < K) (m n : ℕ) (h : Fin n → Fin K × ℝ) : Fin K := have : Nonempty (Fin K) := Fin.pos_iff_nonempty.mp hK - if hn : n < K * m - 1 then RoundRobin.nextAction hK n + if hn : n < K * m then RoundRobin.nextAction hK n else - if hn_eq : n = K * m - 1 then argmax (empMean' n h) - else (h ⟨n, by simp⟩).1 + if hn_eq : n = K * m then argmax (empMean' n h) + else (h ⟨n - 1, by omega⟩).1 /-- The next arm pulled by ETC is chosen in a measurable way. -/ @[fun_prop] lemma ETC.measurable_nextArm (hK : 0 < K) (m n : ℕ) : Measurable (nextArm hK m n) := by have : Nonempty (Fin K) := Fin.pos_iff_nonempty.mp hK unfold nextArm - simp only [dite_eq_ite] - refine Measurable.ite (by simp) (by fun_prop) ?_ - refine Measurable.ite (by simp) ?_ (by fun_prop) + by_cases hn : n < K * m + · simp only [hn, ↓reduceDIte] + fun_prop + simp only [hn, ↓reduceDIte] + by_cases hn_eq : n = K * m + · simp only [hn_eq, ↓reduceDIte] + fun_prop + simp only [hn_eq, ↓reduceDIte] fun_prop /-- The Explore-Then-Commit algorithm: deterministic algorithm that chooses the next arm according to `ETC.nextArm`. -/ noncomputable def etcAlgorithm (hK : 0 < K) (m : ℕ) : Algorithm (Fin K) ℝ := - detAlgorithm (ETC.nextArm hK m) (by fun_prop) ⟨0, hK⟩ + detAlgorithm (ETC.nextArm hK m) (by fun_prop) end AlgorithmDefinition @@ -62,14 +67,12 @@ variable {hK : 0 < K} {m : ℕ} {ν : Kernel (Fin K) ℝ} [IsMarkovKernel ν] {A : ℕ → Ω → Fin K} {R : ℕ → Ω → ℝ} {σ2 : ℝ≥0} -/-- Until round `K * m - 1`, the ETC algorithm behaves like the Round-Robin algorithm. -/ +/-- Before round `K * m`, the ETC algorithm behaves like the Round-Robin algorithm. -/ lemma isAlgEnvSeqUntil_roundRobinAlgorithm (h : IsAlgEnvSeq A R (etcAlgorithm hK m) (stationaryEnv ν) P) : - IsAlgEnvSeqUntil A R (roundRobinAlgorithm hK) (stationaryEnv ν) P (K * m - 1) where + IsAlgEnvSeqUntil A R (roundRobinAlgorithm hK) (stationaryEnv ν) P (K * m) where measurable_action := h.measurable_action measurable_feedback := h.measurable_feedback - hasLaw_action_zero := h.hasLaw_action_zero - hasCondDistrib_feedback_zero := h.hasCondDistrib_feedback_zero hasCondDistrib_action n hn := by convert h.hasCondDistrib_action n using 1 simp only [roundRobinAlgorithm, detAlgorithm_policy, etcAlgorithm] @@ -79,12 +82,8 @@ lemma isAlgEnvSeqUntil_roundRobinAlgorithm section AlgorithmBehavior -lemma arm_zero (h : IsAlgEnvSeq A R (etcAlgorithm hK m) (stationaryEnv ν) P) : - A 0 =ᵐ[P] fun _ ↦ ⟨0, hK⟩ := - RoundRobin.action_zero ((isAlgEnvSeqUntil_roundRobinAlgorithm h).mono zero_le) - lemma arm_ae_eq_etcNextArm (h : IsAlgEnvSeq A R (etcAlgorithm hK m) (stationaryEnv ν) P) (n : ℕ) : - A (n + 1) =ᵐ[P] fun ω ↦ nextArm hK m n (history A R n ω) := by + A n =ᵐ[P] fun ω ↦ nextArm hK m n (history A R n ω) := by have : Nonempty (Fin K) := Fin.pos_iff_nonempty.mp hK exact h.action_detAlgorithm_ae_eq n @@ -92,37 +91,29 @@ lemma arm_ae_eq_etcNextArm (h : IsAlgEnvSeq A R (etcAlgorithm hK m) (stationaryE lemma arm_of_lt (h : IsAlgEnvSeq A R (etcAlgorithm hK m) (stationaryEnv ν) P) {n : ℕ} (hn : n < K * m) : A n =ᵐ[P] fun _ ↦ ⟨n % K, Nat.mod_lt _ hK⟩ := - RoundRobin.action_ae_eq n ((isAlgEnvSeqUntil_roundRobinAlgorithm h).mono (by grind)) + RoundRobin.action_ae_eq n ((isAlgEnvSeqUntil_roundRobinAlgorithm h).mono hn) /-- The arm pulled at time `K * m` is the arm with the highest empirical mean after the exploration phase. -/ lemma arm_mul [Nonempty (Fin K)] - (h : IsAlgEnvSeq A R (etcAlgorithm hK m) (stationaryEnv ν) P) (hm : m ≠ 0) : - A (K * m) =ᵐ[P] - fun ω ↦ argmax (empMean' (K * m - 1) (history A R (K * m - 1) ω)) := by - have : K * m = (K * m - 1) + 1 := by - have : 0 < K * m := Nat.mul_pos hK hm.bot_lt - grind - rw [this] - filter_upwards [arm_ae_eq_etcNextArm h (K * m - 1)] with ω hn_eq + (h : IsAlgEnvSeq A R (etcAlgorithm hK m) (stationaryEnv ν) P) : + A (K * m) =ᵐ[P] fun ω ↦ argmax (empMean' (K * m) (history A R (K * m) ω)) := by + filter_upwards [arm_ae_eq_etcNextArm h (K * m)] with ω hn_eq rw [hn_eq, nextArm, dite_eq_right (by simp), dite_eq_left rfl] - exact this ▸ rfl /-- For `n ≥ K * m`, the arm pulled at time `n + 1` is the same as the arm pulled at time `n`. -/ lemma arm_add_one_of_ge (h : IsAlgEnvSeq A R (etcAlgorithm hK m) (stationaryEnv ν) P) - {n : ℕ} (hm : m ≠ 0) (hn : K * m ≤ n) : + {n : ℕ} (hn : K * m ≤ n) : A (n + 1) =ᵐ[P] fun ω ↦ A n ω := by - filter_upwards [arm_ae_eq_etcNextArm h n] with ω hn_eq - rw [hn_eq, nextArm, dite_eq_right (by grind), dite_eq_right] - · rfl - · have : 0 < K * m := Nat.mul_pos hK hm.bot_lt - grind + filter_upwards [arm_ae_eq_etcNextArm h (n + 1)] with ω hn_eq + rw [hn_eq, nextArm, dite_eq_right (by grind), dite_eq_right (by grind)] + rfl /-- For `n ≥ K * m`, the arm pulled at time `n` is the same as the arm pulled at time `K * m`. -/ lemma arm_of_ge (h : IsAlgEnvSeq A R (etcAlgorithm hK m) (stationaryEnv ν) P) - {n : ℕ} (hm : m ≠ 0) (hn : K * m ≤ n) : + {n : ℕ} (hn : K * m ≤ n) : A n =ᵐ[P] A (K * m) := by - have h_ae n : K * m ≤ n → A (n + 1) =ᵐ[P] fun ω ↦ A n ω := arm_add_one_of_ge h hm + have h_ae n : K * m ≤ n → A (n + 1) =ᵐ[P] fun ω ↦ A n ω := arm_add_one_of_ge h simp_rw [Filter.EventuallyEq, ← ae_all_iff] at h_ae filter_upwards [h_ae] with ω h_ae induction n, hn using Nat.le_induction with @@ -135,22 +126,22 @@ lemma pullCount_mul (h : IsAlgEnvSeq A R (etcAlgorithm hK m) (stationaryEnv ν) RoundRobin.pullCount_mul m (isAlgEnvSeqUntil_roundRobinAlgorithm h) a lemma pullCount_add_one_of_ge (h : IsAlgEnvSeq A R (etcAlgorithm hK m) (stationaryEnv ν) P) - (a : Fin K) (hm : m ≠ 0) {n : ℕ} (hn : K * m ≤ n) : + (a : Fin K) {n : ℕ} (hn : K * m ≤ n) : pullCount A a (n + 1) =ᵐ[P] fun ω ↦ pullCount A a n ω + {ω' | A (K * m) ω' = a}.indicator (fun _ ↦ 1) ω := by simp_rw [Filter.EventuallyEq, pullCount_add_one] - filter_upwards [arm_of_ge h hm hn] with ω h_arm + filter_upwards [arm_of_ge h hn] with ω h_arm congr 3 /-- For `n ≥ K * m`, the number of pulls of each arm `a` at time `n` is equal to `m` plus `n - K * m` if arm `a` is the best arm after the exploration phase. -/ lemma pullCount_of_ge (h : IsAlgEnvSeq A R (etcAlgorithm hK m) (stationaryEnv ν) P) - (a : Fin K) (hm : m ≠ 0) {n : ℕ} (hn : K * m ≤ n) : + (a : Fin K) {n : ℕ} (hn : K * m ≤ n) : pullCount A a n =ᵐ[P] fun ω ↦ m + (n - K * m) * {ω' | A (K * m) ω' = a}.indicator (fun _ ↦ 1) ω := by have h_ae n : K * m ≤ n → pullCount A a (n + 1) =ᵐ[P] fun ω ↦ pullCount A a n ω + {ω' | A (K * m) ω' = a}.indicator (fun _ ↦ 1) ω := - pullCount_add_one_of_ge h a hm + pullCount_add_one_of_ge h a simp_rw [Filter.EventuallyEq, ← ae_all_iff] at h_ae have h_ae_Km : pullCount A a (K * m) =ᵐ[P] fun _ ↦ m := pullCount_mul h a filter_upwards [h_ae_Km, h_ae] with ω h_Km h_ae @@ -167,15 +158,13 @@ lemma sumRewards_bestArm_le_of_arm_mul_eq [Nonempty (Fin K)] (h : IsAlgEnvSeq A R (etcAlgorithm hK m) (stationaryEnv ν) P) (a : Fin K) (hm : m ≠ 0) : ∀ᵐ h ∂P, A (K * m) h = a → sumRewards A R (bestArm ν) (K * m) h ≤ sumRewards A R a (K * m) h := by - filter_upwards [arm_mul h hm, pullCount_mul h a, pullCount_mul h (bestArm ν)] + filter_upwards [arm_mul h, pullCount_mul h a, pullCount_mul h (bestArm ν)] with h h_arm ha h_best h_eq - have h_max := isMaxOn_argmax - (empMean' (K * m - 1) (history A R (K * m - 1) h)) (bestArm ν) + have h_max := isMaxOn_argmax (empMean' (K * m) (history A R (K * m) h)) (bestArm ν) rw [← h_arm, h_eq] at h_max rw [sumRewards_eq_pullCount_mul_empMean, sumRewards_eq_pullCount_mul_empMean, ha, h_best] · gcongr - have : 0 < K * m := Nat.mul_pos hK hm.bot_lt - rwa [empMean_eq_empMean' this.ne', empMean_eq_empMean' this.ne'] + rwa [empMean_eq_empMean', empMean_eq_empMean'] · simp [ha, hm] · simp [h_best, hm] @@ -225,7 +214,7 @@ lemma expectation_pullCount_le [Nonempty (Fin K)] have hA := h.measurable_action have : (fun ω ↦ (pullCount A a n ω : ℝ)) =ᵐ[P] fun ω ↦ m + (n - K * m) * {ω' | A (K * m) ω' = a}.indicator (fun _ ↦ 1) ω := by - filter_upwards [pullCount_of_ge h a hm hn] with ω h + filter_upwards [pullCount_of_ge h a hn] with ω h simp only [h, Set.indicator_apply, Set.mem_ofPred_eq, mul_ite, mul_one, mul_zero, Nat.cast_add, Nat.cast_ite, CharP.cast_eq_zero, add_right_inj] norm_cast diff --git a/LeanMachineLearning/Online/Bandit/Algorithms/Regret/BayesRegretTS.lean b/LeanMachineLearning/Online/Bandit/Algorithms/Regret/BayesRegretTS.lean index 53643f14..8508a313 100644 --- a/LeanMachineLearning/Online/Bandit/Algorithms/Regret/BayesRegretTS.lean +++ b/LeanMachineLearning/Online/Bandit/Algorithms/Regret/BayesRegretTS.lean @@ -24,7 +24,7 @@ and properties are also given in this file. bound `l : ℝ`, reward upper bound `u : ℝ`, sub-Gaussian variance proxy `σ2 : ℝ`, confidence parameter `δ : ℝ`, action `a : Fin K`, and time `n : ℕ`. * `ucb' n h l u σ2 δ a`: clipped upper confidence bound for action `a : Fin K` at time `n : ℕ` given - the history `h : Iic n → Fin K × ℝ` (rather than the entire sequences of actions and rewards). + the history `h : Fin n → Fin K × ℝ` (rather than the entire sequences of actions and rewards). ## Main results @@ -93,21 +93,19 @@ lemma integrable_uncurry_ucb_comp [MeasurableSpace Ω] (hA : ∀ t, Measurable ( /-- Clipped upper confidence bound (history-based version). -/ noncomputable -def ucb' (n : ℕ) (h : Iic n → Fin K × ℝ) (l u σ2 δ : ℝ) (a : Fin K) : ℝ := +def ucb' (n : ℕ) (h : Fin n → Fin K × ℝ) (l u σ2 δ : ℝ) (a : Fin K) : ℝ := if pullCount' n h a = 0 then u else max l (min u (empMean' n h a + √(2 * σ2 * Real.log (1 / δ) / (pullCount' n h a)))) @[fun_prop] lemma measurable_uncurry_ucb' {n : ℕ} : - Measurable (fun p : (Iic n → Fin K × ℝ) × Fin K ↦ ucb' n p.1 l u σ2 δ p.2) := + Measurable (fun p : (Fin n → Fin K × ℝ) × Fin K ↦ ucb' n p.1 l u σ2 δ p.2) := Measurable.ite (by measurability) (by fun_prop) (by fun_prop) -lemma ucb_succ_eq_ucb' {a : Fin K} {n : ℕ} {ω : Ω} : - ucb A R l u σ2 δ a (n + 1) ω = ucb' n (history A R n ω) l u σ2 δ a := by - have hp : pullCount A a (n + 1) ω = pullCount' n (history A R n ω) a := - pullCount_add_one_eq_pullCount' - have he : empMean A R a (n + 1) ω = empMean' n (history A R n ω) a := - empMean_add_one_eq_empMean' +lemma ucb_eq_ucb' {a : Fin K} {n : ℕ} {ω : Ω} : + ucb A R l u σ2 δ a n ω = ucb' n (history A R n ω) l u σ2 δ a := by + have hp : pullCount A a n ω = pullCount' n (history A R n ω) a := pullCount_eq_pullCount' + have he : empMean A R a n ω = empMean' n (history A R n ω) a := empMean_eq_empMean' rw [ucb, ucb', hp, he] /-- Helper for `sum_ucb_sub_mean_le`. -/ @@ -323,21 +321,18 @@ lemma integral_ucb_action_eq_integral_ucb_bestAction (hK : 0 < K) have := h.measurable_action have := h.measurable_param have := h.measurable_feedback - by_cases hn : n = 0 - · simp [hn] - obtain ⟨n, rfl⟩ := Nat.exists_eq_succ_of_ne_zero hn - let uc (ha : (Iic n → Fin K × ℝ) × Fin K) := ucb' n ha.1 l u σ2 δ ha.2 + let uc (ha : (Fin n → Fin K × ℝ) × Fin K) := ucb' n ha.1 l u σ2 δ ha.2 calc - _ = P[fun ω ↦ uc (history A R n ω, A (n + 1) ω)] := by - simp_rw [uc, ucb_succ_eq_ucb'] - _ = ∫ ha, uc ha ∂P.map (fun ω ↦ (history A R n ω, A (n + 1) ω)) := by + _ = P[fun ω ↦ uc (history A R n ω, A n ω)] := by + simp_rw [uc, ucb_eq_ucb'] + _ = ∫ ha, uc ha ∂P.map (fun ω ↦ (history A R n ω, A n ω)) := by rw [← integral_map (by fun_prop) (by fun_prop)] _ = ∫ ha, uc ha ∂P.map (fun ω ↦ (history A R n ω, bestAction κ E ω)) := by rw [← compProd_map_condDistrib (by fun_prop), ← compProd_map_condDistrib (by fun_prop), Measure.compProd_congr (hasCondDistrib_action hK h n).condDistrib_eq] - _ = P[fun ω ↦ ucb A R l u σ2 δ (bestAction κ E ω) (n + 1) ω] := by + _ = P[fun ω ↦ ucb A R l u σ2 δ (bestAction κ E ω) n ω] := by rw [integral_map (by fun_prop) (by fun_prop)] - simp_rw [uc, ucb_succ_eq_ucb'] + simp_rw [uc, ucb_eq_ucb'] lemma integral_regret_eq_add (hK : 0 < K) (h : IsBayesAlgEnvSeq Q κ (tsAlgorithm hK Q κ) E A R P) (hm : ∀ e a, (κ (e, a))[id] ∈ (Set.Icc l u)) (n : ℕ) : diff --git a/LeanMachineLearning/Online/Bandit/Algorithms/TS.lean b/LeanMachineLearning/Online/Bandit/Algorithms/TS.lean index 586502cd..d457ddbe 100644 --- a/LeanMachineLearning/Online/Bandit/Algorithms/TS.lean +++ b/LeanMachineLearning/Online/Bandit/Algorithms/TS.lean @@ -47,7 +47,7 @@ under the posterior over environments given the history so far. The posterior under a uniform algorithm is used to avoid a circular definition. -/ noncomputable def TS.policy (hK : 0 < K) (Q : Measure 𝓔) [IsProbabilityMeasure Q] (κ : Kernel (𝓔 × Fin K) ℝ) - [IsMarkovKernel κ] (n : ℕ) : Kernel (Iic n → (Fin K) × ℝ) (Fin K) := + [IsMarkovKernel κ] (n : ℕ) : Kernel (Fin n → (Fin K) × ℝ) (Fin K) := have : Nonempty (Fin K) := Fin.pos_iff_nonempty.mp hK (IT.bayesTrajMeasurePosterior Q κ uniformAlgorithm n).map (bestAction κ id) @@ -55,17 +55,6 @@ instance {hK : 0 < K} {Q : Measure 𝓔} [IsProbabilityMeasure Q] {κ : Kernel ( [IsMarkovKernel κ] {n : ℕ} : IsMarkovKernel (TS.policy hK Q κ n) := Kernel.IsMarkovKernel.map _ (by fun_prop) -/-- The initial action is sampled according to its probability of being optimal under the prior over -environments. -/ -noncomputable -def TS.initialPolicy (hK : 0 < K) (Q : Measure 𝓔) (κ : Kernel (𝓔 × Fin K) ℝ) : Measure (Fin K) := - have : Nonempty (Fin K) := Fin.pos_iff_nonempty.mp hK - Q.map (bestAction κ id) - -instance {hK : 0 < K} {Q : Measure 𝓔} [IsProbabilityMeasure Q] {κ : Kernel (𝓔 × Fin K) ℝ} : - IsProbabilityMeasure (TS.initialPolicy hK Q κ) := - Measure.isProbabilityMeasure_map (by fun_prop) - /-- The Thompson sampling algorithm with actions in `Fin K`, where `Q : Measure 𝓔` is a prior distribution over parameters, and `κ : Kernel (𝓔 × Fin K) ℝ` is a Markov kernel that defines the stationary environment `stationaryEnv (κ.sectR e)` that corresponds to a parameter `e : 𝓔`. @@ -77,7 +66,6 @@ noncomputable def tsAlgorithm (hK : 0 < K) (Q : Measure 𝓔) [IsProbabilityMeasure Q] (κ : Kernel (𝓔 × Fin K) ℝ) [IsMarkovKernel κ] : Algorithm (Fin K) ℝ where policy := TS.policy hK Q κ - p0 := TS.initialPolicy hK Q κ end Algorithm @@ -88,15 +76,24 @@ variable {E : Ω → 𝓔} {A : ℕ → Ω → Fin K} {R : ℕ → Ω → ℝ} variable {Q : Measure 𝓔} [IsProbabilityMeasure Q] {κ : Kernel (𝓔 × Fin K) ℝ} [IsMarkovKernel κ] variable {P : Measure Ω} [IsProbabilityMeasure P] +/-- The first action of Thompson sampling is sampled according to its probability of being optimal +under the prior over environments. -/ +lemma TS.p0_tsAlgorithm (hK : 0 < K) : + (tsAlgorithm hK Q κ).p0 = Q.map (bestAction κ id) := by + rw [Algorithm.p0_def] + change TS.policy hK Q κ 0 default = _ + rw [TS.policy, Kernel.map_apply _ (by fun_prop), IT.bayesTrajMeasurePosterior_zero, + Kernel.const_apply] + /-- If Thompson sampling has the correct prior over environments, then the conditional distribution of the next action given the history so far is equal to the conditional distribution of the best action given the history so far. -/ lemma TS.hasCondDistrib_action (hK : 0 < K) (h : IsBayesAlgEnvSeq Q κ (tsAlgorithm hK Q κ) E A R P) (n : ℕ) : - HasCondDistrib (A (n + 1)) (history A R n) + HasCondDistrib (A n) (history A R n) (condDistrib (bestAction κ E) (history A R n) P) P where aemeasurable := ((measurable_history h.measurable_action h.measurable_feedback n).prodMk - (h.measurable_action (n + 1))).aemeasurable + (h.measurable_action n)).aemeasurable map_eq := by have hm : Measurable (bestAction κ id) := by fun_prop rw [(h.hasCondDistrib_action' n).map_eq] diff --git a/LeanMachineLearning/Online/Bandit/Algorithms/UCB.lean b/LeanMachineLearning/Online/Bandit/Algorithms/UCB.lean index 19f74348..94ada521 100644 --- a/LeanMachineLearning/Online/Bandit/Algorithms/UCB.lean +++ b/LeanMachineLearning/Online/Bandit/Algorithms/UCB.lean @@ -28,15 +28,15 @@ section Algorithm /-- The exploration bonus of the UCB algorithm, which corresponds to the width of a confidence interval. -/ -noncomputable def ucbWidth' (c : ℝ) (n : ℕ) (h : Iic n → Fin K × ℝ) (a : Fin K) : ℝ := - √(2 * c * log (n + 2) / pullCount' n h a) +noncomputable def ucbWidth' (c : ℝ) (n : ℕ) (h : Fin n → Fin K × ℝ) (a : Fin K) : ℝ := + √(2 * c * log (n + 1) / pullCount' n h a) open Classical in -/-- Arm pulled by the UCB algorithm at time `n + 1`. -/ +/-- Arm pulled by the UCB algorithm at time `n`, as a function of the history before `n`. -/ noncomputable -def UCB.nextArm (hK : 0 < K) (c : ℝ) (n : ℕ) (h : Iic n → Fin K × ℝ) : Fin K := +def UCB.nextArm (hK : 0 < K) (c : ℝ) (n : ℕ) (h : Fin n → Fin K × ℝ) : Fin K := have : Nonempty (Fin K) := Fin.pos_iff_nonempty.mp hK - if n < K - 1 then RoundRobin.nextAction hK n else + if n < K then RoundRobin.nextAction hK n else argmax (fun a ↦ empMean' n h a + ucbWidth' c n h a) @[fun_prop] @@ -49,7 +49,7 @@ lemma UCB.measurable_nextArm (hK : 0 < K) (c : ℝ) (n : ℕ) : Measurable (next /-- The UCB algorithm. -/ noncomputable def ucbAlgorithm (hK : 0 < K) (c : ℝ) : Algorithm (Fin K) ℝ := - detAlgorithm (UCB.nextArm hK c) (by fun_prop) ⟨0, hK⟩ + detAlgorithm (UCB.nextArm hK c) (by fun_prop) end Algorithm namespace UCB @@ -60,14 +60,12 @@ variable {hK : 0 < K} {c : ℝ} {ν : Kernel (Fin K) ℝ} [IsMarkovKernel ν] {A : ℕ → Ω → Fin K} {R : ℕ → Ω → ℝ} {σ2 : ℝ≥0} {n : ℕ} {ω : Ω} -/-- Until round `K - 1`, the UCB algorithm behaves like the Round-Robin algorithm. -/ +/-- Before round `K`, the UCB algorithm behaves like the Round-Robin algorithm. -/ lemma isAlgEnvSeqUntil_roundRobinAlgorithm (h : IsAlgEnvSeq A R (ucbAlgorithm hK c) (stationaryEnv ν) P) : - IsAlgEnvSeqUntil A R (roundRobinAlgorithm hK) (stationaryEnv ν) P (K - 1) where + IsAlgEnvSeqUntil A R (roundRobinAlgorithm hK) (stationaryEnv ν) P K where measurable_action := h.measurable_action measurable_feedback := h.measurable_feedback - hasLaw_action_zero := h.hasLaw_action_zero - hasCondDistrib_feedback_zero := h.hasCondDistrib_feedback_zero hasCondDistrib_action n hn := by convert h.hasCondDistrib_action n using 1 simp only [roundRobinAlgorithm, detAlgorithm_policy, ucbAlgorithm] @@ -88,53 +86,43 @@ lemma measurable_ucbWidth (hA : ∀ n, Measurable (A n)) (c : ℝ) (a : Fin K) : unfold ucbWidth fun_prop -lemma ucbWidth_eq_ucbWidth' (c : ℝ) (a : Fin K) (n : ℕ) (ω : Ω) (hn : n ≠ 0) : - ucbWidth A c a n ω = ucbWidth' c (n - 1) (history A R (n - 1) ω) a := by - simp only [ucbWidth, pullCount_eq_pullCount' (A := A) (R' := R) hn, Nat.cast_nonneg, sqrt_div', - ucbWidth'] - congr 4 - norm_cast - grind +lemma ucbWidth_eq_ucbWidth' (c : ℝ) (a : Fin K) (n : ℕ) (ω : Ω) : + ucbWidth A c a n ω = ucbWidth' c n (history A R n ω) a := by + rw [ucbWidth, ucbWidth', pullCount_eq_pullCount' (A := A) (R' := R)] + rfl lemma arm_zero (h : IsAlgEnvSeq A R (ucbAlgorithm hK c) (stationaryEnv ν) P) : A 0 =ᵐ[P] fun _ ↦ ⟨0, hK⟩ := - RoundRobin.action_zero ((isAlgEnvSeqUntil_roundRobinAlgorithm h).mono zero_le) + RoundRobin.action_zero ((isAlgEnvSeqUntil_roundRobinAlgorithm h).mono hK) lemma arm_ae_eq_ucbNextArm (h : IsAlgEnvSeq A R (ucbAlgorithm hK c) (stationaryEnv ν) P) (n : ℕ) : - A (n + 1) =ᵐ[P] fun ω ↦ nextArm hK c n (history A R n ω) := by + A n =ᵐ[P] fun ω ↦ nextArm hK c n (history A R n ω) := by have : Nonempty (Fin K) := Fin.pos_iff_nonempty.mp hK exact h.action_detAlgorithm_ae_eq n lemma arm_ae_all_eq (h : IsAlgEnvSeq A R (ucbAlgorithm hK c) (stationaryEnv ν) P) : - ∀ᵐ h ∂P, A 0 h = ⟨0, hK⟩ ∧ ∀ n, A (n + 1) h = nextArm hK c n (history A R n h) := by - rw [eventually_and, ae_all_iff] - exact ⟨arm_zero h, arm_ae_eq_ucbNextArm h⟩ + ∀ᵐ h ∂P, ∀ n, A n h = nextArm hK c n (history A R n h) := + ae_all_iff.mpr (arm_ae_eq_ucbNextArm h) lemma ucbIndex_le_ucbIndex_arm (h : IsAlgEnvSeq A R (ucbAlgorithm hK c) (stationaryEnv ν) P) (a : Fin K) (hn : K ≤ n) : ∀ᵐ h ∂P, empMean A R a n h + ucbWidth A c a n h ≤ empMean A R (A n h) n h + ucbWidth A c (A n h) n h := by - filter_upwards [arm_ae_eq_ucbNextArm h (n - 1)] with h h_arm - have : n - 1 + 1 = n := by grind - have h_not_lt : ¬ n - 1 < K - 1 := by grind - simp only [this, nextArm, h_not_lt, ↓reduceIte] at h_arm + filter_upwards [arm_ae_eq_ucbNextArm h n] with h h_arm + have h_not_lt : ¬ n < K := by grind + simp only [nextArm, h_not_lt, ↓reduceIte] at h_arm have : Nonempty (Fin K) := Fin.pos_iff_nonempty.mp hK - simp_rw [h_arm, empMean_eq_empMean' (by grind : n ≠ 0), - ucbWidth_eq_ucbWidth' (A := A) (R := R) _ _ _ _ (by grind : n ≠ 0)] - exact isMaxOn_argmax (fun a ↦ empMean' (n - 1) (history A R (n - 1) h) a - + ucbWidth' c (n - 1) (history A R (n - 1) h) a) _ + simp_rw [h_arm, empMean_eq_empMean', ucbWidth_eq_ucbWidth' (A := A) (R := R)] + exact isMaxOn_argmax (fun a ↦ empMean' n (history A R n h) a + + ucbWidth' c n (history A R n h) a) _ lemma forall_arm_eq_mod_of_lt (h : IsAlgEnvSeq A R (ucbAlgorithm hK c) (stationaryEnv ν) P) : ∀ᵐ h ∂P, ∀ n < K, A n h = ⟨n % K, Nat.mod_lt _ hK⟩ := by simp_rw [ae_all_iff] intro n hn - induction n with - | zero => exact arm_zero h - | succ n _ => - filter_upwards [arm_ae_eq_ucbNextArm h n] with h h_eq - rw [h_eq, nextArm, ite_eq_left] - · rfl - · grind + filter_upwards [arm_ae_eq_ucbNextArm h n] with h h_eq + rw [h_eq] + simp only [nextArm, hn, ↓reduceIte, RoundRobin.nextAction] lemma forall_ucbIndex_le_ucbIndex_arm (h : IsAlgEnvSeq A R (ucbAlgorithm hK c) (stationaryEnv ν) P) (a : Fin K) : diff --git a/LeanMachineLearning/Online/Bandit/ArrayProbSpace.lean b/LeanMachineLearning/Online/Bandit/ArrayProbSpace.lean index 78c77ad4..19459e87 100644 --- a/LeanMachineLearning/Online/Bandit/ArrayProbSpace.lean +++ b/LeanMachineLearning/Online/Bandit/ArrayProbSpace.lean @@ -213,27 +213,21 @@ lemma measurable_truncRow [DecidableEq 𝓐] (a : 𝓐) (m : ℕ) : variable [Nonempty 𝓐] [StandardBorelSpace 𝓐] -/-- The initial action is the image of a uniform random variable by this function. -/ -noncomputable -def initAlgFunction (alg : Algorithm 𝓐 𝓡) : I → 𝓐 := - (Measure.exists_measurable_map_eq alg.p0).choose - -lemma initAlgFunction_map (alg : Algorithm 𝓐 𝓡) : volume.map (initAlgFunction alg) = alg.p0 := - (Measure.exists_measurable_map_eq alg.p0).choose_spec.2 - -@[fun_prop] -lemma measurable_initAlgFunction (alg : Algorithm 𝓐 𝓡) : - Measurable (initAlgFunction alg) := (Measure.exists_measurable_map_eq alg.p0).choose_spec.1 /-- The next action is the image of the history and a uniform random variable by this function. -/ noncomputable def algFunction (alg : Algorithm 𝓐 𝓡) (n : ℕ) : - (Iic n → 𝓐 × 𝓡) → I → 𝓐 := + (Fin n → 𝓐 × 𝓡) → I → 𝓐 := (Kernel.exists_measurable_map_eq_unitInterval (alg.policy n)).choose -lemma algFunction_map (alg : Algorithm 𝓐 𝓡) (n : ℕ) (h : Iic n → 𝓐 × 𝓡) : - volume.map (algFunction alg n h) = alg.policy n h := +lemma algFunction_map (alg : Algorithm 𝓐 𝓡) (n : ℕ) (h : Fin n → 𝓐 × 𝓡) : + volume.map (algFunction alg n h) = alg.policy n h := (Kernel.exists_measurable_map_eq_unitInterval (alg.policy n)).choose_spec.2 h +/-- The initial action is the image of a uniform random variable by `algFunction alg 0 default`. -/ +lemma algFunction_zero_map (alg : Algorithm 𝓐 𝓡) : + volume.map (algFunction alg 0 default) = alg.p0 := + algFunction_map alg 0 default + @[fun_prop] lemma measurable_algFunction (alg : Algorithm 𝓐 𝓡) (n : ℕ) : Measurable (Function.uncurry (algFunction alg n)) := @@ -245,102 +239,103 @@ variable [Nonempty 𝓐] [StandardBorelSpace 𝓐] section HistoryActionReward -/-- History of actions and rewards up to time `n` in the array model. -/ +/-- History of actions and rewards before time `n` in the array model. -/ noncomputable -def hist [DecidableEq 𝓐] (alg : Algorithm 𝓐 𝓡) (ω : probSpace 𝓐 𝓡) : (n : ℕ) → Iic n → 𝓐 × 𝓡 -| 0 => fun _ ↦ (initAlgFunction alg (ω.1 0), ω.2 0 (initAlgFunction alg (ω.1 0))) +def hist [DecidableEq 𝓐] (alg : Algorithm 𝓐 𝓡) (ω : probSpace 𝓐 𝓡) : (n : ℕ) → Fin n → 𝓐 × 𝓡 +| 0 => default | n + 1 => - let hn : Iic n → 𝓐 × 𝓡 := hist alg ω n - let a : 𝓐 := algFunction alg n hn (ω.1 (n + 1)) - fun i ↦ if hin : i ≤ n then hn ⟨i, by simp [hin]⟩ else (a, ω.2 (pullCount' n hn a) a) + let hn : Fin n → 𝓐 × 𝓡 := hist alg ω n + let a : 𝓐 := algFunction alg n hn (ω.1 n) + Fin.snoc hn (a, ω.2 (pullCount' n hn a) a) @[simp] lemma hist_zero [DecidableEq 𝓐] (alg : Algorithm 𝓐 𝓡) (ω : probSpace 𝓐 𝓡) : - hist alg ω 0 = fun _ ↦ (initAlgFunction alg (ω.1 0), ω.2 0 (initAlgFunction alg (ω.1 0))) := - rfl + hist alg ω 0 = default := rfl lemma hist_add_one [DecidableEq 𝓐] (alg : Algorithm 𝓐 𝓡) (ω : probSpace 𝓐 𝓡) (n : ℕ) : - let a : 𝓐 := algFunction alg n (hist alg ω n) (ω.1 (n + 1)) hist alg ω (n + 1) = - fun (i : Iic (n + 1)) ↦ if hin : i ≤ n then hist alg ω n ⟨i, by simp [hin]⟩ - else (a, ω.2 (pullCount' n (hist alg ω n) a) a) := rfl + Fin.snoc (hist alg ω n) (algFunction alg n (hist alg ω n) (ω.1 n), + ω.2 (pullCount' n (hist alg ω n) (algFunction alg n (hist alg ω n) (ω.1 n))) + (algFunction alg n (hist alg ω n) (ω.1 n))) := rfl -lemma hist_eq [DecidableEq 𝓐] (alg : Algorithm 𝓐 𝓡) (ω : probSpace 𝓐 𝓡) (n : ℕ) : - hist alg ω n = fun i : Iic n ↦ hist alg ω i ⟨i.1, by simp⟩ := by - induction n with - | zero => - ext i : 1 - simp only [hist] - rw [Unique.eq_default i] - simp [coe_default_Iic_zero] - | succ n hn => - ext i : 1 - by_cases hin : i ≤ n - · rw [hist_add_one] - simp only [hin, ↓reduceDIte] - rw [funext_iff] at hn - simp_rw [hn] - · grind - -lemma hist_add_one_eq_IicSuccProd' [DecidableEq 𝓐] (alg : Algorithm 𝓐 𝓡) (ω : probSpace 𝓐 𝓡) +lemma hist_add_one_eq_finSuccProd' [DecidableEq 𝓐] (alg : Algorithm 𝓐 𝓡) (ω : probSpace 𝓐 𝓡) (n : ℕ) : - let a : 𝓐 := algFunction alg n (hist alg ω n) (ω.1 (n + 1)) hist alg ω (n + 1) = - (MeasurableEquiv.IicSuccProd (fun _ ↦ 𝓐 × 𝓡) n).symm - (hist alg ω n, (a, ω.2 (pullCount' n (hist alg ω n) a) a)) := by - intro a - rw [hist_add_one] - ext i : 1 - simp only [Kernel.symm_IicSuccProd, MeasurableEquiv.prodCongr, MeasurableEquiv.refl_toEquiv, - MeasurableEquiv.piSingleton, eq_rec_constant, MeasurableEquiv.IicProdIoc, - MeasurableEquiv.trans_apply, MeasurableEquiv.coe_mk, Equiv.prodCongr_apply, Equiv.coe_refl, - Equiv.coe_fn_mk, Prod.map_apply, id_eq] + (MeasurableEquiv.finSuccProd (𝓐 × 𝓡) n).symm + (hist alg ω n, (algFunction alg n (hist alg ω n) (ω.1 n), + ω.2 (pullCount' n (hist alg ω n) (algFunction alg n (hist alg ω n) (ω.1 n))) + (algFunction alg n (hist alg ω n) (ω.1 n)))) := by + rw [MeasurableEquiv.finSuccProd_symm_apply] rfl /-- Action taken at time `n` in the array model. -/ noncomputable def action [DecidableEq 𝓐] (alg : Algorithm 𝓐 𝓡) (n : ℕ) (ω : probSpace 𝓐 𝓡) : 𝓐 := - (hist alg ω n ⟨n, by simp⟩).1 + algFunction alg n (hist alg ω n) (ω.1 n) -lemma action_zero [DecidableEq 𝓐] (alg : Algorithm 𝓐 𝓡) : - action alg 0 = fun ω ↦ initAlgFunction alg (ω.1 0) := by - ext - simp [action, hist_zero] +lemma action_eq [DecidableEq 𝓐] (alg : Algorithm 𝓐 𝓡) (n : ℕ) : + action alg n = fun ω ↦ algFunction alg n (hist alg ω n) (ω.1 n) := rfl -lemma action_add_one_eq [DecidableEq 𝓐] (alg : Algorithm 𝓐 𝓡) (n : ℕ) : - action alg (n + 1) = fun ω ↦ algFunction alg n (hist alg ω n) (ω.1 (n + 1)) := by - ext ω - rw [action, hist_add_one] - simp only [add_le_iff_nonpos_right, nonpos_iff_eq_zero, one_ne_zero, ↓reduceDIte] +lemma action_zero [DecidableEq 𝓐] (alg : Algorithm 𝓐 𝓡) : + action alg 0 = fun ω ↦ algFunction alg 0 default (ω.1 0) := rfl /-- Reward received at time `n` in the array model. -/ noncomputable def reward [DecidableEq 𝓐] (alg : Algorithm 𝓐 𝓡) (n : ℕ) (ω : probSpace 𝓐 𝓡) : 𝓡 := - (hist alg ω n ⟨n, by simp⟩).2 + (hist alg ω (n + 1) (Fin.last n)).2 + +lemma reward_eq' [DecidableEq 𝓐] (alg : Algorithm 𝓐 𝓡) (n : ℕ) (ω : probSpace 𝓐 𝓡) : + reward alg n ω = ω.2 (pullCount' n (hist alg ω n) (action alg n ω)) (action alg n ω) := by + change (hist alg ω (n + 1) (Fin.last n)).2 = _ + rw [hist_add_one, Fin.snoc_last] + rfl + +lemma hist_succ [DecidableEq 𝓐] (alg : Algorithm 𝓐 𝓡) (ω : probSpace 𝓐 𝓡) (n : ℕ) : + hist alg ω (n + 1) = Fin.snoc (hist alg ω n) (action alg n ω, reward alg n ω) := by + rw [hist_add_one, reward_eq'] + rfl -lemma reward_zero [DecidableEq 𝓐] (alg : Algorithm 𝓐 𝓡) : - reward alg 0 = fun ω ↦ ω.2 0 (action alg 0 ω) := by - ext - simp [reward, hist_zero, action_zero] +lemma hist_succ_eq_finSuccProd [DecidableEq 𝓐] (alg : Algorithm 𝓐 𝓡) (ω : probSpace 𝓐 𝓡) + (n : ℕ) : + hist alg ω (n + 1) = + (MeasurableEquiv.finSuccProd (𝓐 × 𝓡) n).symm + (hist alg ω n, (action alg n ω, reward alg n ω)) := by + rw [MeasurableEquiv.finSuccProd_symm_apply, hist_succ] + +lemma hist_apply_last [DecidableEq 𝓐] (alg : Algorithm 𝓐 𝓡) (ω : probSpace 𝓐 𝓡) (n : ℕ) : + hist alg ω (n + 1) (Fin.last n) = (action alg n ω, reward alg n ω) := by + rw [hist_succ, Fin.snoc_last] -lemma reward_add_one [DecidableEq 𝓐] (alg : Algorithm 𝓐 𝓡) (n : ℕ) : - reward alg (n + 1) = - fun ω ↦ ω.2 (pullCount' n (hist alg ω n) (action alg (n + 1) ω)) (action alg (n + 1) ω) := by +lemma hist_eq [DecidableEq 𝓐] (alg : Algorithm 𝓐 𝓡) (ω : probSpace 𝓐 𝓡) (n : ℕ) : + hist alg ω n = fun i : Fin n ↦ (action alg i ω, reward alg i ω) := by + induction n with + | zero => exact Subsingleton.elim _ _ + | succ n hn => + ext i : 1 + refine Fin.lastCases ?_ (fun i ↦ ?_) i + · rw [hist_apply_last] + simp + · rw [hist_succ, Fin.snoc_castSucc, hn] + simp + +/-- The history in the array model is the history of the action and reward processes. -/ +lemma hist_eq_history [DecidableEq 𝓐] (alg : Algorithm 𝓐 𝓡) (ω : probSpace 𝓐 𝓡) (n : ℕ) : + hist alg ω n = history (action alg) (reward alg) n ω := hist_eq alg ω n + +lemma pullCount_action_eq [DecidableEq 𝓐] (alg : Algorithm 𝓐 𝓡) (a : 𝓐) (n : ℕ) + (ω : probSpace 𝓐 𝓡) : + pullCount (action alg) a n ω = pullCount' n (hist alg ω n) a := by + rw [pullCount_eq_pullCount' (R' := reward alg), hist_eq] + +lemma pullCount_action_eq_comp [DecidableEq 𝓐] (alg : Algorithm 𝓐 𝓡) (a : 𝓐) (n : ℕ) : + pullCount (action alg) a n = (fun h ↦ pullCount' n h a) ∘ (hist alg · n) := by ext ω - simp [reward, hist_add_one, action_add_one_eq] + exact pullCount_action_eq alg a n ω lemma reward_eq [DecidableEq 𝓐] (alg : Algorithm 𝓐 𝓡) (n : ℕ) : reward alg n = fun ω ↦ ω.2 (pullCount (action alg) (action alg n ω) n ω) (action alg n ω) := by - cases n with - | zero => ext; simp [reward_zero, action_zero] - | succ n => - ext ω - rw [reward, hist_add_one] - simp only [add_le_iff_nonpos_right, nonpos_iff_eq_zero, one_ne_zero, ↓reduceDIte] - rw [action_add_one_eq, pullCount_eq_pullCount' (R' := reward alg) (by simp)] - simp only [Nat.add_one_sub_one] - rw [hist_eq] - rfl + ext ω + rw [reward_eq', pullCount_action_eq] lemma sumRewards_eq [DecidableEq 𝓐] (alg : Algorithm 𝓐 ℝ) (a : 𝓐) (n : ℕ) (ω : probSpace 𝓐 ℝ) : sumRewards (action alg) (reward alg) a n ω = @@ -354,16 +349,16 @@ lemma sumRewards_eq [DecidableEq 𝓐] (alg : Algorithm 𝓐 ℝ) (a : 𝓐) (n section Measurability -lemma measurable_action_add_one' [DecidableEq 𝓐] {alg : Algorithm 𝓐 𝓡} +lemma measurable_action' [DecidableEq 𝓐] {alg : Algorithm 𝓐 𝓡} (n : ℕ) (h : Measurable (hist alg · n)) : - Measurable (fun x ↦ algFunction alg n (hist alg x n) (x.1 (n + 1))) := by fun_prop + Measurable (fun x ↦ algFunction alg n (hist alg x n) (x.1 n)) := by fun_prop -lemma measurable_pullCount'_action_add_one [DecidableEq 𝓐] {alg : Algorithm 𝓐 𝓡} +lemma measurable_pullCount'_action [DecidableEq 𝓐] {alg : Algorithm 𝓐 𝓡} (n : ℕ) (h_hist : Measurable (hist alg · n)) : Measurable (fun x ↦ - pullCount' n (hist alg x n) (algFunction alg n (hist alg x n) (x.1 (n + 1)))) := by - have h_alg_meas : Measurable (fun x ↦ algFunction alg n (hist alg x n) (x.1 (n + 1))) := - measurable_action_add_one' n h_hist + pullCount' n (hist alg x n) (algFunction alg n (hist alg x n) (x.1 n))) := by + have h_alg_meas : Measurable (fun x ↦ algFunction alg n (hist alg x n) (x.1 n)) := + measurable_action' n h_hist exact (measurable_uncurry_pullCount' (𝓐 := 𝓐) n).comp (h_hist.prodMk h_alg_meas) @[fun_prop] @@ -371,29 +366,26 @@ lemma measurable_hist [DecidableEq 𝓐] [Countable 𝓐] (alg : Algorithm 𝓐 Measurable (fun ω ↦ hist alg ω n) := by induction n with | zero => - simp_rw [hist_zero, measurable_pi_iff] - refine fun _ ↦ Measurable.prodMk (by fun_prop) ?_ - change Measurable ((fun x : 𝓐 × ((ℕ → I) × (ℕ → 𝓐 → 𝓡)) ↦ x.2.2 0 x.1) ∘ - (fun x : (ℕ → I) × (ℕ → 𝓐 → 𝓡) ↦ (initAlgFunction alg (x.1 0), x))) - have : Measurable (fun x : 𝓐 × ((ℕ → I) × (ℕ → 𝓐 → 𝓡)) ↦ x.2.2 0 x.1) := - measurable_from_prod_countable_right fun p ↦ by simp only; fun_prop - exact Measurable.comp (by fun_prop) (Measurable.prodMk (by fun_prop) (by fun_prop)) + simp only [hist_zero] + exact measurable_const | succ n hn => - refine measurable_pi_iff.mpr fun i ↦ ?_ - by_cases hin : i ≤ n - · simp only [hist, hin, ↓reduceDIte] - rw [measurable_pi_iff] at hn - exact hn ⟨i.1, by simp [hin]⟩ - · simp only [hist, hin, ↓reduceDIte] - refine Measurable.prodMk (by fun_prop) ?_ - change Measurable ((fun (x : (ℕ → 𝓐 → 𝓡) × ℕ × 𝓐) ↦ x.1 x.2.1 x.2.2) ∘ - (fun x ↦ (x.2, pullCount' n (hist alg x n) (algFunction alg n (hist alg x n) (x.1 (n + 1))), - (algFunction alg n (hist alg x n) (x.1 (n + 1)))))) - have h1 : Measurable (fun (x : (ℕ → 𝓐 → 𝓡) × ℕ × 𝓐) ↦ x.1 x.2.1 x.2.2) := - measurable_from_prod_countable_left fun p : ℕ × 𝓐 ↦ (by simp only; fun_prop) - refine Measurable.comp (by fun_prop) (Measurable.prodMk (by fun_prop) ?_) - refine Measurable.prodMk ?_ (by fun_prop) - exact measurable_pullCount'_action_add_one n hn + have h_eq : (fun ω ↦ hist alg ω (n + 1)) = + (MeasurableEquiv.finSuccProd (𝓐 × 𝓡) n).symm ∘ + (fun ω ↦ (hist alg ω n, (algFunction alg n (hist alg ω n) (ω.1 n), + ω.2 (pullCount' n (hist alg ω n) (algFunction alg n (hist alg ω n) (ω.1 n))) + (algFunction alg n (hist alg ω n) (ω.1 n))))) := by + ext ω : 1 + exact hist_add_one_eq_finSuccProd' alg ω n + rw [h_eq] + refine (MeasurableEquiv.measurable _).comp (hn.prodMk ?_) + refine Measurable.prodMk (measurable_action' n hn) ?_ + change Measurable ((fun (x : (ℕ → 𝓐 → 𝓡) × ℕ × 𝓐) ↦ x.1 x.2.1 x.2.2) ∘ + (fun x ↦ (x.2, pullCount' n (hist alg x n) (algFunction alg n (hist alg x n) (x.1 n)), + (algFunction alg n (hist alg x n) (x.1 n))))) + have h1 : Measurable (fun (x : (ℕ → 𝓐 → 𝓡) × ℕ × 𝓐) ↦ x.1 x.2.1 x.2.2) := + measurable_from_prod_countable_left fun p : ℕ × 𝓐 ↦ (by simp only; fun_prop) + refine h1.comp (Measurable.prodMk (by fun_prop) ?_) + exact (measurable_pullCount'_action n hn).prodMk (measurable_action' n hn) @[fun_prop] lemma measurable_action [DecidableEq 𝓐] [Countable 𝓐] (alg : Algorithm 𝓐 𝓡) (n : ℕ) : @@ -403,19 +395,12 @@ lemma measurable_action [DecidableEq 𝓐] [Countable 𝓐] (alg : Algorithm lemma measurable_reward [DecidableEq 𝓐] [Countable 𝓐] (alg : Algorithm 𝓐 𝓡) (n : ℕ) : Measurable (reward alg n) := by unfold reward; fun_prop -lemma hist_add_one_eq_IicSuccProd [DecidableEq 𝓐] (alg : Algorithm 𝓐 𝓡) (ω : probSpace 𝓐 𝓡) - (n : ℕ) : - hist alg ω (n + 1) = - (MeasurableEquiv.IicSuccProd (fun _ ↦ 𝓐 × 𝓡) n).symm - (hist alg ω n, (action alg (n + 1) ω, reward alg (n + 1) ω)) := by - rw [hist_add_one_eq_IicSuccProd', reward_add_one, action_add_one_eq] - @[fun_prop] -lemma measurable_pullCount_action_add_one [DecidableEq 𝓐] [Countable 𝓐] (alg : Algorithm 𝓐 𝓡) +lemma measurable_pullCount_action [DecidableEq 𝓐] [Countable 𝓐] (alg : Algorithm 𝓐 𝓡) (n : ℕ) : - Measurable (fun ω ↦ pullCount (action alg) (action alg (n + 1) ω) (n + 1) ω) := by - change Measurable ((fun p : (probSpace 𝓐 𝓡) × 𝓐 ↦ pullCount (action alg) p.2 (n + 1) p.1) ∘ - (fun ω : probSpace 𝓐 𝓡 ↦ (ω, action alg (n + 1) ω))) + Measurable (fun ω ↦ pullCount (action alg) (action alg n ω) n ω) := by + change Measurable ((fun p : (probSpace 𝓐 𝓡) × 𝓐 ↦ pullCount (action alg) p.2 n p.1) ∘ + (fun ω : probSpace 𝓐 𝓡 ↦ (ω, action alg n ω))) exact (measurable_uncurry_pullCount (by fun_prop) _).comp (by fun_prop) end Measurability @@ -428,70 +413,48 @@ section Congruence -- very useful to prove measurability lemma hist_congr (alg : Algorithm 𝓐 𝓡) (n : ℕ) {ω ω' : probSpace 𝓐 𝓡} - (hω1 : ∀ i ≤ n, ω.1 i = ω'.1 i) - (hω2 : ∀ i a, i < pullCount (action alg) a (n + 1) ω → ω.2 i a = ω'.2 i a) : + (hω1 : ∀ i < n, ω.1 i = ω'.1 i) + (hω2 : ∀ i a, i < pullCount (action alg) a n ω → ω.2 i a = ω'.2 i a) : hist alg ω n = hist alg ω' n := by induction n with - | zero => - simp only [zero_add, pullCount_one] at hω2 - simp_rw [hist_zero] - ext i : 1 - simp only [le_refl, hω1, Prod.mk.injEq, true_and] - refine hω2 0 _ ?_ - simp [action, hω1] + | zero => rfl | succ n hn => - simp_rw [hist_add_one_eq_IicSuccProd] - specialize hn fun i hin ↦ hω1 i (by grind) have h_hist : hist alg ω n = hist alg ω' n := by - refine hn fun i a hi ↦ hω2 i a (hi.trans_le ?_) + refine hn (fun i hi ↦ hω1 i (by lia)) fun i a hi ↦ hω2 i a (hi.trans_le ?_) exact pullCount_mono _ (by lia) _ - have h_action : action alg (n + 1) ω = action alg (n + 1) ω' := by - simp_rw [action_add_one_eq] - rw [h_hist, hω1 _ le_rfl] - congr 3 - simp only [reward_add_one, h_hist, h_action] - refine hω2 _ _ ?_ - rw [pullCount_add_one, h_action] - simp only [↓reduceIte] - rw [pullCount_eq_pullCount' (R' := reward alg) (by simp)] - simp only [Nat.add_one_sub_one] - rw [← h_hist, hist_eq] - change pullCount' n (fun i ↦ (action alg i ω, reward alg i ω)) (action alg (n + 1) ω') < - pullCount' n (fun i ↦ (action alg i ω, reward alg i ω)) (action alg (n + 1) ω') + 1 - grind + have h_action : action alg n ω = action alg n ω' := by + simp only [action, h_hist, hω1 n (by lia)] + have h_reward : reward alg n ω = reward alg n ω' := by + rw [reward_eq', reward_eq', ← h_hist, ← h_action] + refine hω2 _ _ ?_ + rw [pullCount_action_eq_pullCount_add_one, pullCount_action_eq] + exact Nat.lt_succ_self _ + rw [hist_succ, hist_succ, h_hist, h_action, h_reward] lemma action_eq_and_pullCount_eq_congr_aux (alg : Algorithm 𝓐 𝓡) (a : 𝓐) (m n : ℕ) {ω ω' : probSpace 𝓐 𝓡} (hω1 : ∀ i, ω.1 i = ω'.1 i) (hω2_ne : ∀ i b, b ≠ a → ω.2 i b = ω'.2 i b) (hω2_eq : ∀ i, i + 1 ≤ m → ω.2 i a = ω'.2 i a) - (h_eq : action alg (n + 1) ω = a ∧ pullCount (action alg) a (n + 1) ω = m) : - action alg (n + 1) ω' = a ∧ pullCount (action alg) a (n + 1) ω' = m := by + (h_eq : action alg n ω = a ∧ pullCount (action alg) a n ω = m) : + action alg n ω' = a ∧ pullCount (action alg) a n ω' = m := by obtain ⟨h_action, h_pc⟩ := h_eq - have h_hist := hist_congr alg n (ω := ω) (ω' := ω') (by grind) fun i b hi ↦ ?_ - swap - · rcases eq_or_ne b a with (rfl | hba) - · refine hω2_eq i ?_ - rw [h_pc] at hi - grind - · grind + have h_hist : hist alg ω n = hist alg ω' n := by + refine hist_congr alg n (fun i _ ↦ hω1 i) fun i b hi ↦ ?_ + rcases eq_or_ne b a with (rfl | hba) + · rw [h_pc] at hi + exact hω2_eq i hi + · exact hω2_ne i b hba constructor - · rw [← h_action, action_add_one_eq] - simp [h_hist, hω1] - · simp_rw [← h_pc, pullCount_eq_sum] - refine Finset.sum_congr rfl fun i hi ↦ ?_ - congr 2 - rw [hist_eq _ _ n, hist_eq _ _ n, funext_iff] at h_hist - unfold action - specialize h_hist ⟨i, by grind⟩ - simp only at h_hist - rw [h_hist] + · rw [← h_action] + simp only [action, h_hist, hω1] + · rw [← h_pc, pullCount_action_eq, pullCount_action_eq, h_hist] lemma action_eq_and_pullCount_eq_congr (alg : Algorithm 𝓐 𝓡) (a : 𝓐) (m n : ℕ) {ω ω' : probSpace 𝓐 𝓡} (hω1 : ∀ i, ω.1 i = ω'.1 i) (hω2_ne : ∀ i b, b ≠ a → ω.2 i b = ω'.2 i b) (hω2_eq : ∀ i, i + 1 ≤ m → ω.2 i a = ω'.2 i a) : - (action alg (n + 1) ω = a ∧ pullCount (action alg) a (n + 1) ω = m) ↔ - (action alg (n + 1) ω' = a ∧ pullCount (action alg) a (n + 1) ω' = m) := + (action alg n ω = a ∧ pullCount (action alg) a n ω = m) ↔ + (action alg n ω' = a ∧ pullCount (action alg) a n ω' = m) := ⟨action_eq_and_pullCount_eq_congr_aux alg a m n hω1 hω2_ne hω2_eq, action_eq_and_pullCount_eq_congr_aux alg a m n (by grind) (by grind) (by grind)⟩ @@ -499,10 +462,8 @@ lemma indicator_action_eq_and_pullCount_eq_congr (alg : Algorithm 𝓐 𝓡) (a {ω ω' : probSpace 𝓐 𝓡} (hω1 : ∀ i, ω.1 i = ω'.1 i) (hω2_ne : ∀ i b, b ≠ a → ω.2 i b = ω'.2 i b) (hω2_eq : ∀ i, i + 1 ≤ m → ω.2 i a = ω'.2 i a) : - {ω | action alg (n + 1) ω = a ∧ pullCount (action alg) a (n + 1) ω = m}.indicator (fun _ ↦ 1) - ω = - {ω | action alg (n + 1) ω = a ∧ pullCount (action alg) a (n + 1) ω = m}.indicator - (fun _ ↦ 1) ω' := by + {ω | action alg n ω = a ∧ pullCount (action alg) a n ω = m}.indicator (fun _ ↦ 1) ω = + {ω | action alg n ω = a ∧ pullCount (action alg) a n ω = m}.indicator (fun _ ↦ 1) ω' := by simp only [Set.indicator_apply, Set.mem_ofPred_eq] simp_rw [action_eq_and_pullCount_eq_congr alg a m n hω1 hω2_ne hω2_eq] @@ -511,29 +472,32 @@ end Congruence section MeasurabilityAdvanced lemma measurable_hist_comap [Countable 𝓐] (alg : Algorithm 𝓐 𝓡) (n : ℕ) : - Measurable[MeasurableSpace.comap (fun ω ↦ (fun (i : Iic n) ↦ ω.1 i, ω.2)) inferInstance] + Measurable[MeasurableSpace.comap (fun ω ↦ (fun (i : Fin n) ↦ ω.1 i, ω.2)) inferInstance] (hist alg · n) := by have h_eq : (hist alg · n) = - ((hist alg · n) ∘ (fun p ↦ (fun i : ℕ ↦ p.1 ⟨min i n, by grind⟩, p.2))) ∘ - (fun ω ↦ (fun (i : Iic n) ↦ ω.1 i, ω.2)) := by + ((hist alg · n) ∘ (fun p ↦ (fun i : ℕ ↦ if hi : i < n then p.1 ⟨i, hi⟩ else 0, p.2))) ∘ + (fun ω ↦ (fun (i : Fin n) ↦ ω.1 i, ω.2)) := by ext ω : 1 - exact hist_congr alg n (by grind) (by simp) + exact hist_congr alg n (fun i hi ↦ by simp [hi]) (fun _ _ _ ↦ rfl) rw [h_eq] refine measurable_comp_comap _ (Measurable.comp (by fun_prop) ?_) refine Measurable.prodMk ?_ (by fun_prop) rw [measurable_pi_iff] intro i - change Measurable ((fun p ↦ p ⟨min i n, by simp⟩) ∘ (fun x : (Iic n → I) × (ℕ → 𝓐 → 𝓡) ↦ x.1)) - exact Measurable.comp (by fun_prop) measurable_fst - -/-- `truncRow` at the number of pulls of `a` up to time `n`: the rewards of action `a` that have -not been observed by time `n` are replaced. The history up to time `n` is measurable with respect -to this function (see `measurable_hist_truncRowPullCount`), and on the event -`pullCount (action alg) a (n + 1) = m` it coincides with `truncRow a m`. -/ + by_cases hi : i < n + · simp only [hi, ↓reduceDIte] + exact (measurable_pi_apply _).comp measurable_fst + · simp only [hi, ↓reduceDIte] + exact measurable_const + +/-- `truncRow` at the number of pulls of `a` before time `n`: the rewards of action `a` that have +not been observed before time `n` are replaced. The history before time `n` is measurable with +respect to this function (see `measurable_hist_truncRowPullCount`), and on the event +`pullCount (action alg) a n = m` it coincides with `truncRow a m`. -/ noncomputable def truncRowPullCount (alg : Algorithm 𝓐 𝓡) (a : 𝓐) (n : ℕ) (ω : probSpace 𝓐 𝓡) : probSpace 𝓐 𝓡 := - truncRow a (pullCount (action alg) a (n + 1) ω) ω + truncRow a (pullCount (action alg) a n ω) ω lemma measurable_hist_truncRowPullCount [Countable 𝓐] (alg : Algorithm 𝓐 𝓡) (a : 𝓐) (n : ℕ) : Measurable[MeasurableSpace.comap (truncRowPullCount alg a n) inferInstance] (hist alg · n) := by @@ -549,15 +513,14 @@ lemma measurable_hist_truncRowPullCount [Countable 𝓐] (alg : Algorithm 𝓐 lemma measurableSet_action_eq_and_pullCount_eq [Countable 𝓐] (alg : Algorithm 𝓐 𝓡) (a : 𝓐) (n m : ℕ) : - MeasurableSet {ω | action alg (n + 1) ω = a ∧ pullCount (action alg) a (n + 1) ω = m} := + MeasurableSet {ω | action alg n ω = a ∧ pullCount (action alg) a n ω = m} := MeasurableSet.inter ((measurableSet_singleton _).preimage (by fun_prop)) ((measurableSet_singleton _).preimage (by fun_prop)) lemma preimage_action_pullCount_eq (alg : Algorithm 𝓐 𝓡) (a : 𝓐) (n m : ℕ) : - (fun ω ↦ (action alg (n + 1) ω, pullCount (action alg) (action alg (n + 1) ω) (n + 1) ω)) - ⁻¹' {(a, m)} = - {ω | action alg (n + 1) ω = a ∧ pullCount (action alg) a (n + 1) ω = m}.indicator - (fun _ ↦ 1) ⁻¹' {1} := by + (fun ω ↦ (action alg n ω, pullCount (action alg) (action alg n ω) n ω)) ⁻¹' {(a, m)} = + {ω | action alg n ω = a ∧ pullCount (action alg) a n ω = m}.indicator (fun _ ↦ 1) + ⁻¹' {1} := by ext ω simp only [Set.mem_preimage, Set.mem_singleton_iff, Prod.mk.injEq, Set.indicator_apply, Set.mem_ofPred_eq, ite_eq_left_iff, not_and, zero_ne_one, imp_false, Classical.not_imp, @@ -568,10 +531,8 @@ lemma preimage_action_pullCount_eq (alg : Algorithm 𝓐 𝓡) (a : 𝓐) (n m : lemma measurable_indicator_action_eq_and_pullCount_eq [Countable 𝓐] (alg : Algorithm 𝓐 𝓡) (a : 𝓐) (m n : ℕ) : Measurable[MeasurableSpace.comap (truncRow a m) inferInstance] - (({ω | action alg (n + 1) ω = a ∧ - pullCount (action alg) a (n + 1) ω = m}).indicator (fun _ ↦ 1)) := by - let f := ({ω | action alg (n + 1) ω = a ∧ pullCount (action alg) a (n + 1) ω = m}).indicator - (fun _ ↦ 1) + (({ω | action alg n ω = a ∧ pullCount (action alg) a n ω = m}).indicator (fun _ ↦ 1)) := by + let f := ({ω | action alg n ω = a ∧ pullCount (action alg) a n ω = m}).indicator (fun _ ↦ 1) have h_eq : f = f ∘ truncRow a m := by ext ω exact indicator_action_eq_and_pullCount_eq_congr alg a m n (fun _ ↦ rfl) @@ -581,22 +542,14 @@ lemma measurable_indicator_action_eq_and_pullCount_eq [Countable 𝓐] (alg : Al exact (Measurable.indicator (by fun_prop) (measurableSet_action_eq_and_pullCount_eq alg a n m)).comp (Measurable.of_comap_le le_rfl) -lemma measurable_pullCount_action_add_one_hist (alg : Algorithm 𝓐 𝓡) (n : ℕ) : - Measurable[MeasurableSpace.comap (fun ω ↦ (hist alg ω n, action alg (n + 1) ω)) inferInstance] - (fun ω ↦ pullCount (action alg) (action alg (n + 1) ω) (n + 1) ω) := by - simp_rw [pullCount_eq_sum] - refine measurable_sum _ fun i hi ↦ Measurable.ite ?_ (by fun_prop) (by fun_prop) - refine measurableSet_eq_fun ?_ (measurable_comp_comap _ measurable_snd) - rw [measurable_iff_comap_le] - simp_rw [hist_eq _ _ n] - rw [← measurable_iff_comap_le] - unfold action - refine Measurable.fst (mγ := inferInstance) ?_ - have : (hist alg · i ⟨i, by grind⟩) = - (fun ω : (Iic n → 𝓐 × 𝓡) × 𝓐 ↦ ω.1 ⟨i, by grind⟩) ∘ - (fun ω ↦ (fun i : Iic n ↦ hist alg ω i ⟨i, by grind⟩, action alg (n + 1) ω)) := rfl - rw [this] - exact measurable_comp_comap _ (Measurable.prodMk (by fun_prop) (by fun_prop)) +lemma measurable_pullCount_action_hist (alg : Algorithm 𝓐 𝓡) (n : ℕ) : + Measurable[MeasurableSpace.comap (fun ω ↦ (hist alg ω n, action alg n ω)) inferInstance] + (fun ω ↦ pullCount (action alg) (action alg n ω) n ω) := by + simp_rw [pullCount_action_eq] + change Measurable[MeasurableSpace.comap (fun ω ↦ (hist alg ω n, action alg n ω)) inferInstance] + ((fun p : (Fin n → 𝓐 × 𝓡) × 𝓐 ↦ pullCount' n p.1 p.2) ∘ + (fun ω ↦ (hist alg ω n, action alg n ω))) + exact measurable_comp_comap _ (measurable_uncurry_pullCount' n) end MeasurabilityAdvanced @@ -614,22 +567,22 @@ lemma indepFun_fst_zero_snd_zero_action (ν : Kernel 𝓐 𝓡) [IsMarkovKernel (by fun_prop) (by fun_prop) omit [Nonempty 𝓐] [StandardBorelSpace 𝓐] [DecidableEq 𝓐] in -lemma indepFun_fst_add_one_aux (ν : Kernel 𝓐 𝓡) [IsMarkovKernel ν] (n : ℕ) : - (fun ω ↦ ω.1 (n + 1)) ⟂ᵢ[arrayMeasure ν] (fun ω ↦ (fun (i : Iic n) ↦ ω.1 i, ω.2)) := by - have h : IndepFun (fun ω : ℕ → I ↦ ω (n + 1)) (fun ω (i : Iic n) ↦ ω i) +lemma indepFun_fst_aux (ν : Kernel 𝓐 𝓡) [IsMarkovKernel ν] (n : ℕ) : + (fun ω ↦ ω.1 n) ⟂ᵢ[arrayMeasure ν] (fun ω ↦ (fun (i : Fin n) ↦ ω.1 i, ω.2)) := by + have h : IndepFun (fun ω : ℕ → I ↦ ω n) (fun ω (i : Fin n) ↦ ω i) (Measure.infinitePi fun _ ↦ volume) := by refine (iIndepFun_infinitePi fun _ ↦ measurable_id).indepFun_of_measurable_iSup_comap - (fun _ ↦ measurable_pi_apply _) (S := Set.Iic n) (by simp) ?_ + (fun _ ↦ measurable_pi_apply _) (S := Set.Iio n) (by simp) ?_ rw [measurable_iff_comap_le, MeasurableSpace.comap_pi] - exact iSup_le fun i ↦ le_iSup₂_of_le (i : ℕ) (Set.mem_Iic.2 (Finset.mem_Iic.1 i.2)) le_rfl + exact iSup_le fun i ↦ le_iSup₂_of_le (i : ℕ) (Set.mem_Iio.2 i.2) le_rfl exact h.fst_prod (ν := streamMeasure ν) (by fun_prop) (by fun_prop) variable [StandardBorelSpace 𝓡] [Nonempty 𝓡] -lemma indepFun_fst_add_one_hist [Countable 𝓐] (alg : Algorithm 𝓐 𝓡) +lemma indepFun_fst_hist [Countable 𝓐] (alg : Algorithm 𝓐 𝓡) (ν : Kernel 𝓐 𝓡) [IsMarkovKernel ν] (n : ℕ) : - IndepFun (fun ω ↦ ω.1 (n + 1)) (hist alg · n) (arrayMeasure ν) := - (indepFun_fst_add_one_aux ν n).of_measurable_right (measurable_hist_comap alg n) + IndepFun (fun ω ↦ ω.1 n) (hist alg · n) (arrayMeasure ν) := + (indepFun_fst_aux ν n).of_measurable_right (measurable_hist_comap alg n) omit [Nonempty 𝓐] [StandardBorelSpace 𝓐] [StandardBorelSpace 𝓡] [Nonempty 𝓡] in /-- The reward `ω.2 m a` is independent of `truncRow a m`, which reads only other coordinates. -/ @@ -660,19 +613,18 @@ omit [StandardBorelSpace 𝓡] [Nonempty 𝓡] in lemma indepFun_snd_apply_pullCount_action [Countable 𝓐] (alg : Algorithm 𝓐 𝓡) (ν : Kernel 𝓐 𝓡) [IsMarkovKernel ν] (a : 𝓐) (m n : ℕ) : (fun ω ↦ ω.2 m a) ⟂ᵢ[arrayMeasure ν] - ({ω | action alg (n + 1) ω = a ∧ - pullCount (action alg) a (n + 1) ω = m}).indicator (fun _ ↦ 1) := + ({ω | action alg n ω = a ∧ pullCount (action alg) a n ω = m}).indicator (fun _ ↦ 1) := (indepFun_snd_apply_truncRow ν a m).of_measurable_right (measurable_indicator_action_eq_and_pullCount_eq alg a m n) lemma indepFun_snd_hist_cond [Countable 𝓐] (alg : Algorithm 𝓐 𝓡) (ν : Kernel 𝓐 𝓡) [IsMarkovKernel ν] (a : 𝓐) (n m : ℕ) : - (fun ω ↦ ω.2 m a) ⟂ᵢ[(arrayMeasure ν)[|(fun ω ↦ (action alg (n + 1) ω, - pullCount (action alg) (action alg (n + 1) ω) (n + 1) ω)) ⁻¹' {(a, m)}]] + (fun ω ↦ ω.2 m a) ⟂ᵢ[(arrayMeasure ν)[|(fun ω ↦ (action alg n ω, + pullCount (action alg) (action alg n ω) n ω)) ⁻¹' {(a, m)}]] (hist alg · n) := by refine IndepFun.of_measurable_right ?_ (measurable_hist_truncRowPullCount alg a n) - have h_ae_eq : truncRowPullCount alg a n =ᵐ[(arrayMeasure ν)[|(fun ω ↦ (action alg (n + 1) ω, - pullCount (action alg) (action alg (n + 1) ω) (n + 1) ω)) ⁻¹' {(a, m)}]] + have h_ae_eq : truncRowPullCount alg a n =ᵐ[(arrayMeasure ν)[|(fun ω ↦ (action alg n ω, + pullCount (action alg) (action alg n ω) n ω)) ⁻¹' {(a, m)}]] truncRow a m := by refine ae_cond_of_forall_mem ((measurableSet_singleton _).preimage (by fun_prop)) fun x hx ↦ ?_ @@ -692,46 +644,14 @@ end Independence section Laws -lemma hasLaw_action_zero (alg : Algorithm 𝓐 𝓡) (ν : Kernel 𝓐 𝓡) [IsMarkovKernel ν] : - HasLaw (action alg 0) alg.p0 (arrayMeasure ν) := by - rw [action_zero] - exact (⟨(measurable_initAlgFunction alg).aemeasurable, initAlgFunction_map alg⟩ : - HasLaw (initAlgFunction alg) alg.p0 volume).comp (hasLaw_fst_apply_arrayMeasure ν 0) - variable [Countable 𝓐] [StandardBorelSpace 𝓡] [Nonempty 𝓡] -lemma hasCondDistrib_reward_zero (alg : Algorithm 𝓐 𝓡) (ν : Kernel 𝓐 𝓡) [IsMarkovKernel ν] : - HasCondDistrib (reward alg 0) (action alg 0) ν (arrayMeasure ν) := by - refine hasCondDistrib_of_condDistrib_eq (by fun_prop) (by fun_prop) ?_ - refine (condDistrib_ae_eq_cond (by fun_prop) (by fun_prop)).trans ?_ - rw [Filter.EventuallyEq, ae_iff_of_countable] - intro a ha - simp only [reward_zero] - calc ((arrayMeasure ν)[|action alg 0 ⁻¹' {a}]).map (fun ω ↦ ω.2 0 (action alg 0 ω)) - _ = ((arrayMeasure ν)[|action alg 0 ⁻¹' {a}]).map (fun ω ↦ ω.2 0 a) := by - refine Measure.map_congr - (ae_cond_of_forall_mem ((measurableSet_singleton _).preimage (by fun_prop)) ?_) - intro x hx - simp only [Set.mem_preimage, Set.mem_singleton_iff] at hx - simp [hx] - _ = ν a := by - rw [cond_of_indepFun] - · exact map_snd_apply_arrayMeasure 0 a - · have : (fun ω ↦ ω.1 0) ⟂ᵢ[arrayMeasure ν] fun ω ↦ ω.2 0 a := - indepFun_fst_zero_snd_zero_action ν a - rw [action_zero] - exact this.comp (φ := initAlgFunction alg) (by fun_prop) measurable_id - · fun_prop - · fun_prop - · simp - · rwa [Measure.map_apply (by fun_prop) (by simp)] at ha - lemma hasCondDistrib_action' (alg : Algorithm 𝓐 𝓡) (ν : Kernel 𝓐 𝓡) [IsMarkovKernel ν] (n : ℕ) : - HasCondDistrib (action alg (n + 1)) (hist alg · n) (alg.policy n) (arrayMeasure ν) := by - have hU : HasCondDistrib (fun ω ↦ ω.1 (n + 1)) (fun ω ↦ (hist alg ω n, hist alg ω n)) + HasCondDistrib (action alg n) (hist alg · n) (alg.policy n) (arrayMeasure ν) := by + have hU : HasCondDistrib (fun ω ↦ ω.1 n) (fun ω ↦ (hist alg ω n, hist alg ω n)) (Kernel.const _ volume) (arrayMeasure ν) := - ((indepFun_fst_add_one_hist alg ν n).symm.comp (measurable_id.prodMk measurable_id) - measurable_id).hasCondDistrib_const (by fun_prop) (hasLaw_fst_apply_arrayMeasure ν (n + 1)) + ((indepFun_fst_hist alg ν n).symm.comp (measurable_id.prodMk measurable_id) + measurable_id).hasCondDistrib_const (by fun_prop) (hasLaw_fst_apply_arrayMeasure ν n) have h := ((hasCondDistrib_self (X := (hist alg · n)) (by fun_prop)).prod hU).comp_left (measurable_algFunction alg n) have h_ker : alg.policy n = (Kernel.id ⊗ₖ Kernel.const _ volume).map @@ -743,35 +663,32 @@ lemma hasCondDistrib_action' (alg : Algorithm 𝓐 𝓡) (ν : Kernel 𝓐 𝓡) rw [lintegral_dirac' _ (measurable_measure_prodMk_left (measurable_algFunction alg n hs)), ← algFunction_map alg n h, Measure.map_apply (by fun_prop) hs] rfl - rw [action_add_one_eq, h_ker] + rw [action_eq, h_ker] exact h omit [StandardBorelSpace 𝓡] [Nonempty 𝓡] in lemma reward_ae_eq_cond (alg : Algorithm 𝓐 𝓡) (ν : Kernel 𝓐 𝓡) (a : 𝓐) (n m : ℕ) : - reward alg (n + 1) =ᵐ[(arrayMeasure ν)[|(fun ω ↦ (action alg (n + 1) ω, - pullCount (action alg) (action alg (n + 1) ω) (n + 1) ω)) ⁻¹' {(a, m)}]] + reward alg n =ᵐ[(arrayMeasure ν)[|(fun ω ↦ (action alg n ω, + pullCount (action alg) (action alg n ω) n ω)) ⁻¹' {(a, m)}]] (fun ω ↦ ω.2 m a) := by rw [reward_eq] - refine ae_cond_of_forall_mem ?_ ?_ - · exact (measurableSet_singleton _).preimage (by fun_prop) - intro ω hω + refine ae_cond_of_forall_mem ((measurableSet_singleton _).preimage (by fun_prop)) fun ω hω ↦ ?_ simp only [Set.mem_preimage, Set.mem_singleton_iff, Prod.mk.injEq] at hω simp only [hω.2] simp [hω.1] -/-- The conditional distribution of the reward at time `n + 1`, given the history up to time `n`, -the action at time `n + 1`, and the number of times that action has been pulled before time `n + 1`, +/-- The conditional distribution of the reward at time `n`, given the history before time `n`, +the action at time `n`, and the number of times that action has been pulled before time `n`, is equal to the kernel `ν`. -/ lemma hasCondDistrib_reward_hist_action_pullCount (alg : Algorithm 𝓐 𝓡) (ν : Kernel 𝓐 𝓡) [IsMarkovKernel ν] (n : ℕ) : - HasCondDistrib (reward alg (n + 1)) - (fun ω ↦ (hist alg ω n, action alg (n + 1) ω, - pullCount (action alg) (action alg (n + 1) ω) (n + 1) ω)) + HasCondDistrib (reward alg n) + (fun ω ↦ (hist alg ω n, action alg n ω, pullCount (action alg) (action alg n ω) n ω)) ((ν.prodMkRight _).prodMkLeft _) (arrayMeasure ν) := by refine hasCondDistrib_of_condDistrib_eq (by fun_prop) (by fun_prop) ?_ refine condDistrib_prod_of_forall_condDistrib_cond (by fun_prop) (by fun_prop) (by fun_prop) _ ?_ intro (a, m) ham - have h_eq : ((ν.prodMkRight _).prodMkLeft _).comap (fun ω : (Iic n → 𝓐 × 𝓡) ↦ (ω, a, m)) + have h_eq : ((ν.prodMkRight _).prodMkLeft _).comap (fun ω : (Fin n → 𝓐 × 𝓡) ↦ (ω, a, m)) (by fun_prop) = Kernel.const _ (ν a) := by ext; simp rw [h_eq, condDistrib_congr_left (reward_ae_eq_cond alg ν a n m)] @@ -785,36 +702,33 @@ lemma hasCondDistrib_reward_hist_action_pullCount · simp · rwa [preimage_action_pullCount_eq] at ham -/-- The reward at time `n + 1` is conditionally independent of the history up to time `n`, -given the action at time `n + 1` and the number of times that action has been pulled before -time `n + 1`. -/ +/-- The reward at time `n` is conditionally independent of the history before time `n`, +given the action at time `n` and the number of times that action has been pulled before +time `n`. -/ lemma condIndepFun_reward_hist (alg : Algorithm 𝓐 𝓡) (ν : Kernel 𝓐 𝓡) [IsMarkovKernel ν] (n : ℕ) : - (reward alg (n + 1)) ⟂ᵢ[(fun ω ↦ (action alg (n + 1) ω, - pullCount (action alg) (action alg (n + 1) ω) (n + 1) ω)), - Measurable.prodMk (by fun_prop) (measurable_pullCount_action_add_one alg n); + (reward alg n) ⟂ᵢ[(fun ω ↦ (action alg n ω, pullCount (action alg) (action alg n ω) n ω)), + Measurable.prodMk (by fun_prop) (measurable_pullCount_action alg n); arrayMeasure ν] (hist alg · n) := by have h_cond := hasCondDistrib_reward_hist_action_pullCount alg ν n refine condIndepFun_of_exists_condDistrib_prod_ae_eq_prodMkLeft (by fun_prop) (by fun_prop) ?_ h_cond.condDistrib_eq - exact Measurable.prodMk (by fun_prop) (measurable_pullCount_action_add_one alg n) + exact Measurable.prodMk (by fun_prop) (measurable_pullCount_action alg n) -/-- The conditional distribution of the reward at time `n + 1`, given the history up to time `n` -and the action at time `n + 1`, is equal to the kernel `ν`. -/ +/-- The conditional distribution of the reward at time `n`, given the history before time `n` +and the action at time `n`, is equal to the kernel `ν`. -/ lemma hasCondDistrib_reward' (alg : Algorithm 𝓐 𝓡) (ν : Kernel 𝓐 𝓡) [IsMarkovKernel ν] (n : ℕ) : - HasCondDistrib (reward alg (n + 1)) (fun ω ↦ (hist alg ω n, action alg (n + 1) ω)) + HasCondDistrib (reward alg n) (fun ω ↦ (hist alg ω n, action alg n ω)) (ν.prodMkLeft _) (arrayMeasure ν) := by have h := (hasCondDistrib_reward_hist_action_pullCount alg ν n).measurableEquiv_comp_right MeasurableEquiv.prodAssoc.symm - obtain ⟨f, hf, hf_eq⟩ := - (measurable_pullCount_action_add_one_hist alg n).exists_eq_measurable_comp - have h_eq : (MeasurableEquiv.prodAssoc.symm ∘ fun ω ↦ (hist alg ω n, action alg (n + 1) ω, - pullCount (action alg) (action alg (n + 1) ω) (n + 1) ω)) = - fun ω ↦ ((hist alg ω n, action alg (n + 1) ω), - f (hist alg ω n, action alg (n + 1) ω)) := by + obtain ⟨f, hf, hf_eq⟩ := (measurable_pullCount_action_hist alg n).exists_eq_measurable_comp + have h_eq : (MeasurableEquiv.prodAssoc.symm ∘ fun ω ↦ (hist alg ω n, action alg n ω, + pullCount (action alg) (action alg n ω) n ω)) = + fun ω ↦ ((hist alg ω n, action alg n ω), f (hist alg ω n, action alg n ω)) := by funext ω exact Prod.ext rfl (congrFun hf_eq ω) - have h_ker : ((ν.prodMkRight ℕ).prodMkLeft (Iic n → 𝓐 × 𝓡)).comap + have h_ker : ((ν.prodMkRight ℕ).prodMkLeft (Fin n → 𝓐 × 𝓡)).comap MeasurableEquiv.prodAssoc.symm.symm MeasurableEquiv.prodAssoc.symm.symm.measurable = (ν.prodMkLeft _).prodMkRight ℕ := by ext p : 1 @@ -823,34 +737,31 @@ lemma hasCondDistrib_reward' (alg : Algorithm 𝓐 𝓡) (ν : Kernel 𝓐 𝓡) exact (hasCondDistrib_prod_right_iff _ _ hf).1 h lemma hasCondDistrib_action (alg : Algorithm 𝓐 𝓡) (ν : Kernel 𝓐 𝓡) [IsMarkovKernel ν] (n : ℕ) : - HasCondDistrib (action alg (n + 1)) - (fun ω (i : Iic n) ↦ (action alg i ω, reward alg i ω)) - (alg.policy n) (arrayMeasure ν) := by - convert hasCondDistrib_action' alg ν n with ω i - · simp only [action] - rw [hist_eq _ _ n] - · simp only [reward] - rw [hist_eq _ _ n] + HasCondDistrib (action alg n) (history (action alg) (reward alg) n) (alg.policy n) + (arrayMeasure ν) := by + simpa only [hist_eq_history] using hasCondDistrib_action' alg ν n lemma hasCondDistrib_reward (alg : Algorithm 𝓐 𝓡) (ν : Kernel 𝓐 𝓡) [IsMarkovKernel ν] (n : ℕ) : - HasCondDistrib (reward alg (n + 1)) - (fun ω ↦ (fun (i : Iic n) ↦ (action alg i ω, reward alg i ω), action alg (n + 1) ω)) + HasCondDistrib (reward alg n) + (fun ω ↦ (history (action alg) (reward alg) n ω, action alg n ω)) ((stationaryEnv ν).feedback n) (arrayMeasure ν) := by - convert hasCondDistrib_reward' alg ν n with ω i - · simp only [action] - rw [hist_eq _ _ n] - · simp only [reward] - rw [hist_eq _ _ n] - · rfl + simpa only [hist_eq_history, feedback_stationaryEnv] using hasCondDistrib_reward' alg ν n lemma isAlgEnvSeq_arrayMeasure (alg : Algorithm 𝓐 𝓡) (ν : Kernel 𝓐 𝓡) [IsMarkovKernel ν] : IsAlgEnvSeq (action alg) (reward alg) alg (stationaryEnv ν) (arrayMeasure ν) where - hasLaw_action_zero := hasLaw_action_zero alg ν - hasCondDistrib_feedback_zero := hasCondDistrib_reward_zero alg ν hasCondDistrib_action := hasCondDistrib_action alg ν hasCondDistrib_feedback := hasCondDistrib_reward alg ν +lemma hasLaw_action_zero (alg : Algorithm 𝓐 𝓡) (ν : Kernel 𝓐 𝓡) [IsMarkovKernel ν] : + HasLaw (action alg 0) alg.p0 (arrayMeasure ν) := + (isAlgEnvSeq_arrayMeasure alg ν).hasLaw_action_zero + +lemma hasCondDistrib_reward_zero (alg : Algorithm 𝓐 𝓡) (ν : Kernel 𝓐 𝓡) [IsMarkovKernel ν] : + HasCondDistrib (reward alg 0) (action alg 0) ν (arrayMeasure ν) := by + have h := (isAlgEnvSeq_arrayMeasure alg ν).hasCondDistrib_feedback_zero + rwa [ν0_stationaryEnv] at h + end Laws end ArrayModel diff --git a/LeanMachineLearning/Online/Bandit/RewardByCountMeasure.lean b/LeanMachineLearning/Online/Bandit/RewardByCountMeasure.lean index 1f60a429..6c75680f 100644 --- a/LeanMachineLearning/Online/Bandit/RewardByCountMeasure.lean +++ b/LeanMachineLearning/Online/Bandit/RewardByCountMeasure.lean @@ -81,16 +81,10 @@ lemma condIndepFun_reward_stepsUntil_action' [StandardBorelSpace Ω] -- on `action n`. have hA := h.measurable_action have hR := h.measurable_feedback - by_cases hn : n = 0 - · have h_indep : R 0 ⟂ᵢ[A 0, hA 0; P] A 0 := - condIndepFun_self_right (by fun_prop) (by fun_prop) - simp only [hn] - refine h_indep.of_measurable_right (hX := hA 0) ?_ - exact measurable_comap_indicator_stepsUntil_eq_zero a m - · have h_indep : R n ⟂ᵢ[A n, hA n; P] fun ω ↦ (history A R (n - 1) ω, A n ω) := - IsAlgEnvSeq.condIndepFun_feedback_history_action_action' h n (by grind) - refine h_indep.of_measurable_right (hX := hA n) ?_ - exact measurable_comap_indicator_stepsUntil_eq h a m n + have h_indep : R n ⟂ᵢ[A n, hA n; P] fun ω ↦ (history A R n ω, A n ω) := + IsAlgEnvSeq.condIndepFun_feedback_history_action_action h n + refine h_indep.of_measurable_right (hX := hA n) ?_ + exact measurable_comap_indicator_stepsUntil_eq R a m n lemma condIndepFun_reward_stepsUntil_action [StandardBorelSpace Ω] [Countable 𝓐] (h : IsAlgEnvSeq A R alg (stationaryEnv ν) P) @@ -287,46 +281,25 @@ lemma indepFun_update_rewardByCountUntil_eval [Countable 𝓐] (hA : ∀ n, Meas exact measurable_comp_comap _ (measurable_update_left.comp (measurable_rewardByCountUntil hA hR t)) -/-- Conditionally on the event that the action at time `n + 1` is `b` and that `b` was pulled `k` -times before, the reward at time `n + 1` is independent of the history up to time `n` and of the -action at time `n + 1`. -/ +/-- Conditionally on the event that the action at time `n` is `b` and that `b` was pulled `k` +times before, the reward at time `n` is independent of the history before time `n` and of the +action at time `n`. -/ lemma indepFun_history_reward_cond (h : IsAlgEnvSeq A R alg (stationaryEnv ν) P) (n : ℕ) (b : 𝓐) (k : ℕ) : - (fun x ↦ (history A R n x, A (n + 1) x)) - ⟂ᵢ[P[|{x | A (n + 1) x = b ∧ pullCount A b (n + 1) x = k}]] R (n + 1) := by + (fun x ↦ (history A R n x, A n x)) + ⟂ᵢ[P[|{x | A n x = b ∧ pullCount A b n x = k}]] R n := by rw [setOf_action_eq_and_pullCount_eq_eq_preimage (R' := R)] exact h.indepFun_history_action_feedback_cond_stationaryEnv n (measurableSet_snd_eq_and_pullCount'_eq n b k) fun u hu ↦ hu.1 -lemma indepFun_action_zero_reward_zero_cond (h : IsAlgEnvSeq A R alg (stationaryEnv ν) P) - (b : 𝓐) (k : ℕ) : - A 0 ⟂ᵢ[P[|{x | A 0 x = b ∧ pullCount A b 0 x = k}]] R 0 := by - rcases eq_or_ne k 0 with rfl | hk - · have h_eq : {x | A 0 x = b ∧ pullCount A b 0 x = 0} = A 0 ⁻¹' {b} := by ext; simp - rw [h_eq] - exact indepFun_cond_preimage_singleton_left (h.measurable_action 0) b _ - · have h_eq : {x | A 0 x = b ∧ pullCount A b 0 x = k} = ∅ := by ext; simp [hk.symm] - rw [h_eq] - simp - /-- Conditionally on the event that the action at time `t` is `b` and that `b` was pulled `k` times before, the reward at time `t` has law `ν b`. -/ lemma hasLaw_reward_cond (h : IsAlgEnvSeq A R alg (stationaryEnv ν) P) (t : ℕ) (b : 𝓐) (k : ℕ) (hP : P {x | A t x = b ∧ pullCount A b t x = k} ≠ 0) : HasLaw (R t) (ν b) (P[|{x | A t x = b ∧ pullCount A b t x = k}]) := by - cases t with - | zero => - rcases eq_or_ne k 0 with rfl | hk - · have h_eq : {x | A 0 x = b ∧ pullCount A b 0 x = 0} = A 0 ⁻¹' {b} := by ext; simp - rw [h_eq] at hP ⊢ - exact h.hasLaw_feedback_zero_cond_stationaryEnv hP - · refine absurd ?_ hP - have h_eq : {x | A 0 x = b ∧ pullCount A b 0 x = k} = ∅ := by ext; simp [hk.symm] - rw [h_eq, measure_empty] - | succ n => - rw [setOf_action_eq_and_pullCount_eq_eq_preimage (R' := R)] at hP ⊢ - exact h.hasLaw_feedback_cond_stationaryEnv n (measurableSet_snd_eq_and_pullCount'_eq n b k) - (fun u hu ↦ hu.1) hP + rw [setOf_action_eq_and_pullCount_eq_eq_preimage (R' := R)] at hP ⊢ + exact h.hasLaw_feedback_cond_stationaryEnv t (measurableSet_snd_eq_and_pullCount'_eq t b k) + (fun u hu ↦ hu.1) hP lemma hasLaw_reward_cond_prod (h : IsAlgEnvSeq A R alg (stationaryEnv ν) P) (t : ℕ) (b : 𝓐) (k : ℕ) (hP : P {x | A t x = b ∧ pullCount A b t x = k} ≠ 0) : @@ -351,32 +324,20 @@ lemma indepFun_update_rewardByCountUntil_reward (h : IsAlgEnvSeq A R alg (statio exact indepFun_zero_measure _ _ have : IsProbabilityMeasure (P[|{x | A t x = b ∧ pullCount A b t x = k}]) := cond_isProbabilityMeasure hP - cases t with - | zero => - have h_indep := (indepFun_action_zero_reward_zero_cond h b k).symm.fst_prod - (ν := streamMeasure ν) (hR 0) (hA 0) - refine (h_indep.of_measurable_right ?_).symm - refine Measurable.comp measurable_update_left ?_ - refine measurable_rewardByCountUntil_of 0 (fun i hi ↦ absurd hi (Nat.not_lt_zero i)) - (fun i hi ↦ absurd hi (Nat.not_lt_zero i)) ?_ - exact measurable_comp_comap (fun ω : Ω × (ℕ → 𝓐 → ℝ) ↦ (A 0 ω.1, ω.2)) measurable_snd - | succ n => - have h_indep := (indepFun_history_reward_cond h n b k).symm.fst_prod - (ν := streamMeasure ν) (hR _) (by fun_prop) - refine (h_indep.of_measurable_right ?_).symm - refine Measurable.comp measurable_update_left ?_ - refine measurable_rewardByCountUntil_of (n + 1) (fun i hi ↦ ?_) (fun i hi ↦ ?_) ?_ - · exact measurable_comp_comap - (fun ω : Ω × (ℕ → 𝓐 → ℝ) ↦ ((history A R n ω.1, A (n + 1) ω.1), ω.2)) - (g := fun v : ((Iic n → 𝓐 × ℝ) × 𝓐) × (ℕ → 𝓐 → ℝ) ↦ - (v.1.1 ⟨i, mem_Iic.2 (Nat.lt_succ_iff.1 hi)⟩).1) (by fun_prop) - · exact measurable_comp_comap - (fun ω : Ω × (ℕ → 𝓐 → ℝ) ↦ ((history A R n ω.1, A (n + 1) ω.1), ω.2)) - (g := fun v : ((Iic n → 𝓐 × ℝ) × 𝓐) × (ℕ → 𝓐 → ℝ) ↦ - (v.1.1 ⟨i, mem_Iic.2 (Nat.lt_succ_iff.1 hi)⟩).2) (by fun_prop) - · exact measurable_comp_comap - (fun ω : Ω × (ℕ → 𝓐 → ℝ) ↦ ((history A R n ω.1, A (n + 1) ω.1), ω.2)) - (g := fun v : ((Iic n → 𝓐 × ℝ) × 𝓐) × (ℕ → 𝓐 → ℝ) ↦ v.2) measurable_snd + have h_indep := (indepFun_history_reward_cond h t b k).symm.fst_prod + (ν := streamMeasure ν) (hR t) (by fun_prop) + refine (h_indep.of_measurable_right ?_).symm + refine Measurable.comp measurable_update_left ?_ + refine measurable_rewardByCountUntil_of t (fun i hi ↦ ?_) (fun i hi ↦ ?_) ?_ + · exact measurable_comp_comap + (fun ω : Ω × (ℕ → 𝓐 → ℝ) ↦ ((history A R t ω.1, A t ω.1), ω.2)) + (g := fun v : ((Fin t → 𝓐 × ℝ) × 𝓐) × (ℕ → 𝓐 → ℝ) ↦ (v.1.1 ⟨i, hi⟩).1) (by fun_prop) + · exact measurable_comp_comap + (fun ω : Ω × (ℕ → 𝓐 → ℝ) ↦ ((history A R t ω.1, A t ω.1), ω.2)) + (g := fun v : ((Fin t → 𝓐 × ℝ) × 𝓐) × (ℕ → 𝓐 → ℝ) ↦ (v.1.1 ⟨i, hi⟩).2) (by fun_prop) + · exact measurable_comp_comap + (fun ω : Ω × (ℕ → 𝓐 → ℝ) ↦ ((history A R t ω.1, A t ω.1), ω.2)) + (g := fun v : ((Fin t → 𝓐 × ℝ) × 𝓐) × (ℕ → 𝓐 → ℝ) ↦ v.2) measurable_snd /-- Conditionally on the event that the action at time `t` is `b` and that `b` was pulled `k` times before, the arrays `rewardByCountUntil A R (t + 1)` and `rewardByCountUntil A R t` have the diff --git a/LeanMachineLearning/SequentialLearning/Algorithm.lean b/LeanMachineLearning/SequentialLearning/Algorithm.lean index cf7cd69f..7f95e258 100644 --- a/LeanMachineLearning/SequentialLearning/Algorithm.lean +++ b/LeanMachineLearning/SequentialLearning/Algorithm.lean @@ -15,15 +15,22 @@ We define structures for stochastic, sequential algorithms and environments, and algorithm-environment sequence, which is a sequence of actions and feedbacks generated by an algorithm interacting with an environment. +At time `n`, the algorithm has observed the `n` action-feedback pairs at times `0, ..., n - 1`. +That history is an element of `Fin n → 𝓐 × 𝓨`. The algorithm then chooses an action `A n` +according to its policy, and the environment returns a feedback `Y n` given the history and `A n`. +In particular, the first action is chosen according to the policy at time `0`, applied to the +empty history (the unique element of `Fin 0 → 𝓐 × 𝓨`). + ## Main definitions * `Algorithm 𝓐 𝓨`: a stochastic, sequential algorithm. * `Environment 𝓐 𝓨`: a stochastic environment. +* `history A Y n`: the history before time `n`, a random variable with values in `Fin n → 𝓐 × 𝓨`. * `IsAlgEnvSeq A 𝓨 alg env P`: an algorithm-environment sequence. That is, a sequence of actions `A` and feedback `Y` that have the correct conditional distributions to be generated by an algorithm `alg` interacting with an environment `env`, defined on a probability space `(Ω, P)`. -* `IsAlgEnvSeqUntil A Y alg env P N`: `A` and `Y` form an algorithm-environment sequence until - time `N`. +* `IsAlgEnvSeqUntil A Y alg env P N`: `A` and `Y` form an algorithm-environment sequence for the + times `n < N`. * `prod_left alg`: an `Algorithm 𝓐 (𝓧 × 𝓨)` obtained from an algorithm `alg : Algorithm 𝓐 𝓨` by ignoring the `𝓧` component of each observation. @@ -41,17 +48,25 @@ variable {𝓐 𝓨 Ω : Type*} {m𝓐 : MeasurableSpace 𝓐} {m𝓨 : Measurab /-- A stochastic, sequential algorithm. -/ structure Algorithm (𝓐 𝓨 : Type*) [MeasurableSpace 𝓐] [MeasurableSpace 𝓨] where - /-- Policy or sampling rule: distribution of the next action. -/ - policy : (n : ℕ) → Kernel (Iic n → 𝓐 × 𝓨) 𝓐 + /-- Policy or sampling rule: distribution of the action at time `n` given the history of the + `n` previous action-feedback pairs. -/ + policy : (n : ℕ) → Kernel (Fin n → 𝓐 × 𝓨) 𝓐 /-- The policy is a Markov kernel. -/ [h_policy : ∀ n, IsMarkovKernel (policy n)] - /-- Distribution of the first action. -/ - p0 : Measure 𝓐 - /-- The first action distribution is a probability measure. -/ - [hp0 : IsProbabilityMeasure p0] instance (alg : Algorithm 𝓐 𝓨) (n : ℕ) : IsMarkovKernel (alg.policy n) := alg.h_policy n -instance (alg : Algorithm 𝓐 𝓨) : IsProbabilityMeasure alg.p0 := alg.hp0 + +/-- Distribution of the first action: the policy at time `0` applied to the empty history. -/ +def Algorithm.p0 (alg : Algorithm 𝓐 𝓨) : Measure 𝓐 := + alg.policy 0 default +deriving IsProbabilityMeasure + +lemma Algorithm.p0_def (alg : Algorithm 𝓐 𝓨) : alg.p0 = alg.policy 0 default := rfl + +lemma Algorithm.policy_zero (alg : Algorithm 𝓐 𝓨) (h : Fin 0 → 𝓐 × 𝓨) : + alg.policy 0 h = alg.p0 := by + rw [Unique.eq_default h] + rfl /-- An algorithm with observations in `𝓧 × 𝓨` obtained from an algorithm with observations in `𝓨` by ignoring the `𝓧` component of each observation. -/ @@ -59,27 +74,43 @@ by ignoring the `𝓧` component of each observation. -/ def Algorithm.prodLeft (𝓧 : Type*) [MeasurableSpace 𝓧] (alg : Algorithm 𝓐 𝓨) : Algorithm 𝓐 (𝓧 × 𝓨) where policy n := (alg.policy n).comap (fun h i ↦ ((h i).1, (h i).2.2)) (by fun_prop) - p0 := alg.p0 + +@[simp] +lemma Algorithm.prodLeft_p0 (𝓧 : Type*) [MeasurableSpace 𝓧] (alg : Algorithm 𝓐 𝓨) : + (alg.prodLeft 𝓧).p0 = alg.p0 := by + rw [p0_def, prodLeft_policy, Kernel.comap_apply, policy_zero] /-- A stochastic environment. -/ structure Environment (𝓐 𝓨 : Type*) [MeasurableSpace 𝓐] [MeasurableSpace 𝓨] where - /-- Distribution of the next observation as function of the past history. -/ - feedback : (n : ℕ) → Kernel ((Iic n → 𝓐 × 𝓨) × 𝓐) 𝓨 + /-- Distribution of the feedback at time `n` as function of the history of the `n` previous + action-feedback pairs and of the action at time `n`. -/ + feedback : (n : ℕ) → Kernel ((Fin n → 𝓐 × 𝓨) × 𝓐) 𝓨 /-- The feedback kernels are Markov kernels. -/ [h_feedback : ∀ n, IsMarkovKernel (feedback n)] - /-- Distribution of the first observation given the first action. -/ - ν0 : Kernel 𝓐 𝓨 - /-- The initial observation kernel is a Markov kernel. -/ - [hp0 : IsMarkovKernel ν0] instance (env : Environment 𝓐 𝓨) (n : ℕ) : IsMarkovKernel (env.feedback n) := env.h_feedback n -instance (env : Environment 𝓐 𝓨) : IsMarkovKernel env.ν0 := env.hp0 -/-- Kernel describing the distribution of the next action-feedback pair given the history -up to `n`. -/ +/-- Distribution of the first feedback given the first action: the feedback kernel at time `0` +applied to the empty history. -/ +noncomputable def Environment.ν0 (env : Environment 𝓐 𝓨) : Kernel 𝓐 𝓨 := + (env.feedback 0).sectR default +deriving IsMarkovKernel + +lemma Environment.ν0_def (env : Environment 𝓐 𝓨) : env.ν0 = (env.feedback 0).sectR default := rfl + +lemma Environment.ν0_apply (env : Environment 𝓐 𝓨) (a : 𝓐) : + env.ν0 a = env.feedback 0 (default, a) := rfl + +lemma Environment.feedback_zero (env : Environment 𝓐 𝓨) (h : Fin 0 → 𝓐 × 𝓨) (a : 𝓐) : + env.feedback 0 (h, a) = env.ν0 a := by + rw [Unique.eq_default h] + rfl + +/-- Kernel describing the distribution of the action-feedback pair at time `n` given the history +before `n`. -/ noncomputable def stepKernel (alg : Algorithm 𝓐 𝓨) (env : Environment 𝓐 𝓨) (n : ℕ) : - Kernel (Iic n → 𝓐 × 𝓨) (𝓐 × 𝓨) := + Kernel (Fin n → 𝓐 × 𝓨) (𝓐 × 𝓨) := alg.policy n ⊗ₖ env.feedback n deriving IsMarkovKernel @@ -91,6 +122,11 @@ lemma fst_stepKernel (alg : Algorithm 𝓐 𝓨) (env : Environment 𝓐 𝓨) ( (stepKernel alg env n).fst = alg.policy n := by rw [stepKernel, Kernel.fst_compProd] +lemma stepKernel_zero (alg : Algorithm 𝓐 𝓨) (env : Environment 𝓐 𝓨) (h : Fin 0 → 𝓐 × 𝓨) : + stepKernel alg env 0 h = alg.p0 ⊗ₘ env.ν0 := by + rw [Unique.eq_default h, stepKernel, Kernel.compProd_apply_eq_compProd_sectR] + rfl + section IsAlgEnvSeq variable {A : ℕ → Ω → 𝓐} {Y : ℕ → Ω → 𝓨} {alg : Algorithm 𝓐 𝓨} {env : Environment 𝓐 𝓨} @@ -115,10 +151,16 @@ lemma measurable_trajectory {A : ℕ → Ω → 𝓐} {Y : ℕ → Ω → 𝓨} unfold trajectory fun_prop -/-- History of the algorithm-environment sequence up to time `n`. -/ -def history (A : ℕ → Ω → 𝓐) (Y : ℕ → Ω → 𝓨) (n : ℕ) (ω : Ω) : Iic n → 𝓐 × 𝓨 := +/-- History of the algorithm-environment sequence before time `n`: the action-feedback pairs at +times `0, ..., n - 1`. -/ +def history (A : ℕ → Ω → 𝓐) (Y : ℕ → Ω → 𝓨) (n : ℕ) (ω : Ω) : Fin n → 𝓐 × 𝓨 := fun i ↦ (A i ω, Y i ω) +lemma history_apply (n : ℕ) (ω : Ω) (i : Fin n) : history A Y n ω i = (A i ω, Y i ω) := rfl + +@[simp] +lemma history_zero : history A Y 0 = fun _ ↦ (default : Fin 0 → 𝓐 × 𝓨) := Unique.eq_default _ + @[fun_prop] lemma measurable_history (hA : ∀ n, Measurable (A n)) (hY : ∀ n, Measurable (Y n)) (n : ℕ) : @@ -127,21 +169,28 @@ lemma measurable_history (hA : ∀ n, Measurable (A n)) fun_prop lemma eval_comp_history (n : ℕ) : - (fun x ↦ x ⟨n, by simp⟩) ∘ (history A Y n) = step A Y n := rfl + (fun x ↦ x (Fin.last n)) ∘ (history A Y (n + 1)) = step A Y n := rfl lemma fst_eval_comp_history (n : ℕ) : - (fun x ↦ (x ⟨n, by simp⟩).1) ∘ (history A Y n) = A n := rfl + (fun x ↦ (x (Fin.last n)).1) ∘ (history A Y (n + 1)) = A n := rfl lemma snd_eval_comp_history (n : ℕ) : - (fun x ↦ (x ⟨n, by simp⟩).2) ∘ (history A Y n) = Y n := rfl + (fun x ↦ (x (Fin.last n)).2) ∘ (history A Y (n + 1)) = Y n := rfl + +/-- The history before time `m` is a restriction of the history before time `n ≥ m`. -/ +lemma history_eq_comp_history {m n : ℕ} (hmn : m ≤ n) : + history A Y m = (fun h (i : Fin m) ↦ h (Fin.castLE hmn i)) ∘ history A Y n := rfl lemma history_succ (n : ℕ) : history A Y (n + 1) = - (MeasurableEquiv.IicSuccProd (fun ℕ ↦ 𝓐 × 𝓨) n).symm ∘ - (fun ω ↦ (history A Y n ω, step A Y (n + 1) ω)) := by + (MeasurableEquiv.finSuccProd (𝓐 × 𝓨) n).symm ∘ + (fun ω ↦ (history A Y n ω, step A Y n ω)) := by funext ω - symm - exact (MeasurableEquiv.IicSuccProd (fun _ ↦ 𝓐 × 𝓨) n).symm_apply_apply (history A Y (n + 1) ω) + simp only [Function.comp_apply, MeasurableEquiv.finSuccProd_symm_apply] + funext i + refine Fin.lastCases ?_ (fun i ↦ ?_) i + · simp [history, step] + · simp [history] /-- An algorithm-environment sequence: a sequence of actions and feedbacks generated by an algorithm interacting with an environment. -/ @@ -152,21 +201,17 @@ structure IsAlgEnvSeq measurable_action n : Measurable (A n) := by fun_prop /-- The feedback sequence is measurable. -/ measurable_feedback n : Measurable (Y n) := by fun_prop - /-- The first action has the correct law. -/ - hasLaw_action_zero : HasLaw (fun ω ↦ (A 0 ω)) alg.p0 P - /-- The first feedback has the correct conditional distribution. -/ - hasCondDistrib_feedback_zero : HasCondDistrib (Y 0) (A 0) env.ν0 P - /-- The next action has the correct conditional distribution given the history. -/ + /-- The action at time `n` has the correct conditional distribution given the history. -/ hasCondDistrib_action n : - HasCondDistrib (A (n + 1)) (history A Y n) (alg.policy n) P - /-- The next feedback has the correct conditional distribution given the history and - next action. -/ + HasCondDistrib (A n) (history A Y n) (alg.policy n) P + /-- The feedback at time `n` has the correct conditional distribution given the history and + the action at time `n`. -/ hasCondDistrib_feedback n : - HasCondDistrib (Y (n + 1)) (fun ω ↦ (history A Y n ω, A (n + 1) ω)) - (env.feedback n) P + HasCondDistrib (Y n) (fun ω ↦ (history A Y n ω, A n ω)) (env.feedback n) P -/-- An algorithm-environment sequence: a sequence of actions and feedbacks generated -by an algorithm interacting with an environment. -/ +/-- An algorithm-environment sequence until time `N`: a sequence of actions and feedbacks such that +the actions and feedbacks at times `n < N` are generated by an algorithm interacting with +an environment. In particular, the law of `history A Y N` is determined. -/ structure IsAlgEnvSeqUntil (A : ℕ → Ω → 𝓐) (Y : ℕ → Ω → 𝓨) (alg : Algorithm 𝓐 𝓨) (env : Environment 𝓐 𝓨) (P : Measure Ω) [IsFiniteMeasure P] (N : ℕ) : Prop where @@ -174,25 +219,18 @@ structure IsAlgEnvSeqUntil measurable_action n : Measurable (A n) := by fun_prop /-- The feedback sequence is measurable. -/ measurable_feedback n : Measurable (Y n) := by fun_prop - /-- The first action has the correct law. -/ - hasLaw_action_zero : HasLaw (fun ω ↦ (A 0 ω)) alg.p0 P - /-- The first feedback has the correct conditional distribution. -/ - hasCondDistrib_feedback_zero : HasCondDistrib (Y 0) (A 0) env.ν0 P - /-- The next action has the correct conditional distribution given the history. -/ + /-- The action at time `n < N` has the correct conditional distribution given the history. -/ hasCondDistrib_action n (hn : n < N) : - HasCondDistrib (A (n + 1)) (history A Y n) (alg.policy n) P - /-- The next feedback has the correct conditional distribution given the history and - next action. -/ + HasCondDistrib (A n) (history A Y n) (alg.policy n) P + /-- The feedback at time `n < N` has the correct conditional distribution given the history and + the action at time `n`. -/ hasCondDistrib_feedback n (hn : n < N) : - HasCondDistrib (Y (n + 1)) (fun ω ↦ (history A Y n ω, A (n + 1) ω)) - (env.feedback n) P + HasCondDistrib (Y n) (fun ω ↦ (history A Y n ω, A n ω)) (env.feedback n) P lemma IsAlgEnvSeqUntil.mono (h : IsAlgEnvSeqUntil A Y alg env P N) {N' : ℕ} (hN : N' ≤ N) : IsAlgEnvSeqUntil A Y alg env P N' where measurable_action := h.measurable_action measurable_feedback := h.measurable_feedback - hasLaw_action_zero := h.hasLaw_action_zero - hasCondDistrib_feedback_zero := h.hasCondDistrib_feedback_zero hasCondDistrib_action n hn := h.hasCondDistrib_action n (hn.trans_le hN) hasCondDistrib_feedback n hn := h.hasCondDistrib_feedback n (hn.trans_le hN) @@ -200,67 +238,116 @@ lemma IsAlgEnvSeq.isAlgEnvSeqUntil (h : IsAlgEnvSeq A Y alg env P) (N : ℕ) : IsAlgEnvSeqUntil A Y alg env P N where measurable_action := h.measurable_action measurable_feedback := h.measurable_feedback - hasLaw_action_zero := h.hasLaw_action_zero - hasCondDistrib_feedback_zero := h.hasCondDistrib_feedback_zero hasCondDistrib_action n _ := h.hasCondDistrib_action n hasCondDistrib_feedback n _ := h.hasCondDistrib_feedback n -@[fun_prop] +lemma isAlgEnvSeq_iff_forall_isAlgEnvSeqUntil : + IsAlgEnvSeq A Y alg env P ↔ ∀ N, IsAlgEnvSeqUntil A Y alg env P N where + mp h N := h.isAlgEnvSeqUntil N + mpr h := { + measurable_action := (h 0).measurable_action + measurable_feedback := (h 0).measurable_feedback + hasCondDistrib_action n := (h (n + 1)).hasCondDistrib_action n n.lt_succ_self + hasCondDistrib_feedback n := (h (n + 1)).hasCondDistrib_feedback n n.lt_succ_self } + lemma IsAlgEnvSeq.measurable_step (h : IsAlgEnvSeq A Y alg env P) (n : ℕ) : Measurable (step A Y n) := by have hA := h.measurable_action have hY := h.measurable_feedback fun_prop -@[fun_prop] lemma IsAlgEnvSeq.measurable_history (h : IsAlgEnvSeq A Y alg env P) (n : ℕ) : Measurable (history A Y n) := by have hA := h.measurable_action have hY := h.measurable_feedback fun_prop -lemma IsAlgEnvSeq.hasLaw_step_zero (h : IsAlgEnvSeq A Y alg env P) : - HasLaw (step A Y 0) (alg.p0 ⊗ₘ env.ν0) P := - HasLaw.prod_of_hasCondDistrib h.hasLaw_action_zero h.hasCondDistrib_feedback_zero +lemma IsAlgEnvSeq.measurable_trajectory (h : IsAlgEnvSeq A Y alg env P) : + Measurable (trajectory A Y) := + Learning.measurable_trajectory h.measurable_action h.measurable_feedback -lemma IsAlgEnvSeqUntil.hasLaw_step_zero (h : IsAlgEnvSeqUntil A Y alg env P N) : - HasLaw (step A Y 0) (alg.p0 ⊗ₘ env.ν0) P := - HasLaw.prod_of_hasCondDistrib h.hasLaw_action_zero h.hasCondDistrib_feedback_zero +lemma IsAlgEnvSeqUntil.measurable_step (h : IsAlgEnvSeqUntil A Y alg env P N) (n : ℕ) : + Measurable (step A Y n) := by + have hA := h.measurable_action + have hY := h.measurable_feedback + fun_prop + +lemma IsAlgEnvSeqUntil.measurable_history (h : IsAlgEnvSeqUntil A Y alg env P N) (n : ℕ) : + Measurable (history A Y n) := by + have hA := h.measurable_action + have hY := h.measurable_feedback + fun_prop lemma IsAlgEnvSeq.hasCondDistrib_step (h : IsAlgEnvSeq A Y alg env P) (n : ℕ) : - HasCondDistrib (step A Y (n + 1)) (history A Y n) (stepKernel alg env n) P := + HasCondDistrib (step A Y n) (history A Y n) (stepKernel alg env n) P := HasCondDistrib.prod (h.hasCondDistrib_action n) (h.hasCondDistrib_feedback n) lemma IsAlgEnvSeqUntil.hasCondDistrib_step (h : IsAlgEnvSeqUntil A Y alg env P N) (n : ℕ) (hn : n < N) : - HasCondDistrib (step A Y (n + 1)) (history A Y n) - (stepKernel alg env n) P := + HasCondDistrib (step A Y n) (history A Y n) (stepKernel alg env n) P := HasCondDistrib.prod (h.hasCondDistrib_action n hn) (h.hasCondDistrib_feedback n hn) -lemma IsAlgEnvSeq.hasLaw_history_zero (h : IsAlgEnvSeq A Y alg env P) : HasLaw (history A Y 0) - ((P.map (step A Y 0)).map (MeasurableEquiv.piUnique (fun _ : Iic 0 ↦ 𝓐 × 𝓨)).symm) P where - aemeasurable := (h.measurable_history 0).aemeasurable - map_eq := by - have he : (MeasurableEquiv.piUnique (fun _ : Iic 0 ↦ 𝓐 × 𝓨)).symm ∘ step A Y 0 = - history A Y 0 := by - funext _ ⟨0, _⟩ - rfl - rw [← he] - have hA := h.measurable_action - have hY := h.measurable_feedback - exact (Measure.map_map (by fun_prop) (by fun_prop)).symm +section Zero + +/-! ### Laws at time `0` + +At time `0` the history is the unique element of `Fin 0 → 𝓐 × 𝓨`: conditioning on it is the same +as not conditioning. Note that those results need `P` to be a probability measure. -/ + +variable [IsProbabilityMeasure P] + +omit [IsFiniteMeasure P] in +/-- The history before time `0` is a constant. -/ +lemma hasLaw_history_zero (A : ℕ → Ω → 𝓐) (Y : ℕ → Ω → 𝓨) : + HasLaw (history A Y 0) (Measure.dirac default) P where + aemeasurable := by rw [history_zero]; exact aemeasurable_const + map_eq := by rw [history_zero, Measure.map_const, measure_univ, one_smul] + +lemma IsAlgEnvSeqUntil.hasLaw_action_zero (h : IsAlgEnvSeqUntil A Y alg env P N) (hN : 0 < N) : + HasLaw (A 0) alg.p0 P := by + have h0 := h.hasCondDistrib_action 0 hN + rw [history_zero] at h0 + exact h0.hasLaw_of_const' + +lemma IsAlgEnvSeq.hasLaw_action_zero (h : IsAlgEnvSeq A Y alg env P) : + HasLaw (A 0) alg.p0 P := + (h.isAlgEnvSeqUntil 1).hasLaw_action_zero zero_lt_one + +omit [IsProbabilityMeasure P] in +lemma IsAlgEnvSeqUntil.hasCondDistrib_feedback_zero (h : IsAlgEnvSeqUntil A Y alg env P N) + (hN : 0 < N) : + HasCondDistrib (Y 0) (A 0) env.ν0 P := + hasCondDistrib_prodMk_left_unique_iff.mp (h.hasCondDistrib_feedback 0 hN) + +omit [IsProbabilityMeasure P] in +lemma IsAlgEnvSeq.hasCondDistrib_feedback_zero (h : IsAlgEnvSeq A Y alg env P) : + HasCondDistrib (Y 0) (A 0) env.ν0 P := + (h.isAlgEnvSeqUntil 1).hasCondDistrib_feedback_zero zero_lt_one + +lemma IsAlgEnvSeqUntil.hasLaw_step_zero (h : IsAlgEnvSeqUntil A Y alg env P N) (hN : 0 < N) : + HasLaw (step A Y 0) (alg.p0 ⊗ₘ env.ν0) P := by + have h0 := h.hasCondDistrib_step 0 hN + rw [history_zero] at h0 + rw [← stepKernel_zero alg env default] + exact h0.hasLaw_of_const' + +lemma IsAlgEnvSeq.hasLaw_step_zero (h : IsAlgEnvSeq A Y alg env P) : + HasLaw (step A Y 0) (alg.p0 ⊗ₘ env.ν0) P := + (h.isAlgEnvSeqUntil 1).hasLaw_step_zero zero_lt_one + +end Zero lemma IsAlgEnvSeq.hasLaw_action_comp (h : IsAlgEnvSeq A Y alg env P) (n : ℕ) : - HasLaw (A (n + 1)) (alg.policy n ∘ₘ (P.map (history A Y n))) P := + HasLaw (A n) (alg.policy n ∘ₘ (P.map (history A Y n))) P := HasCondDistrib.hasLaw_comp (h.hasCondDistrib_action n) lemma IsAlgEnvSeq.hasLaw_feedback_comp (h : IsAlgEnvSeq A Y alg env P) (n : ℕ) : - HasLaw (Y (n + 1)) ((env.feedback n) ∘ₘ (P.map fun ω ↦ (history A Y n ω, A (n + 1) ω))) P := + HasLaw (Y n) ((env.feedback n) ∘ₘ (P.map fun ω ↦ (history A Y n ω, A n ω))) P := HasCondDistrib.hasLaw_comp (h.hasCondDistrib_feedback n) -lemma IsAlgEnvSeq.hasLaw_feedback_zero_comp (h : IsAlgEnvSeq A Y alg env P) : - HasLaw (Y 0) (env.ν0 ∘ₘ (P.map (A 0))) P := - HasCondDistrib.hasLaw_comp (h.hasCondDistrib_feedback_zero) +lemma IsAlgEnvSeq.hasLaw_step_comp (h : IsAlgEnvSeq A Y alg env P) (n : ℕ) : + HasLaw (step A Y n) (stepKernel alg env n ∘ₘ (P.map (history A Y n))) P := + HasCondDistrib.hasLaw_comp (h.hasCondDistrib_step n) /-- Conditionally on the event `A 0 = b`, the first feedback has law `env.ν0 b`. -/ lemma IsAlgEnvSeq.hasLaw_feedback_zero_cond [MeasurableSingletonClass 𝓐] @@ -273,131 +360,100 @@ section Filtration namespace IsAlgEnvSeq -/-- Filtration generated by the history up to time `n`. -/ +/-- Filtration generated by the history up to time `n` (included): `h.filtration n` is the +σ-algebra generated by `history A Y (n + 1)`, that is by the action-feedback pairs at times +`0, ..., n`. -/ def filtration (h : IsAlgEnvSeq A Y alg env P) : Filtration ℕ mΩ where - seq i := MeasurableSpace.comap (history A Y i) inferInstance + seq n := MeasurableSpace.comap (history A Y (n + 1)) inferInstance mono' i j hij := by simp only - rw [← measurable_iff_comap_le] - have : history A Y i = (fun h k ↦ h ⟨k.1, by grind⟩) ∘ history A Y j := rfl - rw [this] + rw [← measurable_iff_comap_le, history_eq_comp_history (Nat.succ_le_succ hij)] exact measurable_comp_comap _ (by fun_prop) le' i := by rw [← measurable_iff_comap_le] - exact Learning.measurable_history h.measurable_action h.measurable_feedback i + exact Learning.measurable_history h.measurable_action h.measurable_feedback _ + +lemma filtration_eq_comap (h : IsAlgEnvSeq A Y alg env P) (n : ℕ) : + h.filtration n = MeasurableSpace.comap (history A Y (n + 1)) inferInstance := rfl + +lemma measurable_history_succ_filtration (h : IsAlgEnvSeq A Y alg env P) (n : ℕ) : + Measurable[h.filtration n] (history A Y (n + 1)) := + measurable_iff_comap_le.mpr le_rfl lemma adapted_history (h : IsAlgEnvSeq A Y alg env P) : - Adapted h.filtration (history A Y) := - fun _ ↦ measurable_iff_comap_le.mpr le_rfl + Adapted h.filtration (history A Y) := by + intro n + rw [history_eq_comp_history n.le_succ] + exact measurable_comp_comap _ (by fun_prop) lemma adapted_step (h : IsAlgEnvSeq A Y alg env P) : Adapted h.filtration (step A Y) := by intro n - have : step A Y n = (fun h ↦ (h ⟨n, by simp⟩)) ∘ (history A Y n) := by - ext ω : 1 - simp [history, step] - rw [this] + rw [← eval_comp_history] exact measurable_comp_comap _ (by fun_prop) lemma adapted_action (h : IsAlgEnvSeq A Y alg env P) : Adapted h.filtration A := by intro n - have : A n = (fun h ↦ (h ⟨n, by simp⟩).1) ∘ (history A Y n) := by - ext ω : 1 - simp [history] - rw [this] + rw [← fst_eval_comp_history (A := A) (Y := Y) n] exact measurable_comp_comap _ (by fun_prop) lemma adapted_feedback (h : IsAlgEnvSeq A Y alg env P) : Adapted h.filtration Y := by intro n - have : Y n = (fun h ↦ (h ⟨n, by simp⟩).2) ∘ (history A Y n) := by - ext ω : 1 - simp [history] - rw [this] + rw [← snd_eval_comp_history (A := A) (Y := Y) n] exact measurable_comp_comap _ (by fun_prop) -/-- Filtration generated by the history at time `n-1` together with the action at time `n`. -/ +/-- Filtration generated by the history before time `n` together with the action at time `n`. -/ def filtrationAction (h : IsAlgEnvSeq A Y alg env P) : Filtration ℕ mΩ where - seq n := if n = 0 then MeasurableSpace.comap (A 0) inferInstance - else h.filtration (n - 1) ⊔ MeasurableSpace.comap (A n) inferInstance + seq n := MeasurableSpace.comap (fun ω ↦ (history A Y n ω, A n ω)) inferInstance mono' n m hnm := by simp only - by_cases hn : n = 0 - · by_cases hm : m = 0 - · simp [hn, hm] - · simp only [hn, ↓reduceIte, hm] - refine le_sup_of_le_left ?_ - rw [← measurable_iff_comap_le] - suffices Measurable[h.filtration 0] (A 0) from - this.mono ((h.filtration).mono zero_le) le_rfl - exact adapted_action h 0 - have hm : m ≠ 0 := by grind - simp only [hn, hm, ↓reduceIte] - have hnm' : n - 1 ≤ m - 1 := by grind - simp only [sup_le_iff] - constructor - · refine le_sup_of_le_left ?_ - exact (h.filtration).mono hnm' - · rcases eq_or_lt_of_le hnm with rfl | hlt - · exact le_sup_of_le_right le_rfl - refine le_sup_of_le_left ?_ - rw [← measurable_iff_comap_le] - have h_le : n ≤ m - 1 := by grind - suffices Measurable[h.filtration n] (A n) from - this.mono ((h.filtration).mono h_le) le_rfl - exact adapted_action h n + rw [← measurable_iff_comap_le] + rcases eq_or_lt_of_le hnm with rfl | hlt + · exact measurable_iff_comap_le.mpr le_rfl + have : (fun ω ↦ (history A Y n ω, A n ω)) = + (fun p : (Fin m → 𝓐 × 𝓨) × 𝓐 ↦ + (fun i : Fin n ↦ p.1 (Fin.castLE hnm i), (p.1 ⟨n, hlt⟩).1)) ∘ + (fun ω ↦ (history A Y m ω, A m ω)) := rfl + rw [this] + exact measurable_comp_comap _ (by fun_prop) le' n := by - by_cases hn : n = 0 - · simp only [hn, ↓reduceIte] - rw [← measurable_iff_comap_le] - exact h.measurable_action 0 - simp only [hn, ↓reduceIte, sup_le_iff] - constructor - · exact (IsAlgEnvSeq.filtration h).le _ - · rw [← measurable_iff_comap_le] - exact h.measurable_action n + rw [← measurable_iff_comap_le] + exact (Learning.measurable_history h.measurable_action h.measurable_feedback n).prodMk + (h.measurable_action n) + +lemma filtrationAction_eq_comap (h : IsAlgEnvSeq A Y alg env P) (n : ℕ) : + h.filtrationAction n = + MeasurableSpace.comap (fun ω ↦ (history A Y n ω, A n ω)) inferInstance := rfl lemma filtrationAction_zero_eq_comap (h : IsAlgEnvSeq A Y alg env P) : h.filtrationAction 0 = MeasurableSpace.comap (A 0) inferInstance := by - simp [filtrationAction] - -lemma filtrationAction_eq_comap (h : IsAlgEnvSeq A Y alg env P) (n : ℕ) (hn : n ≠ 0) : - h.filtrationAction n = - MeasurableSpace.comap (fun ω ↦ (history A Y (n - 1) ω, A n ω)) inferInstance := by - simp only [filtrationAction, filtration, ← MeasurableSpace.comap_prodMk, hn, ↓reduceIte] - rfl + rw [filtrationAction_eq_comap, history_zero] + refine le_antisymm ?_ ?_ + · rw [← measurable_iff_comap_le] + exact measurable_const.prodMk (measurable_iff_comap_le.mpr le_rfl) + · rw [← measurable_iff_comap_le] + exact measurable_snd.comp (measurable_iff_comap_le.mpr le_rfl) -@[fun_prop] -lemma measurable_history_filtrationAction_add_one (h : IsAlgEnvSeq A Y alg env P) (n : ℕ) : - Measurable[h.filtrationAction (n + 1)] (history A Y n) := by - rw [h.filtrationAction_eq_comap (n + 1) (Nat.succ_ne_zero n)] - exact measurable_fst.comp (measurable_iff_comap_le.mpr le_rfl) +lemma measurable_history_filtrationAction (h : IsAlgEnvSeq A Y alg env P) (n : ℕ) : + Measurable[h.filtrationAction n] (history A Y n) := + measurable_fst.comp (measurable_iff_comap_le.mpr le_rfl) lemma filtration_le_filtrationAction_succ (h : IsAlgEnvSeq A Y alg env P) (n : ℕ) : h.filtration n ≤ h.filtrationAction (n + 1) := - measurable_iff_comap_le.mp (h.measurable_history_filtrationAction_add_one n) + measurable_iff_comap_le.mp (h.measurable_history_filtrationAction (n + 1)) lemma filtrationAction_le_filtration (h : IsAlgEnvSeq A Y alg env P) (n : ℕ) : h.filtrationAction n ≤ h.filtration n := by - cases n with - | zero => - rw [h.filtrationAction_zero_eq_comap] - exact measurable_iff_comap_le.mp (h.adapted_action 0) - | succ m => - rw [h.filtrationAction_eq_comap (m + 1) (Nat.succ_ne_zero m), ← measurable_iff_comap_le] - exact ((h.adapted_history m).mono (h.filtration.mono m.le_succ) le_rfl).prodMk - (h.adapted_action (m + 1)) + rw [filtrationAction_eq_comap, ← measurable_iff_comap_le] + exact (h.adapted_history n).prodMk (h.adapted_action n) lemma adapted_action_filtrationAction (h : IsAlgEnvSeq A Y alg env P) : - Adapted h.filtrationAction A := fun n ↦ by - cases n with - | zero => rw [h.filtrationAction_zero_eq_comap]; exact measurable_iff_comap_le.mpr le_rfl - | succ m => - rw [h.filtrationAction_eq_comap (m + 1) (Nat.succ_ne_zero m)] - exact measurable_snd.comp (measurable_iff_comap_le.mpr le_rfl) + Adapted h.filtrationAction A := fun _ ↦ + measurable_snd.comp (measurable_iff_comap_le.mpr le_rfl) lemma measurable_feedback_filtrationAction_of_lt (h : IsAlgEnvSeq A Y alg env P) {m n : ℕ} (hmn : m < n) : diff --git a/LeanMachineLearning/SequentialLearning/AlgorithmDensity.lean b/LeanMachineLearning/SequentialLearning/AlgorithmDensity.lean index 311fd885..de6d7c96 100644 --- a/LeanMachineLearning/SequentialLearning/AlgorithmDensity.lean +++ b/LeanMachineLearning/SequentialLearning/AlgorithmDensity.lean @@ -53,23 +53,30 @@ namespace Algorithm /-- For every time and history, the distribution over actions according to `alg` is absolutely continuous with respect to the distribution over actions according to `alg₀`. -/ structure AbsolutelyContinuous (alg alg₀ : Algorithm 𝓐 𝓨) : Prop where - p0 : alg.p0 ≪ alg₀.p0 policy n h : alg.policy n h ≪ alg₀.policy n h @[inherit_doc AbsolutelyContinuous] scoped notation:50 alg " ≪ₐ " alg₀ => AbsolutelyContinuous alg alg₀ +lemma AbsolutelyContinuous.p0 {alg alg₀ : Algorithm 𝓐 𝓨} (h : alg ≪ₐ alg₀) : alg.p0 ≪ alg₀.p0 := + h.policy 0 default + /-- If the algorithm `alg` is absolutely continuous with respect to the algorithm `alg₀` and they -are both interacting with the same environment, then the law of the history at time `n` under `alg` -is the law of the history at time `n` under `alg₀` with density `alg.density alg₀ n`. -/ +are both interacting with the same environment, then the law of the history before time `n` under +`alg` is the law of the history before time `n` under `alg₀` with density `alg.density alg₀ n`. -/ noncomputable def density [MeasurableSpace.CountablyGenerated 𝓐] (alg alg₀ : Algorithm 𝓐 𝓨) : - (n : ℕ) → (Iic n → 𝓐 × 𝓨) → ℝ≥0∞ - | 0, h => (alg.p0.rnDeriv alg₀.p0 (h ⟨0, by simp⟩).1) + (n : ℕ) → (Fin n → 𝓐 × 𝓨) → ℝ≥0∞ + | 0, _ => 1 | n + 1, h => - let p := MeasurableEquiv.IicSuccProd (fun _ ↦ 𝓐 × 𝓨) n h + let p := MeasurableEquiv.finSuccProd (𝓐 × 𝓨) n h alg.density alg₀ n p.1 * (alg.policy n).rnDeriv (alg₀.policy n) p.1 p.2.1 +@[simp] +lemma density_zero [MeasurableSpace.CountablyGenerated 𝓐] (alg alg₀ : Algorithm 𝓐 𝓨) + (h : Fin 0 → 𝓐 × 𝓨) : + alg.density alg₀ 0 h = 1 := rfl + @[fun_prop] lemma measurable_density [MeasurableSpace.CountablyGenerated 𝓐] (alg alg₀ : Algorithm 𝓐 𝓨) (n : ℕ) : Measurable (alg.density alg₀ n) := by @@ -86,7 +93,7 @@ namespace IsAlgEnvSeq variable {Ω : Type*} [MeasurableSpace Ω] variable {alg : Algorithm 𝓐 𝓨} {env : Environment 𝓐 𝓨} variable {A : ℕ → Ω → 𝓐} {Y : ℕ → Ω → 𝓨} -variable {P : Measure Ω} [IsFiniteMeasure P] +variable {P : Measure Ω} [IsProbabilityMeasure P] variable {Ω₀ : Type*} [MeasurableSpace Ω₀] variable {alg₀ : Algorithm 𝓐 𝓨} @@ -98,16 +105,13 @@ lemma absolutelyContinuous_map_history (h : IsAlgEnvSeq A Y alg env P) P.map (history A Y n) ≪ P₀.map (history A₀ Y₀ n) := by induction n with | zero => - rw [h.hasLaw_history_zero.map_eq, h₀.hasLaw_history_zero.map_eq] - apply Measure.AbsolutelyContinuous.map _ (by fun_prop) - rw [h.hasLaw_step_zero.map_eq, h₀.hasLaw_step_zero.map_eq] - exact Measure.AbsolutelyContinuous.compProd_left hc.p0 _ + rw [(hasLaw_history_zero A Y).map_eq, (hasLaw_history_zero A₀ Y₀).map_eq] | succ n ih => simp_rw [history_succ] rw [← Measure.map_map (by fun_prop), ← Measure.map_map (by fun_prop)] rotate_left - · exact (h₀.measurable_history n).prodMk (h₀.measurable_step (n + 1)) - · exact (h.measurable_history n).prodMk (h.measurable_step (n + 1)) + · exact (h₀.measurable_history n).prodMk (h₀.measurable_step n) + · exact (h.measurable_history n).prodMk (h.measurable_step n) apply Measure.AbsolutelyContinuous.map _ (by fun_prop) rw [(h.hasCondDistrib_step n).map_eq, (h₀.hasCondDistrib_step n).map_eq] apply Measure.AbsolutelyContinuous.compProd ih @@ -122,11 +126,8 @@ lemma hasLaw_history_withDensity (h : IsAlgEnvSeq A Y alg env P) map_eq := by induction n with | zero => - rw [h.hasLaw_history_zero.map_eq, h₀.hasLaw_history_zero.map_eq, h.hasLaw_step_zero.map_eq, - h₀.hasLaw_step_zero.map_eq] - rw [← Measure.withDensity_rnDeriv_eq _ _ hc.p0, - Measure.compProd_withDensity_left (by fun_prop)] - exact map_equiv_withDensity (by fun_prop) + rw [(hasLaw_history_zero A Y).map_eq, (hasLaw_history_zero A₀ Y₀).map_eq, + show alg.density alg₀ 0 = 1 from rfl, withDensity_one] | succ n ih => let ρ h' (ar : 𝓐 × 𝓨) := Kernel.rnDeriv (alg.policy n) (alg₀.policy n) h' ar.1 have hs : stepKernel alg env n = (stepKernel alg₀ env n).withDensity ρ := by @@ -138,8 +139,8 @@ lemma hasLaw_history_withDensity (h : IsAlgEnvSeq A Y alg env P) simp_rw [history_succ] rw [← Measure.map_map (by fun_prop), ← Measure.map_map (by fun_prop)] rotate_left - · exact (h₀.measurable_history n).prodMk (h₀.measurable_step (n + 1)) - · exact (h.measurable_history n).prodMk (h.measurable_step (n + 1)) + · exact (h₀.measurable_history n).prodMk (h₀.measurable_step n) + · exact (h.measurable_history n).prodMk (h.measurable_step n) rw [(h.hasCondDistrib_step n).map_eq, (h₀.hasCondDistrib_step n).map_eq, ih, hs, Measure.compProd_withDensity_withDensity (by fun_prop) (by fun_prop)] exact map_equiv_withDensity (by fun_prop) diff --git a/LeanMachineLearning/SequentialLearning/Algorithms/RandomSampling/Basic.lean b/LeanMachineLearning/SequentialLearning/Algorithms/RandomSampling/Basic.lean index 0a7bfbec..ebaa27d9 100644 --- a/LeanMachineLearning/SequentialLearning/Algorithms/RandomSampling/Basic.lean +++ b/LeanMachineLearning/SequentialLearning/Algorithms/RandomSampling/Basic.lean @@ -44,7 +44,6 @@ measure at each iteration. -/ @[simps] noncomputable def randomSampling (μ : Measure 𝓐) [IsProbabilityMeasure μ] : Algorithm 𝓐 𝓨 where policy _ := Kernel.const _ μ - p0 := μ namespace randomSampling @@ -52,13 +51,8 @@ variable {A : ℕ → Ω → 𝓐} {Y : ℕ → Ω → 𝓨} {env : Environment /-- Each action follows the distribution μ. -/ lemma hasLaw_action (h : IsAlgEnvSeq A Y (randomSampling μ) env P) (n : ℕ) : - HasLaw (A n) μ P := by - by_cases hn : n = 0 - · rw [hn] - exact h.hasLaw_action_zero - · push Not at hn - obtain ⟨k, rfl⟩ := Nat.exists_eq_succ_of_ne_zero hn - exact (h.hasCondDistrib_action k).hasLaw_of_const + HasLaw (A n) μ P := + (h.hasCondDistrib_action n).hasLaw_of_const /-- Actions are mutually independent. -/ lemma iIndep_action (h : IsAlgEnvSeq A Y (randomSampling μ) env P) : @@ -66,13 +60,14 @@ lemma iIndep_action (h : IsAlgEnvSeq A Y (randomSampling μ) env P) : have hA := h.measurable_action rw [iIndepFun_nat_iff_forall_indepFun (by fun_prop)] intro n - have map_eq := (h.hasCondDistrib_action n).map_eq + have map_eq := (h.hasCondDistrib_action (n + 1)).map_eq simp only [randomSampling_policy, Measure.compProd_const] at map_eq have law_eq : P.map (A (n + 1)) = μ := (hasLaw_action h (n + 1)).map_eq rw [← law_eq, ← indepFun_iff_map_prod_eq_prod_map_map] at map_eq - · change A (n + 1) ⟂ᵢ[P] (fun (f : Iic n → 𝓐 × 𝓨) ↦ (fun i ↦ (f i).1))∘ (history A Y n) + · change A (n + 1) ⟂ᵢ[P] (fun (f : Fin (n + 1) → 𝓐 × 𝓨) (i : Iic n) ↦ + (f ⟨i.1, Nat.lt_succ_of_le (mem_Iic.mp i.2)⟩).1) ∘ (history A Y (n + 1)) refine map_eq.symm.comp measurable_id (by fun_prop) - · exact (h.measurable_history n).aemeasurable + · exact (h.measurable_history (n + 1)).aemeasurable · exact (h.measurable_action (n + 1)).aemeasurable end randomSampling diff --git a/LeanMachineLearning/SequentialLearning/Algorithms/RoundRobin.lean b/LeanMachineLearning/SequentialLearning/Algorithms/RoundRobin.lean index dc6e8540..c3ddd464 100644 --- a/LeanMachineLearning/SequentialLearning/Algorithms/RoundRobin.lean +++ b/LeanMachineLearning/SequentialLearning/Algorithms/RoundRobin.lean @@ -74,14 +74,14 @@ variable {𝓨 : Type*} {m𝓨 : MeasurableSpace 𝓨} {K : ℕ} section AlgorithmDefinition -/-- Action chosen by the Round-Robin algorithm at time `n + 1`. This is action `(n + 1) % K`. -/ +/-- Action chosen by the Round-Robin algorithm at time `n`. This is action `n % K`. -/ noncomputable -def RoundRobin.nextAction (hK : 0 < K) (n : ℕ) : Fin K := ⟨(n + 1) % K, Nat.mod_lt _ hK⟩ +def RoundRobin.nextAction (hK : 0 < K) (n : ℕ) : Fin K := ⟨n % K, Nat.mod_lt _ hK⟩ /-- The Round-Robin algorithm: deterministic algorithm that chooses action `n % K` at time `n`. -/ noncomputable def roundRobinAlgorithm (hK : 0 < K) : Algorithm (Fin K) 𝓨 := - detAlgorithm (fun n _ ↦ RoundRobin.nextAction hK n) (by fun_prop) ⟨0, hK⟩ + detAlgorithm (fun n _ ↦ RoundRobin.nextAction hK n) (by fun_prop) end AlgorithmDefinition @@ -92,30 +92,23 @@ variable {hK : 0 < K} {ν : Kernel (Fin K) 𝓨} [IsMarkovKernel ν] {P : Measure Ω} [IsProbabilityMeasure P] {A : ℕ → Ω → Fin K} {Y : ℕ → Ω → 𝓨} -lemma action_zero - (h : IsAlgEnvSeqUntil A Y (roundRobinAlgorithm hK) (stationaryEnv ν) P 0) : - A 0 =ᵐ[P] fun _ ↦ ⟨0, hK⟩ := by - have : Nonempty (Fin K) := Fin.pos_iff_nonempty.mp hK - exact h.action_zero_detAlgorithm - -lemma action_ae_eq_roundRobinNextAction (n : ℕ) - (h : IsAlgEnvSeqUntil A Y (roundRobinAlgorithm hK) (stationaryEnv ν) P (n + 1)) : - A (n + 1) =ᵐ[P] fun _ ↦ nextAction hK n := - h.action_detAlgorithm_ae_eq (by grind) - /-- The action chosen at time `n` is the action `n % K`. -/ lemma action_ae_eq (n : ℕ) - (h : IsAlgEnvSeqUntil A Y (roundRobinAlgorithm hK) (stationaryEnv ν) P n) : + (h : IsAlgEnvSeqUntil A Y (roundRobinAlgorithm hK) (stationaryEnv ν) P (n + 1)) : A n =ᵐ[P] fun _ ↦ ⟨n % K, Nat.mod_lt _ hK⟩ := by - cases n with - | zero => exact action_zero h - | succ n => - filter_upwards [action_ae_eq_roundRobinNextAction n h] with h hn_eq - rw [hn_eq, nextAction] + have : Nonempty (Fin K) := Fin.pos_iff_nonempty.mp hK + exact h.action_detAlgorithm_ae_eq n.lt_succ_self + +lemma action_zero + (h : IsAlgEnvSeqUntil A Y (roundRobinAlgorithm hK) (stationaryEnv ν) P 1) : + A 0 =ᵐ[P] fun _ ↦ ⟨0, hK⟩ := by + filter_upwards [action_ae_eq 0 h] with ω hω + rw [hω] + simp /-- At time `K * m`, the number of times each action is chosen is equal to `m`. -/ lemma pullCount_mul (m : ℕ) - (h : IsAlgEnvSeqUntil A Y (roundRobinAlgorithm hK) (stationaryEnv ν) P (K * m - 1)) + (h : IsAlgEnvSeqUntil A Y (roundRobinAlgorithm hK) (stationaryEnv ν) P (K * m)) (a : Fin K) : pullCount A a (K * m) =ᵐ[P] fun _ ↦ m := by rw [Filter.EventuallyEq] @@ -131,14 +124,14 @@ lemma pullCount_mul (m : ℕ) _ = m := sum_mod_range_mul hK m a lemma pullCount_eq_one - (h : IsAlgEnvSeqUntil A Y (roundRobinAlgorithm hK) (stationaryEnv ν) P (K - 1)) (a : Fin K) : + (h : IsAlgEnvSeqUntil A Y (roundRobinAlgorithm hK) (stationaryEnv ν) P K) (a : Fin K) : pullCount A a K =ᵐ[P] fun _ ↦ 1 := by suffices pullCount A a (K * 1) =ᵐ[P] fun _ ↦ 1 by simpa using this refine pullCount_mul 1 (P := P) (ν := ν) (Y := Y) (hK := hK) ?_ a simpa lemma time_gt_of_pullCount_gt_one - (h : IsAlgEnvSeqUntil A Y (roundRobinAlgorithm hK) (stationaryEnv ν) P (K - 1)) (a : Fin K) : + (h : IsAlgEnvSeqUntil A Y (roundRobinAlgorithm hK) (stationaryEnv ν) P K) (a : Fin K) : ∀ᵐ ω ∂P, ∀ n, 1 < pullCount A a n ω → K < n := by filter_upwards [pullCount_eq_one h a] with h h_eq n hn rw [← h_eq] at hn @@ -146,7 +139,7 @@ lemma time_gt_of_pullCount_gt_one exact hn.not_ge (pullCount_mono _ h_lt _) lemma pullCount_pos_of_time_ge - (h : IsAlgEnvSeqUntil A Y (roundRobinAlgorithm hK) (stationaryEnv ν) P (K - 1)) : + (h : IsAlgEnvSeqUntil A Y (roundRobinAlgorithm hK) (stationaryEnv ν) P K) : ∀ᵐ ω ∂P, ∀ n, K ≤ n → ∀ b : Fin K, 0 < pullCount A b n ω := by have h_ae a := pullCount_eq_one h a simp_rw [Filter.EventuallyEq, ← ae_all_iff] at h_ae @@ -156,7 +149,7 @@ lemma pullCount_pos_of_time_ge exact pullCount_mono _ hn _ lemma pullCount_pos_of_pullCount_gt_one - (h : IsAlgEnvSeqUntil A Y (roundRobinAlgorithm hK) (stationaryEnv ν) P (K - 1)) (a : Fin K) : + (h : IsAlgEnvSeqUntil A Y (roundRobinAlgorithm hK) (stationaryEnv ν) P K) (a : Fin K) : ∀ᵐ ω ∂P, ∀ n, 1 < pullCount A a n ω → ∀ b : Fin K, 0 < pullCount A b n ω := by filter_upwards [time_gt_of_pullCount_gt_one h a, pullCount_pos_of_time_ge h] with ω h1 h2 n h_gt a exact h2 n (h1 n h_gt).le a diff --git a/LeanMachineLearning/SequentialLearning/Algorithms/Uniform.lean b/LeanMachineLearning/SequentialLearning/Algorithms/Uniform.lean index c12d6ff8..a2efb83b 100644 --- a/LeanMachineLearning/SequentialLearning/Algorithms/Uniform.lean +++ b/LeanMachineLearning/SequentialLearning/Algorithms/Uniform.lean @@ -40,8 +40,6 @@ def uniformAlgorithm [Finite 𝓐] [Nonempty 𝓐] : Algorithm 𝓐 𝓨 := rand lemma absolutelyContinuous_uniformAlgorithm [Finite 𝓐] [Nonempty 𝓐] {alg : Algorithm 𝓐 𝓨} : alg ≪ₐ uniformAlgorithm where - p0 := Measure.absolutelyContinuous_of_measure_singleton_ne_zero - (by simp [uniformAlgorithm, uniformOn, ← pos_iff_ne_zero, cond_pos_of_inter_ne_zero]) policy n h := Measure.absolutelyContinuous_of_measure_singleton_ne_zero (by simp [uniformAlgorithm, uniformOn, ← pos_iff_ne_zero, cond_pos_of_inter_ne_zero]) diff --git a/LeanMachineLearning/SequentialLearning/BayesStationaryEnv.lean b/LeanMachineLearning/SequentialLearning/BayesStationaryEnv.lean index 68cc8ffe..97b40c6a 100644 --- a/LeanMachineLearning/SequentialLearning/BayesStationaryEnv.lean +++ b/LeanMachineLearning/SequentialLearning/BayesStationaryEnv.lean @@ -23,9 +23,10 @@ This file defines the structure `IsBayesAlgEnvSeq` and provides its basic proper `κ : Kernel (𝓔 × 𝓐) 𝓨`, and algorithm `alg : Algorithm 𝓐 𝓨`, provides a probability measure `P : Measure (ℕ → 𝓐 × 𝓔 × 𝓨)` on a space that carries `E`, `A`, and `Y` such that `IsBayesAlgEnvSeq Q κ alg E A Y P`. -* `bayesTrajMeasurePosterior Q κ alg n`: a `Kernel (Iic n → 𝓐 × 𝓨) 𝓔` that represents the posterior - over `E` given the history up to time `n` under the prior `Q` and the algorithm `alg`, assuming - that the kernel `κ` specifies how `E` gives rise to the underlying (stationary) environment. +* `bayesTrajMeasurePosterior Q κ alg n`: a `Kernel (Fin n → 𝓐 × 𝓨) 𝓔` that represents the + posterior over `E` given the history before time `n` (the `n` first action-feedback pairs) under + the prior `Q` and the algorithm `alg`, assuming that the kernel `κ` specifies how `E` gives rise + to the underlying (stationary) environment. See also `LeanMachineLearning/SequentialLearning/AlgorithmDensityBayes.lean`. ## Main results @@ -63,14 +64,14 @@ structure IsBayesAlgEnvSeq measurable_action n : Measurable (A n) := by fun_prop measurable_feedback n : Measurable (Y n) := by fun_prop hasLaw_env : HasLaw E Q P - hasCondDistrib_action_zero : HasCondDistrib (A 0) E (Kernel.const _ alg.p0) P - hasCondDistrib_feedback_zero : HasCondDistrib (Y 0) (fun ω ↦ (E ω, A 0 ω)) κ P + /-- The action at time `n` has the correct conditional distribution given the parameter and the + history: it depends only on the history. -/ hasCondDistrib_action n : - HasCondDistrib (A (n + 1)) (fun ω ↦ (E ω, history A Y n ω)) - ((alg.policy n).prodMkLeft _) P + HasCondDistrib (A n) (fun ω ↦ (E ω, history A Y n ω)) ((alg.policy n).prodMkLeft _) P + /-- The feedback at time `n` has the correct conditional distribution given the history, the + parameter and the action at time `n`: it depends only on the parameter and the action. -/ hasCondDistrib_feedback n : - HasCondDistrib (Y (n + 1)) (fun ω ↦ (history A Y n ω, E ω, A (n + 1) ω)) - (κ.prodMkLeft _) P + HasCondDistrib (Y n) (fun ω ↦ (history A Y n ω, E ω, A n ω)) (κ.prodMkLeft _) P namespace IsBayesAlgEnvSeq @@ -78,59 +79,61 @@ variable {Q : Measure 𝓔} {κ : Kernel (𝓔 × 𝓐) 𝓨} {alg : Algorithm variable {E : Ω → 𝓔} {A : ℕ → Ω → 𝓐} {Y : ℕ → Ω → 𝓨} variable {P : Measure Ω} [IsFiniteMeasure P] -lemma hasLaw_action_zero [IsProbabilityMeasure P] (h : IsBayesAlgEnvSeq Q κ alg E A Y P) : - HasLaw (A 0) alg.p0 P := h.hasCondDistrib_action_zero.hasLaw_of_const - lemma hasCondDistrib_action' (h : IsBayesAlgEnvSeq Q κ alg E A Y P) (n : ℕ) : - HasCondDistrib (A (n + 1)) (history A Y n) (alg.policy n) P := + HasCondDistrib (A n) (history A Y n) (alg.policy n) P := (h.hasCondDistrib_action n).comp_right lemma hasCondDistrib_feedback' [IsFiniteKernel κ] (h : IsBayesAlgEnvSeq Q κ alg E A Y P) (n : ℕ) : - HasCondDistrib (Y (n + 1)) (fun ω ↦ (E ω, A (n + 1) ω)) κ P := + HasCondDistrib (Y n) (fun ω ↦ (E ω, A n ω)) κ P := (h.hasCondDistrib_feedback n).comp_right +lemma hasLaw_action_zero [IsProbabilityMeasure P] (h : IsBayesAlgEnvSeq Q κ alg E A Y P) : + HasLaw (A 0) alg.p0 P := by + have h0 := h.hasCondDistrib_action' 0 + rw [history_zero] at h0 + exact h0.hasLaw_of_const' + +/-- The first action is independent of the parameter `E`, and has law `alg.p0`. -/ +lemma hasCondDistrib_action_zero (h : IsBayesAlgEnvSeq Q κ alg E A Y P) : + HasCondDistrib (A 0) E (Kernel.const _ alg.p0) P := + hasCondDistrib_prodMk_right_unique_iff.mp (h.hasCondDistrib_action 0) + variable [StandardBorelSpace 𝓐] [Nonempty 𝓐] [StandardBorelSpace 𝓨] [Nonempty 𝓨] -lemma hasLaw_IT_action_zero (h : IsBayesAlgEnvSeq Q κ alg E A Y P) : - ∀ᵐ e ∂Q, HasLaw (IT.action 0) alg.p0 (condDistrib (trajectory A Y) E P e) := by - rw [← h.hasLaw_env.map_eq] - filter_upwards [condDistrib_comp E - ((measurable_trajectory h.measurable_action h.measurable_feedback).aemeasurable) - (IT.measurable_action (𝓐 := 𝓐) (𝓨 := 𝓨) 0), - h.hasCondDistrib_action_zero.condDistrib_eq] with _ hc hcd - exact ⟨(IT.measurable_action 0).aemeasurable, by - rw [← Kernel.map_apply _ (IT.measurable_action 0), ← hc, - show IT.action 0 ∘ trajectory A Y = A 0 from rfl, hcd, Kernel.const_apply]⟩ - -lemma hasCondDistrib_IT_feedback_zero [IsFiniteKernel κ] (h : IsBayesAlgEnvSeq Q κ alg E A Y P) : - ∀ᵐ e ∂Q, HasCondDistrib (IT.feedback 0) (IT.action 0) (κ.sectR e) - (condDistrib (trajectory A Y) E P e) := by - rw [← h.hasLaw_env.map_eq] - exact h.hasCondDistrib_feedback_zero.hasCondDistrib_sectR - (IT.measurable_action 0) (IT.measurable_feedback 0) - (measurable_trajectory h.measurable_action h.measurable_feedback).aemeasurable +omit [StandardBorelSpace 𝓐] [Nonempty 𝓐] [StandardBorelSpace 𝓨] [Nonempty 𝓨] in +/-- The posterior over the parameter given the empty history is the prior. -/ +lemma condDistrib_param_history_zero [StandardBorelSpace 𝓔] [Nonempty 𝓔] [IsProbabilityMeasure P] + [IsFiniteMeasure Q] (h : IsBayesAlgEnvSeq Q κ alg E A Y P) : + condDistrib E (history A Y 0) P = Kernel.const _ Q := by + ext x : 1 + rw [Unique.eq_default x, history_zero] + have hc : HasCondDistrib E (fun _ : Ω ↦ (default : Fin 0 → 𝓐 × 𝓨)) (Kernel.const _ Q) P := + h.hasLaw_env.hasCondDistrib_const + have h_ae := hc.condDistrib_eq + rw [Measure.map_const, measure_univ, one_smul, Filter.EventuallyEq, + ae_dirac_iff Subsingleton.measurableSet] at h_ae + exact h_ae lemma hasCondDistrib_IT_action (h : IsBayesAlgEnvSeq Q κ alg E A Y P) (n : ℕ) : - ∀ᵐ e ∂Q, HasCondDistrib (IT.action (n + 1)) (IT.hist n) (alg.policy n) + ∀ᵐ e ∂Q, HasCondDistrib (IT.action n) (IT.hist n) (alg.policy n) (condDistrib (trajectory A Y) E P e) := by rw [← h.hasLaw_env.map_eq] filter_upwards [(h.hasCondDistrib_action n).hasCondDistrib_sectR - (IT.measurable_hist n) (IT.measurable_action (n + 1)) + (IT.measurable_hist n) (IT.measurable_action n) (measurable_trajectory h.measurable_action h.measurable_feedback).aemeasurable] with _ he rwa [Kernel.sectR_prodMkLeft] at he lemma hasCondDistrib_IT_feedback [IsFiniteKernel κ] (h : IsBayesAlgEnvSeq Q κ alg E A Y P) (n : ℕ) : - ∀ᵐ e ∂Q, HasCondDistrib (IT.feedback (n + 1)) (fun τ ↦ (IT.hist n τ, IT.action (n + 1) τ)) + ∀ᵐ e ∂Q, HasCondDistrib (IT.feedback n) (fun τ ↦ (IT.hist n τ, IT.action n τ)) ((κ.sectR e).prodMkLeft _) (condDistrib (trajectory A Y) E P e) := by rw [← h.hasLaw_env.map_eq] - have hc : HasCondDistrib (Y (n + 1)) - (fun ω ↦ (E ω, history A Y n ω, A (n + 1) ω)) + have hc : HasCondDistrib (Y n) (fun ω ↦ (E ω, history A Y n ω, A n ω)) (κ.comap (fun (e, _, a) ↦ (e, a)) (by fun_prop)) P := (h.hasCondDistrib_feedback n).measurableEquiv_comp_right (MeasurableEquiv.prodAssoc.symm.trans ((MeasurableEquiv.prodCongr .prodComm (.refl _)).trans .prodAssoc)) - exact hc.hasCondDistrib_sectR ((IT.measurable_hist n).prodMk - (IT.measurable_action (n + 1))) (IT.measurable_feedback (n + 1)) + exact hc.hasCondDistrib_sectR ((IT.measurable_hist n).prodMk (IT.measurable_action n)) + (IT.measurable_feedback n) (measurable_trajectory h.measurable_action h.measurable_feedback).aemeasurable lemma hasLaw_IT_hist (h : IsBayesAlgEnvSeq Q κ alg E A Y P) (n : ℕ) : @@ -146,27 +149,43 @@ lemma hasLaw_IT_hist (h : IsBayesAlgEnvSeq Q κ alg E A Y P) (n : ℕ) : lemma ae_IsAlgEnvSeq [IsMarkovKernel κ] (h : IsBayesAlgEnvSeq Q κ alg E A Y P) : ∀ᵐ e ∂Q, IsAlgEnvSeq IT.action IT.feedback alg (stationaryEnv (κ.sectR e)) (condDistrib (trajectory A Y) E P e) := by - filter_upwards [hasLaw_IT_action_zero h, hasCondDistrib_IT_feedback_zero h, - ae_all_iff.2 (hasCondDistrib_IT_action h), ae_all_iff.2 (hasCondDistrib_IT_feedback h)] - with _ ha0 hr0 hA hR - exact ⟨IT.measurable_action, IT.measurable_feedback, ha0, hr0, hA, hR⟩ + filter_upwards [ae_all_iff.2 (hasCondDistrib_IT_action h), + ae_all_iff.2 (hasCondDistrib_IT_feedback h)] with _ hA hR + exact ⟨IT.measurable_action, IT.measurable_feedback, hA, hR⟩ end IsBayesAlgEnvSeq section IsAlgEnvSeq /-- An environment with observations in `𝓔 × 𝓨`. The first element `e` of an observation is -sampled from `Q` once and remains constant. The second element of an observation is sampled from -`κ (e, a)`, where `a` is the corresponding action. -/ +sampled from `Q` at time `0` and then remains constant: at time `n + 1` it is read from the first +observation in the history. The second element of an observation is sampled from `κ (e, a)`, where +`a` is the corresponding action. -/ noncomputable def bayesStationaryEnv (Q : Measure 𝓔) [IsProbabilityMeasure Q] (κ : Kernel (𝓔 × 𝓐) 𝓨) [IsMarkovKernel κ] : Environment 𝓐 (𝓔 × 𝓨) where - feedback n := - let g : (Iic n → 𝓐 × 𝓔 × 𝓨) × 𝓐 → 𝓔 × 𝓐 := fun (h, a) => ((h ⟨0, by simp⟩).2.1, a) - (Kernel.deterministic (Prod.fst ∘ g) (by fun_prop)) ×ₖ (κ.comap g (by fun_prop)) - ν0 := (Kernel.const _ Q) ⊗ₖ κ.swapLeft + feedback + | 0 => ((Kernel.const _ Q) ⊗ₖ κ.swapLeft).prodMkLeft _ + | n + 1 => + let g : (Fin (n + 1) → 𝓐 × 𝓔 × 𝓨) × 𝓐 → 𝓔 × 𝓐 := fun (h, a) ↦ ((h 0).2.1, a) + (Kernel.deterministic (Prod.fst ∘ g) (by fun_prop)) ×ₖ (κ.comap g (by fun_prop)) + h_feedback n := by cases n <;> infer_instance variable {Q : Measure 𝓔} [IsProbabilityMeasure Q] {κ : Kernel (𝓔 × 𝓐) 𝓨} [IsMarkovKernel κ] + +lemma bayesStationaryEnv_feedback_zero : + (bayesStationaryEnv Q κ).feedback 0 = ((Kernel.const _ Q) ⊗ₖ κ.swapLeft).prodMkLeft _ := rfl + +lemma bayesStationaryEnv_feedback_succ (n : ℕ) : + (bayesStationaryEnv Q κ).feedback (n + 1) = + (Kernel.deterministic (Prod.fst ∘ (fun (p : (Fin (n + 1) → 𝓐 × 𝓔 × 𝓨) × 𝓐) ↦ + ((p.1 0).2.1, p.2))) (by fun_prop)) ×ₖ + (κ.comap (fun p ↦ ((p.1 0).2.1, p.2)) (by fun_prop)) := rfl + +@[simp] +lemma ν0_bayesStationaryEnv : (bayesStationaryEnv Q κ).ν0 = (Kernel.const _ Q) ⊗ₖ κ.swapLeft := by + rw [Environment.ν0_def, bayesStationaryEnv_feedback_zero, Kernel.sectR_prodMkLeft] + variable {alg : Algorithm 𝓐 𝓨} {A : ℕ → Ω → 𝓐} {Y : ℕ → Ω → 𝓔 × 𝓨} variable {P : Measure Ω} [IsProbabilityMeasure P] @@ -178,29 +197,40 @@ lemma IsAlgEnvSeq.isBayesAlgEnvSeq measurable_feedback n := (h.measurable_feedback n).snd hasLaw_env := by apply HasCondDistrib.hasLaw_of_const - simpa [bayesStationaryEnv] using h.hasCondDistrib_feedback_zero.fst - hasCondDistrib_action_zero := by - have hc : HasCondDistrib (fun ω ↦ (Y 0 ω).1) (A 0) (Kernel.const _ Q) P := by - simpa [bayesStationaryEnv] using h.hasCondDistrib_feedback_zero.fst - simpa [h.hasLaw_action_zero.map_eq, Algorithm.prodLeft] using hc.const_map_of_const - hasCondDistrib_feedback_zero := - h.hasCondDistrib_feedback_zero.of_compProd.measurableEquiv_comp_right MeasurableEquiv.prodComm + simpa [Kernel.fst_compProd] using h.hasCondDistrib_feedback_zero.fst hasCondDistrib_action n := by - let f : (Iic n → 𝓐 × 𝓔 × 𝓨) → 𝓔 × (Iic n → 𝓐 × 𝓨) := - fun h ↦ ((h ⟨0, by simp⟩).2.1, fun i ↦ ((h i).1, (h i).2.2)) - have hc : HasCondDistrib (A (n + 1)) (history A Y n) - (((alg.policy n).comap Prod.snd (by fun_prop)).comap f (by fun_prop)) P := - h.hasCondDistrib_action n - exact hc.comp_right (f := f) + cases n with + | zero => + -- At time `0` the history is empty: `E` is the parameter component of `Y 0`, which is + -- independent of `A 0`. + have hc : HasCondDistrib (fun ω ↦ (Y 0 ω).1) (A 0) (Kernel.const _ Q) P := by + simpa [Kernel.fst_compProd] using h.hasCondDistrib_feedback_zero.fst + have hc' : HasCondDistrib (A 0) (fun ω ↦ (Y 0 ω).1) (Kernel.const _ alg.p0) P := by + simpa [h.hasLaw_action_zero.map_eq] using hc.const_map_of_const + exact hasCondDistrib_prodMk_right_unique_iff.mpr hc' + | succ n => + let f : (Fin (n + 1) → 𝓐 × 𝓔 × 𝓨) → 𝓔 × (Fin (n + 1) → 𝓐 × 𝓨) := + fun h ↦ ((h 0).2.1, fun i ↦ ((h i).1, (h i).2.2)) + have hc : HasCondDistrib (A (n + 1)) (history A Y (n + 1)) + (((alg.policy (n + 1)).comap Prod.snd (by fun_prop)).comap f (by fun_prop)) P := + h.hasCondDistrib_action (n + 1) + exact hc.comp_right (f := f) hasCondDistrib_feedback n := by - let f : (Iic n → 𝓐 × 𝓔 × 𝓨) × 𝓐 → (Iic n → 𝓐 × 𝓨) × 𝓔 × 𝓐 := - fun p ↦ ((fun i ↦ ((p.1 i).1, (p.1 i).2.2)), (p.1 ⟨0, by simp⟩).2.1, p.2) - have hc : HasCondDistrib (fun ω ↦ (Y (n + 1) ω).2) - (fun ω ↦ (history A Y n ω, A (n + 1) ω)) - ((Kernel.prodMkLeft ((Iic n) → 𝓐 × 𝓨) κ).comap f (by fun_prop)) P := by - simpa [bayesStationaryEnv, Kernel.prodMkLeft, ← Kernel.comap_comp_right, Function.comp_def] - using (h.hasCondDistrib_feedback n).snd - exact hc.comp_right + cases n with + | zero => + have hc : HasCondDistrib (Y 0) (A 0) ((Kernel.const _ Q) ⊗ₖ κ.swapLeft) P := by + simpa using h.hasCondDistrib_feedback_zero + exact hasCondDistrib_prodMk_left_unique_iff.mpr + (hc.of_compProd.measurableEquiv_comp_right MeasurableEquiv.prodComm) + | succ n => + let f : (Fin (n + 1) → 𝓐 × 𝓔 × 𝓨) × 𝓐 → (Fin (n + 1) → 𝓐 × 𝓨) × 𝓔 × 𝓐 := + fun p ↦ ((fun i ↦ ((p.1 i).1, (p.1 i).2.2)), (p.1 0).2.1, p.2) + have hc : HasCondDistrib (fun ω ↦ (Y (n + 1) ω).2) + (fun ω ↦ (history A Y (n + 1) ω, A (n + 1) ω)) + ((Kernel.prodMkLeft (Fin (n + 1) → 𝓐 × 𝓨) κ).comap f (by fun_prop)) P := by + simpa [bayesStationaryEnv_feedback_succ, Kernel.prodMkLeft, ← Kernel.comap_comp_right, + Function.comp_def] using (h.hasCondDistrib_feedback (n + 1)).snd + exact hc.comp_right end IsAlgEnvSeq @@ -220,15 +250,22 @@ lemma isBayesAlgEnvSeq_bayesTrajMeasure IsBayesAlgEnvSeq Q κ alg (fun ω ↦ (ω 0).2.1) action (fun n ω ↦ (ω n).2.2) (bayesTrajMeasure Q κ alg) := (isAlgEnvSeq_trajMeasure _ _).isBayesAlgEnvSeq -/-- A kernel that represents the posterior over `E` given the history up to time `n`. -/ +/-- A kernel that represents the posterior over `E` given the history before time `n`. -/ noncomputable def bayesTrajMeasurePosterior [StandardBorelSpace 𝓔] [Nonempty 𝓔] (Q : Measure 𝓔) [IsProbabilityMeasure Q] (κ : Kernel (𝓔 × 𝓐) 𝓨) [IsMarkovKernel κ] - (alg : Algorithm 𝓐 𝓨) (n : ℕ) : Kernel (Iic n → 𝓐 × 𝓨) 𝓔 := + (alg : Algorithm 𝓐 𝓨) (n : ℕ) : Kernel (Fin n → 𝓐 × 𝓨) 𝓔 := condDistrib (fun ω ↦ (ω 0).2.1) (history action (fun n ω ↦ (ω n).2.2) n) (bayesTrajMeasure Q κ alg) deriving IsMarkovKernel +/-- The posterior given the empty history is the prior. -/ +lemma bayesTrajMeasurePosterior_zero [StandardBorelSpace 𝓔] [Nonempty 𝓔] + (Q : Measure 𝓔) [IsProbabilityMeasure Q] (κ : Kernel (𝓔 × 𝓐) 𝓨) [IsMarkovKernel κ] + (alg : Algorithm 𝓐 𝓨) : + bayesTrajMeasurePosterior Q κ alg 0 = Kernel.const _ Q := + (isBayesAlgEnvSeq_bayesTrajMeasure Q κ alg).condDistrib_param_history_zero + end IT end Learning diff --git a/LeanMachineLearning/SequentialLearning/Deterministic.lean b/LeanMachineLearning/SequentialLearning/Deterministic.lean index d9a40ac7..913659ce 100644 --- a/LeanMachineLearning/SequentialLearning/Deterministic.lean +++ b/LeanMachineLearning/SequentialLearning/Deterministic.lean @@ -25,17 +25,20 @@ measurable functions. * `IsDeterministicAlg alg`: a typeclass expressing that the algorithm `alg` is deterministic. * `IsDeterministicEnv env`: a typeclass expressing that the environment `env` is deterministic. -* `actionZero alg`: the initial action of a deterministic algorithm `alg`. -* `nextAction alg n`: the function that gives the next action of a deterministic algorithm `alg` - at step `n`, as a function of the history. -* `feedbackFunZero env`: the function that gives the initial feedback of a deterministic - environment `env`. +* `nextAction alg n`: the function that gives the action of a deterministic algorithm `alg` + at step `n`, as a function of the history before `n`. +* `actionZero alg`: the initial action of a deterministic algorithm `alg`. This is + `nextAction alg 0` applied to the empty history. * `feedbackFun env n`: the function that gives the feedback of a deterministic environment `env` at step `n`, as a function of the history and the current action. +* `feedbackFunZero env`: the function that gives the initial feedback of a deterministic + environment `env`. This is `feedbackFun env 0` applied to the empty history. -* `detAlgorithm nextA h_next action0`: a deterministic algorithm that chooses its action - according to the measurable function `nextA` (with proof of measurability `h_next`), - with initial action `action0`. +* `detAlgorithm nextA h_next`: a deterministic algorithm that chooses its action + according to the measurable function `nextA` (with proof of measurability `h_next`). + The initial action is `nextA 0 default`. +* `detEnvironment f hf`: a deterministic environment that gives feedback according to the + measurable function `f` (with proof of measurability `hf`). -/ @@ -49,113 +52,100 @@ namespace Learning variable {𝓐 𝓨 : Type*} {m𝓐 : MeasurableSpace 𝓐} {m𝓨 : MeasurableSpace 𝓨} -/-- An algorithm is deterministic if its initial action and subsequent actions are determined by -measurable functions (and not possibly random kernels). -/ +/-- An algorithm is deterministic if its actions are determined by measurable functions of the +history (and not possibly random kernels). -/ class IsDeterministicAlg (alg : Algorithm 𝓐 𝓨) : Prop where - exists_action0 : ∃ action0, alg.p0 = Measure.dirac action0 - exists_nextAction n : ∃ (nextAction : (Iic n → 𝓐 × 𝓨) → 𝓐) (h_meas : Measurable nextAction), + exists_nextAction n : ∃ (nextAction : (Fin n → 𝓐 × 𝓨) → 𝓐) (h_meas : Measurable nextAction), alg.policy n = Kernel.deterministic nextAction h_meas -/-- The initial action of a deterministic algorithm. -/ -noncomputable -def actionZero (alg : Algorithm 𝓐 𝓨) [h_det : IsDeterministicAlg alg] : 𝓐 := - h_det.exists_action0.choose - -/-- The next action of a deterministic algorithm after step `n`. -/ +/-- The action of a deterministic algorithm at step `n`, as a function of the history before `n`. -/ noncomputable def nextAction (alg : Algorithm 𝓐 𝓨) [h_det : IsDeterministicAlg alg] (n : ℕ) : - (Iic n → 𝓐 × 𝓨) → 𝓐 := + (Fin n → 𝓐 × 𝓨) → 𝓐 := (h_det.exists_nextAction n).choose +/-- The initial action of a deterministic algorithm. -/ +noncomputable +def actionZero (alg : Algorithm 𝓐 𝓨) [IsDeterministicAlg alg] : 𝓐 := + nextAction alg 0 default + @[fun_prop] lemma measurable_nextAction (alg : Algorithm 𝓐 𝓨) [IsDeterministicAlg alg] (n : ℕ) : Measurable (nextAction alg n) := (IsDeterministicAlg.exists_nextAction n).choose_spec.choose -lemma p0_eq_dirac (alg : Algorithm 𝓐 𝓨) [h_det : IsDeterministicAlg alg] : - alg.p0 = Measure.dirac (actionZero alg) := - h_det.exists_action0.choose_spec - lemma policy_eq_deterministic (alg : Algorithm 𝓐 𝓨) [h_det : IsDeterministicAlg alg] (n : ℕ) : alg.policy n = Kernel.deterministic (nextAction alg n) (measurable_nextAction alg n) := (IsDeterministicAlg.exists_nextAction n).choose_spec.choose_spec +lemma nextAction_zero (alg : Algorithm 𝓐 𝓨) [IsDeterministicAlg alg] (h : Fin 0 → 𝓐 × 𝓨) : + nextAction alg 0 h = actionZero alg := by + rw [Unique.eq_default h] + rfl + +lemma p0_eq_dirac (alg : Algorithm 𝓐 𝓨) [IsDeterministicAlg alg] : + alg.p0 = Measure.dirac (actionZero alg) := by + rw [Algorithm.p0_def, policy_eq_deterministic, Kernel.deterministic_apply] + rfl + namespace IsDeterministicAlg variable {Ω : Type*} {mΩ : MeasurableSpace Ω} {alg : Algorithm 𝓐 𝓨} {env : Environment 𝓐 𝓨} {P : Measure Ω} [IsFiniteMeasure P] {A : ℕ → Ω → 𝓐} {Y : ℕ → Ω → 𝓨} {n N : ℕ} -lemma hasLaw_action_zero_of_IsAlgEnvSeqUntil [h_det : IsDeterministicAlg alg] - (h : IsAlgEnvSeqUntil A Y alg env P N) : - HasLaw (A 0) (Measure.dirac (actionZero alg)) P where - aemeasurable := have hA := h.measurable_action; by fun_prop - map_eq := (h.hasLaw_action_zero).map_eq.trans (p0_eq_dirac alg) - -lemma action_zero_of_IsAlgEnvSeqUntil [MeasurableEq 𝓐] [h_det : IsDeterministicAlg alg] - (h : IsAlgEnvSeqUntil A Y alg env P N) : - A 0 =ᵐ[P] fun _ ↦ actionZero alg := by - have h_eq : ∀ᵐ x ∂(P.map (A 0)), x = actionZero alg := by - simp [(hasLaw_action_zero_of_IsAlgEnvSeqUntil h).map_eq] - have hA := h.measurable_action - exact ae_of_ae_map (by fun_prop) h_eq - lemma action_ae_eq_of_IsAlgEnvSeqUntil [MeasurableEq 𝓐] [h_det : IsDeterministicAlg alg] (h : IsAlgEnvSeqUntil A Y alg env P N) (hn : n < N) : - A (n + 1) =ᵐ[P] fun ω ↦ nextAction alg n (history A Y n ω) := by + A n =ᵐ[P] fun ω ↦ nextAction alg n (history A Y n ω) := by have h_eq := (h.hasCondDistrib_action n hn) rw [policy_eq_deterministic alg n] at h_eq exact ae_eq_of_hasCondDistrib_deterministic (measurable_nextAction _ _) (by fun_prop) (by fun_prop) h_eq -lemma hasLaw_action_zero [h_det : IsDeterministicAlg alg] (h : IsAlgEnvSeq A Y alg env P) : +lemma action_zero_of_IsAlgEnvSeqUntil [MeasurableEq 𝓐] [h_det : IsDeterministicAlg alg] + (h : IsAlgEnvSeqUntil A Y alg env P N) (hN : 0 < N) : + A 0 =ᵐ[P] fun _ ↦ actionZero alg := by + filter_upwards [action_ae_eq_of_IsAlgEnvSeqUntil h hN] with ω hω + rw [hω, nextAction_zero] + +lemma hasLaw_action_zero_of_IsAlgEnvSeqUntil [IsProbabilityMeasure P] + [h_det : IsDeterministicAlg alg] (h : IsAlgEnvSeqUntil A Y alg env P N) (hN : 0 < N) : HasLaw (A 0) (Measure.dirac (actionZero alg)) P where aemeasurable := have hA := h.measurable_action; by fun_prop - map_eq := (h.hasLaw_action_zero).map_eq.trans (p0_eq_dirac alg) + map_eq := (h.hasLaw_action_zero hN).map_eq.trans (p0_eq_dirac alg) -lemma action_zero_ae_eq [MeasurableEq 𝓐] [h_det : IsDeterministicAlg alg] +lemma hasLaw_action_zero [IsProbabilityMeasure P] [h_det : IsDeterministicAlg alg] (h : IsAlgEnvSeq A Y alg env P) : - A 0 =ᵐ[P] fun _ ↦ actionZero alg := - action_zero_of_IsAlgEnvSeqUntil (h.isAlgEnvSeqUntil 0) + HasLaw (A 0) (Measure.dirac (actionZero alg)) P := + hasLaw_action_zero_of_IsAlgEnvSeqUntil (h.isAlgEnvSeqUntil 1) zero_lt_one lemma action_ae_eq [MeasurableEq 𝓐] [h_det : IsDeterministicAlg alg] (h : IsAlgEnvSeq A Y alg env P) (n : ℕ) : - A (n + 1) =ᵐ[P] fun ω ↦ nextAction alg n (history A Y n ω) := - action_ae_eq_of_IsAlgEnvSeqUntil (h.isAlgEnvSeqUntil (n + 1)) (by simp) + A n =ᵐ[P] fun ω ↦ nextAction alg n (history A Y n ω) := + action_ae_eq_of_IsAlgEnvSeqUntil (h.isAlgEnvSeqUntil (n + 1)) n.lt_succ_self + +lemma action_zero_ae_eq [MeasurableEq 𝓐] [h_det : IsDeterministicAlg alg] + (h : IsAlgEnvSeq A Y alg env P) : + A 0 =ᵐ[P] fun _ ↦ actionZero alg := + action_zero_of_IsAlgEnvSeqUntil (h.isAlgEnvSeqUntil 1) zero_lt_one lemma action_ae_all_eq [MeasurableEq 𝓐] [h_det : IsDeterministicAlg alg] (h : IsAlgEnvSeq A Y alg env P) : - ∀ᵐ ω ∂P, A 0 ω = actionZero alg ∧ ∀ n, A (n + 1) ω = nextAction alg n (history A Y n ω) := by - rw [eventually_and, ae_all_iff] - exact ⟨action_zero_ae_eq h, action_ae_eq h⟩ + ∀ᵐ ω ∂P, ∀ n, A n ω = nextAction alg n (history A Y n ω) := + ae_all_iff.mpr (action_ae_eq h) end IsDeterministicAlg -/-- An environment is deterministic if its initial feedbacks are determined by -measurable functions (and not possibly random kernels). -/ +/-- An environment is deterministic if its feedbacks are determined by measurable functions of +the history and the action (and not possibly random kernels). -/ class IsDeterministicEnv (env : Environment 𝓐 𝓨) : Prop where - exists_f0 : ∃ (f0 : 𝓐 → 𝓨) (hf0 : Measurable f0), env.ν0 = Kernel.deterministic f0 hf0 - exists_f : ∀ n, ∃ (f : ((Iic n → 𝓐 × 𝓨) × 𝓐) → 𝓨) (hf : Measurable f), + exists_f : ∀ n, ∃ (f : ((Fin n → 𝓐 × 𝓨) × 𝓐) → 𝓨) (hf : Measurable f), env.feedback n = Kernel.deterministic f hf -/-- The initial feedback function of a deterministic environment. -/ -noncomputable -def feedbackFunZero (env : Environment 𝓐 𝓨) [h_det : IsDeterministicEnv env] : 𝓐 → 𝓨 := - h_det.exists_f0.choose - -@[fun_prop] -lemma measurable_feedbackFunZero (env : Environment 𝓐 𝓨) [IsDeterministicEnv env] : - Measurable (feedbackFunZero env) := - (IsDeterministicEnv.exists_f0).choose_spec.choose - -lemma ν0_eq_deterministic (env : Environment 𝓐 𝓨) [IsDeterministicEnv env] : - env.ν0 = Kernel.deterministic (feedbackFunZero env) (measurable_feedbackFunZero env) := - (IsDeterministicEnv.exists_f0).choose_spec.choose_spec - /-- The feedback function of a deterministic environment at step `n`. -/ noncomputable def feedbackFun (env : Environment 𝓐 𝓨) [h_det : IsDeterministicEnv env] (n : ℕ) : - ((Iic n → 𝓐 × 𝓨) × 𝓐) → 𝓨 := + ((Fin n → 𝓐 × 𝓨) × 𝓐) → 𝓨 := (h_det.exists_f n).choose @[fun_prop] @@ -167,13 +157,41 @@ lemma feedback_eq_deterministic (env : Environment 𝓐 𝓨) [IsDeterministicEn env.feedback n = Kernel.deterministic (feedbackFun env n) (measurable_feedbackFun env n) := (IsDeterministicEnv.exists_f n).choose_spec.choose_spec +/-- The initial feedback function of a deterministic environment. -/ +noncomputable +def feedbackFunZero (env : Environment 𝓐 𝓨) [IsDeterministicEnv env] : 𝓐 → 𝓨 := + fun a ↦ feedbackFun env 0 (default, a) + +@[fun_prop] +lemma measurable_feedbackFunZero (env : Environment 𝓐 𝓨) [IsDeterministicEnv env] : + Measurable (feedbackFunZero env) := + (measurable_feedbackFun env 0).comp (measurable_const.prodMk measurable_id) + +lemma feedbackFun_zero (env : Environment 𝓐 𝓨) [IsDeterministicEnv env] (h : Fin 0 → 𝓐 × 𝓨) + (a : 𝓐) : + feedbackFun env 0 (h, a) = feedbackFunZero env a := by + rw [Unique.eq_default h] + rfl + +lemma ν0_eq_deterministic (env : Environment 𝓐 𝓨) [IsDeterministicEnv env] : + env.ν0 = Kernel.deterministic (feedbackFunZero env) (measurable_feedbackFunZero env) := by + ext a : 1 + rw [Environment.ν0_apply, feedback_eq_deterministic, Kernel.deterministic_apply, + Kernel.deterministic_apply] + rfl + namespace IsDeterministicEnv variable {Ω : Type*} {mΩ : MeasurableSpace Ω} {alg : Algorithm 𝓐 𝓨} {env : Environment 𝓐 𝓨} {P : Measure Ω} [IsFiniteMeasure P] {A : ℕ → Ω → 𝓐} {Y : ℕ → Ω → 𝓨} - {f : (n : ℕ) → ((Iic n → 𝓐 × 𝓨) × 𝓐) → 𝓨} {hf : ∀ n, Measurable (f n)} - {f0 : 𝓐 → 𝓨} {hf0 : Measurable f0} + +lemma hasCondDistrib_feedback [h_det : IsDeterministicEnv env] + (h : IsAlgEnvSeq A Y alg env P) (n : ℕ) : + HasCondDistrib (Y n) (fun ω ↦ (history A Y n ω, A n ω)) + (Kernel.deterministic (feedbackFun env n) (measurable_feedbackFun env n)) P := by + rw [← feedback_eq_deterministic] + exact h.hasCondDistrib_feedback n lemma hasCondDistrib_feedback_zero [h_det : IsDeterministicEnv env] (h : IsAlgEnvSeq A Y alg env P) : @@ -182,69 +200,67 @@ lemma hasCondDistrib_feedback_zero [h_det : IsDeterministicEnv env] rw [← ν0_eq_deterministic] exact h.hasCondDistrib_feedback_zero -lemma hasCondDistrib_feedback [h_det : IsDeterministicEnv env] +lemma feedback_ae_eq [MeasurableEq 𝓨] [h_det : IsDeterministicEnv env] (h : IsAlgEnvSeq A Y alg env P) (n : ℕ) : - HasCondDistrib (Y (n + 1)) (fun ω ↦ (history A Y n ω, A (n + 1) ω)) - (Kernel.deterministic (feedbackFun env n) (measurable_feedbackFun env n)) P := by - rw [← feedback_eq_deterministic] - exact h.hasCondDistrib_feedback n + Y n =ᵐ[P] fun ω ↦ feedbackFun env n (history A Y n ω, A n ω) := by + have hA := h.measurable_action + have hY := h.measurable_feedback + exact ae_eq_of_hasCondDistrib_deterministic (measurable_feedbackFun _ _) (by fun_prop) + (by fun_prop) (hasCondDistrib_feedback h n) end IsDeterministicEnv -variable {nextA : (n : ℕ) → (Iic n → 𝓐 × 𝓨) → 𝓐} {h_next : ∀ n, Measurable (nextA n)} - {action0 : 𝓐} {env : Environment 𝓐 𝓨} - {f0 : 𝓐 → 𝓨} {hf0 : Measurable f0} - {f : (n : ℕ) → ((Iic n → 𝓐 × 𝓨) × 𝓐) → 𝓨} {hf : ∀ n, Measurable (f n)} +variable {nextA : (n : ℕ) → (Fin n → 𝓐 × 𝓨) → 𝓐} {h_next : ∀ n, Measurable (nextA n)} + {env : Environment 𝓐 𝓨} + {f : (n : ℕ) → ((Fin n → 𝓐 × 𝓨) × 𝓐) → 𝓨} {hf : ∀ n, Measurable (f n)} -/-- A deterministic algorithm, which chooses the action given by the function `nextAction`. -/ +/-- A deterministic algorithm, which chooses the action given by the function `nextA`. +The initial action is `nextA 0 default`. -/ @[simps] noncomputable -def detAlgorithm (nextA : (n : ℕ) → (Iic n → 𝓐 × 𝓨) → 𝓐) - (h_next : ∀ n, Measurable (nextA n)) (action0 : 𝓐) : +def detAlgorithm (nextA : (n : ℕ) → (Fin n → 𝓐 × 𝓨) → 𝓐) + (h_next : ∀ n, Measurable (nextA n)) : Algorithm 𝓐 𝓨 where policy n := Kernel.deterministic (nextA n) (h_next n) - p0 := Measure.dirac action0 -instance : IsDeterministicAlg (detAlgorithm nextA h_next action0) where - exists_action0 := ⟨action0, rfl⟩ +instance : IsDeterministicAlg (detAlgorithm nextA h_next) where exists_nextAction n := ⟨nextA n, h_next n, rfl⟩ @[simp] -lemma actionZero_detAlgorithm [MeasurableSpace.SeparatesPoints 𝓐] : - actionZero (detAlgorithm nextA h_next action0) = action0 := by - have h_eq := p0_eq_dirac (detAlgorithm nextA h_next action0) - simp only [detAlgorithm] at h_eq - rw [dirac_eq_dirac_iff] at h_eq - exact h_eq.symm +lemma p0_detAlgorithm : (detAlgorithm nextA h_next).p0 = Measure.dirac (nextA 0 default) := by + rw [Algorithm.p0_def, detAlgorithm_policy, Kernel.deterministic_apply] @[simp] lemma nextAction_detAlgorithm [MeasurableSpace.SeparatesPoints 𝓐] (n : ℕ) : - nextAction (detAlgorithm nextA h_next action0) n = nextA n := by - have h_eq := policy_eq_deterministic (detAlgorithm nextA h_next action0) n + nextAction (detAlgorithm nextA h_next) n = nextA n := by + have h_eq := policy_eq_deterministic (detAlgorithm nextA h_next) n simpa [detAlgorithm] using h_eq.symm +@[simp] +lemma actionZero_detAlgorithm [MeasurableSpace.SeparatesPoints 𝓐] : + actionZero (detAlgorithm nextA h_next) = nextA 0 default := by + rw [actionZero, nextAction_detAlgorithm] + /-- A deterministic environment, where the feedback is given by evaluating fixed measurable functions. -/ noncomputable def detEnvironment - (f0 : 𝓐 → 𝓨) (hf0 : Measurable f0) - (f : (n : ℕ) → ((Iic n → 𝓐 × 𝓨) × 𝓐) → 𝓨) (hf : ∀ n, Measurable (f n)) : + (f : (n : ℕ) → ((Fin n → 𝓐 × 𝓨) × 𝓐) → 𝓨) (hf : ∀ n, Measurable (f n)) : Environment 𝓐 𝓨 where feedback n := (Kernel.deterministic (f n) (hf n)) - ν0 := Kernel.deterministic f0 hf0 -instance : IsDeterministicEnv (detEnvironment f0 hf0 f hf) where - exists_f0 := ⟨f0, hf0, rfl⟩ +instance : IsDeterministicEnv (detEnvironment f hf) where exists_f n := ⟨f n, hf n, rfl⟩ @[simp] -lemma feedbackFunZero_detEnvironment [MeasurableSpace.SeparatesPoints 𝓨] : - feedbackFunZero (detEnvironment f0 hf0 f hf) = f0 := by - simpa [detEnvironment] using (ν0_eq_deterministic (detEnvironment f0 hf0 f hf)).symm +lemma feedbackFun_detEnvironment [MeasurableSpace.SeparatesPoints 𝓨] (n : ℕ) : + feedbackFun (detEnvironment f hf) n = f n := by + simpa [detEnvironment] using (feedback_eq_deterministic (detEnvironment f hf) n).symm @[simp] -lemma feedbackFun_detEnvironment [MeasurableSpace.SeparatesPoints 𝓨] (n : ℕ) : - feedbackFun (detEnvironment f0 hf0 f hf) n = f n := by - simpa [detEnvironment] using (feedback_eq_deterministic (detEnvironment f0 hf0 f hf) n).symm +lemma feedbackFunZero_detEnvironment [MeasurableSpace.SeparatesPoints 𝓨] : + feedbackFunZero (detEnvironment f hf) = fun a ↦ f 0 (default, a) := by + unfold feedbackFunZero + rw [feedbackFun_detEnvironment] namespace IsAlgEnvSeq @@ -253,24 +269,24 @@ variable {Ω : Type*} {mΩ : MeasurableSpace Ω} {P : Measure Ω} [IsProbabilityMeasure P] {A : ℕ → Ω → 𝓐} {Y : ℕ → Ω → 𝓨} lemma hasLaw_action_zero_detAlgorithm [MeasurableEq 𝓐] - (h : IsAlgEnvSeq A Y (detAlgorithm nextA h_next action0) env P) : - HasLaw (A 0) (Measure.dirac action0) P := by + (h : IsAlgEnvSeq A Y (detAlgorithm nextA h_next) env P) : + HasLaw (A 0) (Measure.dirac (nextA 0 default)) P := by simpa using IsDeterministicAlg.hasLaw_action_zero h -lemma action_zero_detAlgorithm [MeasurableEq 𝓐] - (h : IsAlgEnvSeq A Y (detAlgorithm nextA h_next action0) env P) : - A 0 =ᵐ[P] fun _ ↦ action0 := - (IsDeterministicAlg.action_zero_ae_eq h).trans (by simp) - lemma action_detAlgorithm_ae_eq [MeasurableEq 𝓐] - (h : IsAlgEnvSeq A Y (detAlgorithm nextA h_next action0) env P) (n : ℕ) : - A (n + 1) =ᵐ[P] fun ω ↦ nextA n (history A Y n ω) := + (h : IsAlgEnvSeq A Y (detAlgorithm nextA h_next) env P) (n : ℕ) : + A n =ᵐ[P] fun ω ↦ nextA n (history A Y n ω) := (IsDeterministicAlg.action_ae_eq h n).trans (by simp) +lemma action_zero_detAlgorithm [MeasurableEq 𝓐] + (h : IsAlgEnvSeq A Y (detAlgorithm nextA h_next) env P) : + A 0 =ᵐ[P] fun _ ↦ nextA 0 default := + (IsDeterministicAlg.action_zero_ae_eq h).trans (by simp) + lemma action_detAlgorithm_ae_all_eq [MeasurableEq 𝓐] - (h : IsAlgEnvSeq A Y (detAlgorithm nextA h_next action0) env P) : - ∀ᵐ ω ∂P, A 0 ω = action0 ∧ ∀ n, A (n + 1) ω = nextA n (history A Y n ω) := by - filter_upwards [IsDeterministicAlg.action_ae_all_eq h] with ω hω using by simp [hω] + (h : IsAlgEnvSeq A Y (detAlgorithm nextA h_next) env P) : + ∀ᵐ ω ∂P, ∀ n, A n ω = nextA n (history A Y n ω) := + ae_all_iff.mpr (action_detAlgorithm_ae_eq h) end IsAlgEnvSeq @@ -281,20 +297,20 @@ variable {Ω : Type*} {mΩ : MeasurableSpace Ω} {P : Measure Ω} [IsProbabilityMeasure P] {A : ℕ → Ω → 𝓐} {Y : ℕ → Ω → 𝓨} {N n : ℕ} lemma hasLaw_action_zero_detAlgorithm [MeasurableEq 𝓐] - (h : IsAlgEnvSeqUntil A Y (detAlgorithm nextA h_next action0) env P N) : - HasLaw (A 0) (Measure.dirac action0) P := by - simpa using IsDeterministicAlg.hasLaw_action_zero_of_IsAlgEnvSeqUntil h - -lemma action_zero_detAlgorithm [MeasurableEq 𝓐] - (h : IsAlgEnvSeqUntil A Y (detAlgorithm nextA h_next action0) env P N) : - A 0 =ᵐ[P] fun _ ↦ action0 := - (IsDeterministicAlg.action_zero_of_IsAlgEnvSeqUntil h).trans (by simp) + (h : IsAlgEnvSeqUntil A Y (detAlgorithm nextA h_next) env P N) (hN : 0 < N) : + HasLaw (A 0) (Measure.dirac (nextA 0 default)) P := by + simpa using IsDeterministicAlg.hasLaw_action_zero_of_IsAlgEnvSeqUntil h hN lemma action_detAlgorithm_ae_eq [MeasurableEq 𝓐] - (h : IsAlgEnvSeqUntil A Y (detAlgorithm nextA h_next action0) env P N) (hn : n < N) : - A (n + 1) =ᵐ[P] fun ω ↦ nextA n (history A Y n ω) := + (h : IsAlgEnvSeqUntil A Y (detAlgorithm nextA h_next) env P N) (hn : n < N) : + A n =ᵐ[P] fun ω ↦ nextA n (history A Y n ω) := (IsDeterministicAlg.action_ae_eq_of_IsAlgEnvSeqUntil h hn).trans (by simp) +lemma action_zero_detAlgorithm [MeasurableEq 𝓐] + (h : IsAlgEnvSeqUntil A Y (detAlgorithm nextA h_next) env P N) (hN : 0 < N) : + A 0 =ᵐ[P] fun _ ↦ nextA 0 default := + (IsDeterministicAlg.action_zero_of_IsAlgEnvSeqUntil h hN).trans (by simp) + end IsAlgEnvSeqUntil end Learning diff --git a/LeanMachineLearning/SequentialLearning/EvaluationEnv.lean b/LeanMachineLearning/SequentialLearning/EvaluationEnv.lean index 3ab4a1f7..9aac0774 100644 --- a/LeanMachineLearning/SequentialLearning/EvaluationEnv.lean +++ b/LeanMachineLearning/SequentialLearning/EvaluationEnv.lean @@ -50,31 +50,29 @@ noncomputable def onlineEvalEnv (g : ℕ → 𝓐 → 𝓨) (hg : ∀ n, Measura obliviousEnv (fun n ↦ Kernel.deterministic (g n) (hg n)) instance : IsObliviousEnv (onlineEvalEnv g hg) := - ⟨⟨fun n ↦ Kernel.deterministic (g n) (hg n), fun _ ↦ inferInstance, rfl, fun _ ↦ rfl⟩⟩ + ⟨⟨fun n ↦ Kernel.deterministic (g n) (hg n), fun _ ↦ inferInstance, fun _ ↦ rfl⟩⟩ instance : IsDeterministicEnv (onlineEvalEnv g hg) where - exists_f0 := ⟨g 0, hg 0, rfl⟩ - exists_f n := ⟨fun p ↦ g (n + 1) p.2, by fun_prop, rfl⟩ + exists_f n := ⟨fun p ↦ g n p.2, by fun_prop, rfl⟩ @[simp] lemma feedbackCondAction_onlineEvalEnv (n : ℕ) : feedbackCondAction (onlineEvalEnv g hg) n = Kernel.deterministic (g n) (hg n) := by simp [onlineEvalEnv] -@[simp] -lemma feedbackFunZero_onlineEvalEnv [MeasurableSpace.SeparatesPoints 𝓨] : - feedbackFunZero (onlineEvalEnv g hg) = g 0 := by - have h_eq := ν0_eq_deterministic (onlineEvalEnv g hg) - simpa only [onlineEvalEnv, ν0_obliviousEnv, Kernel.prodMkLeft_deterministic, - Kernel.deterministic_inj] using h_eq.symm - @[simp] lemma feedbackFun_onlineEvalEnv [MeasurableSpace.SeparatesPoints 𝓨] (n : ℕ) : - feedbackFun (onlineEvalEnv g hg) n = fun p ↦ g (n + 1) p.2 := by + feedbackFun (onlineEvalEnv g hg) n = fun p ↦ g n p.2 := by have h_eq := feedback_eq_deterministic (onlineEvalEnv g hg) n simpa only [onlineEvalEnv, feedback_obliviousEnv, Kernel.prodMkLeft_deterministic, Kernel.deterministic_inj] using h_eq.symm +@[simp] +lemma feedbackFunZero_onlineEvalEnv [MeasurableSpace.SeparatesPoints 𝓨] : + feedbackFunZero (onlineEvalEnv g hg) = g 0 := by + unfold feedbackFunZero + rw [feedbackFun_onlineEvalEnv] + section OnlineEvalEnv variable {Ω : Type*} {mΩ : MeasurableSpace Ω} {alg : Algorithm 𝓐 𝓨} diff --git a/LeanMachineLearning/SequentialLearning/FiniteActions.lean b/LeanMachineLearning/SequentialLearning/FiniteActions.lean index 16c34693..a59ca328 100644 --- a/LeanMachineLearning/SequentialLearning/FiniteActions.lean +++ b/LeanMachineLearning/SequentialLearning/FiniteActions.lean @@ -42,10 +42,10 @@ noncomputable def pullCount (A : ℕ → Ω → 𝓐) (a : 𝓐) (t : ℕ) (ω : Ω) : ℕ := #(filter (fun s ↦ A s ω = a) (range t)) -/-- Number of pulls of arm `a` up to (and including) time `n`. +/-- Number of pulls of arm `a` in the history before time `n`. This is the number of entries in `h` in which the arm is `a`. -/ noncomputable -def pullCount' (n : ℕ) (h : Iic n → 𝓐 × R) (a : 𝓐) := #{s | (h s).1 = a} +def pullCount' (n : ℕ) (h : Fin n → 𝓐 × R) (a : 𝓐) := #{s | (h s).1 = a} @[simp] lemma pullCount_zero (a : 𝓐) : pullCount A a 0 = 0 := by ext; simp [pullCount] @@ -84,27 +84,24 @@ lemma pullCount_add_one : lemma pullCount_eq_sum (a : 𝓐) (t : ℕ) (ω : Ω) : pullCount A a t ω = ∑ s ∈ range t, if A s ω = a then 1 else 0 := by simp [pullCount] -lemma pullCount'_eq_sum (n : ℕ) (h : Iic n → 𝓐 × R) (a : 𝓐) : - pullCount' n h a = ∑ s : Iic n, if (h s).1 = a then 1 else 0 := by simp [pullCount'] +lemma pullCount'_eq_sum (n : ℕ) (h : Fin n → 𝓐 × R) (a : 𝓐) : + pullCount' n h a = ∑ s : Fin n, if (h s).1 = a then 1 else 0 := by simp [pullCount'] -lemma pullCount_add_one_eq_pullCount' {n : ℕ} {ω : Ω} : - pullCount A a (n + 1) ω = pullCount' n (fun i ↦ (A i ω, R' i ω)) a := by +lemma pullCount_eq_pullCount' {n : ℕ} {ω : Ω} : + pullCount A a n ω = pullCount' n (fun i ↦ (A i ω, R' i ω)) a := by rw [pullCount_eq_sum, pullCount'_eq_sum] - rw [Finset.sum_coe_sort (f := fun s ↦ if A s ω = a then 1 else 0) (Iic n)] - congr with m - simp only [mem_range, mem_Iic] - grind + exact (Fin.sum_univ_eq_sum_range (fun i ↦ if A i ω = a then 1 else 0) n).symm -lemma pullCount_eq_pullCount' {n : ℕ} {ω : Ω} (hn : n ≠ 0) : - pullCount A a n ω = pullCount' (n - 1) (fun i ↦ (A i ω, R' i ω)) a := by - cases n with - | zero => exact absurd rfl hn - | succ n => simp [pullCount_add_one_eq_pullCount' (R' := R')] +/-- `pullCount A a n` is a function of the history before time `n`. -/ +lemma pullCount_eq_comp_history (a : 𝓐) (n : ℕ) : + pullCount A a n = (fun h : Fin n → 𝓐 × R ↦ pullCount' n h a) ∘ history A R' n := by + ext ω + exact pullCount_eq_pullCount' lemma pullCount'_mono {n m : ℕ} (hnm : n ≤ m) : pullCount' n (fun i ↦ (A i ω, R' i ω)) a ≤ pullCount' m (fun i ↦ (A i ω, R' i ω)) a := by - rw [← pullCount_add_one_eq_pullCount', ← pullCount_add_one_eq_pullCount'] - exact pullCount_mono a (by lia) _ + rw [← pullCount_eq_pullCount', ← pullCount_eq_pullCount'] + exact pullCount_mono a hnm _ lemma pullCount_le (a : 𝓐) (t : ℕ) (ω : Ω) : pullCount A a t ω ≤ t := (card_filter_le _ _).trans_eq (by simp) @@ -204,18 +201,18 @@ lemma measurable_uncurry_pullCount_comp [Countable 𝓐] [MeasurableSingletonCla @[fun_prop] lemma measurable_pullCount' [MeasurableSingletonClass 𝓐] (n : ℕ) (a : 𝓐) : - Measurable (fun h : Iic n → 𝓐 × R ↦ pullCount' n h a) := by + Measurable (fun h : Fin n → 𝓐 × R ↦ pullCount' n h a) := by simp_rw [pullCount'_eq_sum] - have h_meas s : Measurable (fun (h : Iic n → 𝓐 × R) ↦ if (h s).1 = a then 1 else 0) := by + have h_meas s : Measurable (fun (h : Fin n → 𝓐 × R) ↦ if (h s).1 = a then 1 else 0) := by refine Measurable.ite ?_ (by fun_prop) (by fun_prop) exact (measurableSet_singleton _).preimage (by fun_prop) fun_prop @[fun_prop] lemma measurable_uncurry_pullCount' [MeasurableEq 𝓐] (n : ℕ) : - Measurable (fun p : (Iic n → 𝓐 × R) × 𝓐 ↦ pullCount' n p.1 p.2) := by + Measurable (fun p : (Fin n → 𝓐 × R) × 𝓐 ↦ pullCount' n p.1 p.2) := by simp_rw [pullCount'_eq_sum] - have h_meas s : Measurable (fun h : (Iic n → 𝓐 × R) × 𝓐 ↦ if (h.1 s).1 = h.2 then 1 else 0) := by + have h_meas s : Measurable (fun h : (Fin n → 𝓐 × R) × 𝓐 ↦ if (h.1 s).1 = h.2 then 1 else 0) := by refine Measurable.ite ?_ (by fun_prop) (by fun_prop) exact measurableSet_eq_fun (by fun_prop) (by fun_prop) fun_prop @@ -224,14 +221,10 @@ lemma adapted_pullCount_add_one [MeasurableSingletonClass 𝓐] (h : IsAlgEnvSeq A R' alg env P) (a : 𝓐) : Adapted h.filtration (fun n ↦ pullCount A a (n + 1)) := by intro n - have : pullCount A a (n + 1) = (fun h : Iic n → 𝓐 × R ↦ pullCount' n h a) ∘ - (history A R' n) := by - ext - exact pullCount_add_one_eq_pullCount' - rw [measurable_iff_comap_le] - simp_rw [IsAlgEnvSeq.filtration, this] - rw [← measurable_iff_comap_le] - exact measurable_comp_comap _ (measurable_pullCount' n a) + change Measurable[h.filtration n] (pullCount A a (n + 1)) + rw [measurable_iff_comap_le, h.filtration_eq_comap, pullCount_eq_comp_history (R' := R'), + ← measurable_iff_comap_le] + exact measurable_comp_comap _ (measurable_pullCount' (n + 1) a) lemma stronglyAdapted_pullCount_add_one [MeasurableSingletonClass 𝓐] (h : IsAlgEnvSeq A R' alg env P) (a : 𝓐) : @@ -254,18 +247,18 @@ lemma measurableSet_action_eq_and_pullCount_eq [MeasurableSingletonClass 𝓐] lemma measurableSet_snd_eq_and_pullCount'_eq [MeasurableSingletonClass 𝓐] (n : ℕ) (b : 𝓐) (k : ℕ) : - MeasurableSet {u : (Iic n → 𝓐 × R) × 𝓐 | u.2 = b ∧ pullCount' n u.1 b = k} := + MeasurableSet {u : (Fin n → 𝓐 × R) × 𝓐 | u.2 = b ∧ pullCount' n u.1 b = k} := ((measurableSet_singleton _).preimage measurable_snd).inter ((measurableSet_singleton _).preimage ((measurable_pullCount' n b).comp measurable_fst)) -/-- The event that the action at time `n + 1` is `b` and that `b` was pulled `k` times before is -a preimage by `(history A R' n, A (n + 1))`. -/ +/-- The event that the action at time `n` is `b` and that `b` was pulled `k` times before is +a preimage by `(history A R' n, A n)`. -/ lemma setOf_action_eq_and_pullCount_eq_eq_preimage (n : ℕ) (b : 𝓐) (k : ℕ) : - {x | A (n + 1) x = b ∧ pullCount A b (n + 1) x = k} - = (fun x ↦ (history A R' n x, A (n + 1) x)) ⁻¹' {u | u.2 = b ∧ pullCount' n u.1 b = k} := by + {x | A n x = b ∧ pullCount A b n x = k} + = (fun x ↦ (history A R' n x, A n x)) ⁻¹' {u | u.2 = b ∧ pullCount' n u.1 b = k} := by ext x simp only [Set.mem_ofPred_eq, Set.mem_preimage] - rw [pullCount_add_one_eq_pullCount' (R' := R')] + rw [pullCount_eq_pullCount' (R' := R')] rfl lemma integrable_pullCount [MeasurableSingletonClass 𝓐] @@ -583,9 +576,9 @@ lemma measurable_stepsUntil' [MeasurableSingletonClass 𝓐] (measurable_stepsUntil hA a m).comp measurable_fst lemma measurable_comap_indicator_stepsUntil_eq [MeasurableSingletonClass 𝓐] - (h : IsAlgEnvSeq A R' alg env P) (a : 𝓐) (m n : ℕ) : + (R' : ℕ → Ω → R) (a : 𝓐) (m n : ℕ) : Measurable[MeasurableSpace.comap - (fun ω : Ω ↦ (history A R' (n-1) ω, A n ω)) inferInstance] + (fun ω : Ω ↦ (history A R' n ω, A n ω)) inferInstance] ({ω | stepsUntil A a m ω = ↑n}.indicator fun _ ↦ 1) := by by_cases hm : m = 0 · simp only [hm] @@ -613,67 +606,36 @@ lemma measurable_comap_indicator_stepsUntil_eq [MeasurableSingletonClass 𝓐] exact le_sup_of_le_right le_rfl · rw [measurable_iff_comap_le, Prod.instMeasurableSpace, MeasurableSpace.comap_prodMk] refine le_sup_of_le_left ?_ - rw [← measurable_iff_comap_le] - by_cases hn : n = 0 - · simp only [hn, pullCount_zero] - exact measurable_const - have h_meas := adapted_pullCount_add_one h a (n - 1) - have : 1 ≤ n := by grind - convert h_meas using 1 - · rfl - · simp [Nat.sub_add_cancel this] + rw [← measurable_iff_comap_le, pullCount_eq_comp_history (R' := R')] + exact measurable_comp_comap _ (measurable_pullCount' n a) lemma measurable_indicator_stepsUntil_eq [MeasurableSingletonClass 𝓐] (h : IsAlgEnvSeq A R' alg env P) (a : 𝓐) (m n : ℕ) : Measurable ({ω : Ω | stepsUntil A a m ω = ↑n}.indicator fun _ ↦ 1) := by - refine (measurable_comap_indicator_stepsUntil_eq h a m n).mono ?_ le_rfl + refine (measurable_comap_indicator_stepsUntil_eq (mR := mR) R' a m n).mono ?_ le_rfl refine Measurable.comap_le ?_ have hA := h.measurable_action have hR' := h.measurable_feedback fun_prop -lemma measurableSet_stepsUntil_eq_zero [MeasurableSingletonClass 𝓐] (a : 𝓐) (m : ℕ) : - MeasurableSet[MeasurableSpace.comap (A 0) inferInstance] - {ω : Ω | stepsUntil A a m ω = 0} := by - simp only [stepsUntil_eq_zero_iff (a := a) (m := m), ne_eq] - by_cases hm : m = 0 - · simp only [hm, true_and, zero_ne_one, false_and, or_false] - refine (measurableSet_singleton _).compl.preimage ?_ - rw [measurable_iff_comap_le] - by_cases hm1 : m = 1 - swap; · simp [hm, hm1] - simp only [hm1, one_ne_zero, false_and, true_and, false_or] - refine (measurableSet_singleton _).preimage ?_ - rw [measurable_iff_comap_le] - -lemma measurable_comap_indicator_stepsUntil_eq_zero [MeasurableSingletonClass 𝓐] (a : 𝓐) (m : ℕ) : - Measurable[MeasurableSpace.comap (A 0) inferInstance] - ({ω | stepsUntil A a m ω = 0}.indicator fun _ ↦ 1) := by - rw [measurable_indicator_const_iff] - exact measurableSet_stepsUntil_eq_zero a m - lemma measurableSet_stepsUntil_eq [MeasurableSingletonClass 𝓐] - (h : IsAlgEnvSeq A R' alg env P) (a : 𝓐) (m n : ℕ) : - MeasurableSet[MeasurableSpace.comap (fun ω : Ω ↦ (history A R' (n-1) ω, A n ω)) + (R' : ℕ → Ω → R) (a : 𝓐) (m n : ℕ) : + MeasurableSet[MeasurableSpace.comap (fun ω : Ω ↦ (history A R' n ω, A n ω)) inferInstance] {ω : Ω | stepsUntil A a m ω = ↑n} := by let mProd := MeasurableSpace.comap - (fun ω : Ω ↦ (history A R' (n-1) ω, A n ω)) inferInstance + (fun ω : Ω ↦ (history A R' n ω, A n ω)) inferInstance suffices Measurable[mProd] ({ω | stepsUntil A a m ω = ↑n}.indicator fun x ↦ 1) by rwa [measurable_indicator_const_iff] at this - exact measurable_comap_indicator_stepsUntil_eq h a m n + exact measurable_comap_indicator_stepsUntil_eq R' a m n /-- `stepsUntil a m` is a stopping time with respect to the filtration `filtrationAction`. -/ lemma isStoppingTime_stepsUntil_filtrationAction [MeasurableSingletonClass 𝓐] (h : IsAlgEnvSeq A R' alg env P) (a : 𝓐) (m : ℕ) : IsStoppingTime h.filtrationAction (stepsUntil A a m) := by refine isStoppingTime_of_measurableSet_eq fun n ↦ ?_ - by_cases hn : n = 0 - · subst hn - simp only [WithTop.coe_zero] - exact measurableSet_stepsUntil_eq_zero a m - · rw [h.filtrationAction_eq_comap _ hn] - exact measurableSet_stepsUntil_eq h a m n + rw [h.filtrationAction_eq_comap n] + exact measurableSet_stepsUntil_eq R' a m n end Measurability @@ -912,10 +874,10 @@ lemma sum_comp_pullCount [Fintype 𝓐] [AddCommMonoid R] (f : ℕ → R) (t : · simp [sum_range_succ] · simp -lemma sum_pullCount' [Fintype 𝓐] (n : ℕ) (h : Iic n → 𝓐 × ℝ) : ∑ a, pullCount' n h a = n + 1 := by +lemma sum_pullCount' [Fintype 𝓐] (n : ℕ) (h : Fin n → 𝓐 × ℝ) : ∑ a, pullCount' n h a = n := by simp_rw [pullCount'_eq_sum] rw [Finset.sum_comm] - have hcol (s : Iic n) : ∑ a, (if (h s).1 = a then (1 : ℕ) else 0) = 1 := by + have hcol (s : Fin n) : ∑ a, (if (h s).1 = a then (1 : ℕ) else 0) = 1 := by simp [Finset.sum_ite_eq univ (h s).1 (fun _ ↦ (1 : ℕ))] simp [hcol] diff --git a/LeanMachineLearning/SequentialLearning/IonescuTulceaSpace.lean b/LeanMachineLearning/SequentialLearning/IonescuTulceaSpace.lean index 733e951c..411429af 100644 --- a/LeanMachineLearning/SequentialLearning/IonescuTulceaSpace.lean +++ b/LeanMachineLearning/SequentialLearning/IonescuTulceaSpace.lean @@ -39,66 +39,9 @@ variable {𝓐 𝓨 Ω : Type*} {m𝓐 : MeasurableSpace 𝓐} {m𝓨 : Measurab noncomputable def trajMeasure (alg : Algorithm 𝓐 𝓨) (env : Environment 𝓐 𝓨) : Measure (ℕ → 𝓐 × 𝓨) := - Kernel.trajMeasure (alg.p0 ⊗ₘ env.ν0) (stepKernel alg env) + Kernel.trajMeasureFin (stepKernel alg env) deriving IsProbabilityMeasure -section ModelEquivalence - -variable {Ω Ω' : Type*} {mΩ : MeasurableSpace Ω} {mΩ' : MeasurableSpace Ω'} - {alg : Algorithm 𝓐 𝓨} {env : Environment 𝓐 𝓨} - {P : Measure Ω} [IsProbabilityMeasure P] {P' : Measure Ω'} [IsProbabilityMeasure P'] - {A₁ : ℕ → Ω → 𝓐} {R₁ : ℕ → Ω → 𝓨} {A₂ : ℕ → Ω' → 𝓐} {R₂ : ℕ → Ω' → 𝓨} {N : ℕ} - -lemma IsAlgEnvSeq.map_trajectory (h : IsAlgEnvSeq A₁ R₁ alg env P) : - P.map (trajectory A₁ R₁) = trajMeasure alg env := by - rw [trajMeasure] - have h := (Kernel.hasLaw_trajMeasure (Y := fun n ω ↦ (A₁ n ω, R₁ n ω)) (P := P) - (μ₀ := alg.p0 ⊗ₘ env.ν0) (κ := stepKernel alg env) (fun n ↦ ?_) ?_ (fun n ↦ ?_)).map_eq - · exact h - · have hA := h.measurable_action n - have hR := h.measurable_feedback n - fun_prop - · exact h.hasLaw_step_zero - · exact h.hasCondDistrib_step n - -lemma eq_trajMeasure_map_frestrictLe_of_isAlgEnvSeqUntil - (h : IsAlgEnvSeqUntil A₁ R₁ alg env P N) : - P.map (fun ω (n : Iic N) ↦ (A₁ n ω, R₁ n ω)) = - (trajMeasure alg env).map (Preorder.frestrictLe N) := by - rw [trajMeasure] - have h := Kernel.eq_trajMeasure_map_frestrictLe (Y := fun n ω ↦ (A₁ n ω, R₁ n ω)) - (P := P) (μ₀ := alg.p0 ⊗ₘ env.ν0) (κ := stepKernel alg env) ?_ (fun n hn ↦ ?_) (N := N) - · exact h - · exact h.hasLaw_step_zero - · exact h.hasCondDistrib_step n hn - -/-- The law of the sequence of actions and observations generated by an algorithm-environment pair -is unique: it does not depend on the probability space used. -/ -lemma isAlgEnvSeq_unique (h1 : IsAlgEnvSeq A₁ R₁ alg env P) - (h2 : IsAlgEnvSeq A₂ R₂ alg env P') : - P.map (trajectory A₁ R₁) = P'.map (trajectory A₂ R₂) := by - rw [h1.map_trajectory, h2.map_trajectory] - -/-- The law of the sequence of actions and observations generated by an algorithm-environment pair -is unique: it does not depend on the probability space used. -/ -theorem IsAlgEnvSeq.identDistrib_trajectory (h1 : IsAlgEnvSeq A₁ R₁ alg env P) - (h2 : IsAlgEnvSeq A₂ R₂ alg env P') : - IdentDistrib (trajectory A₁ R₁) (trajectory A₂ R₂) P P' where - aemeasurable_fst := (measurable_pi_iff.2 fun n ↦ (h1.measurable_action n).prodMk - (h1.measurable_feedback n)).aemeasurable - aemeasurable_snd := (measurable_pi_iff.2 fun n ↦ (h2.measurable_action n).prodMk - (h2.measurable_feedback n)).aemeasurable - map_eq := isAlgEnvSeq_unique h1 h2 - -lemma isAlgEnvSeqUntil_unique (h1 : IsAlgEnvSeqUntil A₁ R₁ alg env P N) - (h2 : IsAlgEnvSeqUntil A₂ R₂ alg env P' N) : - P.map (fun ω (n : Iic N) ↦ (A₁ n ω, R₁ n ω)) = - P'.map (fun ω (n : Iic N) ↦ (A₂ n ω, R₂ n ω)) := by - rw [eq_trajMeasure_map_frestrictLe_of_isAlgEnvSeqUntil h1, - eq_trajMeasure_map_frestrictLe_of_isAlgEnvSeqUntil h2] - -end ModelEquivalence - namespace IT /-- Action and feedback at step `n`. -/ @@ -112,12 +55,18 @@ def action (n : ℕ) (h : ℕ → 𝓐 × 𝓨) : 𝓐 := (h n).1 `ℕ → 𝓐 × 𝓨`. -/ def feedback (n : ℕ) (h : ℕ → 𝓐 × 𝓨) : 𝓨 := (h n).2 -/-- `hist n` is the history up to time `n`. This is a random variable on the measurable space -`ℕ → 𝓐 × 𝓨`. -/ -def hist (n : ℕ) (h : ℕ → 𝓐 × 𝓨) : Iic n → 𝓐 × 𝓨 := fun i ↦ h i +/-- `hist n` is the history before time `n`: the action-feedback pairs at times `0, ..., n - 1`. +This is a random variable on the measurable space `ℕ → 𝓐 × 𝓨`. -/ +def hist (n : ℕ) (h : ℕ → 𝓐 × 𝓨) : Fin n → 𝓐 × 𝓨 := fun i ↦ h i lemma fst_comp_step (n : ℕ) : Prod.fst ∘ step (𝓐 := 𝓐) (𝓨 := 𝓨) n = action n := rfl +lemma history_action_feedback (n : ℕ) : + history (action (𝓐 := 𝓐) (𝓨 := 𝓨)) feedback n = hist n := rfl + +@[simp] +lemma hist_zero : hist (𝓐 := 𝓐) (𝓨 := 𝓨) 0 = fun _ ↦ default := Unique.eq_default _ + @[fun_prop] lemma measurable_step (n : ℕ) : Measurable (step n (𝓐 := 𝓐) (𝓨 := 𝓨)) := by unfold step; fun_prop @@ -145,37 +94,111 @@ lemma measurable_feedback_prod : Measurable (fun p : ℕ × (ℕ → 𝓐 × @[fun_prop] lemma measurable_hist (n : ℕ) : Measurable (hist n (𝓐 := 𝓐) (𝓨 := 𝓨)) := by unfold hist; fun_prop -lemma hist_eq_frestrictLe : - hist = Preorder.frestrictLe («π» := fun _ ↦ 𝓐 × 𝓨) := by - ext n h i : 3 - simp [hist, Preorder.frestrictLe] - -/-- Filtration of the algorithm Seq. -/ -protected def filtration (𝓐 𝓨 : Type*) [MeasurableSpace 𝓐] [MeasurableSpace 𝓨] : - Filtration ℕ (inferInstance : MeasurableSpace (ℕ → 𝓐 × 𝓨)) := - MeasureTheory.Filtration.piLE (X := fun _ ↦ 𝓐 × 𝓨) +lemma hist_succ_eq_comp_frestrictLe (n : ℕ) : + hist (𝓐 := 𝓐) (𝓨 := 𝓨) (n + 1) = + (MeasurableEquiv.finSuccPiIic (fun _ ↦ 𝓐 × 𝓨) n).symm ∘ Preorder.frestrictLe n := rfl -lemma filtration_eq_comap (n : ℕ) : - IT.filtration 𝓐 𝓨 n = MeasurableSpace.comap (hist n) inferInstance := by - simp [IT.filtration, Filtration.piLE_eq_comap_frestrictLe, ← hist_eq_frestrictLe] +lemma frestrictLe_eq_comp_hist (n : ℕ) : + Preorder.frestrictLe («π» := fun _ ↦ 𝓐 × 𝓨) n = + (MeasurableEquiv.finSuccPiIic (fun _ ↦ 𝓐 × 𝓨) n) ∘ hist (n + 1) := rfl lemma step_eq_eval_comp_hist (n : ℕ) : - step (𝓐 := 𝓐) (𝓨 := 𝓨) n = (fun x ↦ x ⟨n, by simp⟩) ∘ (hist n) := rfl + step (𝓐 := 𝓐) (𝓨 := 𝓨) n = (fun x ↦ x (Fin.last n)) ∘ (hist (n + 1)) := rfl lemma action_eq_eval_comp_hist (n : ℕ) : - action (𝓐 := 𝓐) (𝓨 := 𝓨) n = (fun x ↦ (x ⟨n, by simp⟩).1) ∘ (hist n) := rfl + action (𝓐 := 𝓐) (𝓨 := 𝓨) n = (fun x ↦ (x (Fin.last n)).1) ∘ (hist (n + 1)) := rfl lemma feedback_eq_eval_comp_hist (n : ℕ) : - feedback (𝓐 := 𝓐) (𝓨 := 𝓨) n = (fun x ↦ (x ⟨n, by simp⟩).2) ∘ (hist n) := rfl + feedback (𝓐 := 𝓐) (𝓨 := 𝓨) n = (fun x ↦ (x (Fin.last n)).2) ∘ (hist (n + 1)) := rfl + +/-- The history before time `m` is a restriction of the history before time `n ≥ m`. -/ +lemma hist_eq_comp_hist {m n : ℕ} (hmn : m ≤ n) : + hist (𝓐 := 𝓐) (𝓨 := 𝓨) m = (fun h (i : Fin m) ↦ h (Fin.castLE hmn i)) ∘ hist n := rfl + +end IT + +section ModelEquivalence + +variable {Ω Ω' : Type*} {mΩ : MeasurableSpace Ω} {mΩ' : MeasurableSpace Ω'} + {alg : Algorithm 𝓐 𝓨} {env : Environment 𝓐 𝓨} + {P : Measure Ω} [IsProbabilityMeasure P] {P' : Measure Ω'} [IsProbabilityMeasure P'] + {A₁ : ℕ → Ω → 𝓐} {R₁ : ℕ → Ω → 𝓨} {A₂ : ℕ → Ω' → 𝓐} {R₂ : ℕ → Ω' → 𝓨} {N : ℕ} + +lemma IsAlgEnvSeq.map_trajectory (h : IsAlgEnvSeq A₁ R₁ alg env P) : + P.map (trajectory A₁ R₁) = trajMeasure alg env := + (Kernel.hasLaw_trajMeasureFin (X := fun _ ↦ 𝓐 × 𝓨) (Y := step A₁ R₁) (P := P) + (κ' := stepKernel alg env) h.measurable_step h.hasCondDistrib_step).map_eq + +lemma IsAlgEnvSeq.hasLaw_trajectory (h : IsAlgEnvSeq A₁ R₁ alg env P) : + HasLaw (trajectory A₁ R₁) (trajMeasure alg env) P where + aemeasurable := h.measurable_trajectory.aemeasurable + map_eq := h.map_trajectory + +lemma IsAlgEnvSeqUntil.map_history (h : IsAlgEnvSeqUntil A₁ R₁ alg env P N) : + P.map (history A₁ R₁ N) = (trajMeasure alg env).map (IT.hist N) := + Kernel.eq_trajMeasureFin_map (X := fun _ ↦ 𝓐 × 𝓨) (Y := step A₁ R₁) (P := P) (N := N) + (κ' := stepKernel alg env) h.measurable_step h.hasCondDistrib_step + +lemma IsAlgEnvSeq.map_history (h : IsAlgEnvSeq A₁ R₁ alg env P) (n : ℕ) : + P.map (history A₁ R₁ n) = (trajMeasure alg env).map (IT.hist n) := + (h.isAlgEnvSeqUntil n).map_history + +/-- The law of the sequence of actions and observations generated by an algorithm-environment pair +is unique: it does not depend on the probability space used. -/ +lemma isAlgEnvSeq_unique (h1 : IsAlgEnvSeq A₁ R₁ alg env P) + (h2 : IsAlgEnvSeq A₂ R₂ alg env P') : + P.map (trajectory A₁ R₁) = P'.map (trajectory A₂ R₂) := by + rw [h1.map_trajectory, h2.map_trajectory] + +/-- The law of the sequence of actions and observations generated by an algorithm-environment pair +is unique: it does not depend on the probability space used. -/ +theorem IsAlgEnvSeq.identDistrib_trajectory (h1 : IsAlgEnvSeq A₁ R₁ alg env P) + (h2 : IsAlgEnvSeq A₂ R₂ alg env P') : + IdentDistrib (trajectory A₁ R₁) (trajectory A₂ R₂) P P' where + aemeasurable_fst := h1.measurable_trajectory.aemeasurable + aemeasurable_snd := h2.measurable_trajectory.aemeasurable + map_eq := isAlgEnvSeq_unique h1 h2 + +/-- The law of the history before time `N` is determined by the algorithm-environment pair, +provided that the sequence is an algorithm-environment sequence until time `N`. -/ +lemma isAlgEnvSeqUntil_unique (h1 : IsAlgEnvSeqUntil A₁ R₁ alg env P N) + (h2 : IsAlgEnvSeqUntil A₂ R₂ alg env P' N) : + P.map (history A₁ R₁ N) = P'.map (history A₂ R₂ N) := by + rw [h1.map_history, h2.map_history] + +end ModelEquivalence + +namespace IT + +/-- Filtration of the algorithm Seq: `IT.filtration 𝓐 𝓨 n` is the σ-algebra generated by the +action-feedback pairs at times `0, ..., n`, that is by `hist (n + 1)`. -/ +protected def filtration (𝓐 𝓨 : Type*) [MeasurableSpace 𝓐] [MeasurableSpace 𝓨] : + Filtration ℕ (inferInstance : MeasurableSpace (ℕ → 𝓐 × 𝓨)) := + MeasureTheory.Filtration.piLE (X := fun _ ↦ 𝓐 × 𝓨) + +lemma filtration_eq_comap (n : ℕ) : + IT.filtration 𝓐 𝓨 n = MeasurableSpace.comap (hist (n + 1)) inferInstance := by + simp only [IT.filtration, Filtration.piLE_eq_comap_frestrictLe] + refine le_antisymm ?_ ?_ + · rw [← measurable_iff_comap_le, frestrictLe_eq_comp_hist] + exact measurable_comp_comap _ (by fun_prop) + · rw [← measurable_iff_comap_le, hist_succ_eq_comp_frestrictLe] + exact measurable_comp_comap _ (by fun_prop) lemma adapted_step : Adapted (IT.filtration 𝓐 𝓨) (step (𝓐 := 𝓐) (𝓨 := 𝓨)) := by intro n rw [filtration_eq_comap, step_eq_eval_comp_hist] exact measurable_comp_comap _ (by fun_prop) +lemma measurable_hist_succ_filtration (n : ℕ) : + Measurable[IT.filtration 𝓐 𝓨 n] (hist (n + 1)) := by + rw [filtration_eq_comap] + exact measurable_iff_comap_le.mpr le_rfl + lemma adapted_hist : Adapted (IT.filtration 𝓐 𝓨) hist := by intro n - simp [filtration_eq_comap, measurable_iff_comap_le] + rw [filtration_eq_comap, hist_eq_comp_hist n.le_succ] + exact measurable_comp_comap _ (by fun_prop) lemma adapted_action : Adapted (IT.filtration 𝓐 𝓨) action := by intro n @@ -189,60 +212,46 @@ lemma adapted_feedback : Adapted (IT.filtration 𝓐 𝓨) feedback := by section FiltrationAction -/-- Filtration generated by the history at time `n-1` together with the action at time `n`. -/ +/-- Filtration generated by the history before time `n` together with the action at time `n`. -/ def filtrationAction (𝓐 𝓨 : Type*) [MeasurableSpace 𝓐] [MeasurableSpace 𝓨] : Filtration ℕ (inferInstance : MeasurableSpace (ℕ → 𝓐 × 𝓨)) where - seq n := if n = 0 then MeasurableSpace.comap (action 0) inferInstance - else IT.filtration 𝓐 𝓨 (n - 1) ⊔ MeasurableSpace.comap (action n) inferInstance + seq n := MeasurableSpace.comap (fun ω ↦ (hist n ω, action n ω)) inferInstance mono' n m hnm := by simp only - by_cases hn : n = 0 - · by_cases hm : m = 0 - · simp [hn, hm] - · simp only [hn, ↓reduceIte, hm] - refine le_sup_of_le_left ?_ - rw [← measurable_iff_comap_le] - suffices Measurable[IT.filtration 𝓐 𝓨 0] (action 0) from - this.mono ((IT.filtration 𝓐 𝓨).mono zero_le) le_rfl - exact adapted_action 0 - have hm : m ≠ 0 := by grind - simp only [hn, hm, ↓reduceIte] - have hnm' : n - 1 ≤ m - 1 := by grind - simp only [sup_le_iff] - constructor - · refine le_sup_of_le_left ?_ - exact (IT.filtration 𝓐 𝓨).mono hnm' - · rcases eq_or_lt_of_le hnm with rfl | hlt - · exact le_sup_of_le_right le_rfl - refine le_sup_of_le_left ?_ - rw [← measurable_iff_comap_le] - have h_le : n ≤ m - 1 := by grind - suffices Measurable[IT.filtration 𝓐 𝓨 n] (action n) from - this.mono ((IT.filtration 𝓐 𝓨).mono h_le) le_rfl - exact adapted_action n + rw [← measurable_iff_comap_le] + rcases eq_or_lt_of_le hnm with rfl | hlt + · exact measurable_iff_comap_le.mpr le_rfl + have : (fun ω ↦ (hist (𝓐 := 𝓐) (𝓨 := 𝓨) n ω, action n ω)) = + (fun p : (Fin m → 𝓐 × 𝓨) × 𝓐 ↦ + (fun i : Fin n ↦ p.1 (Fin.castLE hnm i), (p.1 ⟨n, hlt⟩).1)) ∘ + (fun ω ↦ (hist m ω, action m ω)) := rfl + rw [this] + exact measurable_comp_comap _ (by fun_prop) le' n := by - by_cases hn : n = 0 - · simp only [hn, ↓reduceIte] - rw [← measurable_iff_comap_le] - fun_prop - simp only [hn, ↓reduceIte, sup_le_iff] - constructor - · exact (IT.filtration 𝓐 𝓨).le _ - · rw [← measurable_iff_comap_le] - fun_prop + rw [← measurable_iff_comap_le] + fun_prop + +lemma filtrationAction_eq_comap (n : ℕ) : + filtrationAction 𝓐 𝓨 n = + MeasurableSpace.comap (fun ω ↦ (hist n ω, action n ω)) inferInstance := rfl lemma filtrationAction_zero_eq_comap : filtrationAction 𝓐 𝓨 0 = MeasurableSpace.comap (action 0) inferInstance := by - simp [filtrationAction] + rw [filtrationAction_eq_comap, hist_zero] + refine le_antisymm ?_ ?_ + · rw [← measurable_iff_comap_le] + exact measurable_const.prodMk (measurable_iff_comap_le.mpr le_rfl) + · rw [← measurable_iff_comap_le] + exact measurable_snd.comp (measurable_iff_comap_le.mpr le_rfl) -lemma filtrationAction_eq_comap (n : ℕ) (hn : n ≠ 0) : - filtrationAction 𝓐 𝓨 n = - MeasurableSpace.comap (fun ω ↦ (hist (n - 1) ω, action n ω)) inferInstance := by - simp only [filtrationAction, filtration_eq_comap, ← MeasurableSpace.comap_prodMk, hn, ↓reduceIte] - rfl +lemma measurable_hist_filtrationAction (n : ℕ) : + Measurable[filtrationAction 𝓐 𝓨 n] (hist n) := + measurable_fst.comp (measurable_iff_comap_le.mpr le_rfl) lemma filtration_le_filtrationAction_add_one (n : ℕ) : - IT.filtration 𝓐 𝓨 n ≤ filtrationAction 𝓐 𝓨 (n + 1) := le_sup_of_le_left le_rfl + IT.filtration 𝓐 𝓨 n ≤ filtrationAction 𝓐 𝓨 (n + 1) := by + rw [filtration_eq_comap] + exact measurable_iff_comap_le.mp (measurable_hist_filtrationAction (n + 1)) lemma filtration_le_filtrationAction {m n : ℕ} (h : n < m) : IT.filtration 𝓐 𝓨 n ≤ filtrationAction 𝓐 𝓨 m := by @@ -251,72 +260,32 @@ lemma filtration_le_filtrationAction {m n : ℕ} (h : n < m) : lemma filtrationAction_le_filtration_self (n : ℕ) : filtrationAction 𝓐 𝓨 n ≤ IT.filtration 𝓐 𝓨 n := by - by_cases hn : n = 0 - · simp only [hn, filtrationAction_zero_eq_comap] - rw [← measurable_iff_comap_le] - exact adapted_action 0 - simp only [filtrationAction, hn, ↓reduceIte, sup_le_iff] - constructor - · exact (IT.filtration 𝓐 𝓨).mono (by grind) - · rw [← measurable_iff_comap_le] - exact adapted_action _ + rw [filtrationAction_eq_comap, ← measurable_iff_comap_le] + exact (adapted_hist n).prodMk (adapted_action n) lemma filtrationAction_le_filtration {m n : ℕ} (h : m ≤ n) : filtrationAction 𝓐 𝓨 m ≤ IT.filtration 𝓐 𝓨 n := (filtrationAction_le_filtration_self m).trans ((IT.filtration 𝓐 𝓨).mono h) lemma measurable_action_filtrationAction (n : ℕ) : - Measurable[filtrationAction 𝓐 𝓨 n] (action n) := by - rw [measurable_iff_comap_le] - simp only [filtrationAction] - split_ifs with hn - · simp [hn] - · exact le_sup_of_le_right le_rfl + Measurable[filtrationAction 𝓐 𝓨 n] (action n) := + measurable_snd.comp (measurable_iff_comap_le.mpr le_rfl) end FiltrationAction section Laws -lemma hasLaw_step_zero (alg : Algorithm 𝓐 𝓨) (env : Environment 𝓐 𝓨) : - HasLaw (step 0) (alg.p0 ⊗ₘ env.ν0) (trajMeasure alg env) where - aemeasurable := Measurable.aemeasurable (by fun_prop) - map_eq := by - unfold step - rw [← coe_default_Iic_zero] - simp only [trajMeasure, Kernel.trajMeasure] - rw [← Measure.deterministic_comp_eq_map (by fun_prop), Measure.comp_assoc, - Kernel.deterministic_comp_eq_map, Kernel.traj_zero_map_eval_zero, - Measure.deterministic_comp_eq_map, Measure.map_map (by fun_prop) (by fun_prop)] - exact Measure.map_id - -lemma hasLaw_action_zero (alg : Algorithm 𝓐 𝓨) (env : Environment 𝓐 𝓨) : - HasLaw (action 0) alg.p0 (trajMeasure alg env) where - map_eq := by - rw [← fst_comp_step, ← Measure.map_map (by fun_prop) (by fun_prop), - (hasLaw_step_zero alg env).map_eq, ← Measure.fst, Measure.fst_compProd] - -lemma hasCondDistrib_feedback_zero (alg : Algorithm 𝓐 𝓨) (env : Environment 𝓐 𝓨) : - HasCondDistrib (feedback 0) (action 0) env.ν0 (trajMeasure alg env) := by - have h_step := (hasLaw_step_zero alg env).map_eq - have h_action := (hasLaw_action_zero alg env).map_eq - exact ⟨by fun_prop, by rwa [h_action]⟩ - -lemma _root_.ProbabilityTheory.Kernel.hasCondDistrib_trajMeasure - (alg : Algorithm 𝓐 𝓨) (env : Environment 𝓐 𝓨) (n : ℕ) : - HasCondDistrib (step (n + 1)) (hist n) (stepKernel alg env n) (trajMeasure alg env) := - ⟨by fun_prop, Kernel.map_frestrictLe_trajMeasure_compProd_eq_map_trajMeasure.symm⟩ - lemma hasCondDistrib_step (alg : Algorithm 𝓐 𝓨) (env : Environment 𝓐 𝓨) (n : ℕ) : - HasCondDistrib (step (n + 1)) (hist n) (stepKernel alg env n) (trajMeasure alg env) := - Kernel.hasCondDistrib_trajMeasure alg env n + HasCondDistrib (step n) (hist n) (stepKernel alg env n) (trajMeasure alg env) := + Kernel.hasCondDistrib_trajMeasureFin n lemma hasCondDistrib_action (alg : Algorithm 𝓐 𝓨) (env : Environment 𝓐 𝓨) (n : ℕ) : - HasCondDistrib (action (n + 1)) (hist n) (alg.policy n) (trajMeasure alg env) := by + HasCondDistrib (action n) (hist n) (alg.policy n) (trajMeasure alg env) := by rw [← fst_comp_step, ← fst_stepKernel, Kernel.fst_eq] exact HasCondDistrib.comp_left (hasCondDistrib_step alg env n) measurable_fst lemma hasCondDistrib_feedback (alg : Algorithm 𝓐 𝓨) (env : Environment 𝓐 𝓨) (n : ℕ) : - HasCondDistrib (feedback (n + 1)) (fun ω ↦ (hist n ω, action (n + 1) ω)) (env.feedback n) + HasCondDistrib (feedback n) (fun ω ↦ (hist n ω, action n ω)) (env.feedback n) (trajMeasure alg env) := by have h_step := hasCondDistrib_step alg env n have h_action := hasCondDistrib_action alg env n @@ -325,6 +294,23 @@ lemma hasCondDistrib_feedback (alg : Algorithm 𝓐 𝓨) (env : Environment Measure.map_map (by fun_prop) (by fun_prop)] rfl +lemma isAlgEnvSeq_trajMeasure (alg : Algorithm 𝓐 𝓨) (env : Environment 𝓐 𝓨) : + IsAlgEnvSeq action feedback alg env (trajMeasure alg env) where + hasCondDistrib_action n := hasCondDistrib_action alg env n + hasCondDistrib_feedback n := hasCondDistrib_feedback alg env n + +lemma hasLaw_step_zero (alg : Algorithm 𝓐 𝓨) (env : Environment 𝓐 𝓨) : + HasLaw (step 0) (alg.p0 ⊗ₘ env.ν0) (trajMeasure alg env) := + (isAlgEnvSeq_trajMeasure alg env).hasLaw_step_zero + +lemma hasLaw_action_zero (alg : Algorithm 𝓐 𝓨) (env : Environment 𝓐 𝓨) : + HasLaw (action 0) alg.p0 (trajMeasure alg env) := + (isAlgEnvSeq_trajMeasure alg env).hasLaw_action_zero + +lemma hasCondDistrib_feedback_zero (alg : Algorithm 𝓐 𝓨) (env : Environment 𝓐 𝓨) : + HasCondDistrib (feedback 0) (action 0) env.ν0 (trajMeasure alg env) := + (isAlgEnvSeq_trajMeasure alg env).hasCondDistrib_feedback_zero + lemma condDistrib_feedback_zero [StandardBorelSpace 𝓨] [Nonempty 𝓨] (alg : Algorithm 𝓐 𝓨) (env : Environment 𝓐 𝓨) : condDistrib (feedback 0) (action 0) (trajMeasure alg env) @@ -333,29 +319,22 @@ lemma condDistrib_feedback_zero [StandardBorelSpace 𝓨] [Nonempty 𝓨] lemma condDistrib_step [StandardBorelSpace 𝓐] [Nonempty 𝓐] [StandardBorelSpace 𝓨] [Nonempty 𝓨] (alg : Algorithm 𝓐 𝓨) (env : Environment 𝓐 𝓨) (n : ℕ) : - condDistrib (step (n + 1)) (hist n) (trajMeasure alg env) + condDistrib (step n) (hist n) (trajMeasure alg env) =ᵐ[(trajMeasure alg env).map (hist n)] stepKernel alg env n := (hasCondDistrib_step alg env n).condDistrib_eq lemma condDistrib_action [StandardBorelSpace 𝓐] [Nonempty 𝓐] (alg : Algorithm 𝓐 𝓨) (env : Environment 𝓐 𝓨) (n : ℕ) : - condDistrib (action (n + 1)) (hist n) (trajMeasure alg env) + condDistrib (action n) (hist n) (trajMeasure alg env) =ᵐ[(trajMeasure alg env).map (hist n)] alg.policy n := (hasCondDistrib_action alg env n).condDistrib_eq lemma condDistrib_feedback [StandardBorelSpace 𝓨] [Nonempty 𝓨] (alg : Algorithm 𝓐 𝓨) (env : Environment 𝓐 𝓨) (n : ℕ) : - condDistrib (feedback (n + 1)) (fun ω ↦ (hist n ω, action (n + 1) ω)) (trajMeasure alg env) - =ᵐ[(trajMeasure alg env).map (fun ω ↦ (hist n ω, action (n + 1) ω))] env.feedback n := + condDistrib (feedback n) (fun ω ↦ (hist n ω, action n ω)) (trajMeasure alg env) + =ᵐ[(trajMeasure alg env).map (fun ω ↦ (hist n ω, action n ω))] env.feedback n := (hasCondDistrib_feedback alg env n).condDistrib_eq -lemma isAlgEnvSeq_trajMeasure (alg : Algorithm 𝓐 𝓨) (env : Environment 𝓐 𝓨) : - IsAlgEnvSeq action feedback alg env (trajMeasure alg env) where - hasLaw_action_zero := hasLaw_action_zero alg env - hasCondDistrib_feedback_zero := hasCondDistrib_feedback_zero alg env - hasCondDistrib_action n := hasCondDistrib_action alg env n - hasCondDistrib_feedback n := hasCondDistrib_feedback alg env n - end Laws end IT diff --git a/LeanMachineLearning/SequentialLearning/Means.lean b/LeanMachineLearning/SequentialLearning/Means.lean index 7dad5aee..f583c1f5 100644 --- a/LeanMachineLearning/SequentialLearning/Means.lean +++ b/LeanMachineLearning/SequentialLearning/Means.lean @@ -54,7 +54,7 @@ variable {Ω 𝓐 𝓨 : Type*} {mΩ : MeasurableSpace Ω} {m𝓐 : MeasurableSp chosen at time `n`. -/ noncomputable def Environment.measure (env : Environment 𝓐 𝓨) (A : ℕ → Ω → 𝓐) (Y : ℕ → Ω → 𝓨) (n : ℕ) (ω : Ω) : Kernel 𝓐 𝓨 := - if n = 0 then env.ν0 else (env.feedback (n - 1)).sectR (history A Y (n - 1) ω) + (env.feedback n).sectR (history A Y n ω) /-- The means of the feedback distribution as a function of the action chosen at time `n`. -/ noncomputable def Environment.means (env : Environment 𝓐 𝓨) (A : ℕ → Ω → 𝓐) (Y : ℕ → Ω → 𝓨) @@ -64,17 +64,14 @@ noncomputable def Environment.means (env : Environment 𝓐 𝓨) (A : ℕ → @[simp] lemma means_zero (env : Environment 𝓐 𝓨) (A : ℕ → Ω → 𝓐) (Y : ℕ → Ω → 𝓨) (k : 𝓐) (ω : Ω) : - env.means A Y k 0 ω = (env.ν0 k)[id] := by simp [Environment.means, Environment.measure] + env.means A Y k 0 ω = (env.ν0 k)[id] := by + simp [Environment.means, Environment.measure, Environment.feedback_zero] @[simp] lemma means_of_isObliviousEnv [IsObliviousEnv env] (A : ℕ → Ω → 𝓐) (Y : ℕ → Ω → 𝓨) (k : 𝓐) (n : ℕ) (ω : Ω) : env.means A Y k n ω = (feedbackCondAction env n k)[id] := by - simp only [Environment.means, Environment.measure, ν0_eq_feedbackCondAction, id_eq, - feedback_eq_feedbackCondAction] - split_ifs with hn - · simp [hn] - · simp [Nat.sub_add_cancel (by grind : 1 ≤ n)] + simp [Environment.means, Environment.measure, feedback_eq_feedbackCondAction] lemma means_obliviousEnv (ν : ℕ → Kernel 𝓐 𝓨) [∀ n, IsMarkovKernel (ν n)] (k : 𝓐) (n : ℕ) (ω : Ω) : @@ -89,10 +86,9 @@ lemma IsAlgEnvSeq.stronglyMeasurable_means [SecondCountableTopology 𝓨] [Opens StronglyMeasurable (env.means A Y k n) := by unfold Environment.means have h_eq ω : env.measure A Y n ω k = - (if n = 0 then env.ν0 ∘ₖ (Kernel.deterministic (fun _ ↦ k) (by fun_prop)) - else (env.feedback (n - 1)) ∘ₖ (Kernel.deterministic (fun ω ↦ (history A Y (n - 1) ω, k)) - ((h.measurable_history (n - 1)).prodMk (by fun_prop)))) ω := by - split_ifs with hn <;> simp [hn, Environment.measure, Kernel.comp_deterministic_eq_comap] + (env.feedback n ∘ₖ Kernel.deterministic (fun ω ↦ (history A Y n ω, k)) + ((h.measurable_history n).prodMk (by fun_prop))) ω := by + simp [Environment.measure, Kernel.comp_deterministic_eq_comap] simp_rw [h_eq] fun_prop @@ -106,15 +102,11 @@ lemma IsAlgEnvSeq.adapted_means_filtrationAction [SecondCountableTopology 𝓨] (h : IsAlgEnvSeq A Y alg env P) : Adapted h.filtrationAction (fun n ω ↦ env.means A Y (A n ω) n ω) := by intro n - cases n with - | zero => exact measurable_comp_comap _ stronglyMeasurable_id.integral_kernel.measurable - | succ n => - simp only [Environment.means, Environment.measure, Nat.add_eq_zero_iff, one_ne_zero, and_false, - ↓reduceIte, Nat.add_one_sub_one, Kernel.sectR_apply, id_eq] - change Measurable[h.filtrationAction (n + 1)] - ((fun ω ↦ ∫ x, x ∂(env.feedback n ω)) ∘ (fun ω ↦ (history A Y n ω, A (n + 1) ω))) - rw [IsAlgEnvSeq.filtrationAction_eq_comap _ _ (by grind)] - exact measurable_comp_comap _ stronglyMeasurable_id.integral_kernel.measurable + simp only [Environment.means, Environment.measure, Kernel.sectR_apply, id_eq] + change Measurable[h.filtrationAction n] + ((fun ω ↦ ∫ x, x ∂(env.feedback n ω)) ∘ (fun ω ↦ (history A Y n ω, A n ω))) + rw [IsAlgEnvSeq.filtrationAction_eq_comap] + exact measurable_comp_comap _ stronglyMeasurable_id.integral_kernel.measurable lemma IsAlgEnvSeq.stronglyAdapted_means_filtrationAction [SecondCountableTopology 𝓨] [BorelSpace 𝓨] (h : IsAlgEnvSeq A Y alg env P) : @@ -126,36 +118,23 @@ lemma IsAlgEnvSeq.adapted_means [SecondCountableTopology 𝓨] [BorelSpace 𝓨] Adapted h.filtration (fun n ω ↦ env.means A Y (A n ω) n ω) := fun n ↦ (h.adapted_means_filtrationAction n).mono (h.filtrationAction_le_filtration n) le_rfl -omit [NormedSpace ℝ 𝓨] in -lemma IsAlgEnvSeq.condExp_feedback_zero_comp {𝓩 : Type*} [NormedAddCommGroup 𝓩] [NormedSpace ℝ 𝓩] - [CompleteSpace 𝓩] [StandardBorelSpace 𝓨] - (h : IsAlgEnvSeq A Y alg env P) - {g : 𝓨 → 𝓩} (hg : StronglyMeasurable g) (hint : Integrable (fun ω ↦ g (Y 0 ω)) P) : - P[fun ω ↦ g (Y 0 ω) | h.filtrationAction 0] =ᵐ[P] fun ω ↦ (env.ν0 (A 0 ω))[g] := by - have hX : Measurable (fun ω ↦ (history A Y 0 ω, A 0 ω)) := - (h.measurable_history 0).prodMk (h.measurable_action 0) - rw [h.filtrationAction_zero_eq_comap] - exact h.hasCondDistrib_feedback_zero.condExp_comp_eq (h.measurable_action 0) hg hint - omit [NormedSpace ℝ 𝓨] in lemma IsAlgEnvSeq.condExp_feedback_comp {𝓩 : Type*} [NormedAddCommGroup 𝓩] [NormedSpace ℝ 𝓩] [CompleteSpace 𝓩] [StandardBorelSpace 𝓨] (h : IsAlgEnvSeq A Y alg env P) (n : ℕ) - {g : 𝓨 → 𝓩} (hg : StronglyMeasurable g) (hint : Integrable (fun ω ↦ g (Y (n + 1) ω)) P) : - P[fun ω ↦ g (Y (n + 1) ω) | h.filtrationAction (n + 1)] =ᵐ[P] - fun ω ↦ (env.feedback n (history A Y n ω, A (n + 1) ω))[g] := by - have hX : Measurable (fun ω ↦ (history A Y n ω, A (n + 1) ω)) := - (h.measurable_history n).prodMk (h.measurable_action (n + 1)) - rw [h.filtrationAction_eq_comap (n + 1) (by simp)] + {g : 𝓨 → 𝓩} (hg : StronglyMeasurable g) (hint : Integrable (fun ω ↦ g (Y n ω)) P) : + P[fun ω ↦ g (Y n ω) | h.filtrationAction n] =ᵐ[P] + fun ω ↦ (env.feedback n (history A Y n ω, A n ω))[g] := by + have hX : Measurable (fun ω ↦ (history A Y n ω, A n ω)) := + (h.measurable_history n).prodMk (h.measurable_action n) + rw [h.filtrationAction_eq_comap n] exact (h.hasCondDistrib_feedback n).condExp_comp_eq hX hg hint lemma IsAlgEnvSeq.condExp_feedback [BorelSpace 𝓨] [SecondCountableTopology 𝓨] [CompleteSpace 𝓨] (h : IsAlgEnvSeq A Y alg env P) (n : ℕ) (hint : Integrable (Y n) P) : - P[Y n | h.filtrationAction n] =ᵐ[P] fun ω ↦ env.means A Y (A n ω) n ω := by - cases n with - | zero => exact condExp_feedback_zero_comp h stronglyMeasurable_id hint - | succ n => exact condExp_feedback_comp h n stronglyMeasurable_id hint + P[Y n | h.filtrationAction n] =ᵐ[P] fun ω ↦ env.means A Y (A n ω) n ω := + condExp_feedback_comp h n stronglyMeasurable_id hint lemma IsAlgEnvSeq.memLp_means_action [SecondCountableTopology 𝓨] [BorelSpace 𝓨] (h : IsAlgEnvSeq A Y alg env P) {n : ℕ} {p : ℝ≥0∞} (hp1 : 1 ≤ p) (hp_top : p ≠ ∞) @@ -167,53 +146,28 @@ lemma IsAlgEnvSeq.memLp_means_action [SecondCountableTopology 𝓨] [BorelSpace have hint' : MemLp id p (P.map (Y n)) := by rwa [memLp_map_measure_iff (by fun_prop) (h.measurable_feedback _).aemeasurable] unfold Environment.means Environment.measure - cases n with - | zero => - simp only [↓reduceIte, id_eq] - rw [h.hasLaw_feedback_zero_comp.map_eq, Measure.memLp_comp_iff hp0 hp_top (by fun_prop)] - at hint' - have hint'' := hint'.2.comp_aemeasurable (by fun_prop) - have h_eq ω : env.ν0 (A 0 ω) = (env.ν0 ∘ₖ Kernel.deterministic (A 0) (by fun_prop)) ω := by - simp [Kernel.comp_deterministic_eq_comap] - rw [← integrable_norm_rpow_iff _ hp0 hp_top] - swap - · refine StronglyMeasurable.aestronglyMeasurable ?_ - simp_rw [h_eq] - exact StronglyMeasurable.integral_kernel (by fun_prop) - simp only [id_eq] at hint'' - refine Integrable.mono' hint'' ?_ ?_ - · refine ((AEMeasurable.norm ?_).pow_const _).aestronglyMeasurable - refine (StronglyMeasurable.measurable ?_).aemeasurable - simp_rw [h_eq] - exact StronglyMeasurable.integral_kernel (by fun_prop) - · simp only [Real.norm_eq_abs, Function.comp_apply] - filter_upwards [ae_of_ae_map (hA 0).aemeasurable hint'.1] with ω hω - rw [abs_of_nonneg (by positivity)] - exact norm_integral_rpow_le_integral_norm_rpow hp1 hp_top hω - | succ n => - simp only [Nat.add_eq_zero_iff, one_ne_zero, and_false, ↓reduceIte, Nat.add_one_sub_one, id_eq] - rw [(h.hasLaw_feedback_comp n).map_eq, Measure.memLp_comp_iff hp0 hp_top (by fun_prop)] at hint' - have hint'' := hint'.2.comp_aemeasurable (by fun_prop) - have h_eq ω : (env.feedback n) (history A Y n ω, A (n + 1) ω) = - (env.feedback n ∘ₖ - Kernel.deterministic (fun ω ↦ (history A Y n ω, A (n + 1) ω)) (by fun_prop)) ω := by - simp [Kernel.comp_deterministic_eq_comap] - rw [← integrable_norm_rpow_iff _ hp0 hp_top] - swap - · refine StronglyMeasurable.aestronglyMeasurable ?_ - simp_rw [Kernel.sectR_apply, h_eq] - exact StronglyMeasurable.integral_kernel (by fun_prop) - simp only [id_eq] at hint'' - refine Integrable.mono' hint'' ?_ ?_ - · refine ((AEMeasurable.norm ?_).pow_const _).aestronglyMeasurable - refine (StronglyMeasurable.measurable ?_).aemeasurable - simp_rw [Kernel.sectR_apply, h_eq] - exact StronglyMeasurable.integral_kernel (by fun_prop) - · simp only [Real.norm_eq_abs, Function.comp_apply, Kernel.sectR_apply] - filter_upwards [ae_of_ae_map ((h_hist n).prodMk (hA (n + 1))).aemeasurable hint'.1] - with ω hω - rw [abs_of_nonneg (by positivity)] - exact norm_integral_rpow_le_integral_norm_rpow hp1 hp_top hω + simp only [id_eq] + rw [(h.hasLaw_feedback_comp n).map_eq, Measure.memLp_comp_iff hp0 hp_top (by fun_prop)] at hint' + have hint'' := hint'.2.comp_aemeasurable (by fun_prop) + have h_eq ω : (env.feedback n) (history A Y n ω, A n ω) = + (env.feedback n ∘ₖ + Kernel.deterministic (fun ω ↦ (history A Y n ω, A n ω)) (by fun_prop)) ω := by + simp [Kernel.comp_deterministic_eq_comap] + rw [← integrable_norm_rpow_iff _ hp0 hp_top] + swap + · refine StronglyMeasurable.aestronglyMeasurable ?_ + simp_rw [Kernel.sectR_apply, h_eq] + exact StronglyMeasurable.integral_kernel (by fun_prop) + simp only [id_eq] at hint'' + refine Integrable.mono' hint'' ?_ ?_ + · refine ((AEMeasurable.norm ?_).pow_const _).aestronglyMeasurable + refine (StronglyMeasurable.measurable ?_).aemeasurable + simp_rw [Kernel.sectR_apply, h_eq] + exact StronglyMeasurable.integral_kernel (by fun_prop) + · simp only [Real.norm_eq_abs, Function.comp_apply, Kernel.sectR_apply] + filter_upwards [ae_of_ae_map ((h_hist n).prodMk (hA n)).aemeasurable hint'.1] with ω hω + rw [abs_of_nonneg (by positivity)] + exact norm_integral_rpow_le_integral_norm_rpow hp1 hp_top hω lemma IsAlgEnvSeq.integrable_means_action [SecondCountableTopology 𝓨] [OpensMeasurableSpace 𝓨] (h : IsAlgEnvSeq A Y alg env P) {n : ℕ} (hint : Integrable (Y n) P) : @@ -223,35 +177,19 @@ lemma IsAlgEnvSeq.integrable_means_action [SecondCountableTopology 𝓨] [OpensM have hint' : Integrable id (P.map (Y n)) := by rwa [integrable_map_measure (by fun_prop) (h.measurable_feedback _).aemeasurable] unfold Environment.means Environment.measure - cases n with - | zero => - simp only [↓reduceIte, id_eq] - rw [h.hasLaw_feedback_zero_comp.map_eq, Measure.integrable_comp_iff (by fun_prop)] at hint' - have hint'' := hint'.2.comp_aemeasurable (by fun_prop) - simp only [id_eq] at hint'' - refine Integrable.mono' hint'' ?_ ?_ - · refine StronglyMeasurable.aestronglyMeasurable ?_ - have h_eq ω : env.ν0 (A 0 ω) = - (env.ν0 ∘ₖ Kernel.deterministic (A 0) (by fun_prop)) ω := by - simp [Kernel.comp_deterministic_eq_comap] - simp_rw [h_eq] - exact StronglyMeasurable.integral_kernel (by fun_prop) - · simp only [Function.comp_apply] - filter_upwards with ω using norm_integral_le_integral_norm _ - | succ n => - simp only [Nat.add_eq_zero_iff, one_ne_zero, and_false, ↓reduceIte, Nat.add_one_sub_one, id_eq] - rw [(h.hasLaw_feedback_comp n).map_eq, Measure.integrable_comp_iff (by fun_prop)] at hint' - have hint'' := hint'.2.comp_aemeasurable (by fun_prop) - simp only [id_eq] at hint'' - refine Integrable.mono' hint'' ?_ ?_ - · refine StronglyMeasurable.aestronglyMeasurable ?_ - have h_eq ω : (env.feedback n) (history A Y n ω, A (n + 1) ω) = - (env.feedback n ∘ₖ - Kernel.deterministic (fun ω ↦ (history A Y n ω, A (n + 1) ω)) (by fun_prop)) ω := by - simp [Kernel.comp_deterministic_eq_comap] - simp_rw [Kernel.sectR_apply, h_eq] - exact StronglyMeasurable.integral_kernel (by fun_prop) - · simp only [Function.comp_apply] - filter_upwards with ω using norm_integral_le_integral_norm _ + simp only [id_eq] + rw [(h.hasLaw_feedback_comp n).map_eq, Measure.integrable_comp_iff (by fun_prop)] at hint' + have hint'' := hint'.2.comp_aemeasurable (by fun_prop) + simp only [id_eq] at hint'' + refine Integrable.mono' hint'' ?_ ?_ + · refine StronglyMeasurable.aestronglyMeasurable ?_ + have h_eq ω : (env.feedback n) (history A Y n ω, A n ω) = + (env.feedback n ∘ₖ + Kernel.deterministic (fun ω ↦ (history A Y n ω, A n ω)) (by fun_prop)) ω := by + simp [Kernel.comp_deterministic_eq_comap] + simp_rw [Kernel.sectR_apply, h_eq] + exact StronglyMeasurable.integral_kernel (by fun_prop) + · simp only [Function.comp_apply] + filter_upwards with ω using norm_integral_le_integral_norm _ end Learning diff --git a/LeanMachineLearning/SequentialLearning/StationaryEnv.lean b/LeanMachineLearning/SequentialLearning/StationaryEnv.lean index 57eb7e25..0ac76081 100644 --- a/LeanMachineLearning/SequentialLearning/StationaryEnv.lean +++ b/LeanMachineLearning/SequentialLearning/StationaryEnv.lean @@ -50,7 +50,7 @@ variable {𝓐 𝓨 : Type*} {m𝓐 : MeasurableSpace 𝓐} {m𝓨 : MeasurableS the last action and not on the past history. -/ class IsObliviousEnv (env : Environment 𝓐 𝓨) : Prop where exists_eq_prodMkLeft : ∃ ν : ℕ → Kernel 𝓐 𝓨, (∀ n, IsMarkovKernel (ν n)) ∧ - (env.ν0 = ν 0) ∧ (∀ n, env.feedback n = (ν (n + 1)).prodMkLeft _) + (∀ n, env.feedback n = (ν n).prodMkLeft _) /-- The kernel representing the conditional distribution of the feedback given the action at time `n` in an oblivious environment. -/ @@ -62,13 +62,13 @@ instance (env : Environment 𝓐 𝓨) [IsObliviousEnv env] (n : ℕ) : IsMarkovKernel (feedbackCondAction env n) := IsObliviousEnv.exists_eq_prodMkLeft.choose_spec.1 n -lemma ν0_eq_feedbackCondAction (env : Environment 𝓐 𝓨) [IsObliviousEnv env] : - env.ν0 = feedbackCondAction env 0 := - IsObliviousEnv.exists_eq_prodMkLeft.choose_spec.2.1 - lemma feedback_eq_feedbackCondAction (env : Environment 𝓐 𝓨) [IsObliviousEnv env] (n : ℕ) : - env.feedback n = (feedbackCondAction env (n + 1)).prodMkLeft _ := - IsObliviousEnv.exists_eq_prodMkLeft.choose_spec.2.2 n + env.feedback n = (feedbackCondAction env n).prodMkLeft _ := + IsObliviousEnv.exists_eq_prodMkLeft.choose_spec.2 n + +lemma ν0_eq_feedbackCondAction (env : Environment 𝓐 𝓨) [IsObliviousEnv env] : + env.ν0 = feedbackCondAction env 0 := by + rw [Environment.ν0_def, feedback_eq_feedbackCondAction, Kernel.sectR_prodMkLeft] namespace IsObliviousEnv @@ -79,68 +79,52 @@ variable {Ω : Type*} {mΩ : MeasurableSpace Ω} lemma hasCondDistrib_feedback_history_action [IsObliviousEnv env] (h : IsAlgEnvSeq A Y alg env P) (n : ℕ) : - HasCondDistrib (Y (n + 1)) (fun ω ↦ (history A Y n ω, A (n + 1) ω)) - ((feedbackCondAction env (n + 1)).prodMkLeft _) P := by - have hA := h.measurable_action - have hR' := h.measurable_feedback - refine ⟨by fun_prop, ?_⟩ - have h_eq := (h.hasCondDistrib_feedback n).map_eq - simpa only [feedback_eq_feedbackCondAction] using h_eq + HasCondDistrib (Y n) (fun ω ↦ (history A Y n ω, A n ω)) + ((feedbackCondAction env n).prodMkLeft _) P := by + rw [← feedback_eq_feedbackCondAction] + exact h.hasCondDistrib_feedback n lemma hasCondDistrib_feedback [IsObliviousEnv env] (h : IsAlgEnvSeq A Y alg env P) (n : ℕ) : - HasCondDistrib (Y n) (A n) (feedbackCondAction env n) P := by - have hA := h.measurable_action - have hY := h.measurable_feedback - cases n with - | zero => rw [← ν0_eq_feedbackCondAction]; exact h.hasCondDistrib_feedback_zero - | succ n => - refine ⟨by fun_prop, ?_⟩ - have h_eq := (h.hasCondDistrib_feedback n).map_eq - have : P.map (A (n + 1)) = - (P.map (fun x ↦ (history A Y n x, A (n + 1) x))).snd := by - rw [Measure.snd_map_prodMk (by fun_prop)] - simp only [feedback_eq_feedbackCondAction] at h_eq - rw [this, ← Measure.snd_prodAssoc_compProd_prodMkLeft, ← h_eq, - Measure.snd_map_prodMk (by fun_prop), Measure.map_map (by fun_prop) (by fun_prop)] - congr - -/-- Conditionally on an event determined by the history up to time `n` and the action at time -`n + 1`, on which that action is equal to `b`, the feedback at time `n + 1` has law -`feedbackCondAction env (n + 1) b`. -/ + HasCondDistrib (Y n) (A n) (feedbackCondAction env n) P := + (hasCondDistrib_feedback_history_action h n).comp_right + +/-- Conditionally on an event determined by the history before time `n` and the action at time +`n`, on which that action is equal to `b`, the feedback at time `n` has law +`feedbackCondAction env n b`. -/ lemma hasLaw_feedback_cond [IsObliviousEnv env] (h : IsAlgEnvSeq A Y alg env P) (n : ℕ) - {s : Set ((Iic n → 𝓐 × 𝓨) × 𝓐)} (hs : MeasurableSet s) {b : 𝓐} (hsb : ∀ u ∈ s, u.2 = b) - (hP : P ((fun ω ↦ (history A Y n ω, A (n + 1) ω)) ⁻¹' s) ≠ 0) : - HasLaw (Y (n + 1)) (feedbackCondAction env (n + 1) b) - P[|(fun ω ↦ (history A Y n ω, A (n + 1) ω)) ⁻¹' s] := by + {s : Set ((Fin n → 𝓐 × 𝓨) × 𝓐)} (hs : MeasurableSet s) {b : 𝓐} (hsb : ∀ u ∈ s, u.2 = b) + (hP : P ((fun ω ↦ (history A Y n ω, A n ω)) ⁻¹' s) ≠ 0) : + HasLaw (Y n) (feedbackCondAction env n b) + P[|(fun ω ↦ (history A Y n ω, A n ω)) ⁻¹' s] := by refine (hasCondDistrib_feedback_history_action h n).hasLaw_cond (h.measurable_feedback _) hs (fun u hu ↦ ?_) hP rw [Kernel.prodMkLeft_apply, hsb u hu] -/-- Conditionally on an event determined by the history up to time `n` and the action at time -`n + 1`, on which that action is constant, the feedback at time `n + 1` is independent of the -history up to time `n` and of the action at time `n + 1`. -/ +/-- Conditionally on an event determined by the history before time `n` and the action at time +`n`, on which that action is constant, the feedback at time `n` is independent of the +history before time `n` and of the action at time `n`. -/ lemma indepFun_history_action_feedback_cond [IsObliviousEnv env] (h : IsAlgEnvSeq A Y alg env P) (n : ℕ) - {s : Set ((Iic n → 𝓐 × 𝓨) × 𝓐)} (hs : MeasurableSet s) {b : 𝓐} (hsb : ∀ u ∈ s, u.2 = b) : - (fun ω ↦ (history A Y n ω, A (n + 1) ω)) - ⟂ᵢ[P[|(fun ω ↦ (history A Y n ω, A (n + 1) ω)) ⁻¹' s]] Y (n + 1) := by + {s : Set ((Fin n → 𝓐 × 𝓨) × 𝓐)} (hs : MeasurableSet s) {b : 𝓐} (hsb : ∀ u ∈ s, u.2 = b) : + (fun ω ↦ (history A Y n ω, A n ω)) + ⟂ᵢ[P[|(fun ω ↦ (history A Y n ω, A n ω)) ⁻¹' s]] Y n := by have hA := h.measurable_action have hY := h.measurable_feedback refine (hasCondDistrib_feedback_history_action h n).indepFun_cond (by fun_prop) hs - (η := feedbackCondAction env (n + 1) b) fun u hu ↦ ?_ + (η := feedbackCondAction env n b) fun u hu ↦ ?_ rw [Kernel.prodMkLeft_apply, hsb u hu] variable [StandardBorelSpace 𝓐] [Nonempty 𝓐] [StandardBorelSpace 𝓨] [Nonempty 𝓨] -/-- The feedback at time `n + 1` is conditionally independent of the history up to time `n` -given the action at time `n + 1`. -/ +/-- The feedback at time `n` is conditionally independent of the history before time `n` +given the action at time `n`. -/ lemma condIndepFun_feedback_history_action [StandardBorelSpace Ω] [IsObliviousEnv env] (h : IsAlgEnvSeq A Y alg env P) (n : ℕ) : - Y (n + 1) ⟂ᵢ[A (n + 1), h.measurable_action _ ; P] history A Y n := by + Y n ⟂ᵢ[A n, h.measurable_action _ ; P] history A Y n := by have hA := h.measurable_action have hY := h.measurable_feedback refine condIndepFun_of_exists_condDistrib_prod_ae_eq_prodMkLeft - (η := feedbackCondAction env (n + 1)) + (η := feedbackCondAction env n) (by fun_prop) (by fun_prop) (by fun_prop) ?_ refine HasCondDistrib.condDistrib_eq ?_ rw [← feedback_eq_feedbackCondAction] @@ -148,38 +132,32 @@ lemma condIndepFun_feedback_history_action [StandardBorelSpace Ω] lemma condIndepFun_feedback_history_action_action [StandardBorelSpace Ω] [IsObliviousEnv env] (h : IsAlgEnvSeq A Y alg env P) (n : ℕ) : - Y (n + 1) ⟂ᵢ[A (n + 1), h.measurable_action (n + 1); P] - (fun ω ↦ (history A Y n ω, A (n + 1) ω)) := by - have h_indep : Y (n + 1) ⟂ᵢ[A (n + 1), h.measurable_action (n + 1); P] history A Y n := + Y n ⟂ᵢ[A n, h.measurable_action n; P] (fun ω ↦ (history A Y n ω, A n ω)) := by + have h_indep : Y n ⟂ᵢ[A n, h.measurable_action n; P] history A Y n := condIndepFun_feedback_history_action h n have hA := h.measurable_action have hY := h.measurable_feedback exact h_indep.prod_right (by fun_prop) (by fun_prop) (by fun_prop) -lemma condIndepFun_feedback_history_action_action' [StandardBorelSpace Ω] - [IsObliviousEnv env] (h : IsAlgEnvSeq A Y alg env P) (n : ℕ) (hn : n ≠ 0) : - Y n ⟂ᵢ[A n, h.measurable_action n; P] (fun ω ↦ (history A Y (n - 1) ω, A n ω)) := by - have := condIndepFun_feedback_history_action_action h (n - 1) - grind - end IsObliviousEnv /-- An oblivious environment, in which the distribution of the next feedback depends only on the last action, but in a possibly time-dependent manner. -/ @[simps] def obliviousEnv (ν : ℕ → Kernel 𝓐 𝓨) [∀ n, IsMarkovKernel (ν n)] : Environment 𝓐 𝓨 where - feedback n := (ν (n + 1)).prodMkLeft _ - ν0 := ν 0 + feedback n := (ν n).prodMkLeft _ lemma feedback_obliviousEnv (ν : ℕ → Kernel 𝓐 𝓨) [∀ n, IsMarkovKernel (ν n)] (n : ℕ) : - (obliviousEnv ν).feedback n = (ν (n + 1)).prodMkLeft _ := by simp [obliviousEnv] + (obliviousEnv ν).feedback n = (ν n).prodMkLeft _ := rfl +@[simp] lemma ν0_obliviousEnv (ν : ℕ → Kernel 𝓐 𝓨) [∀ n, IsMarkovKernel (ν n)] : - (obliviousEnv ν).ν0 = ν 0 := by simp [obliviousEnv] + (obliviousEnv ν).ν0 = ν 0 := by + rw [Environment.ν0_def, feedback_obliviousEnv, Kernel.sectR_prodMkLeft] instance (ν : ℕ → Kernel 𝓐 𝓨) [∀ n, IsMarkovKernel (ν n)] : IsObliviousEnv (obliviousEnv ν) where - exists_eq_prodMkLeft := ⟨fun n ↦ ν n, inferInstance,rfl, fun _ ↦ rfl⟩ + exists_eq_prodMkLeft := ⟨ν, inferInstance, fun _ ↦ rfl⟩ @[simp] lemma feedbackCondAction_obliviousEnv (ν : ℕ → Kernel 𝓐 𝓨) [hν : ∀ n, IsMarkovKernel (ν n)] @@ -192,15 +170,10 @@ lemma feedbackCondAction_obliviousEnv (ν : ℕ → Kernel 𝓐 𝓨) [hν : ∀ · refine absurd (hν 0) ?_ simp only [Subsingleton.eq_zero ν, Pi.zero_apply] exact Kernel.not_isMarkovKernel_zero - have : Nonempty (Iic n → 𝓐 × 𝓨) := ⟨fun _ ↦ (h𝓐.some, hR.some)⟩ - have h_eq_zero := ν0_eq_feedbackCondAction (obliviousEnv ν) - have h_eq := feedback_eq_feedbackCondAction (obliviousEnv ν) (n - 1) - cases n with - | zero => exact h_eq_zero.symm - | succ n => - simp only [Nat.add_one_sub_one, obliviousEnv_feedback, add_tsub_cancel_right] at h_eq - rw [← Kernel.prodMkLeft_inj (γ := Iic n → 𝓐 × 𝓨)] - exact h_eq.symm + have : Nonempty (Fin n → 𝓐 × 𝓨) := ⟨fun _ ↦ (h𝓐.some, hR.some)⟩ + have h_eq := feedback_eq_feedbackCondAction (obliviousEnv ν) n + rw [feedback_obliviousEnv, Kernel.prodMkLeft_inj] at h_eq + exact h_eq.symm /-- A stationary environment, in which the distribution of the next feedback depends only on the last action. -/ @@ -208,14 +181,14 @@ def stationaryEnv (ν : Kernel 𝓐 𝓨) [IsMarkovKernel ν] : Environment 𝓐 @[simp] lemma feedback_stationaryEnv (ν : Kernel 𝓐 𝓨) [IsMarkovKernel ν] (n : ℕ) : - (stationaryEnv ν).feedback n = ν.prodMkLeft _ := by simp [stationaryEnv] + (stationaryEnv ν).feedback n = ν.prodMkLeft _ := rfl @[simp] -lemma ν0_stationaryEnv (ν : Kernel 𝓐 𝓨) [IsMarkovKernel ν] : (stationaryEnv ν).ν0 = ν := by - simp [stationaryEnv] +lemma ν0_stationaryEnv (ν : Kernel 𝓐 𝓨) [IsMarkovKernel ν] : (stationaryEnv ν).ν0 = ν := + ν0_obliviousEnv _ instance (ν : Kernel 𝓐 𝓨) [IsMarkovKernel ν] : IsObliviousEnv (stationaryEnv ν) where - exists_eq_prodMkLeft := ⟨fun _ ↦ ν, inferInstance, rfl, fun _ ↦ rfl⟩ + exists_eq_prodMkLeft := ⟨fun _ ↦ ν, inferInstance, fun _ ↦ rfl⟩ @[simp] lemma feedbackCondAction_stationaryEnv (ν : Kernel 𝓐 𝓨) [hν : IsMarkovKernel ν] (n : ℕ) : @@ -246,51 +219,38 @@ lemma condDistrib_feedback_stationaryEnv [StandardBorelSpace 𝓨] [Nonempty condDistrib (Y n) (A n) P =ᵐ[P.map (A n)] ν := (hasCondDistrib_feedback_stationaryEnv h n).condDistrib_eq -/-- Conditionally on the event `A 0 = b`, the first feedback has law `ν b`. -/ -lemma hasLaw_feedback_zero_cond_stationaryEnv [MeasurableSingletonClass 𝓐] - (h : IsAlgEnvSeq A Y alg (stationaryEnv ν) P) {b : 𝓐} (hP : P (A 0 ⁻¹' {b}) ≠ 0) : - HasLaw (Y 0) (ν b) P[|A 0 ⁻¹' {b}] := by - simpa using h.hasLaw_feedback_zero_cond hP - -/-- Conditionally on an event determined by the history up to time `n` and the action at time -`n + 1`, on which that action is equal to `b`, the feedback at time `n + 1` has law `ν b`. -/ +/-- Conditionally on an event determined by the history before time `n` and the action at time +`n`, on which that action is equal to `b`, the feedback at time `n` has law `ν b`. -/ lemma hasLaw_feedback_cond_stationaryEnv (h : IsAlgEnvSeq A Y alg (stationaryEnv ν) P) (n : ℕ) - {s : Set ((Iic n → 𝓐 × 𝓨) × 𝓐)} (hs : MeasurableSet s) {b : 𝓐} (hsb : ∀ u ∈ s, u.2 = b) - (hP : P ((fun ω ↦ (history A Y n ω, A (n + 1) ω)) ⁻¹' s) ≠ 0) : - HasLaw (Y (n + 1)) (ν b) P[|(fun ω ↦ (history A Y n ω, A (n + 1) ω)) ⁻¹' s] := by + {s : Set ((Fin n → 𝓐 × 𝓨) × 𝓐)} (hs : MeasurableSet s) {b : 𝓐} (hsb : ∀ u ∈ s, u.2 = b) + (hP : P ((fun ω ↦ (history A Y n ω, A n ω)) ⁻¹' s) ≠ 0) : + HasLaw (Y n) (ν b) P[|(fun ω ↦ (history A Y n ω, A n ω)) ⁻¹' s] := by simpa using IsObliviousEnv.hasLaw_feedback_cond h n hs hsb hP -/-- Conditionally on an event determined by the history up to time `n` and the action at time -`n + 1`, on which that action is constant, the feedback at time `n + 1` is independent of the -history up to time `n` and of the action at time `n + 1`. -/ +/-- Conditionally on an event determined by the history before time `n` and the action at time +`n`, on which that action is constant, the feedback at time `n` is independent of the +history before time `n` and of the action at time `n`. -/ lemma indepFun_history_action_feedback_cond_stationaryEnv (h : IsAlgEnvSeq A Y alg (stationaryEnv ν) P) (n : ℕ) - {s : Set ((Iic n → 𝓐 × 𝓨) × 𝓐)} (hs : MeasurableSet s) {b : 𝓐} (hsb : ∀ u ∈ s, u.2 = b) : - (fun ω ↦ (history A Y n ω, A (n + 1) ω)) - ⟂ᵢ[P[|(fun ω ↦ (history A Y n ω, A (n + 1) ω)) ⁻¹' s]] Y (n + 1) := + {s : Set ((Fin n → 𝓐 × 𝓨) × 𝓐)} (hs : MeasurableSet s) {b : 𝓐} (hsb : ∀ u ∈ s, u.2 = b) : + (fun ω ↦ (history A Y n ω, A n ω)) + ⟂ᵢ[P[|(fun ω ↦ (history A Y n ω, A n ω)) ⁻¹' s]] Y n := IsObliviousEnv.indepFun_history_action_feedback_cond h n hs hsb -/-- The feedback at time `n + 1` is conditionally independent of the history up to time `n` -given the action at time `n + 1`. -/ +/-- The feedback at time `n` is conditionally independent of the history before time `n` +given the action at time `n`. -/ lemma condIndepFun_feedback_history_action [StandardBorelSpace Ω] [StandardBorelSpace 𝓐] [Nonempty 𝓐] [StandardBorelSpace 𝓨] [Nonempty 𝓨] (h : IsAlgEnvSeq A Y alg (stationaryEnv ν) P) (n : ℕ) : - Y (n + 1) ⟂ᵢ[A (n + 1), h.measurable_action _ ; P] history A Y n := + Y n ⟂ᵢ[A n, h.measurable_action _ ; P] history A Y n := IsObliviousEnv.condIndepFun_feedback_history_action h n lemma condIndepFun_feedback_history_action_action [StandardBorelSpace Ω] [StandardBorelSpace 𝓐] [Nonempty 𝓐] [StandardBorelSpace 𝓨] [Nonempty 𝓨] (h : IsAlgEnvSeq A Y alg (stationaryEnv ν) P) (n : ℕ) : - Y (n + 1) ⟂ᵢ[A (n + 1), h.measurable_action (n + 1); P] - (fun ω ↦ (history A Y n ω, A (n + 1) ω)) := + Y n ⟂ᵢ[A n, h.measurable_action n; P] (fun ω ↦ (history A Y n ω, A n ω)) := IsObliviousEnv.condIndepFun_feedback_history_action_action h n -lemma condIndepFun_feedback_history_action_action' [StandardBorelSpace Ω] - [StandardBorelSpace 𝓐] [Nonempty 𝓐] [StandardBorelSpace 𝓨] [Nonempty 𝓨] - (h : IsAlgEnvSeq A Y alg (stationaryEnv ν) P) (n : ℕ) (hn : n ≠ 0) : - Y n ⟂ᵢ[A n, h.measurable_action n; P] (fun ω ↦ (history A Y (n - 1) ω, A n ω)) := - IsObliviousEnv.condIndepFun_feedback_history_action_action' h n hn - end IsAlgEnvSeq end Learning diff --git a/LeanMachineLearning/SequentialLearning/SumRewards.lean b/LeanMachineLearning/SequentialLearning/SumRewards.lean index 804e62f3..b02ad3d3 100644 --- a/LeanMachineLearning/SequentialLearning/SumRewards.lean +++ b/LeanMachineLearning/SequentialLearning/SumRewards.lean @@ -27,9 +27,9 @@ variable {𝓐 𝓨 Ω : Type*} {m𝓐 : MeasurableSpace 𝓐} {m𝓨 : Measurab noncomputable def sumRewards (A : ℕ → Ω → 𝓐) (R : ℕ → Ω → 𝓨) (a : 𝓐) (t : ℕ) (ω : Ω) : 𝓨 := ∑ s ∈ range t, if A s ω = a then (R s) ω else 0 -/-- Sum of rewards of arm `a` up to (and including) time `n`. -/ +/-- Sum of rewards of arm `a` in the history before time `n`. -/ noncomputable -def sumRewards' (n : ℕ) (h : Iic n → 𝓐 × 𝓨) (a : 𝓐) := +def sumRewards' (n : ℕ) (h : Fin n → 𝓐 × 𝓨) (a : 𝓐) := ∑ s, if (h s).1 = a then (h s).2 else 0 /-- Empirical mean reward obtained when pulling action `a` up to time `t` (exclusive). -/ @@ -37,9 +37,9 @@ noncomputable def empMean (A : ℕ → Ω → 𝓐) (R : ℕ → Ω → ℝ) (a : 𝓐) (t : ℕ) (ω : Ω) : ℝ := sumRewards A R a t ω / pullCount A a t ω -/-- Empirical mean of arm `a` at time `n`. -/ +/-- Empirical mean of arm `a` in the history before time `n`. -/ noncomputable -def empMean' (n : ℕ) (h : Iic n → 𝓐 × ℝ) (a : 𝓐) := +def empMean' (n : ℕ) (h : Fin n → 𝓐 × ℝ) (a : 𝓐) := sumRewards' n h a / pullCount' n h a @[simp] @@ -85,29 +85,26 @@ lemma sum_rewardByCount_eq_sumRewards {R : ℕ → Ω → 𝓨} (a : 𝓐) (t : · unfold sumRewards rwa [pullCount_eq_pullCount_of_action_ne hta, sum_range_succ, ite_eq_right hta, add_zero] -lemma sumRewards_add_one_eq_sumRewards' {R : ℕ → Ω → 𝓨} {n : ℕ} {ω : Ω} : - sumRewards A R a (n + 1) ω = sumRewards' n (fun i ↦ (A i ω, R i ω)) a := by - unfold sumRewards sumRewards' - rw [Finset.sum_coe_sort (f := fun s ↦ if A s ω = a then R s ω else 0) (Iic n)] - congr with m - simp only [mem_range, mem_Iic] - grind - -lemma sumRewards_eq_sumRewards' {R : ℕ → Ω → 𝓨} {n : ℕ} {ω : Ω} (hn : n ≠ 0) : - sumRewards A R a n ω = sumRewards' (n - 1) (fun i ↦ (A i ω, R i ω)) a := by - cases n with - | zero => exact absurd rfl hn - | succ n => simp [sumRewards_add_one_eq_sumRewards'] - -lemma empMean_add_one_eq_empMean' {R : ℕ → Ω → ℝ} {n : ℕ} {ω : Ω} : - empMean A R a (n + 1) ω = empMean' n (fun i ↦ (A i ω, R i ω)) a := by - unfold empMean empMean' - rw [sumRewards_add_one_eq_sumRewards', pullCount_add_one_eq_pullCount'] +lemma sumRewards_eq_sumRewards' {R : ℕ → Ω → 𝓨} {n : ℕ} {ω : Ω} : + sumRewards A R a n ω = sumRewards' n (fun i ↦ (A i ω, R i ω)) a := + (Fin.sum_univ_eq_sum_range (fun i ↦ if A i ω = a then R i ω else 0) n).symm + +/-- `sumRewards A R a n` is a function of the history before time `n`. -/ +lemma sumRewards_eq_comp_history {R : ℕ → Ω → 𝓨} (a : 𝓐) (n : ℕ) : + sumRewards A R a n = (fun h : Fin n → 𝓐 × 𝓨 ↦ sumRewards' n h a) ∘ history A R n := by + ext ω + exact sumRewards_eq_sumRewards' -lemma empMean_eq_empMean' {R : ℕ → Ω → ℝ} {n : ℕ} {ω : Ω} (hn : n ≠ 0) : - empMean A R a n ω = empMean' (n - 1) (fun i ↦ (A i ω, R i ω)) a := by +lemma empMean_eq_empMean' {R : ℕ → Ω → ℝ} {n : ℕ} {ω : Ω} : + empMean A R a n ω = empMean' n (fun i ↦ (A i ω, R i ω)) a := by unfold empMean empMean' - rw [sumRewards_eq_sumRewards' hn, pullCount_eq_pullCount' hn] + rw [sumRewards_eq_sumRewards', pullCount_eq_pullCount'] + +/-- `empMean A R a n` is a function of the history before time `n`. -/ +lemma empMean_eq_comp_history {R : ℕ → Ω → ℝ} (a : 𝓐) (n : ℕ) : + empMean A R a n = (fun h : Fin n → 𝓐 × ℝ ↦ empMean' n h a) ∘ history A R n := by + ext ω + exact empMean_eq_empMean' lemma sumRewards_sub_pullCount_smul_eq_sum {R : ℕ → Ω → 𝓨} (c : 𝓐 → 𝓨) : sumRewards A R a (n + 1) ω - pullCount A a (n + 1) ω • c a = @@ -163,22 +160,20 @@ lemma measurable_uncurry_empMean_comp [Countable 𝓐] [MeasurableSingletonClass lemma measurable_sumRewards' [MeasurableSingletonClass 𝓐] [MeasurableAdd₂ 𝓨] (n : ℕ) (a : 𝓐) : Measurable (sumRewards' (𝓨 := 𝓨) n · a) := by simp_rw [sumRewards'] - have h_meas s : Measurable (fun (h : Iic n → 𝓐 × ℝ) ↦ if (h s).1 = a then (h s).2 else 0) := by + have h_meas s : Measurable (fun (h : Fin n → 𝓐 × 𝓨) ↦ if (h s).1 = a then (h s).2 else 0) := by refine Measurable.ite ?_ (by fun_prop) (by fun_prop) exact (measurableSet_singleton _).preimage (by fun_prop) - refine Finset.measurable_fun_sum _ fun s hs ↦ ?_ - exact Measurable.ite (by measurability) (by fun_prop) (by fun_prop) + fun_prop @[fun_prop] lemma measurable_uncurry_sumRewards' [MeasurableEq 𝓐] [MeasurableAdd₂ 𝓨] (n : ℕ) : - Measurable (fun p : (Iic n → 𝓐 × 𝓨) × 𝓐 ↦ sumRewards' n p.1 p.2) := by + Measurable (fun p : (Fin n → 𝓐 × 𝓨) × 𝓐 ↦ sumRewards' n p.1 p.2) := by simp_rw [sumRewards'] - have h_meas s : Measurable (fun p : (Iic n → 𝓐 × ℝ) × 𝓐 ↦ + have h_meas s : Measurable (fun p : (Fin n → 𝓐 × 𝓨) × 𝓐 ↦ if (p.1 s).1 = p.2 then (p.1 s).2 else 0) := by refine Measurable.ite ?_ (by fun_prop) (by fun_prop) exact measurableSet_eq_fun (by fun_prop) (by fun_prop) - refine Finset.measurable_fun_sum _ fun s hs ↦ ?_ - exact Measurable.ite (by measurability) (by fun_prop) (by fun_prop) + fun_prop @[fun_prop] lemma measurable_empMean' [MeasurableSingletonClass 𝓐] (n : ℕ) (a : 𝓐) : @@ -186,7 +181,7 @@ lemma measurable_empMean' [MeasurableSingletonClass 𝓐] (n : ℕ) (a : 𝓐) : @[fun_prop] lemma measurable_uncurry_empMean' [MeasurableEq 𝓐] (n : ℕ) : - Measurable (fun p : (Iic n → 𝓐 × ℝ) × 𝓐 ↦ empMean' n p.1 p.2) := by unfold empMean'; fun_prop + Measurable (fun p : (Fin n → 𝓐 × ℝ) × 𝓐 ↦ empMean' n p.1 p.2) := by unfold empMean'; fun_prop variable [MeasurableSingletonClass 𝓐]