Fluctuation complexity, restrict possibilites to formally defined self-informations

docs/src/information_measures.md

@@ -15,6 +15,7 @@ information(::InformationMeasure, ::OutcomeSpace, ::Any)
 information(::DifferentialInfoEstimator, ::Any)
 information_maximum
 information_normalized
+self_information
 ```
 
 ## Entropies

src/core/information_functions.jl

@@ -1,4 +1,5 @@
 export information, information_maximum, information_normalized, convert_logunit
+export self_information
 export entropy
 
 ###########################################################################################
@@ -279,6 +280,39 @@ function information(::InformationMeasure, ::DifferentialInfoEstimator, args...)
     ))
 end
 
+"""
+    self_information(measure::InformationMeasure, pᵢ)
+
+Compute the "self-information"/"surprisal" of a single probability `pᵢ` under the given 
+information measure. 
+
+This function assumes `pᵢ > 0`, so make sure to pre-filter your probabilities.
+
+## Definition
+
+We here use the definition "self-information" very loosely, and 
+define it as the functional ``I_M(p_i)`` that satisfies ``\\sum_i p_i I_M(p_i) = I_M``,
+where `I_M` is the given information measure. 
+
+If `measure` is [`Shannon`](@ref), then this is the 
+[Shannon self-information](https://en.wikipedia.org/wiki/Information_content), which 
+fulfils a set of axioms. If `measure` is some other information, then it is not guaranteed
+that these axioms are fulfilled. We *only* guarantee that the probability-weighted 
+sum of the self-information equals the information measure.
+
+!!! note "Motivation for this definition"
+    This definition is motivated by the desire to compute generalized 
+    [`FluctuationComplexity`](@ref), which is a measure of fluctuations of local self-information
+    relative to some information-theoretic summary statistic of a distribution. Defining 
+    these self-information functions has, as far as we know, not been treated in the literature 
+    before, and will be part of an upcoming paper we're writing! 
+"""
+function self_information(measure::InformationMeasure, pᵢ)
+    throw(ArgumentError(
+        """`InformationMeasure` $(typeof(measure)) does not implement `self_information`."""
+    ))
+end
+
 
 ###########################################################################################
 # Utils

src/information_measure_definitions/fluctuation_complexity.jl

@@ -2,27 +2,31 @@ export FluctuationComplexity
 
 """
     FluctuationComplexity <: InformationMeasure
-    FluctuationComplexity(; definition = Shannon(; base = 2), base = 2)
+    FluctuationComplexity(; definition = Shannon())
 
 The "fluctuation complexity" quantifies the standard deviation of the information content of the states 
 ``\\omega_i`` around some summary statistic ([`InformationMeasure`](@ref)) of a PMF. Specifically, given some 
 outcome space ``\\Omega`` with outcomes ``\\omega_i \\in \\Omega`` 
 and a probability mass function ``p(\\Omega) = \\{ p(\\omega_i) \\}_{i=1}^N``, it is defined as
 
 ```math
-\\sigma_I(p) := \\sqrt{\\sum_{i=1}^N p_i(I_i - H_*)^2}
+\\sigma_I_Q(p) := \\sqrt{\\sum_{i=1}^N p_i(I_Q(p_i) - H_*)^2}
 ```
 
-where ``I_i = -\\log_{base}(p_i)`` is the information content of the i-th outcome. The type of information measure
-``*`` is controlled by `definition`. 
+where ``I_Q(p_i)`` is the [`self_information`](@ref) of the i-th outcome with respect to the information 
+measure of type ``Q`` (controlled by `definition`).
 
-The `base` controls the base of the logarithm that goes into the information content terms. Make sure that 
-you pick a `base` that is consistent with the base chosen for the `definition` (relevant for e.g. [`Shannon`](@ref)).
+## Compatible with
+
+- [`Shannon`](@ref)
+- [`Tsallis`](@ref)
+- [`Curado`](@ref)
+- [`ShannonExtropy`](@ref)
 
 ## Properties 
 
-If `definition` is the [`Shannon`](@ref) entropy, then we recover 
-the [Shannon-type information fluctuation complexity](https://en.wikipedia.org/wiki/Information_fluctuation_complexity) 
+If `definition` is the [`Shannon`](@ref) entropy, then we recover the 
+[Shannon-type information fluctuation complexity](https://en.wikipedia.org/wiki/Information_fluctuation_complexity) 
 from [Bates1993](@cite). Then the fluctuation complexity is zero for PMFs with only a single non-zero element, or 
 for the uniform distribution.
 
@@ -32,7 +36,8 @@ properties [Bates1993](@cite).
 !!! note "Potential for new research" 
     As far as we know, using other information measures besides Shannon entropy for the 
     fluctuation complexity hasn't been explored in the literature yet. Our implementation, however, allows for it.
-    Please inform us if you try some new combinations!
+    We're currently writing a paper outlining the generalizations to other measures. For now, we verify 
+    correctness of the measure through numerical examples in out test-suite.
 """
 struct FluctuationComplexity{M <: InformationMeasure, I <: Integer} <: InformationMeasure
     definition::M
@@ -50,9 +55,8 @@ end
 function information(e::FluctuationComplexity, probs::Probabilities)
     def = e.definition
     h = information(def, probs)
-    non0_probs = Iterators.filter(!iszero, vec(probs))
-    logf = log_with_base(e.base)
-    return sqrt(sum(pᵢ * (-logf(pᵢ) - h) ^ 2 for pᵢ in non0_probs))
+    s = self_information(def, probs)
+    return sqrt(sum(pᵢ * (s - h)^2 for pᵢ in non0_probs))
 end
 
 # The maximum is not generally known.

src/information_measure_definitions/kaniadakis.jl

@@ -40,3 +40,8 @@ end
 function information_maximum(e::Kaniadakis, L::Int)
     throw(ErrorException("information_maximum not implemeted for Kaniadakis entropy yet"))
 end
+
+function self_information(e::Kaniadakis, pᵢ)
+    κ = e.κ
+    return (pᵢ^(-κ) - pᵢ^κ) / (2κ)
+end

src/information_measure_definitions/shannon.jl

@@ -24,4 +24,8 @@ function information(e::Shannon, probs::Probabilities)
     return -sum(x*logf(x) for x in non0_probs)
 end
 
+function self_information(e::Shannon, pᵢ)
+    return -log(e.base, pᵢ)
+end
+
 information_maximum(e::Shannon, L::Int) = log_with_base(e.base)(L)

src/information_measure_definitions/shannon_extropy.jl

@@ -33,3 +33,7 @@ function information_maximum(e::ShannonExtropy, L::Int)
 
     return (L - 1) * log(e.base, L / (L - 1))
 end
+
+function self_information(e::ShannonExtropy, pᵢ)
+    return -log(e.base, 1 - pᵢ)
+end

src/information_measure_definitions/streched_exponential.jl

@@ -39,7 +39,6 @@ function stretched_exponential(pᵢ, η, base)
     # integral used in Anteneodo & Plastino (1999). See
     # https://specialfunctions.juliamath.org/stable/functions_list/#SpecialFunctions.gamma_inc
     Γx = gamma(x)
-
     return gamma_inc(x, -log(base, pᵢ))[2] * Γx - pᵢ * Γx
 end
 
@@ -56,3 +55,11 @@ function information_maximum(e::StretchedExponential, L::Int)
     # entry in the tuple returned from `gamma_inc`.
     L * gamma_inc(x, log(e.base, L))[2] * Γx - Γx
 end
+
+function self_information(e::StretchedExponential, pᵢ) 
+    η, base = e.η, e.base 
+    Γ₁ = gamma((η + 1) / η, -log(base, pᵢ))
+    Γ₂ = gamma((η + 1) / η)
+    # NB! Filter for pᵢ != 0 before calling this method.
+    return Γ₁/pᵢ - Γ₂
+end

src/information_measure_definitions/tsallis.jl

@@ -50,3 +50,7 @@ function information_maximum(e::Tsallis, L::Int)
         return k*(L^(1 - q) - 1) / (1 - q)
     end
 end
+
+function self_information(e::Tsallis, pᵢ)
+    return (1 - pᵢ^(e.q- 1)) / (e.q - 1)
+end

src/information_measure_definitions/tsallis_extropy.jl

@@ -57,3 +57,8 @@ function information_maximum(e::TsallisExtropy, L::Int)
 
     return ((L - 1) * L^(q - 1) - (L - 1)^q) / ((q - 1) * L^(q - 1))
 end
+
+function self_information(e::TsallisExtropy, pᵢ, N) #must have N
+    k, q = e.k, e.q
+    return (N - 1)/(q - 1) - (1 - pᵢ)^q / (q-1)
+end

test/infomeasures/infomeasure_types/kaniadakis.jl

@@ -11,3 +11,18 @@ p = Probabilities([0, 1])
 
 # Kaniadakis does not state for which distribution for which Kaniadakis entropy is maximised
 @test_throws ErrorException information_maximum(Kaniadakis(), 2)
+
+
+# ---------------------------------------------------------------------------------------------------------
+# Self-information tests
+# ---------------------------------------------------------------------------------------------------------
+# Check experimentally that the self-information expressions are correct by comparing to the 
+# regular computation of the measure from a set of probabilities.
+function information_from_selfinfo(e::Kaniadakis, probs::Probabilities)
+    e.κ ≈ 1.0 && return information_wm(Shannon(; base = e.base ), probs)
+    non0_probs = collect(Iterators.filter(!iszero, vec(probs)))
+    return sum(pᵢ * self_information(e, pᵢ) for pᵢ in non0_probs)
+end
+p = Probabilities([1//5, 1//5, 1//5, 0, 1//5])
+Hk = Kaniadakis(κ = 2)
+@test round(information_from_selfinfo(Hk, p), digits = 5) ≈ round(information(Hk, p), digits = 5)

test/infomeasures/infomeasure_types/shannon.jl

@@ -15,3 +15,18 @@ xp = Probabilities(x)
 
 # or minimal when only one probability is nonzero and equal to 1.0
 @test information(Shannon(), Probabilities([1.0, 0.0, 0.0, 0.0])) ≈ 0.0
+
+
+
+# ---------------------------------------------------------------------------------------------------------
+# Self-information tests
+# ---------------------------------------------------------------------------------------------------------
+# Check experimentally that the self-information expressions are correct by comparing to the 
+# regular computation of the measure from a set of probabilities.
+function information_from_selfinfo(e::Shannon, probs::Probabilities)
+    non0_probs = collect(Iterators.filter(!iszero, vec(probs)))
+    return sum(pᵢ * self_information(e, pᵢ) for pᵢ in non0_probs)
+end
+p = Probabilities([1//10, 1//5, 1//7, 1//5, 0])
+Hs = Shannon()
+@test round(information_from_selfinfo(Hs, p), digits = 5) ≈ round(information(Hs, p), digits = 5)

test/infomeasures/infomeasure_types/shannon_extropy.jl

@@ -20,3 +20,16 @@ js = information_normalized(ShannonExtropy(), UniqueElements(), x)
 x = [0.2, 0.2, 0.4, 0.4, 0.5, 0.5]
 js = information(ShannonExtropy(base = 2), UniqueElements(), x)
 @test js ≈ (3 - 1)*log(2, (3 / (3 - 1)))
+
+# ---------------------------------------------------------------------------------------------------------
+# Self-information tests
+# ---------------------------------------------------------------------------------------------------------
+# Check experimentally that the self-information expressions are correct by comparing to the 
+# regular computation of the measure from a set of probabilities.
+function information_from_selfinfo(e::ShannonExtropy, probs::Probabilities)
+    non0_probs = collect(Iterators.filter(!iszero, vec(probs)))
+    return sum((1 - pᵢ) * self_information(e, pᵢ) for pᵢ in non0_probs)
+end
+p = Probabilities([1//5, 1//5, 1//5, 1//2, 0])
+Js = ShannonExtropy()
+@test round(information_from_selfinfo(Js, p), digits = 5) ≈ round(information(Js, p), digits = 5)

test/infomeasures/infomeasure_types/stretched_exponential.jl

@@ -21,3 +21,19 @@ x = [repeat([0, 1], 5); 0]
 est = OrdinalPatterns(m = 2)
 @test information(StretchedExponential(η = η, base = b), est, x) ≈
     information_maximum(StretchedExponential(η = η, base = b), total_outcomes(est))
+
+
+# ---------------------------------------------------------------------------------------------------------
+# Self-information tests
+# ---------------------------------------------------------------------------------------------------------
+# Check experimentally that the self-information expressions are correct by comparing to the 
+# regular computation of the measure from a set of probabilities.
+function information_from_selfinfo(e::StretchedExponential, probs::Probabilities)
+    non0_probs = collect(Iterators.filter(!iszero, vec(probs)))
+    return sum(pᵢ * self_information(e, pᵢ) for pᵢ in non0_probs)
+end
+η = 2
+base = 2
+p = Probabilities([1//10, 1//5, 1//7, 1//5, 0])
+H_AP = StretchedExponential(η = η, base = base)
+@test round(information_from_selfinfo(H_AP, p), digits = 5) ≈ round(information(H_AP, p), digits = 5)

test/infomeasures/infomeasure_types/tsallis.jl

@@ -34,3 +34,19 @@ maxvals = [information_maximum(Tsallis(q = q, k = k), N) for q in q_cases]
 
 # Reduces to Shannon entropy for q → 1.0
 @test information(Tsallis(base = 2, q = 1.0), ps) ≈ log(2, N)
+
+# ---------------------------------------------------------------------------------------------------------
+# Self-information tests
+# ---------------------------------------------------------------------------------------------------------
+# Check experimentally that the self-information expressions are correct by comparing to the 
+# regular computation of the measure from a set of probabilities.
+function information_from_selfinfo(e::Tsallis, probs::Probabilities)
+    e.q ≈ 1.0 && return information_wm(Shannon(; base = e.base ), probs)
+    non0_probs = collect(Iterators.filter(!iszero, vec(probs)))
+    return sum(pᵢ * self_information(e, pᵢ) for pᵢ in non0_probs)
+end
+p = Probabilities([1//5, 1//5, 1//5, 1//2, 0])
+Ht = Tsallis(q = 2)
+@test round(information_from_selfinfo(Ht, p), digits = 5) ≈ round(information(Ht, p), digits = 5)
+
+

test/infomeasures/infomeasure_types/tsallis_extropy.jl

@@ -40,3 +40,16 @@ jn = information_normalized(TsallisExtropy(; q = 2), UniqueElements(), x)
 # Equivalent to Shannon extropy for q == 1
 @test information(TsallisExtropy(q = 1), UniqueElements(), x) ≈
     information(ShannonExtropy(), UniqueElements(), x)
+
+# ---------------------------------------------------------------------------------------------------------
+# Self-information tests
+# ---------------------------------------------------------------------------------------------------------
+# Check experimentally that the self-information expressions are correct by comparing to the 
+# regular computation of the measure from a set of probabilities.
+function information_from_selfinfo(e::TsallisExtropy, probs::Probabilities)
+    non0_probs = collect(Iterators.filter(!iszero, vec(probs)))
+    return sum((1 - pᵢ) * self_information(e, pᵢ, length(non0_probs)) for pᵢ in non0_probs)
+end
+p = Probabilities([1//5, 1//5, 1//5, 1//2, 0])
+Jt = TsallisExtropy(q = 2)
+@test round(information_from_selfinfo(Jt, p), digits = 5) ≈ round(information(Jt, p), digits = 5)

test/infomeasures/interface.jl

@@ -31,6 +31,11 @@ end
     @test_throws ArgumentError information(AlizadehArghami(Tsallis()), x)
 
     @test_throws ArgumentError entropy(ShannonExtropy(), OrdinalPatterns(), x)
+
+    # Error when measure doesn't implement `self_information`.
+    struct MyNewInfoMeasure <: InformationMeasure end
+    p = 0.1
+    @test_throws ArgumentError self_information(MyNewInfoMeasure(), p)
 
 
     # some new measure