Information theory: entropy, divergence and dependence measures.
entropy(probabilities, base=2.0)
Shannon entropy of a discrete distribution in units of base.
Source code in src/cds2/infotheory.py
| def entropy(probabilities: Sequence[float] | FloatArray, base: float = 2.0) -> float:
"""Shannon entropy of a discrete distribution in units of ``base``."""
if base <= 0 or math.isclose(base, 1.0):
msg = "base must be positive and different from 1"
raise ValueError(msg)
probs = _as_probabilities(probabilities)
positive = probs[probs > 0]
return float(-(positive * np.log(positive)).sum() / math.log(base))
|
joint_entropy(joint, base=2.0)
Entropy of a joint distribution given as a 2-D probability table.
Source code in src/cds2/infotheory.py
50
51
52
53
54
55
56
57
58 | def joint_entropy(joint: Sequence[Sequence[float]] | FloatArray, base: float = 2.0) -> float:
"""Entropy of a joint distribution given as a 2-D probability table."""
matrix = np.asarray(joint, dtype=float)
if matrix.ndim != 2 or matrix.size == 0:
msg = "joint distribution must be a non-empty 2-D table"
raise ValueError(msg)
flat = _as_probabilities(matrix.ravel())
positive = flat[flat > 0]
return float(-(positive * np.log(positive)).sum() / math.log(base))
|
conditional_entropy(joint, marginal=None, base=2.0)
H(Y|X) from a joint P(X, Y) table (rows indexed by X).
Source code in src/cds2/infotheory.py
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87 | def conditional_entropy(
joint: Sequence[Sequence[float]] | FloatArray,
marginal: Sequence[float] | FloatArray | None = None,
base: float = 2.0,
) -> float:
"""H(Y|X) from a joint P(X, Y) table (rows indexed by X)."""
matrix = np.asarray(joint, dtype=float)
if matrix.ndim != 2 or matrix.size == 0:
msg = "joint distribution must be a non-empty 2-D table"
raise ValueError(msg)
row_sums = matrix.sum(axis=1)
if marginal is None:
marginal_values = row_sums
else:
marginal_values = np.asarray(marginal, dtype=float)
if marginal_values.shape != (matrix.shape[0],):
msg = "marginal shape does not match the joint table rows"
raise ValueError(msg)
total = float(row_sums.sum())
entropy_value = 0.0
for row, p_x in zip(matrix, marginal_values, strict=True):
if p_x <= 0:
continue
conditional = row / p_x
positive = conditional[conditional > 0]
entropy_value += p_x / total * -(positive * np.log(positive)).sum() / math.log(base)
return float(entropy_value)
|
kl_divergence(p, q, base=2.0)
Kullback-Leibler divergence D_KL(P || Q); zero terms where P is 0.
Source code in src/cds2/infotheory.py
90
91
92
93
94
95
96
97
98
99
100
101 | def kl_divergence(
p: Sequence[float] | FloatArray, q: Sequence[float] | FloatArray, base: float = 2.0
) -> float:
"""Kullback-Leibler divergence D_KL(P || Q); zero terms where P is 0."""
p_array = _as_probabilities(p)
q_array = _as_probabilities(q)
if p_array.shape != q_array.shape:
msg = "p and q must have the same length"
raise ValueError(msg)
support = (q_array > 0) & (p_array > 0)
ratio_terms = p_array[support] * (np.log(p_array[support]) - np.log(q_array[support]))
return float(ratio_terms.sum() / math.log(base))
|
js_divergence(p, q, base=2.0)
Jensen-Shannon divergence between P and Q (symmetric, bounded).
Source code in src/cds2/infotheory.py
104
105
106
107
108
109
110
111
112
113
114 | def js_divergence(
p: Sequence[float] | FloatArray, q: Sequence[float] | FloatArray, base: float = 2.0
) -> float:
"""Jensen-Shannon divergence between P and Q (symmetric, bounded)."""
p_array = _as_probabilities(p)
q_array = _as_probabilities(q)
if p_array.shape != q_array.shape:
msg = "p and q must have the same length"
raise ValueError(msg)
mixture = 0.5 * (p_array + q_array)
return 0.5 * kl_divergence(p_array, mixture, base) + 0.5 * kl_divergence(q_array, mixture, base)
|
cross_entropy(p, q, base=2.0)
Cross-entropy H(P, Q) = H(P) + D_KL(P || Q).
Source code in src/cds2/infotheory.py
117
118
119
120
121
122
123
124
125
126
127
128 | def cross_entropy(
p: Sequence[float] | FloatArray, q: Sequence[float] | FloatArray, base: float = 2.0
) -> float:
"""Cross-entropy H(P, Q) = H(P) + D_KL(P || Q)."""
p_array = _as_probabilities(p)
q_array = _as_probabilities(q)
if p_array.shape != q_array.shape:
msg = "p and q must have the same length"
raise ValueError(msg)
support = (q_array > 0) & (p_array > 0)
terms = -p_array[support] * np.log(q_array[support])
return float(terms.sum() / math.log(base))
|
Mutual information I(X; Y) from a joint P(X, Y) table.
Source code in src/cds2/infotheory.py
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146 | def mutual_information(joint: Sequence[Sequence[float]] | FloatArray, base: float = 2.0) -> float:
"""Mutual information I(X; Y) from a joint P(X, Y) table."""
matrix = np.asarray(joint, dtype=float)
if matrix.ndim != 2 or matrix.size == 0:
msg = "joint distribution must be a non-empty 2-D table"
raise ValueError(msg)
total = float(matrix.sum())
if not math.isclose(total, 1.0, abs_tol=1e-9):
msg = "joint probabilities must sum to 1"
raise ValueError(msg)
row_sums = matrix.sum(axis=1, keepdims=True)
col_sums = matrix.sum(axis=0, keepdims=True)
independent = row_sums @ col_sums
support = matrix > 0
terms = matrix[support] * np.log(matrix[support] / independent[support])
return float(terms.sum() / math.log(base))
|
I(X; Y) scaled into [0, 1] by the square-root entropy product.
Source code in src/cds2/infotheory.py
149
150
151
152
153
154
155
156
157
158
159 | def normalized_mutual_information(
joint: Sequence[Sequence[float]] | FloatArray, base: float = 2.0
) -> float:
"""I(X; Y) scaled into [0, 1] by the square-root entropy product."""
matrix = np.asarray(joint, dtype=float)
h_x = entropy(matrix.sum(axis=1), base)
h_y = entropy(matrix.sum(axis=0), base)
denominator = math.sqrt(h_x * h_y)
if denominator == 0:
return 0.0
return mutual_information(matrix, base) / denominator
|
permutation_entropy(signal, order=3, delay=1, base=2.0)
Bandt-Pompe permutation entropy of an ordinal-pattern histogram.
Source code in src/cds2/infotheory.py
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184 | def permutation_entropy(
signal: Sequence[float] | FloatArray, order: int = 3, delay: int = 1, base: float = 2.0
) -> float:
"""Bandt-Pompe permutation entropy of an ordinal-pattern histogram."""
series = np.asarray(signal, dtype=float)
if order < 2:
msg = "order must be at least 2"
raise ValueError(msg)
if delay < 1:
msg = "delay must be at least 1"
raise ValueError(msg)
needed = (order - 1) * delay + 1
if series.size < needed:
msg = f"signal needs at least {needed} samples for order={order}, delay={delay}"
raise ValueError(msg)
windows = np.lib.stride_tricks.sliding_window_view(series, needed)[::delay]
embedded = windows[:, ::delay][:, :order]
codes = np.argsort(embedded, axis=1, kind="stable")
weights = np.array([order**i for i in range(order)], dtype=np.int64)
patterns = codes @ weights
_, counts = np.unique(patterns, return_counts=True)
probabilities = counts / counts.sum()
return entropy(probabilities, base)
|