Skip to content

Distributions

This module contains functionality related to distributions.

base

Distribution

Base class for all distributions.

This class should be subclassed by other distributions when you want to use ground truth scores, denoisers, noise predictors, or velocity estimators.

Each distribution implementation provides methods to compute various vector fields related to the diffusion process, such as denoising (x0), noise prediction (eps), velocity estimation (v), and score estimation.

Source code in src/diffusionlab/distributions/base.py
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
class Distribution:
    """
    Base class for all distributions.

    This class should be subclassed by other distributions when you want to use ground truth
    scores, denoisers, noise predictors, or velocity estimators.

    Each distribution implementation provides methods to compute various vector fields
    related to the diffusion process, such as denoising (x0), noise prediction (eps),
    velocity estimation (v), and score estimation.
    """

    @classmethod
    def validate_hparams(cls, dist_hparams: Dict[str, Any]) -> None:
        """
        Validate the hyperparameters for the distribution.

        Arguments:
            dist_hparams: A dictionary of hyperparameters for the distribution.

        Returns:
            None

        Throws:
            AssertionError: If the parameters are invalid, the assertion fails at exactly the point of failure.
        """
        assert len(dist_hparams) == 0

    @classmethod
    def validate_params(
        cls, possibly_batched_dist_params: Dict[str, torch.Tensor]
    ) -> None:
        """
        Validate the parameters for the distribution.

        Arguments:
            possibly_batched_dist_params: A dictionary of parameters for the distribution.
                Each value is a PyTorch tensor, possibly having a batch dimension.

        Returns:
            None

        Throws:
            AssertionError: If the parameters are invalid, the assertion fails at exactly the point of failure.
        """
        assert len(possibly_batched_dist_params) == 0

    @classmethod
    def x0(
        cls,
        x_t: torch.Tensor,
        t: torch.Tensor,
        diffusion_process: DiffusionProcess,
        batched_dist_params: Dict[str, torch.Tensor],
        dist_hparams: Dict[str, Any],
    ) -> torch.Tensor:
        """
        Computes the denoiser E[x_0 | x_t] at a given time t and input x_t, under the data model

        x_t = alpha(t) * x_0 + sigma(t) * eps

        where x_0 is drawn from the data distribution, and eps is drawn independently from N(0, I).

        Arguments:
            x_t: The input tensor, of shape (N, *D), where *D is the shape of each data.
            t: The time tensor, of shape (N, ).
            diffusion_process: The diffusion process whose forward and reverse dynamics determine
                the time-evolution of the vector fields corresponding to the distribution.
            batched_dist_params: A dictionary of batched parameters for the distribution.
                Each parameter is of shape (N, *P) where P is the shape of the parameter.
            dist_hparams: A dictionary of hyperparameters for the distribution.

        Returns:
            The prediction of x_0, of shape (N, *D).

        Note:
            The batched_dist_params dictionary contains BATCHED tensors, i.e., the first dimension is the batch dimension.
        """
        raise NotImplementedError

    @classmethod
    def eps(
        cls,
        x_t: torch.Tensor,
        t: torch.Tensor,
        diffusion_process: DiffusionProcess,
        batched_dist_params: Dict[str, torch.Tensor],
        dist_hparams: Dict[str, Any],
    ) -> torch.Tensor:
        """
        Computes the noise predictor E[eps | x_t] at a given time t and input x_t, under the data model

        x_t = alpha(t) * x_0 + sigma(t) * eps

        where x_0 is drawn from the data distribution, and eps is drawn independently from N(0, I).
        This is stateless for the same reason as the denoiser method.

        Arguments:
            x_t: The input tensor, of shape (N, *D), where *D is the shape of each data.
            t: The time tensor, of shape (N, ).
            diffusion_process: The diffusion process whose forward and reverse dynamics determine
                the time-evolution of the vector fields corresponding to the distribution.
            batched_dist_params: A dictionary of batched parameters for the distribution.
                Each parameter is of shape (N, *P) where P is the shape of the parameter.
            dist_hparams: A dictionary of hyperparameters for the distribution.

        Returns:
            The prediction of eps, of shape (N, *D).

        Note:
            The batched_dist_params dictionary contains BATCHED tensors, i.e., the first dimension is the batch dimension.
        """
        x0_hat = cls.x0(x_t, t, diffusion_process, batched_dist_params, dist_hparams)
        eps_hat = convert_vector_field_type(
            x_t,
            x0_hat,
            diffusion_process.alpha(t),
            diffusion_process.sigma(t),
            diffusion_process.alpha_prime(t),
            diffusion_process.sigma_prime(t),
            in_type=VectorFieldType.X0,
            out_type=VectorFieldType.EPS,
        )
        return eps_hat

    @classmethod
    def v(
        cls,
        x_t: torch.Tensor,
        t: torch.Tensor,
        diffusion_process: DiffusionProcess,
        batched_dist_params: Dict[str, torch.Tensor],
        dist_hparams: Dict[str, Any],
    ) -> torch.Tensor:
        """
        Computes the velocity estimator E[d/dt x_t | x_t] at a given time t and input x_t, under the data model

        x_t = alpha(t) * x_0 + sigma(t) * eps

        where x_0 is drawn from the data distribution, and eps is drawn independently from N(0, I).
        This is stateless for the same reason as the denoiser method.

        Arguments:
            x_t: The input tensor, of shape (N, *D), where *D is the shape of each data.
            t: The time tensor, of shape (N, ).
            diffusion_process: The diffusion process whose forward and reverse dynamics determine
                the time-evolution of the vector fields corresponding to the distribution.
            batched_dist_params: A dictionary of batched parameters for the distribution.
                Each parameter is of shape (N, *P) where P is the shape of the parameter.
            dist_hparams: A dictionary of hyperparameters for the distribution.

        Returns:
            The prediction of d/dt x_t, of shape (N, *D).

        Note:
            The batched_dist_params dictionary contains BATCHED tensors, i.e., the first dimension is the batch dimension.
        """
        x0_hat = cls.x0(x_t, t, diffusion_process, batched_dist_params, dist_hparams)
        v_hat = convert_vector_field_type(
            x_t,
            x0_hat,
            diffusion_process.alpha(t),
            diffusion_process.sigma(t),
            diffusion_process.alpha_prime(t),
            diffusion_process.sigma_prime(t),
            in_type=VectorFieldType.X0,
            out_type=VectorFieldType.V,
        )
        return v_hat

    @classmethod
    def score(
        cls,
        x_t: torch.Tensor,
        t: torch.Tensor,
        diffusion_process: DiffusionProcess,
        batched_dist_params: Dict[str, torch.Tensor],
        dist_hparams: Dict[str, Any],
    ) -> torch.Tensor:
        """
        Computes the score estimator grad_x log p(x_t, t) at a given time t and input x_t, under the data model

        x_t = alpha(t) * x_0 + sigma(t) * eps

        where x_0 is drawn from the data distribution, and eps is drawn independently from N(0, I).
        This is stateless for the same reason as the denoiser method.

        Arguments:
            x_t: The input tensor, of shape (N, *D), where *D is the shape of each data.
            t: The time tensor, of shape (N, ).
            diffusion_process: The diffusion process whose forward and reverse dynamics determine
                the time-evolution of the vector fields corresponding to the distribution.
            batched_dist_params: A dictionary of batched parameters for the distribution.
                Each parameter is of shape (N, *P) where P is the shape of the parameter.
            dist_hparams: A dictionary of hyperparameters for the distribution.

        Returns:
            The prediction of grad_x log p(x_t, t), of shape (N, *D).

        Note:
            The batched_dist_params dictionary contains BATCHED tensors, i.e., the first dimension is the batch dimension.
        """
        x0_hat = cls.x0(x_t, t, diffusion_process, batched_dist_params, dist_hparams)
        score_hat = convert_vector_field_type(
            x_t,
            x0_hat,
            diffusion_process.alpha(t),
            diffusion_process.sigma(t),
            diffusion_process.alpha_prime(t),
            diffusion_process.sigma_prime(t),
            in_type=VectorFieldType.X0,
            out_type=VectorFieldType.SCORE,
        )
        return score_hat

    @classmethod
    def sample(
        cls,
        N: int,
        dist_params: Dict[str, torch.Tensor],
        dist_hparams: Dict[str, Any],
    ) -> Tuple[torch.Tensor, Any]:
        """
        Draws N i.i.d. samples from the data distribution.

        Arguments:
            N: The number of samples to draw.
            dist_params: A dictionary of parameters for the distribution.
            dist_hparams: A dictionary of hyperparameters for the distribution.

        Returns:
            A tuple (samples, metadata), where samples is a tensor of shape (N, *D) and metadata is any additional information.
            For example, if the distribution has labels, the metadata is a tensor of shape (N, ) containing the labels.
            Note that the samples are always placed on the CPU.
        """
        raise NotImplementedError

    @staticmethod
    def batch_dist_params(
        N: int, dist_params: Dict[str, torch.Tensor]
    ) -> Dict[str, torch.Tensor]:
        """
        Add a batch dimension to the distribution parameters.

        Arguments:
            N: The number of samples in the batch.
            dist_params: A dictionary of parameters for the distribution.

        Returns:
            A dictionary of parameters for the distribution, with a batch dimension added.
        """
        return {k: v.unsqueeze(0).expand(N, *v.shape) for k, v in dist_params.items()}

batch_dist_params(N, dist_params) staticmethod

Add a batch dimension to the distribution parameters.

Parameters:

Name Type Description Default
N int

The number of samples in the batch.

required
dist_params Dict[str, Tensor]

A dictionary of parameters for the distribution.

required

Returns:

Type Description
Dict[str, Tensor]

A dictionary of parameters for the distribution, with a batch dimension added.

Source code in src/diffusionlab/distributions/base.py
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
@staticmethod
def batch_dist_params(
    N: int, dist_params: Dict[str, torch.Tensor]
) -> Dict[str, torch.Tensor]:
    """
    Add a batch dimension to the distribution parameters.

    Arguments:
        N: The number of samples in the batch.
        dist_params: A dictionary of parameters for the distribution.

    Returns:
        A dictionary of parameters for the distribution, with a batch dimension added.
    """
    return {k: v.unsqueeze(0).expand(N, *v.shape) for k, v in dist_params.items()}

eps(x_t, t, diffusion_process, batched_dist_params, dist_hparams) classmethod

Computes the noise predictor E[eps | x_t] at a given time t and input x_t, under the data model

x_t = alpha(t) * x_0 + sigma(t) * eps

where x_0 is drawn from the data distribution, and eps is drawn independently from N(0, I). This is stateless for the same reason as the denoiser method.

Parameters:

Name Type Description Default
x_t Tensor

The input tensor, of shape (N, D), where D is the shape of each data.

required
t Tensor

The time tensor, of shape (N, ).

required
diffusion_process DiffusionProcess

The diffusion process whose forward and reverse dynamics determine the time-evolution of the vector fields corresponding to the distribution.

required
batched_dist_params Dict[str, Tensor]

A dictionary of batched parameters for the distribution. Each parameter is of shape (N, *P) where P is the shape of the parameter.

required
dist_hparams Dict[str, Any]

A dictionary of hyperparameters for the distribution.

required

Returns:

Type Description
Tensor

The prediction of eps, of shape (N, *D).

Note

The batched_dist_params dictionary contains BATCHED tensors, i.e., the first dimension is the batch dimension.

Source code in src/diffusionlab/distributions/base.py
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
@classmethod
def eps(
    cls,
    x_t: torch.Tensor,
    t: torch.Tensor,
    diffusion_process: DiffusionProcess,
    batched_dist_params: Dict[str, torch.Tensor],
    dist_hparams: Dict[str, Any],
) -> torch.Tensor:
    """
    Computes the noise predictor E[eps | x_t] at a given time t and input x_t, under the data model

    x_t = alpha(t) * x_0 + sigma(t) * eps

    where x_0 is drawn from the data distribution, and eps is drawn independently from N(0, I).
    This is stateless for the same reason as the denoiser method.

    Arguments:
        x_t: The input tensor, of shape (N, *D), where *D is the shape of each data.
        t: The time tensor, of shape (N, ).
        diffusion_process: The diffusion process whose forward and reverse dynamics determine
            the time-evolution of the vector fields corresponding to the distribution.
        batched_dist_params: A dictionary of batched parameters for the distribution.
            Each parameter is of shape (N, *P) where P is the shape of the parameter.
        dist_hparams: A dictionary of hyperparameters for the distribution.

    Returns:
        The prediction of eps, of shape (N, *D).

    Note:
        The batched_dist_params dictionary contains BATCHED tensors, i.e., the first dimension is the batch dimension.
    """
    x0_hat = cls.x0(x_t, t, diffusion_process, batched_dist_params, dist_hparams)
    eps_hat = convert_vector_field_type(
        x_t,
        x0_hat,
        diffusion_process.alpha(t),
        diffusion_process.sigma(t),
        diffusion_process.alpha_prime(t),
        diffusion_process.sigma_prime(t),
        in_type=VectorFieldType.X0,
        out_type=VectorFieldType.EPS,
    )
    return eps_hat

sample(N, dist_params, dist_hparams) classmethod

Draws N i.i.d. samples from the data distribution.

Parameters:

Name Type Description Default
N int

The number of samples to draw.

required
dist_params Dict[str, Tensor]

A dictionary of parameters for the distribution.

required
dist_hparams Dict[str, Any]

A dictionary of hyperparameters for the distribution.

required

Returns:

Type Description
Tensor

A tuple (samples, metadata), where samples is a tensor of shape (N, *D) and metadata is any additional information.

Any

For example, if the distribution has labels, the metadata is a tensor of shape (N, ) containing the labels.

Tuple[Tensor, Any]

Note that the samples are always placed on the CPU.

Source code in src/diffusionlab/distributions/base.py
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
@classmethod
def sample(
    cls,
    N: int,
    dist_params: Dict[str, torch.Tensor],
    dist_hparams: Dict[str, Any],
) -> Tuple[torch.Tensor, Any]:
    """
    Draws N i.i.d. samples from the data distribution.

    Arguments:
        N: The number of samples to draw.
        dist_params: A dictionary of parameters for the distribution.
        dist_hparams: A dictionary of hyperparameters for the distribution.

    Returns:
        A tuple (samples, metadata), where samples is a tensor of shape (N, *D) and metadata is any additional information.
        For example, if the distribution has labels, the metadata is a tensor of shape (N, ) containing the labels.
        Note that the samples are always placed on the CPU.
    """
    raise NotImplementedError

score(x_t, t, diffusion_process, batched_dist_params, dist_hparams) classmethod

Computes the score estimator grad_x log p(x_t, t) at a given time t and input x_t, under the data model

x_t = alpha(t) * x_0 + sigma(t) * eps

where x_0 is drawn from the data distribution, and eps is drawn independently from N(0, I). This is stateless for the same reason as the denoiser method.

Parameters:

Name Type Description Default
x_t Tensor

The input tensor, of shape (N, D), where D is the shape of each data.

required
t Tensor

The time tensor, of shape (N, ).

required
diffusion_process DiffusionProcess

The diffusion process whose forward and reverse dynamics determine the time-evolution of the vector fields corresponding to the distribution.

required
batched_dist_params Dict[str, Tensor]

A dictionary of batched parameters for the distribution. Each parameter is of shape (N, *P) where P is the shape of the parameter.

required
dist_hparams Dict[str, Any]

A dictionary of hyperparameters for the distribution.

required

Returns:

Type Description
Tensor

The prediction of grad_x log p(x_t, t), of shape (N, *D).

Note

The batched_dist_params dictionary contains BATCHED tensors, i.e., the first dimension is the batch dimension.

Source code in src/diffusionlab/distributions/base.py
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
@classmethod
def score(
    cls,
    x_t: torch.Tensor,
    t: torch.Tensor,
    diffusion_process: DiffusionProcess,
    batched_dist_params: Dict[str, torch.Tensor],
    dist_hparams: Dict[str, Any],
) -> torch.Tensor:
    """
    Computes the score estimator grad_x log p(x_t, t) at a given time t and input x_t, under the data model

    x_t = alpha(t) * x_0 + sigma(t) * eps

    where x_0 is drawn from the data distribution, and eps is drawn independently from N(0, I).
    This is stateless for the same reason as the denoiser method.

    Arguments:
        x_t: The input tensor, of shape (N, *D), where *D is the shape of each data.
        t: The time tensor, of shape (N, ).
        diffusion_process: The diffusion process whose forward and reverse dynamics determine
            the time-evolution of the vector fields corresponding to the distribution.
        batched_dist_params: A dictionary of batched parameters for the distribution.
            Each parameter is of shape (N, *P) where P is the shape of the parameter.
        dist_hparams: A dictionary of hyperparameters for the distribution.

    Returns:
        The prediction of grad_x log p(x_t, t), of shape (N, *D).

    Note:
        The batched_dist_params dictionary contains BATCHED tensors, i.e., the first dimension is the batch dimension.
    """
    x0_hat = cls.x0(x_t, t, diffusion_process, batched_dist_params, dist_hparams)
    score_hat = convert_vector_field_type(
        x_t,
        x0_hat,
        diffusion_process.alpha(t),
        diffusion_process.sigma(t),
        diffusion_process.alpha_prime(t),
        diffusion_process.sigma_prime(t),
        in_type=VectorFieldType.X0,
        out_type=VectorFieldType.SCORE,
    )
    return score_hat

v(x_t, t, diffusion_process, batched_dist_params, dist_hparams) classmethod

Computes the velocity estimator E[d/dt x_t | x_t] at a given time t and input x_t, under the data model

x_t = alpha(t) * x_0 + sigma(t) * eps

where x_0 is drawn from the data distribution, and eps is drawn independently from N(0, I). This is stateless for the same reason as the denoiser method.

Parameters:

Name Type Description Default
x_t Tensor

The input tensor, of shape (N, D), where D is the shape of each data.

required
t Tensor

The time tensor, of shape (N, ).

required
diffusion_process DiffusionProcess

The diffusion process whose forward and reverse dynamics determine the time-evolution of the vector fields corresponding to the distribution.

required
batched_dist_params Dict[str, Tensor]

A dictionary of batched parameters for the distribution. Each parameter is of shape (N, *P) where P is the shape of the parameter.

required
dist_hparams Dict[str, Any]

A dictionary of hyperparameters for the distribution.

required

Returns:

Type Description
Tensor

The prediction of d/dt x_t, of shape (N, *D).

Note

The batched_dist_params dictionary contains BATCHED tensors, i.e., the first dimension is the batch dimension.

Source code in src/diffusionlab/distributions/base.py
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
@classmethod
def v(
    cls,
    x_t: torch.Tensor,
    t: torch.Tensor,
    diffusion_process: DiffusionProcess,
    batched_dist_params: Dict[str, torch.Tensor],
    dist_hparams: Dict[str, Any],
) -> torch.Tensor:
    """
    Computes the velocity estimator E[d/dt x_t | x_t] at a given time t and input x_t, under the data model

    x_t = alpha(t) * x_0 + sigma(t) * eps

    where x_0 is drawn from the data distribution, and eps is drawn independently from N(0, I).
    This is stateless for the same reason as the denoiser method.

    Arguments:
        x_t: The input tensor, of shape (N, *D), where *D is the shape of each data.
        t: The time tensor, of shape (N, ).
        diffusion_process: The diffusion process whose forward and reverse dynamics determine
            the time-evolution of the vector fields corresponding to the distribution.
        batched_dist_params: A dictionary of batched parameters for the distribution.
            Each parameter is of shape (N, *P) where P is the shape of the parameter.
        dist_hparams: A dictionary of hyperparameters for the distribution.

    Returns:
        The prediction of d/dt x_t, of shape (N, *D).

    Note:
        The batched_dist_params dictionary contains BATCHED tensors, i.e., the first dimension is the batch dimension.
    """
    x0_hat = cls.x0(x_t, t, diffusion_process, batched_dist_params, dist_hparams)
    v_hat = convert_vector_field_type(
        x_t,
        x0_hat,
        diffusion_process.alpha(t),
        diffusion_process.sigma(t),
        diffusion_process.alpha_prime(t),
        diffusion_process.sigma_prime(t),
        in_type=VectorFieldType.X0,
        out_type=VectorFieldType.V,
    )
    return v_hat

validate_hparams(dist_hparams) classmethod

Validate the hyperparameters for the distribution.

Parameters:

Name Type Description Default
dist_hparams Dict[str, Any]

A dictionary of hyperparameters for the distribution.

required

Returns:

Type Description
None

None

Throws

AssertionError: If the parameters are invalid, the assertion fails at exactly the point of failure.

Source code in src/diffusionlab/distributions/base.py
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
@classmethod
def validate_hparams(cls, dist_hparams: Dict[str, Any]) -> None:
    """
    Validate the hyperparameters for the distribution.

    Arguments:
        dist_hparams: A dictionary of hyperparameters for the distribution.

    Returns:
        None

    Throws:
        AssertionError: If the parameters are invalid, the assertion fails at exactly the point of failure.
    """
    assert len(dist_hparams) == 0

validate_params(possibly_batched_dist_params) classmethod

Validate the parameters for the distribution.

Parameters:

Name Type Description Default
possibly_batched_dist_params Dict[str, Tensor]

A dictionary of parameters for the distribution. Each value is a PyTorch tensor, possibly having a batch dimension.

required

Returns:

Type Description
None

None

Throws

AssertionError: If the parameters are invalid, the assertion fails at exactly the point of failure.

Source code in src/diffusionlab/distributions/base.py
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
@classmethod
def validate_params(
    cls, possibly_batched_dist_params: Dict[str, torch.Tensor]
) -> None:
    """
    Validate the parameters for the distribution.

    Arguments:
        possibly_batched_dist_params: A dictionary of parameters for the distribution.
            Each value is a PyTorch tensor, possibly having a batch dimension.

    Returns:
        None

    Throws:
        AssertionError: If the parameters are invalid, the assertion fails at exactly the point of failure.
    """
    assert len(possibly_batched_dist_params) == 0

x0(x_t, t, diffusion_process, batched_dist_params, dist_hparams) classmethod

Computes the denoiser E[x_0 | x_t] at a given time t and input x_t, under the data model

x_t = alpha(t) * x_0 + sigma(t) * eps

where x_0 is drawn from the data distribution, and eps is drawn independently from N(0, I).

Parameters:

Name Type Description Default
x_t Tensor

The input tensor, of shape (N, D), where D is the shape of each data.

required
t Tensor

The time tensor, of shape (N, ).

required
diffusion_process DiffusionProcess

The diffusion process whose forward and reverse dynamics determine the time-evolution of the vector fields corresponding to the distribution.

required
batched_dist_params Dict[str, Tensor]

A dictionary of batched parameters for the distribution. Each parameter is of shape (N, *P) where P is the shape of the parameter.

required
dist_hparams Dict[str, Any]

A dictionary of hyperparameters for the distribution.

required

Returns:

Type Description
Tensor

The prediction of x_0, of shape (N, *D).

Note

The batched_dist_params dictionary contains BATCHED tensors, i.e., the first dimension is the batch dimension.

Source code in src/diffusionlab/distributions/base.py
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
@classmethod
def x0(
    cls,
    x_t: torch.Tensor,
    t: torch.Tensor,
    diffusion_process: DiffusionProcess,
    batched_dist_params: Dict[str, torch.Tensor],
    dist_hparams: Dict[str, Any],
) -> torch.Tensor:
    """
    Computes the denoiser E[x_0 | x_t] at a given time t and input x_t, under the data model

    x_t = alpha(t) * x_0 + sigma(t) * eps

    where x_0 is drawn from the data distribution, and eps is drawn independently from N(0, I).

    Arguments:
        x_t: The input tensor, of shape (N, *D), where *D is the shape of each data.
        t: The time tensor, of shape (N, ).
        diffusion_process: The diffusion process whose forward and reverse dynamics determine
            the time-evolution of the vector fields corresponding to the distribution.
        batched_dist_params: A dictionary of batched parameters for the distribution.
            Each parameter is of shape (N, *P) where P is the shape of the parameter.
        dist_hparams: A dictionary of hyperparameters for the distribution.

    Returns:
        The prediction of x_0, of shape (N, *D).

    Note:
        The batched_dist_params dictionary contains BATCHED tensors, i.e., the first dimension is the batch dimension.
    """
    raise NotImplementedError

empirical

EmpiricalDistribution

Bases: Distribution

An empirical distribution, i.e., the uniform distribution over a dataset. Formally, the distribution is defined as:

mu(B) = (1/N) * sum_(i=1)^(N) delta(x_i in B)

where x_i is the ith data point in the dataset, and N is the number of data points.

Distribution Parameters
  • None
Distribution Hyperparameters
  • labeled_data: A DataLoader of data which spawns the empirical distribution, where each data sample is a (data, label) tuple. Both data and label are PyTorch tensors.
Note
  • This class has no sample() method as it's difficult to sample randomly from a DataLoader. In practice, you can sample directly from the DataLoader and apply filtering there.
Source code in src/diffusionlab/distributions/empirical.py
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
class EmpiricalDistribution(Distribution):
    """
    An empirical distribution, i.e., the uniform distribution over a dataset.
    Formally, the distribution is defined as:

    mu(B) = (1/N) * sum_(i=1)^(N) delta(x_i in B)

    where x_i is the ith data point in the dataset, and N is the number of data points.

    Distribution Parameters:
        - None

    Distribution Hyperparameters:
        - labeled_data: A DataLoader of data which spawns the empirical distribution, where each data sample is a (data, label) tuple. Both data and label are PyTorch tensors.

    Note:
        - This class has no sample() method as it's difficult to sample randomly from a DataLoader. In practice, you can sample directly from the DataLoader and apply filtering there.
    """

    @classmethod
    def validate_hparams(cls, dist_hparams: Dict[str, Any]) -> None:
        """
        Validate the hyperparameters for the empirical distribution.

        Arguments:
            dist_hparams: A dictionary of hyperparameters for the distribution.
                Must contain 'labeled_data' which is a DataLoader.

        Returns:
            None

        Throws:
            AssertionError: If the parameters are invalid.
        """
        assert "labeled_data" in dist_hparams
        labeled_data = dist_hparams["labeled_data"]
        assert isinstance(labeled_data, DataLoader)
        assert len(labeled_data) > 0

    @classmethod
    def x0(
        cls,
        x_t: torch.Tensor,
        t: torch.Tensor,
        diffusion_process: DiffusionProcess,
        batched_dist_params: Dict[str, torch.Tensor],
        dist_hparams: Dict[str, Any],
    ) -> torch.Tensor:
        """
        Computes the denoiser E[x_0 | x_t] for an empirical distribution.

        This method computes the denoiser by performing a weighted average of the
        dataset samples, where the weights are determined by the likelihood of x_t
        given each sample.

        Arguments:
            x_t: The input tensor, of shape (N, *D), where *D is the shape of each data.
            t: The time tensor, of shape (N, ).
            diffusion_process: The diffusion process.
            batched_dist_params: A dictionary of batched parameters for the distribution.
                Not used for empirical distribution.
            dist_hparams: A dictionary of hyperparameters for the distribution.
                Must contain 'labeled_data' which is a DataLoader.

        Returns:
            The prediction of x_0, of shape (N, *D).
        """
        data = dist_hparams["labeled_data"]

        x_flattened = torch.flatten(x_t, start_dim=1, end_dim=-1)  # (N, *D)

        alpha = diffusion_process.alpha(t)  # (N, )
        sigma = diffusion_process.sigma(t)  # (N, )

        softmax_denom = torch.zeros_like(t)  # (N, )
        x0_hat = torch.zeros_like(x_t)  # (N, *D)
        for X_batch, y_batch in data:
            X_batch = X_batch.to(x_t.device, non_blocking=True)  # (B, *D)
            X_batch_flattened = torch.flatten(X_batch, start_dim=1, end_dim=-1)[
                None, ...
            ]  # (1, B, D*)
            alpha_X_batch_flattened = (
                pad_shape_back(alpha, X_batch_flattened.shape) * X_batch_flattened
            )  # (N, B, D*)
            dists = (
                torch.cdist(x_flattened[:, None, ...], alpha_X_batch_flattened)[
                    :, 0, ...
                ]
                ** 2
            )  # (N, B)
            exp_dists = torch.exp(
                -dists / (2 * pad_shape_back(sigma, dists.shape) ** 2)
            )  # (N, B)
            softmax_denom += torch.sum(exp_dists, dim=-1)  # (N, )
            x0_hat += torch.sum(
                pad_shape_back(exp_dists, X_batch[None, ...].shape)
                * X_batch[None, ...],  # (N, B, *D)
                dim=1,
            )  # (N, *D)

        softmax_denom = torch.maximum(
            softmax_denom,
            torch.tensor(
                torch.finfo(softmax_denom.dtype).eps, device=softmax_denom.device
            ),
        )
        x0_hat = x0_hat / pad_shape_back(softmax_denom, x0_hat.shape)  # (N, *D)
        return x0_hat

validate_hparams(dist_hparams) classmethod

Validate the hyperparameters for the empirical distribution.

Parameters:

Name Type Description Default
dist_hparams Dict[str, Any]

A dictionary of hyperparameters for the distribution. Must contain 'labeled_data' which is a DataLoader.

required

Returns:

Type Description
None

None

Throws

AssertionError: If the parameters are invalid.

Source code in src/diffusionlab/distributions/empirical.py
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
@classmethod
def validate_hparams(cls, dist_hparams: Dict[str, Any]) -> None:
    """
    Validate the hyperparameters for the empirical distribution.

    Arguments:
        dist_hparams: A dictionary of hyperparameters for the distribution.
            Must contain 'labeled_data' which is a DataLoader.

    Returns:
        None

    Throws:
        AssertionError: If the parameters are invalid.
    """
    assert "labeled_data" in dist_hparams
    labeled_data = dist_hparams["labeled_data"]
    assert isinstance(labeled_data, DataLoader)
    assert len(labeled_data) > 0

x0(x_t, t, diffusion_process, batched_dist_params, dist_hparams) classmethod

Computes the denoiser E[x_0 | x_t] for an empirical distribution.

This method computes the denoiser by performing a weighted average of the dataset samples, where the weights are determined by the likelihood of x_t given each sample.

Parameters:

Name Type Description Default
x_t Tensor

The input tensor, of shape (N, D), where D is the shape of each data.

required
t Tensor

The time tensor, of shape (N, ).

required
diffusion_process DiffusionProcess

The diffusion process.

required
batched_dist_params Dict[str, Tensor]

A dictionary of batched parameters for the distribution. Not used for empirical distribution.

required
dist_hparams Dict[str, Any]

A dictionary of hyperparameters for the distribution. Must contain 'labeled_data' which is a DataLoader.

required

Returns:

Type Description
Tensor

The prediction of x_0, of shape (N, *D).

Source code in src/diffusionlab/distributions/empirical.py
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
@classmethod
def x0(
    cls,
    x_t: torch.Tensor,
    t: torch.Tensor,
    diffusion_process: DiffusionProcess,
    batched_dist_params: Dict[str, torch.Tensor],
    dist_hparams: Dict[str, Any],
) -> torch.Tensor:
    """
    Computes the denoiser E[x_0 | x_t] for an empirical distribution.

    This method computes the denoiser by performing a weighted average of the
    dataset samples, where the weights are determined by the likelihood of x_t
    given each sample.

    Arguments:
        x_t: The input tensor, of shape (N, *D), where *D is the shape of each data.
        t: The time tensor, of shape (N, ).
        diffusion_process: The diffusion process.
        batched_dist_params: A dictionary of batched parameters for the distribution.
            Not used for empirical distribution.
        dist_hparams: A dictionary of hyperparameters for the distribution.
            Must contain 'labeled_data' which is a DataLoader.

    Returns:
        The prediction of x_0, of shape (N, *D).
    """
    data = dist_hparams["labeled_data"]

    x_flattened = torch.flatten(x_t, start_dim=1, end_dim=-1)  # (N, *D)

    alpha = diffusion_process.alpha(t)  # (N, )
    sigma = diffusion_process.sigma(t)  # (N, )

    softmax_denom = torch.zeros_like(t)  # (N, )
    x0_hat = torch.zeros_like(x_t)  # (N, *D)
    for X_batch, y_batch in data:
        X_batch = X_batch.to(x_t.device, non_blocking=True)  # (B, *D)
        X_batch_flattened = torch.flatten(X_batch, start_dim=1, end_dim=-1)[
            None, ...
        ]  # (1, B, D*)
        alpha_X_batch_flattened = (
            pad_shape_back(alpha, X_batch_flattened.shape) * X_batch_flattened
        )  # (N, B, D*)
        dists = (
            torch.cdist(x_flattened[:, None, ...], alpha_X_batch_flattened)[
                :, 0, ...
            ]
            ** 2
        )  # (N, B)
        exp_dists = torch.exp(
            -dists / (2 * pad_shape_back(sigma, dists.shape) ** 2)
        )  # (N, B)
        softmax_denom += torch.sum(exp_dists, dim=-1)  # (N, )
        x0_hat += torch.sum(
            pad_shape_back(exp_dists, X_batch[None, ...].shape)
            * X_batch[None, ...],  # (N, B, *D)
            dim=1,
        )  # (N, *D)

    softmax_denom = torch.maximum(
        softmax_denom,
        torch.tensor(
            torch.finfo(softmax_denom.dtype).eps, device=softmax_denom.device
        ),
    )
    x0_hat = x0_hat / pad_shape_back(softmax_denom, x0_hat.shape)  # (N, *D)
    return x0_hat

gmm

GMMDistribution

Bases: Distribution

A Gaussian Mixture Model (GMM) with K components. Formally, the distribution is defined as:

mu(B) = sum_(i=1)^(K) pi_i * N(mu_i, Sigma_i)(B)

where mu_i is the mean of the ith component, Sigma_i is the covariance matrix of the ith component, and pi_i is the prior probability of the ith component.

Distribution Parameters
  • means: A tensor of shape (K, D) containing the means of the components.
  • covs: A tensor of shape (K, D, D) containing the covariance matrices of the components.
  • priors: A tensor of shape (K, ) containing the prior probabilities of the components.
Distribution Hyperparameters
  • None
Source code in src/diffusionlab/distributions/gmm.py
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
class GMMDistribution(Distribution):
    """
    A Gaussian Mixture Model (GMM) with K components.
    Formally, the distribution is defined as:

    mu(B) = sum_(i=1)^(K) pi_i * N(mu_i, Sigma_i)(B)

    where mu_i is the mean of the ith component, Sigma_i is the covariance matrix of the ith component,
    and pi_i is the prior probability of the ith component.

    Distribution Parameters:
        - means: A tensor of shape (K, D) containing the means of the components.
        - covs: A tensor of shape (K, D, D) containing the covariance matrices of the components.
        - priors: A tensor of shape (K, ) containing the prior probabilities of the components.

    Distribution Hyperparameters:
        - None
    """

    @classmethod
    def validate_params(
        cls, possibly_batched_dist_params: Dict[str, torch.Tensor]
    ) -> None:
        assert (
            "means" in possibly_batched_dist_params
            and "covs" in possibly_batched_dist_params
            and "priors" in possibly_batched_dist_params
        )
        means = possibly_batched_dist_params["means"]
        covs = possibly_batched_dist_params["covs"]
        priors = possibly_batched_dist_params["priors"]

        if len(means.shape) == 2:
            assert len(covs.shape) == 3
            assert len(priors.shape) == 1
            means = means[None, :, :]
            covs = covs[None, :, :, :]
            priors = priors[None, :]

        assert len(means.shape) == 3
        assert len(covs.shape) == 4
        assert len(priors.shape) == 2

        N, K, D = means.shape
        assert (
            len(covs.shape) == 4
            and covs.shape[0] == N
            and covs.shape[1] == K
            and covs.shape[2] == D
            and covs.shape[3] == D
        )
        assert len(priors.shape) == 2 and priors.shape[0] == N and priors.shape[1] == K
        assert means.device == covs.device == priors.device

        assert torch.all(priors >= 0)
        sum_priors = torch.sum(priors, dim=-1)
        assert torch.allclose(sum_priors, torch.ones_like(sum_priors))

        evals = torch.linalg.eigvalsh(covs)
        assert torch.all(
            evals >= -D * torch.finfo(evals.dtype).eps
        )  # Allow for numerical errors

    @classmethod
    def x0(
        cls,
        x_t: torch.Tensor,
        t: torch.Tensor,
        diffusion_process: DiffusionProcess,
        batched_dist_params: Dict[str, torch.Tensor],
        dist_hparams: Dict[str, Any],
    ) -> torch.Tensor:
        """
        Computes the denoiser E[x_0 | x_t] for a GMM distribution.

        Arguments:
            x_t: The input tensor, of shape (N, D).
            t: The time tensor, of shape (N, ).
            diffusion_process: The diffusion process.
            batched_dist_params: A dictionary containing the batched parameters of the distribution.
                - means: A tensor of shape (N, K, D) containing the means of the components.
                - covs: A tensor of shape (N, K, D, D) containing the covariance matrices of the components.
                - priors: A tensor of shape (N, K) containing the prior probabilities of the components.
            dist_hparams: A dictionary of hyperparameters for the distribution.

        Returns:
            The prediction of x_0, of shape (N, D).

        Note:
            The batched_dist_params dictionary contains BATCHED tensors, i.e., the first dimension is the batch dimension.
        """
        means = batched_dist_params["means"]  # (N, K, D)
        covs = batched_dist_params["covs"]  # (N, K, D, D)
        priors = batched_dist_params["priors"]  # (N, K)

        N, K, D = means.shape

        alpha = diffusion_process.alpha(t)  # (N, )
        sigma = diffusion_process.sigma(t)  # (N, )

        covs_t = (alpha[:, None, None, None] ** 2) * covs + (
            sigma[:, None, None, None] ** 2
        ) * torch.eye(D, device=x_t.device)[None, None, :, :]  # (N, K, D, D)
        centered_x = x_t[:, None, :] - alpha[:, None, None] * means  # (N, K, D)
        covs_t_inv_centered_x = torch.linalg.lstsq(
            covs_t,  # (N, K, D, D)
            centered_x[..., None],  # (N, K, D, 1)
        ).solution[..., 0]  # (N, K, D, 1) -> (N, K, D)

        mahalanobis_dists = torch.sum(
            centered_x * covs_t_inv_centered_x, dim=-1
        )  # (N, K)
        logdets_covs_t = logdet_pd(covs_t)  # (N, K)
        w = (
            torch.log(priors) - 1 / 2 * logdets_covs_t - 1 / 2 * mahalanobis_dists
        )  # (N, K)
        softmax_w = torch.softmax(w, dim=-1)  # (N, K)

        weighted_normalized_x = torch.sum(
            softmax_w[:, :, None] * covs_t_inv_centered_x, dim=-2
        )  # (N, D)
        x0_hat = (1 / alpha[:, None]) * (
            x_t - (sigma[:, None] ** 2) * weighted_normalized_x
        )  # (N, D)

        return x0_hat

    @classmethod
    def sample(
        cls,
        N: int,
        dist_params: Dict[str, torch.Tensor],
        dist_hparams: Dict[str, Any],
    ) -> Tuple[torch.Tensor, torch.Tensor]:
        means = dist_params["means"]  # (K, D)
        covs = dist_params["covs"]  # (K, D, D)
        priors = dist_params["priors"]  # (K, )

        K, D = means.shape

        device = priors.device
        y = torch.multinomial(priors, N, replacement=True)  # (N, )
        X = torch.empty((N, D), device=device)
        for k in range(K):
            idx = y == k
            X[idx] = (
                torch.randn((X[idx].shape[0], D), device=device) @ sqrt_psd(covs[k])
                + means[k][None, :]
            )
        return X.to("cpu"), y.to("cpu")

sample(N, dist_params, dist_hparams) classmethod

Source code in src/diffusionlab/distributions/gmm.py
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
@classmethod
def sample(
    cls,
    N: int,
    dist_params: Dict[str, torch.Tensor],
    dist_hparams: Dict[str, Any],
) -> Tuple[torch.Tensor, torch.Tensor]:
    means = dist_params["means"]  # (K, D)
    covs = dist_params["covs"]  # (K, D, D)
    priors = dist_params["priors"]  # (K, )

    K, D = means.shape

    device = priors.device
    y = torch.multinomial(priors, N, replacement=True)  # (N, )
    X = torch.empty((N, D), device=device)
    for k in range(K):
        idx = y == k
        X[idx] = (
            torch.randn((X[idx].shape[0], D), device=device) @ sqrt_psd(covs[k])
            + means[k][None, :]
        )
    return X.to("cpu"), y.to("cpu")

validate_params(possibly_batched_dist_params) classmethod

Source code in src/diffusionlab/distributions/gmm.py
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
@classmethod
def validate_params(
    cls, possibly_batched_dist_params: Dict[str, torch.Tensor]
) -> None:
    assert (
        "means" in possibly_batched_dist_params
        and "covs" in possibly_batched_dist_params
        and "priors" in possibly_batched_dist_params
    )
    means = possibly_batched_dist_params["means"]
    covs = possibly_batched_dist_params["covs"]
    priors = possibly_batched_dist_params["priors"]

    if len(means.shape) == 2:
        assert len(covs.shape) == 3
        assert len(priors.shape) == 1
        means = means[None, :, :]
        covs = covs[None, :, :, :]
        priors = priors[None, :]

    assert len(means.shape) == 3
    assert len(covs.shape) == 4
    assert len(priors.shape) == 2

    N, K, D = means.shape
    assert (
        len(covs.shape) == 4
        and covs.shape[0] == N
        and covs.shape[1] == K
        and covs.shape[2] == D
        and covs.shape[3] == D
    )
    assert len(priors.shape) == 2 and priors.shape[0] == N and priors.shape[1] == K
    assert means.device == covs.device == priors.device

    assert torch.all(priors >= 0)
    sum_priors = torch.sum(priors, dim=-1)
    assert torch.allclose(sum_priors, torch.ones_like(sum_priors))

    evals = torch.linalg.eigvalsh(covs)
    assert torch.all(
        evals >= -D * torch.finfo(evals.dtype).eps
    )  # Allow for numerical errors

x0(x_t, t, diffusion_process, batched_dist_params, dist_hparams) classmethod

Computes the denoiser E[x_0 | x_t] for a GMM distribution.

Parameters:

Name Type Description Default
x_t Tensor

The input tensor, of shape (N, D).

required
t Tensor

The time tensor, of shape (N, ).

required
diffusion_process DiffusionProcess

The diffusion process.

required
batched_dist_params Dict[str, Tensor]

A dictionary containing the batched parameters of the distribution. - means: A tensor of shape (N, K, D) containing the means of the components. - covs: A tensor of shape (N, K, D, D) containing the covariance matrices of the components. - priors: A tensor of shape (N, K) containing the prior probabilities of the components.

required
dist_hparams Dict[str, Any]

A dictionary of hyperparameters for the distribution.

required

Returns:

Type Description
Tensor

The prediction of x_0, of shape (N, D).

Note

The batched_dist_params dictionary contains BATCHED tensors, i.e., the first dimension is the batch dimension.

Source code in src/diffusionlab/distributions/gmm.py
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
@classmethod
def x0(
    cls,
    x_t: torch.Tensor,
    t: torch.Tensor,
    diffusion_process: DiffusionProcess,
    batched_dist_params: Dict[str, torch.Tensor],
    dist_hparams: Dict[str, Any],
) -> torch.Tensor:
    """
    Computes the denoiser E[x_0 | x_t] for a GMM distribution.

    Arguments:
        x_t: The input tensor, of shape (N, D).
        t: The time tensor, of shape (N, ).
        diffusion_process: The diffusion process.
        batched_dist_params: A dictionary containing the batched parameters of the distribution.
            - means: A tensor of shape (N, K, D) containing the means of the components.
            - covs: A tensor of shape (N, K, D, D) containing the covariance matrices of the components.
            - priors: A tensor of shape (N, K) containing the prior probabilities of the components.
        dist_hparams: A dictionary of hyperparameters for the distribution.

    Returns:
        The prediction of x_0, of shape (N, D).

    Note:
        The batched_dist_params dictionary contains BATCHED tensors, i.e., the first dimension is the batch dimension.
    """
    means = batched_dist_params["means"]  # (N, K, D)
    covs = batched_dist_params["covs"]  # (N, K, D, D)
    priors = batched_dist_params["priors"]  # (N, K)

    N, K, D = means.shape

    alpha = diffusion_process.alpha(t)  # (N, )
    sigma = diffusion_process.sigma(t)  # (N, )

    covs_t = (alpha[:, None, None, None] ** 2) * covs + (
        sigma[:, None, None, None] ** 2
    ) * torch.eye(D, device=x_t.device)[None, None, :, :]  # (N, K, D, D)
    centered_x = x_t[:, None, :] - alpha[:, None, None] * means  # (N, K, D)
    covs_t_inv_centered_x = torch.linalg.lstsq(
        covs_t,  # (N, K, D, D)
        centered_x[..., None],  # (N, K, D, 1)
    ).solution[..., 0]  # (N, K, D, 1) -> (N, K, D)

    mahalanobis_dists = torch.sum(
        centered_x * covs_t_inv_centered_x, dim=-1
    )  # (N, K)
    logdets_covs_t = logdet_pd(covs_t)  # (N, K)
    w = (
        torch.log(priors) - 1 / 2 * logdets_covs_t - 1 / 2 * mahalanobis_dists
    )  # (N, K)
    softmax_w = torch.softmax(w, dim=-1)  # (N, K)

    weighted_normalized_x = torch.sum(
        softmax_w[:, :, None] * covs_t_inv_centered_x, dim=-2
    )  # (N, D)
    x0_hat = (1 / alpha[:, None]) * (
        x_t - (sigma[:, None] ** 2) * weighted_normalized_x
    )  # (N, D)

    return x0_hat

IsoGMMDistribution

Bases: Distribution

An isotropic (i.e., spherical variances) Gaussian Mixture Model (GMM) with K components. Formally, the distribution is defined as:

mu(B) = sum_(i=1)^(K) pi_i * N(mu_i, tau_i^2 * I_D)(B)

where mu_i is the mean of the ith component, tau is the standard deviation of the spherical variances, and pi_i is the prior probability of the ith component.

Distribution Parameters
  • means: A tensor of shape (K, D) containing the means of the components.
  • vars: A tensor of shape (K, ) containing the variances of the components.
  • priors: A tensor of shape (K, ) containing the prior probabilities of the components.
Distribution Hyperparameters
  • None
Source code in src/diffusionlab/distributions/gmm.py
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
class IsoGMMDistribution(Distribution):
    """
    An isotropic (i.e., spherical variances) Gaussian Mixture Model (GMM) with K components.
    Formally, the distribution is defined as:

    mu(B) = sum_(i=1)^(K) pi_i * N(mu_i, tau_i^2 * I_D)(B)

    where mu_i is the mean of the ith component, tau is the standard deviation of the spherical variances,
    and pi_i is the prior probability of the ith component.

    Distribution Parameters:
        - means: A tensor of shape (K, D) containing the means of the components.
        - vars: A tensor of shape (K, ) containing the variances of the components.
        - priors: A tensor of shape (K, ) containing the prior probabilities of the components.

    Distribution Hyperparameters:
        - None
    """

    @classmethod
    def validate_params(
        cls, possibly_batched_dist_params: Dict[str, torch.Tensor]
    ) -> None:
        assert (
            "means" in possibly_batched_dist_params
            and "vars" in possibly_batched_dist_params
            and "priors" in possibly_batched_dist_params
        )
        means = possibly_batched_dist_params["means"]
        vars_ = possibly_batched_dist_params["vars"]
        priors = possibly_batched_dist_params["priors"]

        if len(means.shape) == 2:
            assert len(vars_.shape) == 1
            assert len(priors.shape) == 1
            means = means[None, :, :]
            vars_ = vars_[None, :]
            priors = priors[None, :]

        assert len(means.shape) == 3
        N, K, D = means.shape
        assert len(vars_.shape) == 2 and vars_.shape[0] == N and vars_.shape[1] == K
        assert len(priors.shape) == 2 and priors.shape[0] == N and priors.shape[1] == K
        assert means.device == vars_.device == priors.device

        priors_sum = torch.sum(priors, dim=-1)
        assert torch.all(priors_sum >= 0)
        assert torch.allclose(priors_sum, torch.ones_like(priors_sum))
        assert torch.all(
            vars_ >= -D * torch.finfo(vars_.dtype).eps
        )  # Allow for numerical errors

    @classmethod
    def x0(
        cls,
        x_t: torch.Tensor,
        t: torch.Tensor,
        diffusion_process: DiffusionProcess,
        batched_dist_params: Dict[str, torch.Tensor],
        dist_hparams: Dict[str, Any],
    ) -> torch.Tensor:
        """
        Computes the denoiser E[x_0 | x_t] for an isotropic GMM distribution.

        Arguments:
            x_t: The input tensor, of shape (N, D).
            t: The time tensor, of shape (N, ).
            diffusion_process: The diffusion process whose forward and reverse dynamics determine
                the time-evolution of the vector fields corresponding to the distribution.
            batched_dist_params: A dictionary containing the batched parameters of the distribution.
                - means: A tensor of shape (N, K, D) containing the means of the components.
                - vars: A tensor of shape (N, K) containing the variances of the components.
                - priors: A tensor of shape (N, K) containing the prior probabilities of the components.
            dist_hparams: A dictionary of hyperparameters for the distribution.

        Returns:
            The prediction of x_0, of shape (N, D).

        Note:
            The batched_dist_params dictionary contains BATCHED tensors, i.e., the first dimension is the batch dimension.
        """
        means = batched_dist_params["means"]  # (N, K, D)
        vars_ = batched_dist_params["vars"]  # (N, K)
        priors = batched_dist_params["priors"]  # (N, K)

        N, K, D = means.shape

        alpha = diffusion_process.alpha(t)  # (N, )
        sigma = diffusion_process.sigma(t)  # (N, )

        vars_t = (alpha[:, None] ** 2) * vars_ + (sigma[:, None] ** 2)  # (N, K)
        centered_x = x_t[:, None, :] - alpha[:, None, None] * means  # (N, K, D)
        vars_t_inv_centered_x = centered_x / vars_t[:, :, None]  # (N, K, D)

        mahalanobis_dists = torch.sum(
            centered_x * vars_t_inv_centered_x, dim=-1
        )  # (N, K)
        w = (
            torch.log(priors) - D / 2 * torch.log(vars_t) - 1 / 2 * mahalanobis_dists
        )  # (N, K)
        softmax_w = torch.softmax(w, dim=-1)  # (N, K)

        weighted_normalized_x = torch.sum(
            softmax_w[:, :, None] * vars_t_inv_centered_x, dim=-2
        )  # (N, D)
        x0_hat = (1 / alpha[:, None]) * (
            x_t - (sigma[:, None] ** 2) * weighted_normalized_x
        )  # (N, D)

        return x0_hat

    @classmethod
    def sample(
        cls,
        N: int,
        dist_params: Dict[str, torch.Tensor],
        dist_hparams: Dict[str, Any],
    ) -> Tuple[torch.Tensor, torch.Tensor]:
        """
        Draws N i.i.d. samples from the isotropic GMM distribution.

        Arguments:
            N: The number of samples to draw.
            dist_params: A dictionary of parameters for the distribution.
                - means: A tensor of shape (K, D) containing the means of the components.
                - vars: A tensor of shape (K, ) containing the variances of the components.
                - priors: A tensor of shape (K, ) containing the prior probabilities of the components.
            dist_hparams: A dictionary of hyperparameters for the distribution.

        Returns:
            A tuple (samples, labels), where samples is a tensor of shape (N, D) and labels is a tensor of shape (N, )
            containing the component indices from which each sample was drawn.
            Note that the samples are always placed on the CPU.
        """
        means = dist_params["means"]  # (K, D)
        vars_ = dist_params["vars"]  # (K, )
        priors = dist_params["priors"]  # (K, )

        K, D = means.shape
        covs = (
            torch.eye(D, device=vars_.device)[None, :, :].expand(K, -1, -1)
            * vars_[:, None, None]
        )
        return GMMDistribution.sample(
            N, {"means": means, "covs": covs, "priors": priors}, dict()
        )

sample(N, dist_params, dist_hparams) classmethod

Draws N i.i.d. samples from the isotropic GMM distribution.

Parameters:

Name Type Description Default
N int

The number of samples to draw.

required
dist_params Dict[str, Tensor]

A dictionary of parameters for the distribution. - means: A tensor of shape (K, D) containing the means of the components. - vars: A tensor of shape (K, ) containing the variances of the components. - priors: A tensor of shape (K, ) containing the prior probabilities of the components.

required
dist_hparams Dict[str, Any]

A dictionary of hyperparameters for the distribution.

required

Returns:

Type Description
Tensor

A tuple (samples, labels), where samples is a tensor of shape (N, D) and labels is a tensor of shape (N, )

Tensor

containing the component indices from which each sample was drawn.

Tuple[Tensor, Tensor]

Note that the samples are always placed on the CPU.

Source code in src/diffusionlab/distributions/gmm.py
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
@classmethod
def sample(
    cls,
    N: int,
    dist_params: Dict[str, torch.Tensor],
    dist_hparams: Dict[str, Any],
) -> Tuple[torch.Tensor, torch.Tensor]:
    """
    Draws N i.i.d. samples from the isotropic GMM distribution.

    Arguments:
        N: The number of samples to draw.
        dist_params: A dictionary of parameters for the distribution.
            - means: A tensor of shape (K, D) containing the means of the components.
            - vars: A tensor of shape (K, ) containing the variances of the components.
            - priors: A tensor of shape (K, ) containing the prior probabilities of the components.
        dist_hparams: A dictionary of hyperparameters for the distribution.

    Returns:
        A tuple (samples, labels), where samples is a tensor of shape (N, D) and labels is a tensor of shape (N, )
        containing the component indices from which each sample was drawn.
        Note that the samples are always placed on the CPU.
    """
    means = dist_params["means"]  # (K, D)
    vars_ = dist_params["vars"]  # (K, )
    priors = dist_params["priors"]  # (K, )

    K, D = means.shape
    covs = (
        torch.eye(D, device=vars_.device)[None, :, :].expand(K, -1, -1)
        * vars_[:, None, None]
    )
    return GMMDistribution.sample(
        N, {"means": means, "covs": covs, "priors": priors}, dict()
    )

validate_params(possibly_batched_dist_params) classmethod

Source code in src/diffusionlab/distributions/gmm.py
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
@classmethod
def validate_params(
    cls, possibly_batched_dist_params: Dict[str, torch.Tensor]
) -> None:
    assert (
        "means" in possibly_batched_dist_params
        and "vars" in possibly_batched_dist_params
        and "priors" in possibly_batched_dist_params
    )
    means = possibly_batched_dist_params["means"]
    vars_ = possibly_batched_dist_params["vars"]
    priors = possibly_batched_dist_params["priors"]

    if len(means.shape) == 2:
        assert len(vars_.shape) == 1
        assert len(priors.shape) == 1
        means = means[None, :, :]
        vars_ = vars_[None, :]
        priors = priors[None, :]

    assert len(means.shape) == 3
    N, K, D = means.shape
    assert len(vars_.shape) == 2 and vars_.shape[0] == N and vars_.shape[1] == K
    assert len(priors.shape) == 2 and priors.shape[0] == N and priors.shape[1] == K
    assert means.device == vars_.device == priors.device

    priors_sum = torch.sum(priors, dim=-1)
    assert torch.all(priors_sum >= 0)
    assert torch.allclose(priors_sum, torch.ones_like(priors_sum))
    assert torch.all(
        vars_ >= -D * torch.finfo(vars_.dtype).eps
    )  # Allow for numerical errors

x0(x_t, t, diffusion_process, batched_dist_params, dist_hparams) classmethod

Computes the denoiser E[x_0 | x_t] for an isotropic GMM distribution.

Parameters:

Name Type Description Default
x_t Tensor

The input tensor, of shape (N, D).

required
t Tensor

The time tensor, of shape (N, ).

required
diffusion_process DiffusionProcess

The diffusion process whose forward and reverse dynamics determine the time-evolution of the vector fields corresponding to the distribution.

required
batched_dist_params Dict[str, Tensor]

A dictionary containing the batched parameters of the distribution. - means: A tensor of shape (N, K, D) containing the means of the components. - vars: A tensor of shape (N, K) containing the variances of the components. - priors: A tensor of shape (N, K) containing the prior probabilities of the components.

required
dist_hparams Dict[str, Any]

A dictionary of hyperparameters for the distribution.

required

Returns:

Type Description
Tensor

The prediction of x_0, of shape (N, D).

Note

The batched_dist_params dictionary contains BATCHED tensors, i.e., the first dimension is the batch dimension.

Source code in src/diffusionlab/distributions/gmm.py
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
@classmethod
def x0(
    cls,
    x_t: torch.Tensor,
    t: torch.Tensor,
    diffusion_process: DiffusionProcess,
    batched_dist_params: Dict[str, torch.Tensor],
    dist_hparams: Dict[str, Any],
) -> torch.Tensor:
    """
    Computes the denoiser E[x_0 | x_t] for an isotropic GMM distribution.

    Arguments:
        x_t: The input tensor, of shape (N, D).
        t: The time tensor, of shape (N, ).
        diffusion_process: The diffusion process whose forward and reverse dynamics determine
            the time-evolution of the vector fields corresponding to the distribution.
        batched_dist_params: A dictionary containing the batched parameters of the distribution.
            - means: A tensor of shape (N, K, D) containing the means of the components.
            - vars: A tensor of shape (N, K) containing the variances of the components.
            - priors: A tensor of shape (N, K) containing the prior probabilities of the components.
        dist_hparams: A dictionary of hyperparameters for the distribution.

    Returns:
        The prediction of x_0, of shape (N, D).

    Note:
        The batched_dist_params dictionary contains BATCHED tensors, i.e., the first dimension is the batch dimension.
    """
    means = batched_dist_params["means"]  # (N, K, D)
    vars_ = batched_dist_params["vars"]  # (N, K)
    priors = batched_dist_params["priors"]  # (N, K)

    N, K, D = means.shape

    alpha = diffusion_process.alpha(t)  # (N, )
    sigma = diffusion_process.sigma(t)  # (N, )

    vars_t = (alpha[:, None] ** 2) * vars_ + (sigma[:, None] ** 2)  # (N, K)
    centered_x = x_t[:, None, :] - alpha[:, None, None] * means  # (N, K, D)
    vars_t_inv_centered_x = centered_x / vars_t[:, :, None]  # (N, K, D)

    mahalanobis_dists = torch.sum(
        centered_x * vars_t_inv_centered_x, dim=-1
    )  # (N, K)
    w = (
        torch.log(priors) - D / 2 * torch.log(vars_t) - 1 / 2 * mahalanobis_dists
    )  # (N, K)
    softmax_w = torch.softmax(w, dim=-1)  # (N, K)

    weighted_normalized_x = torch.sum(
        softmax_w[:, :, None] * vars_t_inv_centered_x, dim=-2
    )  # (N, D)
    x0_hat = (1 / alpha[:, None]) * (
        x_t - (sigma[:, None] ** 2) * weighted_normalized_x
    )  # (N, D)

    return x0_hat

IsoHomoGMMDistribution

Bases: Distribution

An isotropic homoscedastic (i.e., equal spherical variances) Gaussian Mixture Model (GMM) with K components. Formally, the distribution is defined as:

mu(B) = sum_(i=1)^(K) pi_i * N(mu_i, tau^2 * I_D)(B)

where mu_i is the mean of the ith component, tau is the standard deviation of the spherical variances, and pi_i is the prior probability of the ith component.

Distribution Parameters
  • means: A tensor of shape (K, D) containing the means of the components.
  • var: A tensor of shape () containing the variances of the components.
  • priors: A tensor of shape (K, ) containing the prior probabilities of the components.
Distribution Hyperparameters
  • None
Source code in src/diffusionlab/distributions/gmm.py
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
class IsoHomoGMMDistribution(Distribution):
    """
    An isotropic homoscedastic (i.e., equal spherical variances) Gaussian Mixture Model (GMM) with K components.
    Formally, the distribution is defined as:

    mu(B) = sum_(i=1)^(K) pi_i * N(mu_i, tau^2 * I_D)(B)

    where mu_i is the mean of the ith component, tau is the standard deviation of the spherical variances,
    and pi_i is the prior probability of the ith component.

    Distribution Parameters:
        - means: A tensor of shape (K, D) containing the means of the components.
        - var: A tensor of shape () containing the variances of the components.
        - priors: A tensor of shape (K, ) containing the prior probabilities of the components.

    Distribution Hyperparameters:
        - None
    """

    @classmethod
    def validate_params(
        cls, possibly_batched_dist_params: Dict[str, torch.Tensor]
    ) -> None:
        assert (
            "means" in possibly_batched_dist_params
            and "var" in possibly_batched_dist_params
            and "priors" in possibly_batched_dist_params
        )
        means = possibly_batched_dist_params["means"]
        var = possibly_batched_dist_params["var"]
        priors = possibly_batched_dist_params["priors"]

        if len(means.shape) == 2:
            assert len(var.shape) == 0
            assert len(priors.shape) == 1
            means = means[None, :, :]
            var = var[None]
            priors = priors[None, :]

        assert len(means.shape) == 3
        N, K, D = means.shape
        assert len(var.shape) == 1 and var.shape[0] == N
        assert len(priors.shape) == 2 and priors.shape[0] == N and priors.shape[1] == K
        assert means.device == var.device == priors.device

        priors_sum = torch.sum(priors, dim=-1)
        assert torch.all(priors_sum >= 0)
        assert torch.allclose(priors_sum, torch.ones_like(priors_sum))
        assert torch.all(
            var >= -D * torch.finfo(var.dtype).eps
        )  # Allow for numerical errors

    @classmethod
    def x0(
        cls,
        x_t: torch.Tensor,
        t: torch.Tensor,
        diffusion_process: DiffusionProcess,
        batched_dist_params: Dict[str, torch.Tensor],
        dist_hparams: Dict[str, Any],
    ) -> torch.Tensor:
        """
        Computes the denoiser E[x_0 | x_t] for an isotropic homoscedastic GMM distribution.

        Arguments:
            x_t: The input tensor, of shape (N, D).
            t: The time tensor, of shape (N, ).
            diffusion_process: The diffusion process whose forward and reverse dynamics determine
                the time-evolution of the vector fields corresponding to the distribution.
            batched_dist_params: A dictionary containing the batched parameters of the distribution.
                - means: A tensor of shape (N, K, D) containing the means of the components.
                - var: A tensor of shape (N, ) containing the shared variance of all components.
                - priors: A tensor of shape (N, K) containing the prior probabilities of the components.
            dist_hparams: A dictionary of hyperparameters for the distribution.

        Returns:
            The prediction of x_0, of shape (N, D).

        Note:
            The batched_dist_params dictionary contains BATCHED tensors, i.e., the first dimension is the batch dimension.
        """
        means = batched_dist_params["means"]  # (N, K, D)
        var = batched_dist_params["var"]  # (N, )
        priors = batched_dist_params["priors"]  # (N, K)

        N, K, D = means.shape

        alpha = diffusion_process.alpha(t)  # (N, )
        sigma = diffusion_process.sigma(t)  # (N, )

        var_t = (alpha**2) * var + (sigma**2)  # (N, )
        centered_x = x_t[:, None, :] - alpha[:, None, None] * means  # (N, K, D)
        vars_t_inv_centered_x = centered_x / var_t[:, None, None]  # (N, K, D)

        mahalanobis_dists = torch.sum(
            centered_x * vars_t_inv_centered_x, dim=-1
        )  # (N, K)
        w = torch.log(priors) - 1 / 2 * mahalanobis_dists  # (N, K)
        softmax_w = torch.softmax(w, dim=-1)  # (N, K)

        weighted_normalized_x = torch.sum(
            softmax_w[:, :, None] * vars_t_inv_centered_x, dim=-2
        )  # (N, D)
        x0_hat = (1 / alpha[:, None]) * (
            x_t - (sigma[:, None] ** 2) * weighted_normalized_x
        )  # (N, D)

        return x0_hat

    @classmethod
    def sample(
        cls,
        N: int,
        dist_params: Dict[str, torch.Tensor],
        dist_hparams: Dict[str, Any],
    ) -> Tuple[torch.Tensor, torch.Tensor]:
        """
        Draws N i.i.d. samples from the isotropic homoscedastic GMM distribution.

        Arguments:
            N: The number of samples to draw.
            dist_params: A dictionary of parameters for the distribution.
                - means: A tensor of shape (K, D) containing the means of the components.
                - var: A tensor of shape () containing the shared variance of all components.
                - priors: A tensor of shape (K, ) containing the prior probabilities of the components.
            dist_hparams: A dictionary of hyperparameters for the distribution.

        Returns:
            A tuple (samples, labels), where samples is a tensor of shape (N, D) and labels is a tensor of shape (N, )
            containing the component indices from which each sample was drawn.
            Note that the samples are always placed on the CPU.
        """
        means = dist_params["means"]  # (K, D)
        var = dist_params["var"]  # ()
        priors = dist_params["priors"]  # (K, )

        K, D = means.shape
        covs = torch.eye(D, device=var.device)[None, :, :].expand(K, -1, -1) * var
        return GMMDistribution.sample(
            N, {"means": means, "covs": covs, "priors": priors}, dict()
        )

sample(N, dist_params, dist_hparams) classmethod

Draws N i.i.d. samples from the isotropic homoscedastic GMM distribution.

Parameters:

Name Type Description Default
N int

The number of samples to draw.

required
dist_params Dict[str, Tensor]

A dictionary of parameters for the distribution. - means: A tensor of shape (K, D) containing the means of the components. - var: A tensor of shape () containing the shared variance of all components. - priors: A tensor of shape (K, ) containing the prior probabilities of the components.

required
dist_hparams Dict[str, Any]

A dictionary of hyperparameters for the distribution.

required

Returns:

Type Description
Tensor

A tuple (samples, labels), where samples is a tensor of shape (N, D) and labels is a tensor of shape (N, )

Tensor

containing the component indices from which each sample was drawn.

Tuple[Tensor, Tensor]

Note that the samples are always placed on the CPU.

Source code in src/diffusionlab/distributions/gmm.py
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
@classmethod
def sample(
    cls,
    N: int,
    dist_params: Dict[str, torch.Tensor],
    dist_hparams: Dict[str, Any],
) -> Tuple[torch.Tensor, torch.Tensor]:
    """
    Draws N i.i.d. samples from the isotropic homoscedastic GMM distribution.

    Arguments:
        N: The number of samples to draw.
        dist_params: A dictionary of parameters for the distribution.
            - means: A tensor of shape (K, D) containing the means of the components.
            - var: A tensor of shape () containing the shared variance of all components.
            - priors: A tensor of shape (K, ) containing the prior probabilities of the components.
        dist_hparams: A dictionary of hyperparameters for the distribution.

    Returns:
        A tuple (samples, labels), where samples is a tensor of shape (N, D) and labels is a tensor of shape (N, )
        containing the component indices from which each sample was drawn.
        Note that the samples are always placed on the CPU.
    """
    means = dist_params["means"]  # (K, D)
    var = dist_params["var"]  # ()
    priors = dist_params["priors"]  # (K, )

    K, D = means.shape
    covs = torch.eye(D, device=var.device)[None, :, :].expand(K, -1, -1) * var
    return GMMDistribution.sample(
        N, {"means": means, "covs": covs, "priors": priors}, dict()
    )

validate_params(possibly_batched_dist_params) classmethod

Source code in src/diffusionlab/distributions/gmm.py
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
@classmethod
def validate_params(
    cls, possibly_batched_dist_params: Dict[str, torch.Tensor]
) -> None:
    assert (
        "means" in possibly_batched_dist_params
        and "var" in possibly_batched_dist_params
        and "priors" in possibly_batched_dist_params
    )
    means = possibly_batched_dist_params["means"]
    var = possibly_batched_dist_params["var"]
    priors = possibly_batched_dist_params["priors"]

    if len(means.shape) == 2:
        assert len(var.shape) == 0
        assert len(priors.shape) == 1
        means = means[None, :, :]
        var = var[None]
        priors = priors[None, :]

    assert len(means.shape) == 3
    N, K, D = means.shape
    assert len(var.shape) == 1 and var.shape[0] == N
    assert len(priors.shape) == 2 and priors.shape[0] == N and priors.shape[1] == K
    assert means.device == var.device == priors.device

    priors_sum = torch.sum(priors, dim=-1)
    assert torch.all(priors_sum >= 0)
    assert torch.allclose(priors_sum, torch.ones_like(priors_sum))
    assert torch.all(
        var >= -D * torch.finfo(var.dtype).eps
    )  # Allow for numerical errors

x0(x_t, t, diffusion_process, batched_dist_params, dist_hparams) classmethod

Computes the denoiser E[x_0 | x_t] for an isotropic homoscedastic GMM distribution.

Parameters:

Name Type Description Default
x_t Tensor

The input tensor, of shape (N, D).

required
t Tensor

The time tensor, of shape (N, ).

required
diffusion_process DiffusionProcess

The diffusion process whose forward and reverse dynamics determine the time-evolution of the vector fields corresponding to the distribution.

required
batched_dist_params Dict[str, Tensor]

A dictionary containing the batched parameters of the distribution. - means: A tensor of shape (N, K, D) containing the means of the components. - var: A tensor of shape (N, ) containing the shared variance of all components. - priors: A tensor of shape (N, K) containing the prior probabilities of the components.

required
dist_hparams Dict[str, Any]

A dictionary of hyperparameters for the distribution.

required

Returns:

Type Description
Tensor

The prediction of x_0, of shape (N, D).

Note

The batched_dist_params dictionary contains BATCHED tensors, i.e., the first dimension is the batch dimension.

Source code in src/diffusionlab/distributions/gmm.py
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
@classmethod
def x0(
    cls,
    x_t: torch.Tensor,
    t: torch.Tensor,
    diffusion_process: DiffusionProcess,
    batched_dist_params: Dict[str, torch.Tensor],
    dist_hparams: Dict[str, Any],
) -> torch.Tensor:
    """
    Computes the denoiser E[x_0 | x_t] for an isotropic homoscedastic GMM distribution.

    Arguments:
        x_t: The input tensor, of shape (N, D).
        t: The time tensor, of shape (N, ).
        diffusion_process: The diffusion process whose forward and reverse dynamics determine
            the time-evolution of the vector fields corresponding to the distribution.
        batched_dist_params: A dictionary containing the batched parameters of the distribution.
            - means: A tensor of shape (N, K, D) containing the means of the components.
            - var: A tensor of shape (N, ) containing the shared variance of all components.
            - priors: A tensor of shape (N, K) containing the prior probabilities of the components.
        dist_hparams: A dictionary of hyperparameters for the distribution.

    Returns:
        The prediction of x_0, of shape (N, D).

    Note:
        The batched_dist_params dictionary contains BATCHED tensors, i.e., the first dimension is the batch dimension.
    """
    means = batched_dist_params["means"]  # (N, K, D)
    var = batched_dist_params["var"]  # (N, )
    priors = batched_dist_params["priors"]  # (N, K)

    N, K, D = means.shape

    alpha = diffusion_process.alpha(t)  # (N, )
    sigma = diffusion_process.sigma(t)  # (N, )

    var_t = (alpha**2) * var + (sigma**2)  # (N, )
    centered_x = x_t[:, None, :] - alpha[:, None, None] * means  # (N, K, D)
    vars_t_inv_centered_x = centered_x / var_t[:, None, None]  # (N, K, D)

    mahalanobis_dists = torch.sum(
        centered_x * vars_t_inv_centered_x, dim=-1
    )  # (N, K)
    w = torch.log(priors) - 1 / 2 * mahalanobis_dists  # (N, K)
    softmax_w = torch.softmax(w, dim=-1)  # (N, K)

    weighted_normalized_x = torch.sum(
        softmax_w[:, :, None] * vars_t_inv_centered_x, dim=-2
    )  # (N, D)
    x0_hat = (1 / alpha[:, None]) * (
        x_t - (sigma[:, None] ** 2) * weighted_normalized_x
    )  # (N, D)

    return x0_hat

LowRankGMMDistribution

Bases: Distribution

A Gaussian Mixture Model (GMM) with K low-rank components. Formally, the distribution is defined as:

mu(B) = sum_(i=1)^(K) pi_i * N(mu_i, Sigma_i)(B)

where mu_i is the mean of the ith component, Sigma_i is the covariance matrix of the ith component, and pi_i is the prior probability of the ith component. Notably, Sigma_i is a low-rank matrix of the form

Sigma_i = A_i @ A_i^T

Distribution Parameters
  • means: A tensor of shape (K, D) containing the means of the components.
  • covs_factors: A tensor of shape (K, D, P) containing the tall factors of the covariance matrices of the components.
  • priors: A tensor of shape (K, ) containing the prior probabilities of the components.
Distribution Hyperparameters
  • None
Note
  • The covariance matrices are not explicitly stored, but rather computed as Sigma_i = A_i @ A_i^T.
  • The time and memory complexity is much lower in this class compared to the full GMM class, if and only if each covariance is low-rank (P << D).
Source code in src/diffusionlab/distributions/gmm.py
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
class LowRankGMMDistribution(Distribution):
    """
    A Gaussian Mixture Model (GMM) with K low-rank components.
    Formally, the distribution is defined as:

    mu(B) = sum_(i=1)^(K) pi_i * N(mu_i, Sigma_i)(B)

    where mu_i is the mean of the ith component, Sigma_i is the covariance matrix of the ith component,
    and pi_i is the prior probability of the ith component. Notably, Sigma_i is a low-rank matrix of the form

    Sigma_i =  A_i @ A_i^T

    Distribution Parameters:
        - means: A tensor of shape (K, D) containing the means of the components.
        - covs_factors: A tensor of shape (K, D, P) containing the tall factors of the covariance matrices of the components.
        - priors: A tensor of shape (K, ) containing the prior probabilities of the components.

    Distribution Hyperparameters:
        - None

    Note:
        - The covariance matrices are not explicitly stored, but rather computed as Sigma_i = A_i @ A_i^T.
        - The time and memory complexity is much lower in this class compared to the full GMM class, if and only if each covariance is low-rank (P << D).
    """

    @classmethod
    def validate_params(
        cls, possibly_batched_dist_params: Dict[str, torch.Tensor]
    ) -> None:
        assert (
            "means" in possibly_batched_dist_params
            and "covs_factors" in possibly_batched_dist_params
            and "priors" in possibly_batched_dist_params
        )
        means = possibly_batched_dist_params["means"]
        covs_factors = possibly_batched_dist_params["covs_factors"]
        priors = possibly_batched_dist_params["priors"]

        if len(means.shape) == 2:
            assert len(covs_factors.shape) == 3
            assert len(priors.shape) == 1
            means = means[None, :, :]
            covs_factors = covs_factors[None, :, :, :]
            priors = priors[None, :]

        assert len(means.shape) == 3
        assert len(covs_factors.shape) == 4
        assert len(priors.shape) == 2

        N, K, D, P = covs_factors.shape
        assert means.shape[0] == N and means.shape[1] == K and means.shape[2] == D
        assert len(priors.shape) == 2 and priors.shape[0] == N and priors.shape[1] == K
        assert means.device == covs_factors.device == priors.device

        assert torch.all(priors >= 0)
        sum_priors = torch.sum(priors, dim=-1)
        assert torch.allclose(sum_priors, torch.ones_like(sum_priors))

    @classmethod
    def x0(
        cls,
        x_t: torch.Tensor,
        t: torch.Tensor,
        diffusion_process: DiffusionProcess,
        batched_dist_params: Dict[str, torch.Tensor],
        dist_hparams: Dict[str, Any],
    ) -> torch.Tensor:
        """
        Computes the denoiser E[x_0 | x_t] for a low-rank GMM distribution.

        Arguments:
            x_t: The input tensor, of shape (N, D).
            t: The time tensor, of shape (N, ).
            diffusion_process: The diffusion process whose forward and reverse dynamics determine
                the time-evolution of the vector fields corresponding to the distribution.
            batched_dist_params: A dictionary containing the batched parameters of the distribution.
                - means: A tensor of shape (N, K, D) containing the means of the components.
                - covs_factors: A tensor of shape (N, K, D, P) containing the tall factors of the covariance matrices.
                - priors: A tensor of shape (N, K) containing the prior probabilities of the components.
            dist_hparams: A dictionary of hyperparameters for the distribution.

        Returns:
            The prediction of x_0, of shape (N, D).

        Note:
            The batched_dist_params dictionary contains BATCHED tensors, i.e., the first dimension is the batch dimension.
            The covariance matrices are implicitly defined as Sigma_i = A_i @ A_i^T, where A_i is the ith factor.
        """
        means = batched_dist_params["means"]  # (N, K, D)
        covs_factors = batched_dist_params["covs_factors"]  # (N, K, D, R)
        priors = batched_dist_params["priors"]  # (N, K)

        N, K, D, P = covs_factors.shape
        covs_factors_T = covs_factors.transpose(-1, -2)  # (N, K, R, D)

        alpha = diffusion_process.alpha(t)  # (N, )
        sigma = diffusion_process.sigma(t)  # (N, )
        alpha_sigma_ratio_sq = (alpha / sigma) ** 2  # (N, )
        sigma_alpha_ratio_sq = 1 / alpha_sigma_ratio_sq  # (N, )

        internal_covs = covs_factors_T @ covs_factors  # (N, K, R, R)
        logdets_covs_t = 2 * D * torch.log(sigma[:, None]) + logdet_pd(
            torch.eye(P, device=covs_factors.device)[None, None, :, :]  # (1, 1, P, P)
            + alpha_sigma_ratio_sq[:, None, None, None] * internal_covs  # (N, K, P, P)
        )  # (N, K)

        centered_x = x_t[:, None, :] - alpha[:, None, None] * means  # (N, K, D)
        covs_t_inv_centered_x = (1 / sigma[:, None, None] ** 2) * (
            centered_x  # (N, K, D)
            - (
                covs_factors  # (N, K, D, P)
                @ torch.linalg.lstsq(  # (N, K, P, 1)
                    internal_covs  # (N, K, P, P)
                    + sigma_alpha_ratio_sq[:, None, None, None]  # (N, K, 1, 1)
                    * torch.eye(P, device=internal_covs.device)[
                        None, None, :, :
                    ],  # (1, 1, P, P)
                    covs_factors_T @ centered_x[:, :, :, None],  # (N, K, P, 1)
                ).solution  # (N, K, P, 1)
            )[:, :, :, 0]  # (N, K, D, 1) -> (N, K, D)
        )  # (N, K, D)

        mahalanobis_dists = torch.sum(
            centered_x * covs_t_inv_centered_x, dim=-1
        )  # (N, K)
        w = (
            torch.log(priors) - 1 / 2 * logdets_covs_t - 1 / 2 * mahalanobis_dists
        )  # (N, K)
        softmax_w = torch.softmax(w, dim=-1)  # (N, K)

        weighted_normalized_x = torch.sum(
            softmax_w[:, :, None] * covs_t_inv_centered_x, dim=-2
        )  # (N, D)
        x0_hat = (1 / alpha[:, None]) * (
            x_t - (sigma[:, None] ** 2) * weighted_normalized_x
        )  # (N, D)

        return x0_hat

    @classmethod
    def sample(
        cls,
        N: int,
        dist_params: Dict[str, torch.Tensor],
        dist_hparams: Dict[str, Any],
    ) -> Tuple[torch.Tensor, torch.Tensor]:
        """
        Draws N i.i.d. samples from the low-rank GMM distribution.

        Arguments:
            N: The number of samples to draw.
            dist_params: A dictionary of parameters for the distribution.
                - means: A tensor of shape (K, D) containing the means of the components.
                - covs_factors: A tensor of shape (K, D, P) containing the tall factors of the covariance matrices.
                - priors: A tensor of shape (K, ) containing the prior probabilities of the components.
            dist_hparams: A dictionary of hyperparameters for the distribution.

        Returns:
            A tuple (samples, labels), where samples is a tensor of shape (N, D) and labels is a tensor of shape (N, )
            containing the component indices from which each sample was drawn.
            Note that the samples are always placed on the CPU.
        """
        means = dist_params["means"]  # (K, D)
        covs_factors = dist_params["covs_factors"]  # (K, D, P)
        priors = dist_params["priors"]  # (K, )

        K, D, P = covs_factors.shape

        device = priors.device
        y = torch.multinomial(priors, N, replacement=True)  # (N, )
        X = torch.empty((N, D), device=device)
        for k in range(K):
            idx = y == k
            X[idx] = (
                torch.randn((X[idx].shape[0], P), device=device) @ covs_factors[k].T
                + means[k][None, :]
            )
        return X.to("cpu"), y.to("cpu")

sample(N, dist_params, dist_hparams) classmethod

Draws N i.i.d. samples from the low-rank GMM distribution.

Parameters:

Name Type Description Default
N int

The number of samples to draw.

required
dist_params Dict[str, Tensor]

A dictionary of parameters for the distribution. - means: A tensor of shape (K, D) containing the means of the components. - covs_factors: A tensor of shape (K, D, P) containing the tall factors of the covariance matrices. - priors: A tensor of shape (K, ) containing the prior probabilities of the components.

required
dist_hparams Dict[str, Any]

A dictionary of hyperparameters for the distribution.

required

Returns:

Type Description
Tensor

A tuple (samples, labels), where samples is a tensor of shape (N, D) and labels is a tensor of shape (N, )

Tensor

containing the component indices from which each sample was drawn.

Tuple[Tensor, Tensor]

Note that the samples are always placed on the CPU.

Source code in src/diffusionlab/distributions/gmm.py
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
@classmethod
def sample(
    cls,
    N: int,
    dist_params: Dict[str, torch.Tensor],
    dist_hparams: Dict[str, Any],
) -> Tuple[torch.Tensor, torch.Tensor]:
    """
    Draws N i.i.d. samples from the low-rank GMM distribution.

    Arguments:
        N: The number of samples to draw.
        dist_params: A dictionary of parameters for the distribution.
            - means: A tensor of shape (K, D) containing the means of the components.
            - covs_factors: A tensor of shape (K, D, P) containing the tall factors of the covariance matrices.
            - priors: A tensor of shape (K, ) containing the prior probabilities of the components.
        dist_hparams: A dictionary of hyperparameters for the distribution.

    Returns:
        A tuple (samples, labels), where samples is a tensor of shape (N, D) and labels is a tensor of shape (N, )
        containing the component indices from which each sample was drawn.
        Note that the samples are always placed on the CPU.
    """
    means = dist_params["means"]  # (K, D)
    covs_factors = dist_params["covs_factors"]  # (K, D, P)
    priors = dist_params["priors"]  # (K, )

    K, D, P = covs_factors.shape

    device = priors.device
    y = torch.multinomial(priors, N, replacement=True)  # (N, )
    X = torch.empty((N, D), device=device)
    for k in range(K):
        idx = y == k
        X[idx] = (
            torch.randn((X[idx].shape[0], P), device=device) @ covs_factors[k].T
            + means[k][None, :]
        )
    return X.to("cpu"), y.to("cpu")

validate_params(possibly_batched_dist_params) classmethod

Source code in src/diffusionlab/distributions/gmm.py
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
@classmethod
def validate_params(
    cls, possibly_batched_dist_params: Dict[str, torch.Tensor]
) -> None:
    assert (
        "means" in possibly_batched_dist_params
        and "covs_factors" in possibly_batched_dist_params
        and "priors" in possibly_batched_dist_params
    )
    means = possibly_batched_dist_params["means"]
    covs_factors = possibly_batched_dist_params["covs_factors"]
    priors = possibly_batched_dist_params["priors"]

    if len(means.shape) == 2:
        assert len(covs_factors.shape) == 3
        assert len(priors.shape) == 1
        means = means[None, :, :]
        covs_factors = covs_factors[None, :, :, :]
        priors = priors[None, :]

    assert len(means.shape) == 3
    assert len(covs_factors.shape) == 4
    assert len(priors.shape) == 2

    N, K, D, P = covs_factors.shape
    assert means.shape[0] == N and means.shape[1] == K and means.shape[2] == D
    assert len(priors.shape) == 2 and priors.shape[0] == N and priors.shape[1] == K
    assert means.device == covs_factors.device == priors.device

    assert torch.all(priors >= 0)
    sum_priors = torch.sum(priors, dim=-1)
    assert torch.allclose(sum_priors, torch.ones_like(sum_priors))

x0(x_t, t, diffusion_process, batched_dist_params, dist_hparams) classmethod

Computes the denoiser E[x_0 | x_t] for a low-rank GMM distribution.

Parameters:

Name Type Description Default
x_t Tensor

The input tensor, of shape (N, D).

required
t Tensor

The time tensor, of shape (N, ).

required
diffusion_process DiffusionProcess

The diffusion process whose forward and reverse dynamics determine the time-evolution of the vector fields corresponding to the distribution.

required
batched_dist_params Dict[str, Tensor]

A dictionary containing the batched parameters of the distribution. - means: A tensor of shape (N, K, D) containing the means of the components. - covs_factors: A tensor of shape (N, K, D, P) containing the tall factors of the covariance matrices. - priors: A tensor of shape (N, K) containing the prior probabilities of the components.

required
dist_hparams Dict[str, Any]

A dictionary of hyperparameters for the distribution.

required

Returns:

Type Description
Tensor

The prediction of x_0, of shape (N, D).

Note

The batched_dist_params dictionary contains BATCHED tensors, i.e., the first dimension is the batch dimension. The covariance matrices are implicitly defined as Sigma_i = A_i @ A_i^T, where A_i is the ith factor.

Source code in src/diffusionlab/distributions/gmm.py
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
@classmethod
def x0(
    cls,
    x_t: torch.Tensor,
    t: torch.Tensor,
    diffusion_process: DiffusionProcess,
    batched_dist_params: Dict[str, torch.Tensor],
    dist_hparams: Dict[str, Any],
) -> torch.Tensor:
    """
    Computes the denoiser E[x_0 | x_t] for a low-rank GMM distribution.

    Arguments:
        x_t: The input tensor, of shape (N, D).
        t: The time tensor, of shape (N, ).
        diffusion_process: The diffusion process whose forward and reverse dynamics determine
            the time-evolution of the vector fields corresponding to the distribution.
        batched_dist_params: A dictionary containing the batched parameters of the distribution.
            - means: A tensor of shape (N, K, D) containing the means of the components.
            - covs_factors: A tensor of shape (N, K, D, P) containing the tall factors of the covariance matrices.
            - priors: A tensor of shape (N, K) containing the prior probabilities of the components.
        dist_hparams: A dictionary of hyperparameters for the distribution.

    Returns:
        The prediction of x_0, of shape (N, D).

    Note:
        The batched_dist_params dictionary contains BATCHED tensors, i.e., the first dimension is the batch dimension.
        The covariance matrices are implicitly defined as Sigma_i = A_i @ A_i^T, where A_i is the ith factor.
    """
    means = batched_dist_params["means"]  # (N, K, D)
    covs_factors = batched_dist_params["covs_factors"]  # (N, K, D, R)
    priors = batched_dist_params["priors"]  # (N, K)

    N, K, D, P = covs_factors.shape
    covs_factors_T = covs_factors.transpose(-1, -2)  # (N, K, R, D)

    alpha = diffusion_process.alpha(t)  # (N, )
    sigma = diffusion_process.sigma(t)  # (N, )
    alpha_sigma_ratio_sq = (alpha / sigma) ** 2  # (N, )
    sigma_alpha_ratio_sq = 1 / alpha_sigma_ratio_sq  # (N, )

    internal_covs = covs_factors_T @ covs_factors  # (N, K, R, R)
    logdets_covs_t = 2 * D * torch.log(sigma[:, None]) + logdet_pd(
        torch.eye(P, device=covs_factors.device)[None, None, :, :]  # (1, 1, P, P)
        + alpha_sigma_ratio_sq[:, None, None, None] * internal_covs  # (N, K, P, P)
    )  # (N, K)

    centered_x = x_t[:, None, :] - alpha[:, None, None] * means  # (N, K, D)
    covs_t_inv_centered_x = (1 / sigma[:, None, None] ** 2) * (
        centered_x  # (N, K, D)
        - (
            covs_factors  # (N, K, D, P)
            @ torch.linalg.lstsq(  # (N, K, P, 1)
                internal_covs  # (N, K, P, P)
                + sigma_alpha_ratio_sq[:, None, None, None]  # (N, K, 1, 1)
                * torch.eye(P, device=internal_covs.device)[
                    None, None, :, :
                ],  # (1, 1, P, P)
                covs_factors_T @ centered_x[:, :, :, None],  # (N, K, P, 1)
            ).solution  # (N, K, P, 1)
        )[:, :, :, 0]  # (N, K, D, 1) -> (N, K, D)
    )  # (N, K, D)

    mahalanobis_dists = torch.sum(
        centered_x * covs_t_inv_centered_x, dim=-1
    )  # (N, K)
    w = (
        torch.log(priors) - 1 / 2 * logdets_covs_t - 1 / 2 * mahalanobis_dists
    )  # (N, K)
    softmax_w = torch.softmax(w, dim=-1)  # (N, K)

    weighted_normalized_x = torch.sum(
        softmax_w[:, :, None] * covs_t_inv_centered_x, dim=-2
    )  # (N, D)
    x0_hat = (1 / alpha[:, None]) * (
        x_t - (sigma[:, None] ** 2) * weighted_normalized_x
    )  # (N, D)

    return x0_hat