Skip to content

min_distance module

Estimates semilinear separable models with a given entropy function. The entropy function and the surplus matrix must both be linear in the parameters.

estimate_semilinear_mde(muhat, phi_bases, entropy, additional_parameters=None, initial_weights=None)

Estimates the parameters of the distributions and of the base functions.

Parameters:

Name Type Description Default
muhat Matching

the observed Matching

required
phi_bases np.ndarray

an (X, Y, K) array of bases

required
entropy EntropyFunctions

an EntropyFunctions object

required
additional_parameters Optional[list]

additional parameters of the distribution of errors, if any

None
initial_weights Optional[np.ndarray]

if specified, used as the weighting matrix for the first step when entropy.param_dependent is True

None

Returns:

Type Description
MDEResults

an MDEResults instance

Example
 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
# We simulate a Choo and Siow homoskedastic marriage market
#  and we estimate a gender-heteroskedastic model on the simulated data.
X, Y, K = 10, 20, 2
n_households = int(1e6)
lambda_true = np.random.randn(K)
phi_bases = np.random.randn(X, Y, K)
n = np.ones(X)
m = np.ones(Y)
Phi = phi_bases @ lambda_true
choo_siow_instance = ChooSiowPrimitives(Phi, n, m)
mus_sim = choo_siow_instance.simulate(n_households)
choo_siow_instance.describe()

entropy_model =  entropy_choo_siow_gender_heteroskedastic_numeric
n_alpha = 1
true_alpha = np.ones(n_alpha)
true_coeffs = np.concatenate((true_alpha, lambda_true))

print_stars(entropy_model.description)

mde_results = estimate_semilinear_mde(
    mus_sim, phi_bases, entropy_model)

mde_results.print_results(true_coeffs=true_coeffs, n_alpha=1)
Source code in cupid_matching/min_distance.py
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
def estimate_semilinear_mde(
    muhat: Matching,
    phi_bases: np.ndarray,
    entropy: EntropyFunctions,
    additional_parameters: Optional[list] = None,
    initial_weights: Optional[np.ndarray] = None,
) -> MDEResults:
    """
    Estimates the parameters of the distributions and of the base functions.

    Args:
        muhat: the observed Matching
        phi_bases: an (X, Y, K) array of bases
        entropy: an `EntropyFunctions` object
        additional_parameters: additional parameters of the distribution of errors,
            if any
        initial_weights: if specified, used as the weighting matrix
            for the first step when `entropy.param_dependent` is `True`

    Returns:
        an `MDEResults` instance

    Example:
        ```py
        # We simulate a Choo and Siow homoskedastic marriage market
        #  and we estimate a gender-heteroskedastic model on the simulated data.
        X, Y, K = 10, 20, 2
        n_households = int(1e6)
        lambda_true = np.random.randn(K)
        phi_bases = np.random.randn(X, Y, K)
        n = np.ones(X)
        m = np.ones(Y)
        Phi = phi_bases @ lambda_true
        choo_siow_instance = ChooSiowPrimitives(Phi, n, m)
        mus_sim = choo_siow_instance.simulate(n_households)
        choo_siow_instance.describe()

        entropy_model =  entropy_choo_siow_gender_heteroskedastic_numeric
        n_alpha = 1
        true_alpha = np.ones(n_alpha)
        true_coeffs = np.concatenate((true_alpha, lambda_true))

        print_stars(entropy_model.description)

        mde_results = estimate_semilinear_mde(
            mus_sim, phi_bases, entropy_model)

        mde_results.print_results(true_coeffs=true_coeffs, n_alpha=1)
        ```

    """
    muxyhat, _, _, nhat, mhat = muhat.unpack()
    X, Y = muxyhat.shape
    XY = X * Y
    ndims_phi = phi_bases.ndim
    if ndims_phi != 3:
        bs_error_abort(f"phi_bases should have 3 dimensions, not {ndims_phi}")
    Xp, Yp, K = phi_bases.shape
    if Xp != X or Yp != Y:
        bs_error_abort(
            f"phi_bases should have shape ({X}, {Y}, {K}) not ({Xp}, {Yp}, {K})"
        )
    parameterized_entropy = entropy.parameter_dependent
    if parameterized_entropy:
        if initial_weights is None:
            print_stars(
                "Using the identity matrix as weighting matrix in the first step."
            )
            S_mat = np.eye(XY)
        else:
            S_mat = initial_weights

    phi_mat = _make_XY_K_mat(phi_bases)
    e0_fun = entropy.e0_fun
    if additional_parameters is None:
        e0_fun = cast(MatchingFunction, e0_fun)
        e0_vals = e0_fun(muhat)
    else:
        e0_fun = cast(MatchingFunctionParam, e0_fun)
        e0_vals = e0_fun(muhat, additional_parameters)
    e0_hat = e0_vals.ravel()

    if not parameterized_entropy:  # we only have e0(mu,r)
        n_pars = K
        hessian = entropy.hessian
        if hessian == "provided":
            e0_derivative = cast(EntropyHessians, entropy.e0_derivative)
            if additional_parameters is None:
                hessian_components_mumu = e0_derivative[0](muhat)
                hessian_components_mur = e0_derivative[1](muhat)
            else:
                e0_derivative1 = cast(EntropyHessiansParam, entropy.e0_derivative)
                hessian_components_mumu = e0_derivative1[0](
                    muhat, additional_parameters
                )
                hessian_components_mur = e0_derivative1[1](muhat, additional_parameters)
        else:
            if additional_parameters is None:
                hessian_components = _numeric_hessian(entropy, muhat)
            else:
                hessian_components = _numeric_hessian(
                    entropy,
                    muhat,
                    additional_parameters=additional_parameters,
                )
            (
                hessian_components_mumu,
                hessian_components_mur,
            ) = hessian_components
        hessian_mumu = _fill_hessianMuMu_from_components(hessian_components_mumu)
        hessian_mur = _fill_hessianMuR_from_components(hessian_components_mur)
        hessians_both = np.concatenate((hessian_mumu, hessian_mur), axis=1)

        _, var_munm = _variance_muhat(muhat)
        var_entropy_gradient = hessians_both @ var_munm @ hessians_both.T
        S_mat = spla.inv(var_entropy_gradient)
        estimated_coefficients, varcov_coefficients = _compute_estimates(
            phi_mat, S_mat, e0_hat
        )
        stderrs_coefficients = np.sqrt(np.diag(varcov_coefficients))
        est_Phi = phi_mat @ estimated_coefficients
        residuals = est_Phi + e0_hat
    else:  # parameterized entropy: e0(mu,r) + e(mu,r) . alpha
        # create the F matrix
        if additional_parameters is None:
            e_fun = cast(MatchingFunction, entropy.e_fun)
            e_vals = e_fun(muhat)
        else:
            e_fun1 = cast(MatchingFunctionParam, entropy.e_fun)
            e_vals = e_fun1(muhat, additional_parameters)
        e_hat = _make_XY_K_mat(e_vals)

        F_hat = np.column_stack((e_hat, phi_mat))
        n_pars = e_hat.shape[1] + K
        # first pass with an initial weighting matrix
        first_coeffs, _ = _compute_estimates(F_hat, S_mat, e0_hat)
        first_alpha = first_coeffs[:-K]

        # compute the efficient weighting matrix
        hessian = entropy.hessian
        if hessian == "provided":
            if additional_parameters is None:
                e0_derivative = cast(EntropyHessians, entropy.e0_derivative)
                e_derivative = cast(EntropyHessians, entropy.e_derivative)
                e0_derivative_mumu = cast(EntropyHessianMuMu, e0_derivative[0])
                hessian_components_mumu_e0 = e0_derivative_mumu(muhat)
                e0_derivative_mur = cast(EntropyHessianMuR, e0_derivative[1])
                hessian_components_mur_e0 = e0_derivative_mur(muhat)
                e_derivative_mumu = cast(EntropyHessianMuMu, e_derivative[0])
                hessian_components_mumu_e = e_derivative_mumu(muhat)
                e_derivative_mur = cast(EntropyHessianMuR, e_derivative[1])
                hessian_components_mur_e = e_derivative_mur(muhat)
            else:
                e0_derivative1 = cast(EntropyHessiansParam, entropy.e0_derivative)
                e_derivative1 = cast(EntropyHessiansParam, entropy.e_derivative)
                e0_derivative_mumu1 = cast(EntropyHessianMuMuParam, e0_derivative1[0])
                e0_derivative_mur1 = cast(EntropyHessianMuRParam, e0_derivative1[1])
                e_derivative_mumu1 = cast(EntropyHessianMuMuParam, e_derivative1[0])
                e_derivative_mur1 = cast(EntropyHessianMuRParam, e_derivative1[1])
                hessian_components_mumu_e0 = e0_derivative_mumu1(
                    muhat, additional_parameters
                )
                hessian_components_mur_e0 = e0_derivative_mur1(
                    muhat, additional_parameters
                )
                hessian_components_mumu_e = e_derivative_mumu1(
                    muhat, additional_parameters
                )
                hessian_components_mur_e = e_derivative_mur1(
                    muhat, additional_parameters
                )

            # print_stars("First-stage estimates:")
            # print(first_coeffs)

            hessian_components_mumu1 = (
                hessian_components_mumu_e0[0]
                + hessian_components_mumu_e[0] @ first_alpha,
                hessian_components_mumu_e0[1]
                + hessian_components_mumu_e[1] @ first_alpha,
                hessian_components_mumu_e0[2]
                + hessian_components_mumu_e[2] @ first_alpha,
            )
            hessian_components_mur1 = (
                hessian_components_mur_e0[0]
                + hessian_components_mur_e[0] @ first_alpha,
                hessian_components_mur_e0[1]
                + hessian_components_mur_e[1] @ first_alpha,
            )
            hessian_mumu = _fill_hessianMuMu_from_components(hessian_components_mumu1)
            hessian_mur = _fill_hessianMuR_from_components(hessian_components_mur1)
        else:  # numeric hessian
            if additional_parameters is None:
                hessian_components = _numeric_hessian(entropy, muhat, alpha=first_alpha)
            else:
                hessian_components = _numeric_hessian(
                    entropy,
                    muhat,
                    alpha=first_alpha,
                    additional_parameters=additional_parameters,
                )
            (
                hessian_components_mumu,
                hessian_components_mur,
            ) = hessian_components
            hessian_mumu = _fill_hessianMuMu_from_components(hessian_components_mumu)
            hessian_mur = _fill_hessianMuR_from_components(hessian_components_mur)

        hessians_both = np.concatenate((hessian_mumu, hessian_mur), axis=1)

        _, var_munm = _variance_muhat(muhat)
        var_entropy_gradient = hessians_both @ var_munm @ hessians_both.T
        S_mat = spla.inv(var_entropy_gradient)

        # second pass
        estimated_coefficients, varcov_coefficients = _compute_estimates(
            F_hat, S_mat, e0_hat
        )
        est_alpha, est_beta = (
            estimated_coefficients[:-K],
            estimated_coefficients[-K:],
        )
        stderrs_coefficients = np.sqrt(np.diag(varcov_coefficients))
        est_Phi = phi_mat @ est_beta
        residuals = est_Phi + e0_hat + e_hat @ est_alpha

    value_obj = residuals.T @ S_mat @ residuals
    ndf = X * Y - n_pars
    test_stat = value_obj
    n_individuals = np.sum(nhat) + np.sum(mhat)
    n_households = n_individuals - np.sum(muxyhat)

    results = MDEResults(
        X=X,
        Y=Y,
        K=K,
        number_households=n_households,
        estimated_coefficients=estimated_coefficients,
        varcov_coefficients=varcov_coefficients,
        stderrs_coefficients=stderrs_coefficients,
        estimated_Phi=est_Phi.reshape((X, Y)),
        test_statistic=test_stat,
        ndf=ndf,
        test_pvalue=sts.chi2.sf(test_stat, ndf),
        parameterized_entropy=parameterized_entropy,
    )
    return results