From a28cea19e93d996457e104041e0467507538b907 Mon Sep 17 00:00:00 2001 From: dickbird <20980982+cafeTechne@users.noreply.github.com> Date: Sun, 15 Nov 2026 20:18:25 -0500 Subject: [PATCH 1/2] Tests: write WONDER fixture with newline='' so its SHA-256 matches on Windows (text mode turned LF into CRLF; 3 Windows-only failures since Round 21) --- census/health_sources.py | 2 +- census/manifest/methods_text.py | 32 ++---------- census/manifest/run.py | 69 ++++---------------------- census/manifest/schema.py | 17 ++----- census/manifest/validate.py | 88 +++------------------------------ tests/test_round21_part2.py | 2 +- 6 files changed, 24 insertions(+), 186 deletions(-) diff --git a/census/health_sources.py b/census/health_sources.py index 2dd7dae..d7ee59d 100644 --- a/census/health_sources.py +++ b/census/health_sources.py @@ -40,7 +40,7 @@ "wonder": ("file", "sha256")} # where an external variable may appear: only as the outcome of these methods (its error is independent # of the ACS sampling error, so it enters the residual; as a covariate it would go uncorrected) -EXTERNAL_OUTCOME_METHODS = ("fit_eiv", "fit_replicate", "fh_regression") +from .manifest.methods import EXTERNAL_OUTCOME_METHODS JOIN_MISSING_LIMIT = 0.05 KINDS = ("direct", "model_based", "partly_model_based") diff --git a/census/manifest/methods_text.py b/census/manifest/methods_text.py index 84858b7..af75378 100644 --- a/census/manifest/methods_text.py +++ b/census/manifest/methods_text.py @@ -34,35 +34,9 @@ # Methods text # --------------------------------------------------------------------------- -_METHOD_TEXT = { - "fit_replicate": ("Coefficients were estimated with {model} regression (statsmodels) on the full-sample " - "estimates; sampling standard errors came from refitting on each of {rep_source}, Var = 4/80 sum (b_r - b)^2, combined with the model standard error as " - "se_total = sqrt(se_model^2 + se_sampling^2). Reported standard errors, intervals and p-values use " - "the {target_se} ({target} target: {estimand}), with normal-theory intervals. Without " - "replicates only model standard errors are reported.", ["vre", "design", "statsmodels"]), - "regression_calibration": ("Covariates measured with known ACS sampling error ({cov}) were corrected by " - "regression calibration with heteroscedastic known error variances; standard " - "errors (se_total, used for normal-theory intervals and p-values) by row bootstrap " - "re-estimating the calibration. Coefficients are {estimand}.", ["carroll"]), - "fit_eiv": ("Coefficients were estimated by moment-corrected least squares ({model}; Fuller 1987) with " - "per-unit sampling-error covariances of every design column and of the outcome, including the " - "outcome-covariate error covariance, estimated from {rep_source} (4/80 sum over replicates). " - "Standard errors are {se_basis} (se_total, used for " - "normal-theory intervals and p-values); naive coefficients are reported alongside. Coefficients " - "are {estimand}.", ["fuller", "vre", "design"]), - "simex": ("Covariates measured with known ACS sampling error ({cov}) were corrected by " - "simulation-extrapolation (SIMEX, quadratic extrapolation) with the measurement-error jackknife " - "variance (se_total, used for normal-theory intervals and p-values). Coefficients are " - "{estimand}.", ["simex", "carroll"]), - "fh_regression": ("The outcome's known sampling variance was modelled with an area-level Fay-Herriot model " - "(GLS coefficients, REML variance component).", ["fh"]), - "fay_herriot": ("Small-area estimates are Fay-Herriot EBLUPs (REML) with Prasad-Rao/Datta-Lahiri MSE.", - ["fh"]), - "correlation_matrix": ("Correlations with Fisher-z intervals{rep}.", ["fisher", "vre"]), - "linear_trend_test": ("Linear trends across non-overlapping 5-year periods were tested by GLS with known " - "sampling variances.", ["testing"]), - "batch_z_test": ("Differences were tested with the Census two-sample z test, SE = MOE/1.645.", ["testing"]), -} +from .methods import METHODS + +_METHOD_TEXT = {name: spec.text for name, spec in sorted(METHODS.items(), key=lambda item: {'fit_replicate': 0, 'regression_calibration': 1, 'fit_eiv': 2, 'simex': 3, 'fh_regression': 4, 'fay_herriot': 5, 'correlation_matrix': 6, 'linear_trend_test': 7, 'batch_z_test': 8}.get(item[0], len(METHODS))) if spec.text is not None} def _age_standardized_text(d: Mapping[str, Any]) -> str: diff --git a/census/manifest/run.py b/census/manifest/run.py index 1543502..cb5c393 100644 --- a/census/manifest/run.py +++ b/census/manifest/run.py @@ -136,70 +136,19 @@ def _run_analysis(a: Mapping[str, Any], frame: pd.DataFrame, level: float, repli if args.get("spatial_se") is not None: args["coords"] = coords held_external = [v for v in _formula_names(str(a.get("formula", "")).split("~")[0]) if v in set(external) or (v.endswith("_numerator") and v[:-10] in set(external))] - if method == "fit_replicate": - fixed_vars = list(held_external) - if a.get("replicate_scope") == "covariates_only": # pre-registered: the outcome is held fixed - fixed_vars += [v for v in _formula_names(a["formula"].split("~")[0]) if v not in fixed_vars] - if fixed_vars: - args["fixed_variables"] = fixed_vars - return models.fit_replicate(a["formula"], frame, replicates=replicates, confidence_level=level, **args) - if method == "fit_eiv": - if held_external: - args["fixed_variables"] = held_external - return models.fit_eiv(a["formula"], frame, replicates=replicates, confidence_level=level, **args) - if method in ("regression_calibration", "simex"): - return getattr(models, method)(a["formula"], frame, confidence_level=level, **args) - if method in ("fay_herriot", "fh_regression"): - y = a["formula"].split("~")[0].strip() - args.setdefault("se_col", f"{y}_se") - return getattr(sae, method)(a["formula"], frame, confidence_level=level, **args) - if method == "correlation_matrix": - return correlation.correlation_matrix(frame, args.pop("variables"), confidence_level=level, - replicates=replicates, correction=correction, **args) - if method == "linear_trend_test": - v, g = args.pop("variable"), str(args.pop("geoid")) - sub = frame[frame["GEOID"] == g].sort_values("Year") - return trends.linear_trend_test(sub["Year"].tolist(), sub[v].tolist(), ses=sub[f"{v}_se"].tolist(), - confidence_level=level, **args) - if method == "batch_z_test": - v1, v2 = args.pop("variable_1"), args.pop("variable_2") - df = frame[["GEOID"]].copy() - for v, tag in ((v1, "1"), (v2, "2")): - df[f"e{tag}"] = frame[v].to_numpy(float) - df[f"m{tag}"] = frame[f"{v}_se"].to_numpy(float) * 1.645 - out = multitest.batch_z_test(df, "e1", "m1", "e2", "m2", confidence_level=level, - correction=correction, **args) - return out - raise ValueError(f"unsupported method {method}") + from .methods import METHODS + if method not in METHODS: + raise ValueError(f"unsupported method {method}") + return METHODS[method].run(a, args, frame, level, replicates, correction, + {"held_external": held_external, "coords": coords}) def _statistic(h: Mapping[str, Any], res: Any) -> Dict[str, Any]: + from .methods import STATISTIC_EXTRACTORS stat = h["statistic"] - if stat == "coefficient": - c = res.coefficients.set_index("term") - if h["term"] not in c.index: - raise KeyError(f"term {h['term']!r} not in model (terms: {list(c.index)})") - row = c.loc[h["term"]] - from ..models import interval_se_column - se_col = interval_se_column(res) # the SE behind p_value and the interval - return {"estimate": float(row["estimate"]), "se": float(row[se_col]), "se_basis": se_col, - "p_two_sided": float(row["p_value"]), - "ci_lower": float(row["ci_lower"]), "ci_upper": float(row["ci_upper"])} - if stat == "correlation": - a, b = h["variables"] - return {"estimate": float(res.matrix.loc[a, b]), "se": None, "p_two_sided": float(res.p_values.loc[a, b]), - "ci_lower": float(res.ci_lower.loc[a, b]), "ci_upper": float(res.ci_upper.loc[a, b])} - if stat == "trend_slope": - return {"estimate": res["slope"], "se": res["se_slope"], "p_two_sided": res["p_value"], - "ci_lower": res["ci_lower"], "ci_upper": res["ci_upper"]} - if stat == "z_difference": - row = res[res["GEOID"] == str(h["geoid"])] - if row.empty: - raise KeyError(f"GEOID {h['geoid']} not in the test frame") - r = row.iloc[0] - diff = float(r.get("difference", r["e1"] - r["e2"])) - return {"estimate": diff, "se": None, "p_two_sided": float(r["p_value"]), "ci_lower": None, "ci_upper": None} - raise ValueError(stat) + if stat not in STATISTIC_EXTRACTORS: + raise ValueError(stat) + return STATISTIC_EXTRACTORS[stat](h, res) def _tag(result: Any, h: str, analysis_id: str, status: str = STATUS_CONFIRMATORY) -> Any: diff --git a/census/manifest/schema.py b/census/manifest/schema.py index 10b209f..92bd821 100644 --- a/census/manifest/schema.py +++ b/census/manifest/schema.py @@ -34,19 +34,8 @@ DATA_SOURCE_KEYS = ("synthetic", "description", "generator") STATUS_EXPLORATORY = "exploratory" -# method -> (required args, statistics it supports) -ANALYSIS_METHODS: Dict[str, Tuple[Tuple[str, ...], Tuple[str, ...]]] = { - "fit_replicate": (("formula",), ("coefficient",)), - "fit_eiv": (("formula",), ("coefficient",)), - "regression_calibration": (("formula", "error_covariates"), ("coefficient",)), - "simex": (("formula", "error_covariates"), ("coefficient",)), - "fh_regression": (("formula",), ("coefficient",)), - "fay_herriot": (("formula",), ("coefficient",)), - "correlation_matrix": (("variables",), ("correlation",)), - "linear_trend_test": (("variable", "geoid"), ("trend_slope",)), - "batch_z_test": (("variable_1", "variable_2"), ("z_difference",)), -} -STATISTICS = ("coefficient", "correlation", "trend_slope", "z_difference") +from .methods import ANALYSIS_METHODS, STATISTICS, REPLICATE_METHODS + EXCLUSION_RULES = ("cv_screen", "geoid_exclude") ADJUSTMENT_TYPES = ("rpp",) RPP_ADJUSTMENT_KEYS = ("type", "variables", "rpp_year", "areas", "rpp_years", "dollar_year", "price_index") @@ -253,4 +242,4 @@ def _formula_names(formula: str) -> List[str]: # Manifest methods that use replicates; each runs with replicate_policy="strict". # (Spatial, multilevel, DiD and PUMS analyses are not manifest methods; if added # they must join this set so confirmatory runs refuse incomplete replicates.) -REPLICATE_METHODS = ("fit_replicate", "correlation_matrix", "fit_eiv") + diff --git a/census/manifest/validate.py b/census/manifest/validate.py index c01d471..11df819 100644 --- a/census/manifest/validate.py +++ b/census/manifest/validate.py @@ -5,6 +5,7 @@ from datetime import datetime from typing import Any, Dict, List, Mapping import numpy as np +from .methods import METHODS from .adjustments import ( _check_adjustments, ) @@ -315,8 +316,7 @@ def _check_analysis(i, a, manifest, names, ext_names, ids, folded_ids, used.append(a[key]) aargs = a.get("args") or {} outcome_name = str(a.get("formula", "")).split("~")[0].strip() - count_outcome = (outcome_name if method == "fit_replicate" and aargs.get("model") == "poisson" - and outcome_name.endswith("_numerator") and outcome_name[:-10] in ext_names else None) + count_outcome = METHODS[method].check("count", locals()) if count_outcome and count_outcome in _formula_names(str(a.get("formula", "")).split("~", 1)[1]): errors.append("external count columns are allowed as the Poisson outcome only") for u in used: @@ -333,86 +333,12 @@ def _check_analysis(i, a, manifest, names, ext_names, ids, folded_ids, elif [u for u in ext_used if u not in outcome] or len(outcome) != 1: errors.append(f"analyses[{i}] ({a.get('id')}): external variables {ext_used} may only be the outcome " "(alone): as covariates their model-based error would go uncorrected") - elif method == "fit_replicate" and (a.get("args") or {}).get("target", "population") == "these_areas": - errors.append(f"analyses[{i}] ({a.get('id')}): an external outcome is held fixed in the refits, so " - "target 'these_areas' (replicate SE alone) is refused; use 'population' or " - "'conservative'") - if method == "fit_replicate": - allowed = {"model", "weights", "exposure", "trials", "cv_threshold", "max_failed_replicates", - "target", "scale", "spatial_se"} - unknown = sorted(set(aargs) - allowed) - if unknown: - errors.append(f"fit_replicate does not take args {unknown}") - glm = aargs.get("model", "ols") - if aargs.get("scale") is not None and (aargs["scale"] not in ("X2", "robust") - or glm not in ("binomial", "poisson")): - errors.append("scale must be 'robust' or 'X2' for binomial or poisson models only") - if aargs.get("scale") == "robust" and aargs.get("spatial_se") is not None: - errors.append("scale 'robust' and spatial_se are alternatives; choose one") - parent = outcome_name if outcome_name in ext_names else count_outcome[:-10] if count_outcome else None - if parent is not None: - if glm not in ("ols", "binomial", "poisson"): - errors.append(f"{glm} not validated for external outcomes (PLAN_ROUND_22 out of scope)") - if glm == "binomial" and aargs.get("trials") != parent + "_denominator": - errors.append(f"external binomial outcome requires trials '{parent}_denominator'; effective trials refused") - if glm == "poisson" and (count_outcome is None or aargs.get("exposure") != parent + "_denominator"): - errors.append(f"external poisson needs {parent}_numerator outcome and exposure '{parent}_denominator'") - if glm in ("binomial", "poisson") and aargs.get("scale") is None and aargs.get("spatial_se") is None: - errors.append("overdispersion would be ignored: set scale 'robust'") - elif (glm in ("binomial", "poisson") and aargs.get("scale") == "X2" - and manifest_hash(manifest) not in LEGACY_X2_MANIFESTS): - errors.append("external count outcome with scale 'X2' refused: Pearson X2 SEs under-cover when " - "areas differ beyond binomial/Poisson chance (validated coverage 0.69-0.79 at " - "nominal 0.90, docs/audits/robust_se_validation.md); set scale 'robust' (HC3)") - for role in ("weights", "exposure", "trials"): - column = aargs.get(role) - valid_count = parent and ((glm == "binomial" and role == "trials") or - (glm == "poisson" and role == "exposure")) and column == parent+"_denominator" - if column is not None and column not in names and column not in FORMULA_ID_COLUMNS and not valid_count: - errors.append(f"fit_replicate {role} {column!r} is not a manifest variable in this role") + else: + METHODS[method].check("external", locals()) + METHODS[method].check("before_spatial", locals()) spatial = (a.get("args") or {}).get("spatial_se") - if spatial is not None: - valid_spatial = (isinstance(spatial, Mapping) and set(spatial) == {"cutoff_km"} - and type(spatial["cutoff_km"]) in (int, float) - and np.isfinite(spatial["cutoff_km"]) and spatial["cutoff_km"] > 0) - if not valid_spatial: - errors.append("spatial_se must be an object with exactly cutoff_km (finite number > 0)") - from ..spatial_hac import SPATIAL_HAC_VALIDATED - if not SPATIAL_HAC_VALIDATED and not manifest.get("exploratory", False): - errors.append("spatial_se failed its validation (docs/audits/spatial_hac_validation.md); exploratory only") - if (a.get("args") or {}).get("cluster"): - errors.append("spatial_se and cluster are mutually exclusive") - if method not in ("fit_eiv", "fit_replicate") or (method == "fit_replicate" and - (a.get("args") or {}).get("model", "ols") not in ("binomial", "poisson")): - errors.append("spatial_se is supported for fit_eiv and fit_replicate binomial/poisson") - if method == "fit_eiv": - eargs = a.get("args") or {} - unknown = sorted(set(eargs) - {"model", "weights", "cv_threshold", "target", "cluster", "cluster_se", "spatial_se"}) - if unknown: - errors.append(f"analyses[{i}] ({a.get('id')}): fit_eiv does not take args {unknown} (allowed: " - "model, weights, cv_threshold, cluster, cluster_se)") - if "cluster_se" in eargs and (eargs["cluster_se"] not in ("cr1", "jackknife") or "cluster" not in eargs): - errors.append(f"analyses[{i}] ({a.get('id')}): fit_eiv args.cluster_se must be 'cr1' or 'jackknife' " - "and needs args.cluster") - if "target" in eargs: - errors.append(f"analyses[{i}] ({a.get('id')}): fit_eiv takes no 'target': its intervals use the " - "sandwich se_total (a measurement-error correction is a claim about the population " - "relationship, not about these areas)") - if "cluster" in eargs: - cl = eargs["cluster"] - from ..models import CLUSTER_PREFIX_RE - mm = CLUSTER_PREFIX_RE.match(str(cl)) if isinstance(cl, str) else None - if not isinstance(cl, str) or not (cl in names or cl in FORMULA_ID_COLUMNS - or (mm and 1 <= int(mm.group(1)) <= 11)): - errors.append(f"analyses[{i}] ({a.get('id')}): fit_eiv args.cluster must be a manifest variable " - "or GEOID.str[:N] (e.g. GEOID.str[:5] for counties of tracts)") - if str(eargs.get("model", "ols")).lower() not in ("ols", "wls"): - errors.append(f"analyses[{i}] ({a.get('id')}): fit_eiv supports model 'ols' or 'wls'") - if str(eargs.get("model", "ols")).lower() == "wls" and not eargs.get("weights"): - errors.append(f"analyses[{i}] ({a.get('id')}): fit_eiv with model 'wls' needs args.weights") - if not census_vre_source(manifest) and a.get("replicates") != "required": - errors.append(f"analyses[{i}] ({a.get('id')}): fit_eiv needs replicates: declare " - "replicates.source 'census_vre' or \"replicates\": \"required\"") + METHODS[method].check("spatial", locals()) + METHODS[method].check("after_spatial", locals()) scope = a.get("replicate_scope", "all") if scope not in REPLICATE_SCOPES: errors.append(f"analyses[{i}] ({a.get('id')}): replicate_scope must be one of {REPLICATE_SCOPES}") diff --git a/tests/test_round21_part2.py b/tests/test_round21_part2.py index 8f69caf..5c14614 100644 --- a/tests/test_round21_part2.py +++ b/tests/test_round21_part2.py @@ -285,7 +285,7 @@ def test_usaleep_education_term(self): class TestWonder(Tmp): def _file(self, text=WONDER): p = os.path.join(self.dir, "export.txt") - with open(p, "w", encoding="utf-8") as f: + with open(p, "w", encoding="utf-8", newline="") as f: # hash the exact bytes on Windows too f.write(text) return p, hashlib.sha256(text.encode()).hexdigest() From bb3c365b23d2a224e9502dc19ab0975fe2cac5b3 Mon Sep 17 00:00:00 2001 From: dickbird <20980982+cafeTechne@users.noreply.github.com> Date: Sun, 15 Nov 2026 21:23:54 -0500 Subject: [PATCH 2/2] Restore census/ to d1e4443: the previous commit swept in unrelated in-progress Round 24 edits from the working tree; this PR is only the one-line test fix --- census/health_sources.py | 2 +- census/manifest/methods_text.py | 32 ++++++++++-- census/manifest/run.py | 69 ++++++++++++++++++++++---- census/manifest/schema.py | 17 +++++-- census/manifest/validate.py | 88 ++++++++++++++++++++++++++++++--- 5 files changed, 185 insertions(+), 23 deletions(-) diff --git a/census/health_sources.py b/census/health_sources.py index d7ee59d..2dd7dae 100644 --- a/census/health_sources.py +++ b/census/health_sources.py @@ -40,7 +40,7 @@ "wonder": ("file", "sha256")} # where an external variable may appear: only as the outcome of these methods (its error is independent # of the ACS sampling error, so it enters the residual; as a covariate it would go uncorrected) -from .manifest.methods import EXTERNAL_OUTCOME_METHODS +EXTERNAL_OUTCOME_METHODS = ("fit_eiv", "fit_replicate", "fh_regression") JOIN_MISSING_LIMIT = 0.05 KINDS = ("direct", "model_based", "partly_model_based") diff --git a/census/manifest/methods_text.py b/census/manifest/methods_text.py index af75378..84858b7 100644 --- a/census/manifest/methods_text.py +++ b/census/manifest/methods_text.py @@ -34,9 +34,35 @@ # Methods text # --------------------------------------------------------------------------- -from .methods import METHODS - -_METHOD_TEXT = {name: spec.text for name, spec in sorted(METHODS.items(), key=lambda item: {'fit_replicate': 0, 'regression_calibration': 1, 'fit_eiv': 2, 'simex': 3, 'fh_regression': 4, 'fay_herriot': 5, 'correlation_matrix': 6, 'linear_trend_test': 7, 'batch_z_test': 8}.get(item[0], len(METHODS))) if spec.text is not None} +_METHOD_TEXT = { + "fit_replicate": ("Coefficients were estimated with {model} regression (statsmodels) on the full-sample " + "estimates; sampling standard errors came from refitting on each of {rep_source}, Var = 4/80 sum (b_r - b)^2, combined with the model standard error as " + "se_total = sqrt(se_model^2 + se_sampling^2). Reported standard errors, intervals and p-values use " + "the {target_se} ({target} target: {estimand}), with normal-theory intervals. Without " + "replicates only model standard errors are reported.", ["vre", "design", "statsmodels"]), + "regression_calibration": ("Covariates measured with known ACS sampling error ({cov}) were corrected by " + "regression calibration with heteroscedastic known error variances; standard " + "errors (se_total, used for normal-theory intervals and p-values) by row bootstrap " + "re-estimating the calibration. Coefficients are {estimand}.", ["carroll"]), + "fit_eiv": ("Coefficients were estimated by moment-corrected least squares ({model}; Fuller 1987) with " + "per-unit sampling-error covariances of every design column and of the outcome, including the " + "outcome-covariate error covariance, estimated from {rep_source} (4/80 sum over replicates). " + "Standard errors are {se_basis} (se_total, used for " + "normal-theory intervals and p-values); naive coefficients are reported alongside. Coefficients " + "are {estimand}.", ["fuller", "vre", "design"]), + "simex": ("Covariates measured with known ACS sampling error ({cov}) were corrected by " + "simulation-extrapolation (SIMEX, quadratic extrapolation) with the measurement-error jackknife " + "variance (se_total, used for normal-theory intervals and p-values). Coefficients are " + "{estimand}.", ["simex", "carroll"]), + "fh_regression": ("The outcome's known sampling variance was modelled with an area-level Fay-Herriot model " + "(GLS coefficients, REML variance component).", ["fh"]), + "fay_herriot": ("Small-area estimates are Fay-Herriot EBLUPs (REML) with Prasad-Rao/Datta-Lahiri MSE.", + ["fh"]), + "correlation_matrix": ("Correlations with Fisher-z intervals{rep}.", ["fisher", "vre"]), + "linear_trend_test": ("Linear trends across non-overlapping 5-year periods were tested by GLS with known " + "sampling variances.", ["testing"]), + "batch_z_test": ("Differences were tested with the Census two-sample z test, SE = MOE/1.645.", ["testing"]), +} def _age_standardized_text(d: Mapping[str, Any]) -> str: diff --git a/census/manifest/run.py b/census/manifest/run.py index cb5c393..1543502 100644 --- a/census/manifest/run.py +++ b/census/manifest/run.py @@ -136,19 +136,70 @@ def _run_analysis(a: Mapping[str, Any], frame: pd.DataFrame, level: float, repli if args.get("spatial_se") is not None: args["coords"] = coords held_external = [v for v in _formula_names(str(a.get("formula", "")).split("~")[0]) if v in set(external) or (v.endswith("_numerator") and v[:-10] in set(external))] - from .methods import METHODS - if method not in METHODS: - raise ValueError(f"unsupported method {method}") - return METHODS[method].run(a, args, frame, level, replicates, correction, - {"held_external": held_external, "coords": coords}) + if method == "fit_replicate": + fixed_vars = list(held_external) + if a.get("replicate_scope") == "covariates_only": # pre-registered: the outcome is held fixed + fixed_vars += [v for v in _formula_names(a["formula"].split("~")[0]) if v not in fixed_vars] + if fixed_vars: + args["fixed_variables"] = fixed_vars + return models.fit_replicate(a["formula"], frame, replicates=replicates, confidence_level=level, **args) + if method == "fit_eiv": + if held_external: + args["fixed_variables"] = held_external + return models.fit_eiv(a["formula"], frame, replicates=replicates, confidence_level=level, **args) + if method in ("regression_calibration", "simex"): + return getattr(models, method)(a["formula"], frame, confidence_level=level, **args) + if method in ("fay_herriot", "fh_regression"): + y = a["formula"].split("~")[0].strip() + args.setdefault("se_col", f"{y}_se") + return getattr(sae, method)(a["formula"], frame, confidence_level=level, **args) + if method == "correlation_matrix": + return correlation.correlation_matrix(frame, args.pop("variables"), confidence_level=level, + replicates=replicates, correction=correction, **args) + if method == "linear_trend_test": + v, g = args.pop("variable"), str(args.pop("geoid")) + sub = frame[frame["GEOID"] == g].sort_values("Year") + return trends.linear_trend_test(sub["Year"].tolist(), sub[v].tolist(), ses=sub[f"{v}_se"].tolist(), + confidence_level=level, **args) + if method == "batch_z_test": + v1, v2 = args.pop("variable_1"), args.pop("variable_2") + df = frame[["GEOID"]].copy() + for v, tag in ((v1, "1"), (v2, "2")): + df[f"e{tag}"] = frame[v].to_numpy(float) + df[f"m{tag}"] = frame[f"{v}_se"].to_numpy(float) * 1.645 + out = multitest.batch_z_test(df, "e1", "m1", "e2", "m2", confidence_level=level, + correction=correction, **args) + return out + raise ValueError(f"unsupported method {method}") def _statistic(h: Mapping[str, Any], res: Any) -> Dict[str, Any]: - from .methods import STATISTIC_EXTRACTORS stat = h["statistic"] - if stat not in STATISTIC_EXTRACTORS: - raise ValueError(stat) - return STATISTIC_EXTRACTORS[stat](h, res) + if stat == "coefficient": + c = res.coefficients.set_index("term") + if h["term"] not in c.index: + raise KeyError(f"term {h['term']!r} not in model (terms: {list(c.index)})") + row = c.loc[h["term"]] + from ..models import interval_se_column + se_col = interval_se_column(res) # the SE behind p_value and the interval + return {"estimate": float(row["estimate"]), "se": float(row[se_col]), "se_basis": se_col, + "p_two_sided": float(row["p_value"]), + "ci_lower": float(row["ci_lower"]), "ci_upper": float(row["ci_upper"])} + if stat == "correlation": + a, b = h["variables"] + return {"estimate": float(res.matrix.loc[a, b]), "se": None, "p_two_sided": float(res.p_values.loc[a, b]), + "ci_lower": float(res.ci_lower.loc[a, b]), "ci_upper": float(res.ci_upper.loc[a, b])} + if stat == "trend_slope": + return {"estimate": res["slope"], "se": res["se_slope"], "p_two_sided": res["p_value"], + "ci_lower": res["ci_lower"], "ci_upper": res["ci_upper"]} + if stat == "z_difference": + row = res[res["GEOID"] == str(h["geoid"])] + if row.empty: + raise KeyError(f"GEOID {h['geoid']} not in the test frame") + r = row.iloc[0] + diff = float(r.get("difference", r["e1"] - r["e2"])) + return {"estimate": diff, "se": None, "p_two_sided": float(r["p_value"]), "ci_lower": None, "ci_upper": None} + raise ValueError(stat) def _tag(result: Any, h: str, analysis_id: str, status: str = STATUS_CONFIRMATORY) -> Any: diff --git a/census/manifest/schema.py b/census/manifest/schema.py index 92bd821..10b209f 100644 --- a/census/manifest/schema.py +++ b/census/manifest/schema.py @@ -34,8 +34,19 @@ DATA_SOURCE_KEYS = ("synthetic", "description", "generator") STATUS_EXPLORATORY = "exploratory" -from .methods import ANALYSIS_METHODS, STATISTICS, REPLICATE_METHODS - +# method -> (required args, statistics it supports) +ANALYSIS_METHODS: Dict[str, Tuple[Tuple[str, ...], Tuple[str, ...]]] = { + "fit_replicate": (("formula",), ("coefficient",)), + "fit_eiv": (("formula",), ("coefficient",)), + "regression_calibration": (("formula", "error_covariates"), ("coefficient",)), + "simex": (("formula", "error_covariates"), ("coefficient",)), + "fh_regression": (("formula",), ("coefficient",)), + "fay_herriot": (("formula",), ("coefficient",)), + "correlation_matrix": (("variables",), ("correlation",)), + "linear_trend_test": (("variable", "geoid"), ("trend_slope",)), + "batch_z_test": (("variable_1", "variable_2"), ("z_difference",)), +} +STATISTICS = ("coefficient", "correlation", "trend_slope", "z_difference") EXCLUSION_RULES = ("cv_screen", "geoid_exclude") ADJUSTMENT_TYPES = ("rpp",) RPP_ADJUSTMENT_KEYS = ("type", "variables", "rpp_year", "areas", "rpp_years", "dollar_year", "price_index") @@ -242,4 +253,4 @@ def _formula_names(formula: str) -> List[str]: # Manifest methods that use replicates; each runs with replicate_policy="strict". # (Spatial, multilevel, DiD and PUMS analyses are not manifest methods; if added # they must join this set so confirmatory runs refuse incomplete replicates.) - +REPLICATE_METHODS = ("fit_replicate", "correlation_matrix", "fit_eiv") diff --git a/census/manifest/validate.py b/census/manifest/validate.py index 11df819..c01d471 100644 --- a/census/manifest/validate.py +++ b/census/manifest/validate.py @@ -5,7 +5,6 @@ from datetime import datetime from typing import Any, Dict, List, Mapping import numpy as np -from .methods import METHODS from .adjustments import ( _check_adjustments, ) @@ -316,7 +315,8 @@ def _check_analysis(i, a, manifest, names, ext_names, ids, folded_ids, used.append(a[key]) aargs = a.get("args") or {} outcome_name = str(a.get("formula", "")).split("~")[0].strip() - count_outcome = METHODS[method].check("count", locals()) + count_outcome = (outcome_name if method == "fit_replicate" and aargs.get("model") == "poisson" + and outcome_name.endswith("_numerator") and outcome_name[:-10] in ext_names else None) if count_outcome and count_outcome in _formula_names(str(a.get("formula", "")).split("~", 1)[1]): errors.append("external count columns are allowed as the Poisson outcome only") for u in used: @@ -333,12 +333,86 @@ def _check_analysis(i, a, manifest, names, ext_names, ids, folded_ids, elif [u for u in ext_used if u not in outcome] or len(outcome) != 1: errors.append(f"analyses[{i}] ({a.get('id')}): external variables {ext_used} may only be the outcome " "(alone): as covariates their model-based error would go uncorrected") - else: - METHODS[method].check("external", locals()) - METHODS[method].check("before_spatial", locals()) + elif method == "fit_replicate" and (a.get("args") or {}).get("target", "population") == "these_areas": + errors.append(f"analyses[{i}] ({a.get('id')}): an external outcome is held fixed in the refits, so " + "target 'these_areas' (replicate SE alone) is refused; use 'population' or " + "'conservative'") + if method == "fit_replicate": + allowed = {"model", "weights", "exposure", "trials", "cv_threshold", "max_failed_replicates", + "target", "scale", "spatial_se"} + unknown = sorted(set(aargs) - allowed) + if unknown: + errors.append(f"fit_replicate does not take args {unknown}") + glm = aargs.get("model", "ols") + if aargs.get("scale") is not None and (aargs["scale"] not in ("X2", "robust") + or glm not in ("binomial", "poisson")): + errors.append("scale must be 'robust' or 'X2' for binomial or poisson models only") + if aargs.get("scale") == "robust" and aargs.get("spatial_se") is not None: + errors.append("scale 'robust' and spatial_se are alternatives; choose one") + parent = outcome_name if outcome_name in ext_names else count_outcome[:-10] if count_outcome else None + if parent is not None: + if glm not in ("ols", "binomial", "poisson"): + errors.append(f"{glm} not validated for external outcomes (PLAN_ROUND_22 out of scope)") + if glm == "binomial" and aargs.get("trials") != parent + "_denominator": + errors.append(f"external binomial outcome requires trials '{parent}_denominator'; effective trials refused") + if glm == "poisson" and (count_outcome is None or aargs.get("exposure") != parent + "_denominator"): + errors.append(f"external poisson needs {parent}_numerator outcome and exposure '{parent}_denominator'") + if glm in ("binomial", "poisson") and aargs.get("scale") is None and aargs.get("spatial_se") is None: + errors.append("overdispersion would be ignored: set scale 'robust'") + elif (glm in ("binomial", "poisson") and aargs.get("scale") == "X2" + and manifest_hash(manifest) not in LEGACY_X2_MANIFESTS): + errors.append("external count outcome with scale 'X2' refused: Pearson X2 SEs under-cover when " + "areas differ beyond binomial/Poisson chance (validated coverage 0.69-0.79 at " + "nominal 0.90, docs/audits/robust_se_validation.md); set scale 'robust' (HC3)") + for role in ("weights", "exposure", "trials"): + column = aargs.get(role) + valid_count = parent and ((glm == "binomial" and role == "trials") or + (glm == "poisson" and role == "exposure")) and column == parent+"_denominator" + if column is not None and column not in names and column not in FORMULA_ID_COLUMNS and not valid_count: + errors.append(f"fit_replicate {role} {column!r} is not a manifest variable in this role") spatial = (a.get("args") or {}).get("spatial_se") - METHODS[method].check("spatial", locals()) - METHODS[method].check("after_spatial", locals()) + if spatial is not None: + valid_spatial = (isinstance(spatial, Mapping) and set(spatial) == {"cutoff_km"} + and type(spatial["cutoff_km"]) in (int, float) + and np.isfinite(spatial["cutoff_km"]) and spatial["cutoff_km"] > 0) + if not valid_spatial: + errors.append("spatial_se must be an object with exactly cutoff_km (finite number > 0)") + from ..spatial_hac import SPATIAL_HAC_VALIDATED + if not SPATIAL_HAC_VALIDATED and not manifest.get("exploratory", False): + errors.append("spatial_se failed its validation (docs/audits/spatial_hac_validation.md); exploratory only") + if (a.get("args") or {}).get("cluster"): + errors.append("spatial_se and cluster are mutually exclusive") + if method not in ("fit_eiv", "fit_replicate") or (method == "fit_replicate" and + (a.get("args") or {}).get("model", "ols") not in ("binomial", "poisson")): + errors.append("spatial_se is supported for fit_eiv and fit_replicate binomial/poisson") + if method == "fit_eiv": + eargs = a.get("args") or {} + unknown = sorted(set(eargs) - {"model", "weights", "cv_threshold", "target", "cluster", "cluster_se", "spatial_se"}) + if unknown: + errors.append(f"analyses[{i}] ({a.get('id')}): fit_eiv does not take args {unknown} (allowed: " + "model, weights, cv_threshold, cluster, cluster_se)") + if "cluster_se" in eargs and (eargs["cluster_se"] not in ("cr1", "jackknife") or "cluster" not in eargs): + errors.append(f"analyses[{i}] ({a.get('id')}): fit_eiv args.cluster_se must be 'cr1' or 'jackknife' " + "and needs args.cluster") + if "target" in eargs: + errors.append(f"analyses[{i}] ({a.get('id')}): fit_eiv takes no 'target': its intervals use the " + "sandwich se_total (a measurement-error correction is a claim about the population " + "relationship, not about these areas)") + if "cluster" in eargs: + cl = eargs["cluster"] + from ..models import CLUSTER_PREFIX_RE + mm = CLUSTER_PREFIX_RE.match(str(cl)) if isinstance(cl, str) else None + if not isinstance(cl, str) or not (cl in names or cl in FORMULA_ID_COLUMNS + or (mm and 1 <= int(mm.group(1)) <= 11)): + errors.append(f"analyses[{i}] ({a.get('id')}): fit_eiv args.cluster must be a manifest variable " + "or GEOID.str[:N] (e.g. GEOID.str[:5] for counties of tracts)") + if str(eargs.get("model", "ols")).lower() not in ("ols", "wls"): + errors.append(f"analyses[{i}] ({a.get('id')}): fit_eiv supports model 'ols' or 'wls'") + if str(eargs.get("model", "ols")).lower() == "wls" and not eargs.get("weights"): + errors.append(f"analyses[{i}] ({a.get('id')}): fit_eiv with model 'wls' needs args.weights") + if not census_vre_source(manifest) and a.get("replicates") != "required": + errors.append(f"analyses[{i}] ({a.get('id')}): fit_eiv needs replicates: declare " + "replicates.source 'census_vre' or \"replicates\": \"required\"") scope = a.get("replicate_scope", "all") if scope not in REPLICATE_SCOPES: errors.append(f"analyses[{i}] ({a.get('id')}): replicate_scope must be one of {REPLICATE_SCOPES}")