Experiment & Causal Study Design Workbook

{ "Title": "Experiment & Causal Study Design Workbook", "IntroductionHtml": "

Use this guided workbook to pre-specify a defensible experiment or causal study that answers a practical decision question. Pre-specification reduces researcher degrees of freedom, avoids selective reporting, and speeds operational adoption. Fill each section with the best available information and save a stable plan you can reference during analysis and rollout.

", "SubmitLabel": "Save plan", "SuccessMessage": "Your experiment plan has been saved. You can return to update it, export it for pre-registration, or share it with reviewers.", "DataType": "ExperimentDesignWorkbook", "SchemaVersion": "1.0", "Fields": [ { "Key": "problem_context", "Label": "Problem & decision context", "FieldType": "textarea", "Required": true, "HelpText": "Describe the operational problem, the decision that the experiment will inform, affected stakeholders, timing constraints, and why this question matters. Keep it concise (3–6 sentences)." }, { "Key": "primary_decision", "Label": "Primary decision or action to be taken if the intervention 'works'", "FieldType": "text", "Required": true, "HelpText": "Example: 'Roll out change to all customers', 'Delay launch', 'Increase inspection frequency'. Specify who decides and the success threshold." }, { "Key": "hypothesis", "Label": "Hypothesis (If — then — because)", "FieldType": "textarea", "Required": true, "HelpText": "Write a measurable hypothesis: 'If we [intervention], then [measurable outcome will change] because [mechanism].' Example: 'If we add a confirmation step, then error rate will drop by X% because people notice mistakes.'" }, { "Key": "intervention_description", "Label": "Intervention(s) / treatment arms", "FieldType": "textarea", "Required": true, "HelpText": "List each arm clearly and what users/units experience in each. Include any implementation constraints, rollout windows, and dependencies." }, { "Key": "unit_of_randomization", "Label": "Unit of randomization", "FieldType": "select", "Required": true, "HelpText": "Choose the level at which assignment will occur. This affects sample size and inference.", "Options": [ {"Value": "individual", "Label": "Individual / user"}, {"Value": "account", "Label": "Account / customer"}, {"Value": "day", "Label": "Day / time window"}, {"Value": "cluster", "Label": "Cluster (e.g., store, site)"}, {"Value": "other", "Label": "Other (describe below)"} ] }, { "Key": "randomization_method", "Label": "Randomization and assignment plan", "FieldType": "textarea", "Required": true, "HelpText": "Describe how you'll assign units to arms (simple random, stratified, block randomization, geo-based, deterministic rule). Mention any blocking/stratification variables and implementation details." }, { "Key": "primary_outcome", "Label": "Primary outcome metric (name)", "FieldType": "text", "Required": true, "HelpText": "Name the single primary metric that will decide success. Use a clear operational definition." }, { "Key": "primary_outcome_definition", "Label": "Primary outcome: precise definition and measurement plan", "FieldType": "textarea", "Required": true, "HelpText": "Define numerator/denominator, time window, data source, units, smoothing or aggregation, and any filters. Example: '30-day retention = % users with any activity between day 1 and day 30 post-signup, measured in production DB table X.'" }, { "Key": "guardrail_metrics", "Label": "Guardrail / safety metrics", "FieldType": "textarea", "Required": false, "HelpText": "List metrics to monitor for unintended harms (e.g., error rates, complaint volume, latency, cost). Define thresholds that would pause or stop the experiment." }, { "Key": "baseline_rate", "Label": "Baseline rate / mean for primary outcome (estimate)", "FieldType": "number", "Required": false, "HelpText": "Enter the best available baseline estimate (proportion or mean). If unknown, note how you'll estimate it." }, { "Key": "min_detectable_effect", "Label": "Minimum detectable effect (MDE) you care about", "FieldType": "number", "Required": false, "HelpText": "Enter the smallest practical effect size (absolute or relative) that would change the decision. Be realistic: smaller MDEs require much larger sample sizes." }, { "Key": "alpha", "Label": "Significance level (alpha)", "FieldType": "number", "Required": true, "HelpText": "Common defaults: 0.05. Choose lower if multiple comparisons are expected." }, { "Key": "power", "Label": "Desired statistical power", "FieldType": "number", "Required": true, "HelpText": "Common default: 0.8 (80%). Higher power increases sample requirements." }, { "Key": "sample_size_plan", "Label": "Sample size and power calculation summary", "FieldType": "textarea", "Required": true, "HelpText": "Record the sample size per arm, calculation method, assumptions (baseline, MDE, alpha, power, variance), clustering or ICC if applicable, and the calculator or script used (link or file name). If you will run a pilot, describe it here." }, { "Key": "sample_size_attachment", "Label": "Optional: Link or note for attached sample-size calculation", "FieldType": "text", "Required": false, "HelpText": "Provide a link to a spreadsheet, script, or internal tool, or paste the calculation reference." }, { "Key": "stopping_rules", "Label": "Stopping rules and interim checks", "FieldType": "textarea", "Required": true, "HelpText": "Specify if you'll do interim analyses, what triggers pausing or stopping (efficacy, futility, safety), who reviews interim results, and the statistical correction or alpha-spending plan. If no interim analyses, state that explicitly." }, { "Key": "ethics_and_risk", "Label": "Ethics, privacy, and safety considerations", "FieldType": "textarea", "Required": true, "HelpText": "List potential risks to participants, data privacy controls, informed consent if required, regulatory/IRB review status, and mitigation steps. Note any vulnerable groups affected." }, { "Key": "analysis_model", "Label": "Primary analysis method / model", "FieldType": "select", "Required": true, "HelpText": "Choose the principal statistical test or model you'll use for the primary outcome. Specify link function, adjustments, and software or script locations.", "Options": [ {"Value": "difference_means_ttest", "Label": "Difference in means / t-test"}, {"Value": "proportion_test", "Label": "Difference in proportions / chi-square or z-test"}, {"Value": "regression", "Label": "Regression adjustment (OLS / logistic / Poisson)"}, {"Value": "survival", "Label": "Survival / time-to-event model"}, {"Value": "other", "Label": "Other (describe)"} ] }, { "Key": "analysis_covariates", "Label": "Pre-specified covariates and subgroup analyses", "FieldType": "textarea", "Required": false, "HelpText": "List covariates for adjusted analyses and the exact subgroups you will test. For subgroups, specify if tests are exploratory and how you'll report multiple comparisons." }, { "Key": "missing_data_plan", "Label": "Missing data and data quality plan", "FieldType": "textarea", "Required": true, "HelpText": "Describe how you'll handle missing outcomes (imputation, complete-case), data validation steps, and who owns data quality during the experiment." }, { "Key": "multiple_comparisons", "Label": "Multiple comparisons / multiplicity plan", "FieldType": "select", "Required": true, "HelpText": "State whether you'll adjust for multiple tests (Bonferroni, Holm, FDR, pre-specify primary endpoint only, etc.).", "Options": [ {"Value": "no_adjustment_primary_only", "Label": "No adjustment (primary endpoint only)"}, {"Value": "bonferroni", "Label": "Bonferroni or similar"}, {"Value": "fdr", "Label": "False Discovery Rate (FDR)"}, {"Value": "other", "Label": "Other (describe)"} ] }, { "Key": "rollout_plan", "Label": "Operational rollout and monitoring plan", "FieldType": "textarea", "Required": true, "HelpText": "Explain how you'll deploy the experiment, monitor technical health, observe guardrail metrics, and who will receive alerts. Include dashboards or logs to watch." }, { "Key": "monitoring_thresholds", "Label": "Key monitoring thresholds and owners", "FieldType": "textarea", "Required": false, "HelpText": "List thresholds that trigger investigation (e.g., error rate > X%, latency > Y ms) and the responsible person or team." }, { "Key": "analysis_timeline", "Label": "Planned analysis timeline and publication", "FieldType": "text", "Required": true, "HelpText": "When will you lock data and run the final analysis? Will results be communicated in a post-mortem or pre-registered report?" }, { "Key": "result_document_template", "Label": "Result summary and decision template (fill after analysis)", "FieldType": "textarea", "Required": false, "HelpText": "Use this template after analysis: observed effect and CI, p-value, direction relative to MDE, guardrail outcomes, robustness checks, recommended action (adopt / iterate / stop), owner and timeline." }, { "Key": "decision_options", "Label": "Pre-specified decision rules", "FieldType": "textarea", "Required": true, "HelpText": "Describe precise decision criteria (e.g., 'Adopt if estimated lift >= MDE and no guardrail exceeded; Iterate if effect positive but CI includes zero; Stop if negative effect or guardrail threshold exceeded')." }, { "Key": "owners_and_contacts", "Label": "Experiment owner(s) and contact information", "FieldType": "textarea", "Required": true, "HelpText": "List primary owner, analytics lead, engineering contacts, and governance reviewers. Include names, roles, and emails." }, { "Key": "notes_and_assumptions", "Label": "Key assumptions, known risks, and open questions", "FieldType": "textarea", "Required": false, "HelpText": "Record assumptions that could invalidate inference (noncompliance, interference, seasonality) and how you'll check them." } ] }