Synthetic Data Risk & Utility Assessment Template

{ "Title": "Synthetic Data Risk & Utility Assessment", "IntroductionHtml": "

This assessment helps teams decide whether synthetic data (alone or combined with other techniques) is appropriate for a given dataset and use case. Use it to document use-case fit, privacy goals, regulatory constraints, utility priorities, planned tests, operational considerations, and a clear recommendation with next steps. Save the form and iterate as you run pilots and tests.

", "SubmitLabel": "Save Assessment", "SuccessMessage": "Assessment saved. Use the results to plan pilots, tests, and governance steps.", "DataType": "SyntheticDataRiskUtilityAssessment_v1", "SchemaVersion": 1, "Fields": [ { "Key": "assessor_name", "Label": "Assessor name", "FieldType": "text", "HelpText": "Who is completing this assessment?", "Required": true }, { "Key": "assessment_date", "Label": "Date", "FieldType": "text", "HelpText": "YYYY-MM-DD or free text date", "Required": true }, { "Key": "project_name", "Label": "Project / dataset name", "FieldType": "text", "HelpText": "Short name that identifies the dataset, model, or initiative", "Required": true }, { "Key": "dataset_description", "Label": "Dataset description", "FieldType": "textarea", "HelpText": "What does the dataset contain? Key tables, record counts, time span, and sensitive attributes.", "Required": true }, { "Key": "intended_use_cases", "Label": "Intended use cases (select all that apply)", "FieldType": "checkbox", "Options": [ {"Value": "model_training", "Label": "Model training"}, {"Value": "share_with_partners", "Label": "Share with partners / vendors"}, {"Value": "public_release", "Label": "Public release"}, {"Value": "testing_validation", "Label": "Testing / validation"}, {"Value": "analytics_exploration", "Label": "Exploratory analytics / research"}, {"Value": "benchmarking", "Label": "Benchmarking"}, {"Value": "other", "Label": "Other"} ], "HelpText": "How will the data be used? Different uses change acceptable fidelity and risk thresholds.", "Required": true }, { "Key": "primary_analytic_tasks", "Label": "Primary analytic tasks (select all that apply)", "FieldType": "checkbox", "Options": [ {"Value": "classification", "Label": "Classification"}, {"Value": "regression", "Label": "Regression"}, {"Value": "clustering", "Label": "Clustering"}, {"Value": "anomaly_detection", "Label": "Anomaly detection"}, {"Value": "time_series", "Label": "Time series / forecasting"}, {"Value": "descriptive_stats", "Label": "Descriptive statistics / reporting"}, {"Value": "causal_inference", "Label": "Causal inference / experimentation analysis"}, {"Value": "other", "Label": "Other"} ], "HelpText": "Select the analytic tasks most important for success; use these later to define fidelity tests.", "Required": true }, { "Key": "sensitivity_level", "Label": "Data sensitivity level", "FieldType": "select", "Options": [ {"Value": "public", "Label": "Public"}, {"Value": "internal", "Label": "Internal"}, {"Value": "confidential", "Label": "Confidential (personal / business-sensitive)"}, {"Value": "highly_confidential", "Label": "Highly confidential (PII, health, financial)"} ], "HelpText": "Estimate sensitivity to guide privacy requirements.", "Required": true }, { "Key": "regulatory_constraints", "Label": "Regulatory / contractual constraints (select all that apply)", "FieldType": "checkbox", "Options": [ {"Value": "gdpr", "Label": "GDPR"}, {"Value": "hipaa", "Label": "HIPAA"}, {"Value": "ccpa", "Label": "CCPA or similar"}, {"Value": "sector_specific", "Label": "Sector-specific rules (finance, telecom, etc.)"}, {"Value": "contractual_restrictions", "Label": "Contractual restrictions with partners"}, {"Value": "none_identified", "Label": "None identified"}, {"Value": "other", "Label": "Other"} ], "HelpText": "Identify laws or contracts that affect allowable transformations and sharing.", "Required": true }, { "Key": "privacy_goals", "Label": "Primary privacy goals (select all that apply)", "FieldType": "checkbox", "Options": [ {"Value": "prevent_reid", "Label": "Prevent re-identification"}, {"Value": "limit_attribute_disclosure", "Label": "Limit attribute disclosure"}, {"Value": "prevent_membership_inference", "Label": "Prevent membership inference"}, {"Value": "meet_regulatory_minimums", "Label": "Meet regulatory minimums"}, {"Value": "reduce_risk_below_threshold", "Label": "Reduce overall risk below internal threshold"} ], "HelpText": "These goals will determine acceptable techniques and evaluation rigor.", "Required": true }, { "Key": "techniques_considered", "Label": "Privacy-preserving techniques under consideration", "FieldType": "checkbox", "Options": [ {"Value": "synthetic_generation", "Label": "Synthetic data generation"}, {"Value": "k_anonymity", "Label": "k-anonymity / generalization"}, {"Value": "l_diversity_t_closeness", "Label": "l-diversity / t-closeness"}, {"Value": "differential_privacy", "Label": "Differential privacy (DP)"}, {"Value": "masking_tokenization", "Label": "Masking / tokenization / pseudonymization"}, {"Value": "sampling_subsetting", "Label": "Sampling / subsetting"}, {"Value": "hybrid", "Label": "Hybrid approach (synthetic + anonymization)"}, {"Value": "other", "Label": "Other"} ], "HelpText": "Select all methods you're evaluating. Later fields collect evaluation plans.", "Required": true }, { "Key": "estimated_reidentification_risk", "Label": "Estimated baseline re-identification risk (1 = very low, 5 = very high)", "FieldType": "scale", "HelpText": "Estimate risk before transformations based on auxiliary data availability and uniqueness of records.", "Required": true }, { "Key": "privacy_likelihood", "Label": "Likelihood of privacy breach without mitigation (1-5)", "FieldType": "scale", "HelpText": "How likely is an attack or accidental disclosure?", "Required": true }, { "Key": "privacy_impact", "Label": "Impact of privacy breach (1 = low, 5 = severe)", "FieldType": "scale", "HelpText": "Consider reputational, regulatory, individual harm, and contractual consequences.", "Required": true }, { "Key": "utility_priority_classification", "Label": "Importance: Classification / predictive accuracy (1 low – 5 high)", "FieldType": "scale", "Required": false }, { "Key": "utility_priority_regression", "Label": "Importance: Regression / numeric accuracy (1 low – 5 high)", "FieldType": "scale", "Required": false }, { "Key": "utility_priority_distributional", "Label": "Importance: Preserving distributions and descriptive stats (1 low – 5 high)", "FieldType": "scale", "Required": false }, { "Key": "utility_priority_correlation", "Label": "Importance: Feature relationships / correlations (1 low – 5 high)", "FieldType": "scale", "Required": false }, { "Key": "utility_priority_rare_events", "Label": "Importance: Rare-event / tail behavior (1 low – 5 high)", "FieldType": "scale", "Required": false }, { "Key": "fidelity_acceptance_criteria", "Label": "Fidelity acceptance criteria", "FieldType": "textarea", "HelpText": "Define numeric thresholds that would be acceptable for key tasks (examples: max AUC drop = 0.02, MAE increase <= 10%, JS divergence < 0.05 for key features). Be specific per task.", "Required": true }, { "Key": "planned_evaluation_tests", "Label": "Planned fidelity tests (select all that apply)", "FieldType": "checkbox", "Options": [ {"Value": "model_metric_comparison", "Label": "Compare model metrics on real vs synthetic (AUC, accuracy, MAE)"}, {"Value": "distributional_tests", "Label": "Distributional distance tests (KS, JS, Wasserstein)"}, {"Value": "correlation_matrix_check", "Label": "Correlation / covariance preservation checks"}, {"Value": "rare_event_coverage", "Label": "Rare event / tail representation tests"}, {"Value": "temporal_consistency", "Label": "Temporal / sequential consistency tests"}, {"Value": "downstream_e2e_tests", "Label": "End-to-end downstream task tests (business KPIs)"}, {"Value": "other", "Label": "Other"} ], "HelpText": "Choose tests that map to your primary analytic tasks.", "Required": true }, { "Key": "planned_privacy_tests", "Label": "Planned adversarial / privacy tests (select all that apply)", "FieldType": "checkbox", "Options": [ {"Value": "membership_inference", "Label": "Membership inference tests"}, {"Value": "linkage_attack", "Label": "Linkage / record linkage attacks using auxiliary data"}, {"Value": "attribute_inference", "Label": "Attribute inference attacks"}, {"Value": "reidentification_simulation", "Label": "Re-identification simulation / risk scoring"}, {"Value": "privacy_budget_analysis", "Label": "Privacy budget and DP parameter checks (if DP used)"}, {"Value": "other", "Label": "Other"} ], "HelpText": "Adversarial tests help validate claimed guarantees.", "Required": true }, { "Key": "operational_lineage_access", "Label": "Operational considerations: do you have lineage and access controls for the original data?", "FieldType": "radio", "Options": [ {"Value": "yes", "Label": "Yes"}, {"Value": "partial", "Label": "Partial"}, {"Value": "no", "Label": "No"} ], "HelpText": "Lineage, provenance and access controls are important for safe generation and accountable reuse.", "Required": true }, { "Key": "generator_tooling", "Label": "Planned generator / tool name", "FieldType": "text", "HelpText": "Name the synthetic generator, library, or service you plan to use (or 'TBD').", "Required": false }, { "Key": "privacy_budget_estimate_epsilon", "Label": "If using differential privacy, planned privacy budget (epsilon) or range", "FieldType": "text", "HelpText": "Enter epsilon or note 'N/A' if DP is not used. Lower epsilon = stronger privacy but lower utility.", "Required": false }, { "Key": "estimated_cost_hours", "Label": "Estimated effort for pilot (hours)", "FieldType": "number", "HelpText": "Rough estimate for running a fidelity & privacy pilot (engineering + evaluation).", "Required": false }, { "Key": "stakeholders_notified", "Label": "Key stakeholders to notify or consult", "FieldType": "textarea", "HelpText": "List privacy officer, legal, data owners, security, and business owners.", "Required": false }, { "Key": "decision_recommendation", "Label": "Preliminary recommendation (select one)", "FieldType": "select", "Options": [ {"Value": "proceed_synthetic_pilot", "Label": "Proceed with synthetic-data pilot (closed environment)"}, {"Value": "synthetic_with_hybrid", "Label": "Use synthetic + anonymization hybrid"}, {"Value": "anonymization_only", "Label": "Prefer strict anonymization / k-anonymity approach"}, {"Value": "limit_sharing_use_real_small_sample", "Label": "Limit sharing; use smaller real-data sample under controls"}, {"Value": "do_not_share", "Label": "Do not share / do not use synthetic for this purpose"} ], "HelpText": "Choose the option that best fits current risk/utility trade-offs. Use tests to validate or change this recommendation.", "Required": true }, { "Key": "recommendation_justification", "Label": "Recommendation justification", "FieldType": "textarea", "HelpText": "Briefly explain why this option was chosen, referencing risks, utility priorities, and acceptance criteria.", "Required": true }, { "Key": "next_actions", "Label": "Next actions (select all that apply)", "FieldType": "checkbox", "Options": [ {"Value": "run_fidelity_pilot", "Label": "Run fidelity pilot comparing real vs synthetic"}, {"Value": "run_privacy_attacks", "Label": "Run adversarial privacy tests"}, {"Value": "consult_privacy_officer", "Label": "Consult privacy officer / legal"}, {"Value": "choose_generator", "Label": "Select and configure generator tool / library"}, {"Value": "define_monitoring", "Label": "Define post-release monitoring and logging"}, {"Value": "establish_governance", "Label": "Establish data governance and reuse policies"} ], "HelpText": "Practical next steps to validate and operationalize the recommendation.", "Required": true }, { "Key": "final_confidence", "Label": "Final confidence in recommendation (1 low – 5 high)", "FieldType": "scale", "HelpText": "How confident are you in this recommendation given current information?", "Required": true }, { "Key": "notes_and_attachments_reference", "Label": "Notes, references, and attachments (reference links or internal ticket IDs)", "FieldType": "textarea", "HelpText": "Link to notebooks, evaluation results, security tickets, or storage location of artifacts.", "Required": false } ] }