Synthetic Data & Privacy-Preserving Methods Evaluation Checklist

{ "Title": "Synthetic Data & Privacy-Preserving Methods Evaluation", "IntroductionHtml": "

Use this guided evaluation to decide whether a privacy-preserving approach (synthetic data, anonymization, differential privacy, or hybrid methods) is appropriate for a dataset and to record the trade-offs, tests, governance approvals, and next steps. Save the results to retain an auditable decision record and help teams validate real-world utility before relying on synthetic-only training or sharing.

", "SubmitLabel": "Save Evaluation", "SuccessMessage": "Evaluation saved. You can return to update results or export the record for governance review.", "DataType": "SyntheticDataEvaluation", "SchemaVersion": "1.0", "Fields": [ { "Key": "evaluation_date", "FieldType": "text", "Label": "Evaluation date", "HelpText": "YYYY-MM-DD or free-text date." }, { "Key": "objective", "FieldType": "textarea", "Label": "Objective for synthetic/data-sharing", "HelpText": "Describe the measurable purpose (e.g., model training, analytics testing, partner sharing) and success criteria. Be specific about which analyses or model behaviors must be preserved.", "Required": true }, { "Key": "data_sensitivity", "FieldType": "select", "Label": "Data sensitivity / classification", "HelpText": "Choose the most appropriate classification for this dataset according to your policies.", "Options": [ { "Value": "public", "Label": "Public" }, { "Value": "internal", "Label": "Internal" }, { "Value": "confidential", "Label": "Confidential" }, { "Value": "highly_confidential", "Label": "Highly confidential / regulated (PII/PHI)" } ], "Required": true }, { "Key": "policy_aligned", "FieldType": "yesno", "Label": "Is dataset handling aligned with internal data classification and legal/privacy policy?", "HelpText": "Confirm classification, retention, and permitted uses are documented and approved." }, { "Key": "privacy_approaches", "FieldType": "checkbox", "Label": "Privacy-preserving approaches under consideration", "HelpText": "Check all that apply. If hybrid, describe below.", "Options": [ { "Value": "synthetic_generation", "Label": "Synthetic data generation" }, { "Value": "anonymization", "Label": "Anonymization / pseudonymization" }, { "Value": "k_anonymity", "Label": "k-anonymity / l-diversity / t-closeness techniques" }, { "Value": "differential_privacy", "Label": "Differential privacy (DP)" }, { "Value": "tokenization", "Label": "Tokenization" }, { "Value": "hybrid", "Label": "Hybrid approach (mix of methods)" }, { "Value": "other", "Label": "Other (describe below)" } ] }, { "Key": "privacy_approach_details", "FieldType": "textarea", "Label": "Details on selected approach(es)", "HelpText": "Explain generator type, anonymization rules, DP mechanism, or hybrid design. Include links to implementation repo or config if available." }, { "Key": "privacy_metric_selected", "FieldType": "checkbox", "Label": "Privacy metrics chosen for evaluation", "HelpText": "Select metrics you will measure to assess privacy guarantees and re-identification risk.", "Options": [ { "Value": "k_anonymity_metric", "Label": "k-anonymity (report k)" }, { "Value": "differential_privacy_metric", "Label": "Differential privacy (report epsilon, delta)" }, { "Value": "reid_simulation", "Label": "Empirical re-identification simulations / linkage attacks" }, { "Value": "distance_metrics", "Label": "Record-level distance / nearest neighbor checks" }, { "Value": "attribute_disclosure", "Label": "Attribute disclosure risk tests" }, { "Value": "other_metric", "Label": "Other (describe below)" } ] }, { "Key": "dp_epsilon", "FieldType": "number", "Label": "Differential privacy epsilon (if applicable)", "HelpText": "Enter numeric epsilon value used or planned. Leave blank if DP not used. Note: smaller epsilon indicates stronger privacy but lower utility." }, { "Key": "reid_risk_score", "FieldType": "scale", "Label": "Empirical re-identification risk (team estimate)", "HelpText": "1 = negligible, 5 = high. Attach details in Known limitations and test results.", "ScaleMin": 1, "ScaleMax": 5 }, { "Key": "utility_tests", "FieldType": "checkbox", "Label": "Utility tests planned / executed", "HelpText": "Select the utility tests used to verify analytical value remains acceptable.", "Options": [ { "Value": "statistical_similarity", "Label": "Statistical similarity (marginals, moments)" }, { "Value": "distribution_checks", "Label": "Feature distribution and joint-distribution checks" }, { "Value": "correlation_checks", "Label": "Correlation / dependency preservation" }, { "Value": "downstream_model_perf", "Label": "Downstream model performance comparison" }, { "Value": "aggregate_queries", "Label": "Aggregate query / dashboard comparison" }, { "Value": "edge_case_coverage", "Label": "Edge-case and rare-event coverage" }, { "Value": "time_series_behaviour", "Label": "Time-series behaviour and sequence integrity" }, { "Value": "other_utility_test", "Label": "Other (describe below)" } ] }, { "Key": "utility_test_results", "FieldType": "textarea", "Label": "Summary of utility test results", "HelpText": "Summarize statistical differences, model metric deltas (AUC, accuracy, MAPE, etc.), and any observed failures or regime shifts. Include links to detailed test reports." }, { "Key": "downstream_performance_delta_pct", "FieldType": "number", "Label": "Representative downstream performance delta (%)", "HelpText": "Enter percent change (positive or negative) between models trained on real vs synthetic data for key metrics. Use absolute percent where possible." }, { "Key": "tooling_compatibility", "FieldType": "checkbox", "Label": "Tooling and format compatibility", "HelpText": "Which tools, environments, or pipelines will this synthetic/anonymized dataset need to work with?", "Options": [ { "Value": "python_pandas", "Label": "Python / pandas" }, { "Value": "spark", "Label": "Spark / PySpark" }, { "Value": "sql_data_warehouse", "Label": "SQL / Data warehouse" }, { "Value": "ml_frameworks", "Label": "ML frameworks (scikit-learn, TensorFlow, PyTorch)" }, { "Value": "external_partners", "Label": "External partner systems / APIs" }, { "Value": "other_tooling", "Label": "Other (describe below)" } ] }, { "Key": "governance_approvals_obtained", "FieldType": "yesno", "Label": "Have required governance and legal approvals been obtained?", "HelpText": "Includes Data Protection Officer, Legal, Information Security, and business data owner approvals as required." }, { "Key": "approver_names_and_links", "FieldType": "textarea", "Label": "Approver names, roles, and links to approval artifacts", "HelpText": "List people who approved and where approvals are recorded (tickets, signed forms, policy records)." }, { "Key": "release_controls", "FieldType": "checkbox", "Label": "Planned release controls", "HelpText": "Select controls that will accompany data release.", "Options": [ { "Value": "access_control", "Label": "Access controls / RBAC" }, { "Value": "audit_logging", "Label": "Audit logging and usage monitoring" }, { "Value": "usage_agreements", "Label": "Data use agreements / contracts" }, { "Value": "retention_limits", "Label": "Retention and deletion limits" }, { "Value": "versioning", "Label": "Versioning and change logs" }, { "Value": "monitoring_alerts", "Label": "Monitoring and alerts for anomalous queries" } ] }, { "Key": "release_notes_and_controls_detail", "FieldType": "textarea", "Label": "Details on release controls and distribution plan", "HelpText": "Describe how the data will be shared, with whom, and any contractual or technical safeguards." }, { "Key": "known_limitations", "FieldType": "textarea", "Label": "Known limitations, caveats, and unresolved risks", "HelpText": "Be explicit about any analytical weaknesses, rare-event failures, potential bias amplification, or untested slices." }, { "Key": "minimal_test_scripts_included", "FieldType": "yesno", "Label": "Are minimal test scripts included for reproducible utility and privacy checks?", "HelpText": "Scripts or notebooks that reproduce the utility/privacy checks should be provided where practical." }, { "Key": "test_scripts_location", "FieldType": "text", "Label": "Location of test scripts / notebooks (URL or repo path)", "HelpText": "Provide a link to CI, repo, or artifact store where tests live." }, { "Key": "recommendation", "FieldType": "select", "Label": "Recommendation / next step", "HelpText": "Select the team recommendation based on the evidence collected.", "Options": [ { "Value": "proceed_synthetic_with_controls", "Label": "Proceed with synthetic release with listed controls" }, { "Value": "use_synthetic_for_testing_only", "Label": "Use synthetic only for testing / development; require real validation for production" }, { "Value": "further_testing_required", "Label": "Further testing / harder privacy evaluation required before release" }, { "Value": "do_not_use_synthetic", "Label": "Do not use synthetic for this dataset (use other approaches)" } ], "Required": true }, { "Key": "recommendation_rationale", "FieldType": "textarea", "Label": "Rationale and acceptance criteria for recommendation", "HelpText": "Explain why the recommendation was chosen and what acceptance criteria must be met to change it." }, { "Key": "responsible_team", "FieldType": "text", "Label": "Responsible team / owner", "HelpText": "Team or person accountable for follow-through." }, { "Key": "risk_owner", "FieldType": "text", "Label": "Risk owner", "HelpText": "Person responsible for monitoring re-identification risk and utility drift." }, { "Key": "priority_score", "FieldType": "scale", "Label": "Priority for follow-up", "HelpText": "1 = low priority, 5 = urgent. Use to plan governance and testing resource allocation.", "ScaleMin": 1, "ScaleMax": 5 }, { "Key": "reviewer_signature", "FieldType": "text", "Label": "Reviewer name / signature", "HelpText": "Name and role of the person completing this evaluation." }, { "Key": "notes_for_audit", "FieldType": "textarea", "Label": "Notes for compliance/audit reviewers", "HelpText": "Optional notes that auditors or governance reviewers will find useful." } ] }