From 37385723a7fffac68cced2c5472dacb7d35715ac Mon Sep 17 00:00:00 2001 From: dustinvannoy-db Date: Sun, 15 Feb 2026 18:09:08 -0800 Subject: [PATCH 01/24] Rewrite synthetic-data-generation for improved performance and features --- .../synthetic-data-generation/.gitignore | 7 + .../synthetic-data-generation/README.md | 50 - .../synthetic-data-generation/SKILL.md | 947 +++++++++++++----- .../scripts/example_faker_udf.py | 167 +++ .../scripts/example_pandas.py | 94 ++ .../scripts/example_polars.py | 157 +++ .../scripts/generate_ecommerce_data.py | 217 ++++ 7 files changed, 1364 insertions(+), 275 deletions(-) create mode 100644 databricks-skills/synthetic-data-generation/.gitignore delete mode 100644 databricks-skills/synthetic-data-generation/README.md create mode 100644 databricks-skills/synthetic-data-generation/scripts/example_faker_udf.py create mode 100644 databricks-skills/synthetic-data-generation/scripts/example_pandas.py create mode 100644 databricks-skills/synthetic-data-generation/scripts/example_polars.py create mode 100644 databricks-skills/synthetic-data-generation/scripts/generate_ecommerce_data.py diff --git a/databricks-skills/synthetic-data-generation/.gitignore b/databricks-skills/synthetic-data-generation/.gitignore new file mode 100644 index 00000000..96dceadf --- /dev/null +++ b/databricks-skills/synthetic-data-generation/.gitignore @@ -0,0 +1,7 @@ +# Generated data +output/ + +# Python +__pycache__/ +*.pyc +.venv/ diff --git a/databricks-skills/synthetic-data-generation/README.md b/databricks-skills/synthetic-data-generation/README.md deleted file mode 100644 index 55419a59..00000000 --- a/databricks-skills/synthetic-data-generation/README.md +++ /dev/null @@ -1,50 +0,0 @@ -# Synthetic Data Generation - -Generate realistic synthetic data using Faker and Spark, with non-linear distributions, integrity constraints, and save to Databricks. - -## Overview - -This skill guides the generation of realistic, story-driven synthetic data for Databricks using Python with Faker, NumPy, and Spark. It activates when users need test data, demo datasets, or synthetic tables with believable distributions and referential integrity. The generated data is saved as raw Parquet files to Unity Catalog Volumes, ready for downstream processing through Spark Declarative Pipelines (bronze/silver/gold layers). - -## What's Included - -``` -synthetic-data-generation/ -└── SKILL.md -``` - -## Key Topics - -- Workflow: write Python scripts locally, execute on Databricks via MCP tools, iterate on failures -- Context reuse pattern for faster execution (`cluster_id` and `context_id` persistence) -- Storage destination: Unity Catalog Volumes with `ai_dev_kit` catalog default -- Raw transactional data only (no pre-aggregated fields) to feed downstream SDP pipelines -- Referential integrity: generate master tables first, then child tables with valid foreign keys -- Non-linear distributions: log-normal for prices, exponential for durations, weighted categoricals -- Time-based patterns: weekday/weekend effects, holiday calendars, seasonality, event spikes -- Row coherence: correlated attributes (tier affects priority, priority affects resolution time, resolution affects CSAT) -- Data volume guidelines: 10K-50K minimum rows so patterns survive GROUP BY aggregation -- Dynamic date ranges: last 6 months from current date -- Script structure with configuration variables at top and validation at bottom -- Pandas for generation, Spark for saving to Volumes - -## When to Use - -- Creating test or demo datasets for Databricks -- Generating synthetic data with realistic distributions -- Building data that preserves referential integrity across multiple tables -- Preparing raw data for a medallion architecture pipeline -- Needing reproducible datasets with configurable seeds and volumes -- Prototyping dashboards or analytics with believable data - -## Related Skills - -- [Spark Declarative Pipelines](../spark-declarative-pipelines/) -- for building bronze/silver/gold pipelines on top of generated data -- [Databricks AI/BI Dashboards](../databricks-aibi-dashboards/) -- for visualizing the generated data in dashboards -- [Databricks Unity Catalog](../databricks-unity-catalog/) -- for managing catalogs, schemas, and volumes where data is stored - -## Resources - -- [Unity Catalog Volumes](https://docs.databricks.com/en/connect/unity-catalog/volumes.html) -- [Faker Library Documentation](https://faker.readthedocs.io/) -- [Databricks Execution Context API](https://docs.databricks.com/api/workspace/commandexecution) diff --git a/databricks-skills/synthetic-data-generation/SKILL.md b/databricks-skills/synthetic-data-generation/SKILL.md index 6d029941..72c74869 100644 --- a/databricks-skills/synthetic-data-generation/SKILL.md +++ b/databricks-skills/synthetic-data-generation/SKILL.md @@ -1,77 +1,452 @@ --- name: synthetic-data-generation -description: "Generate realistic synthetic data using Faker and Spark, with non-linear distributions, integrity constraints, and save to Databricks. Use when creating test data, demo datasets, or synthetic tables." +description: "Generate realistic synthetic data using Spark + Faker or Polars. Supports serverless execution, multiple output formats (Parquet/JSON/CSV/Delta), and scales from thousands to millions of rows. Use for test data, demo datasets, or synthetic tables." --- # Synthetic Data Generation -Generate realistic, story-driven synthetic data for Databricks using Python with Faker and Spark. +Generate realistic, story-driven synthetic data for Databricks using Spark + Faker or Polars. +Always present a generation plan with assumptions before generating code. -## Common Libraries +## Generation Planning Workflow -These libraries are useful for generating realistic synthetic data: +**Before generating any code, you MUST present a plan for user approval.** Give them a "Surprise Me" option if they don't want to specify details. -- **faker**: Generates realistic names, addresses, emails, companies, dates, etc. -- **holidays**: Provides country-specific holiday calendars for realistic date patterns +### Step 1: Gather Requirements -These are typically NOT pre-installed on Databricks. Install them using `execute_databricks_command` tool: -- `code`: "%pip install faker holidays" +Ask the user about: +- What domain/scenario? (e-commerce, support tickets, IoT sensors, etc.) +- How many tables? What relationships between them? +- Approximate row counts per table? +- Output format preference? (Parquet to Volume is default) +- One-time generation or scheduled job? -Save the returned `cluster_id` and `context_id` for subsequent calls. +### Step 2: Present Table Specification -## Workflow +Show a clear specification with **YOUR ASSUMPTIONS surfaced**: -1. **Write Python code to a local file** in the project (e.g., `scripts/generate_data.py`) -2. **Execute on Databricks** using the `run_python_file_on_databricks` MCP tool -3. **If execution fails**: Edit the local file to fix the error, then re-execute -4. **Reuse the context** for follow-up executions by passing the returned `cluster_id` and `context_id` +| Table | Columns | Rows | Key Assumptions | +|-------|---------|------|-----------------| +| customers | customer_id, name, email, tier, region, created_at | 5,000 | Tier weighted: Free 60%, Pro 30%, Enterprise 10% | +| orders | order_id, customer_id (FK), amount, order_date, status | 15,000 | Enterprise customers generate 5x more orders than Free | -**Always work with local files first, then execute.** This makes debugging easier - you can see and edit the code. +**Assumptions I'm making:** +- Amount distribution: log-normal by tier (Enterprise avg ~$1800, Pro ~$245, Free ~$55) +- Date range: last 6 months from today +- Status distribution: 65% delivered, 15% shipped, 10% processing, 5% pending, 5% cancelled -### Context Reuse Pattern +**Ask user**: "Does this look correct? Any adjustments needed?" -The first execution auto-selects a running cluster and creates an execution context. **Reuse this context for follow-up calls** - it's much faster (~1s vs ~15s) and shares variables/imports: +### Step 3: Ask About Data Features -**First execution** - use `run_python_file_on_databricks` tool: -- `file_path`: "scripts/generate_data.py" +Prompt user with options (enabled by default unless otherwise noted): +- [x] Skew (non-uniform distributions) - **Enabled by default** +- [x] Joins (referential integrity between tables) - **Enabled by default** +- [ ] Bad data injection (for data quality testing) + - Nulls in required fields + - Outliers/impossible values (house price $1, age 500) + - Duplicate primary keys + - Orphan foreign keys (referencing non-existent parents) +- [ ] Multi-language text (non-English names/addresses) +- [ ] Incremental mode (append vs overwrite) - for scheduled jobs + +### Pre-Generation Checklist + +Before writing any generation code, verify: + +- [ ] User confirmed compute preference (serverless vs cluster) +- [ ] Table specification shown and approved +- [ ] Assumptions about distributions surfaced and confirmed +- [ ] Output location confirmed (catalog.schema) +- [ ] Data features selected (skew, joins, bad data, etc.) +- [ ] Row counts appropriate for use case + +**Do NOT proceed to code generation until user approves the plan.** + +## Execution Options + +Choose your execution mode based on your needs: + +### Option 1: Databricks Connect with Serverless (Recommended) + +Run code locally while Spark operations execute on serverless compute. Best for development and interactive work. + +**When user requests data generation:** +1. Confirm serverless is acceptable: "I'll use serverless compute. Is that OK?" +2. If they request classic cluster: "Serverless is recommended for cost efficiency. Are you sure you need a classic cluster?" + +**Setup:** +```bash +# Install locally - IMPORTANT: Use version 17.3.x (NOT 18.x which has serverless issues) +pip install "databricks-connect>=17.3,<18" faker polars numpy pandas -Returns: `{ success, output, error, cluster_id, context_id, ... }` +# Configure ~/.databrickscfg +[DEFAULT] +host = https://your-workspace.cloud.databricks.com/ +serverless_compute_id = auto +auth_type = databricks-cli +``` + +**In your script:** +```python +from databricks.connect import DatabricksSession + +spark = DatabricksSession.builder.serverless(True).getOrCreate() +# Now Spark operations execute on serverless compute +``` + +**Benefits:** +- Instant start (no cluster spin-up) +- Local debugging with IDE integration +- Dependencies installed locally via pip +- Iterate quickly: edit file, re-run immediately -Save `cluster_id` and `context_id` for follow-up calls. +### Option 2: Serverless Job (Production/Scheduled) -**If execution fails:** -1. Read the error from the result -2. Edit the local Python file to fix the issue -3. Re-execute with same context using `run_python_file_on_databricks` tool: +Submit jobs to serverless compute with dependencies managed via the `environments` parameter. Best for production workloads and scheduled jobs. + +**Use `create_job` MCP tool with environments:** +- `name`: "generate_synthetic_data" +- `tasks`: [{ task with `environment_key` reference }] +- `environments`: [{ + "environment_key": "datagen_env", + "spec": { + "client": "4", + "dependencies": ["faker", "polars", "numpy", "pandas", "holidays"] + } + }] + +**Benefits:** +- No local environment needed +- Automatic dependency management +- Scheduled execution support +- Production-ready scaling + +### Option 3: Classic Cluster (Fallback) + +Execute on a classic all-purpose cluster. Use only if serverless is unavailable or you need specific cluster features. + +**Warning:** Classic clusters take 3-8 minutes to start if not already running. Prefer serverless for faster iteration. + +**Workflow:** +1. Install dependencies using `execute_databricks_command` tool: + - `code`: "%pip install faker polars numpy pandas holidays" + - Save returned `cluster_id` and `context_id` + +2. Execute script using `run_python_file_on_databricks` tool: - `file_path`: "scripts/generate_data.py" - `cluster_id`: "" - `context_id`: "" -**Follow-up executions** reuse the context (faster, shares state): -- `file_path`: "scripts/validate_data.py" -- `cluster_id`: "" -- `context_id`: "" +**When to use:** Only when serverless is not available, or you need specific cluster configurations (GPUs, custom init scripts, etc.) + +## Common Libraries + +These libraries are useful for generating realistic synthetic data: + +- **faker**: Generates realistic names, addresses, emails, companies, dates, etc. (100+ providers) +- **polars**: Fast local DataFrame library for small/medium datasets +- **numpy/pandas**: Statistical distributions and data manipulation +- **holidays**: Provides country-specific holiday calendars for realistic date patterns + +**For Databricks Connect:** Install locally with `pip install "databricks-connect>=17.3,<18" faker polars numpy pandas holidays` + +**For Serverless Jobs:** Include in `environments.spec.dependencies`: `["faker", "polars", "numpy", "pandas", "holidays"]` + +**For Classic Clusters:** Install using `execute_databricks_command` tool: +- `code`: "%pip install faker polars numpy pandas holidays" +- Save the returned `cluster_id` and `context_id` for subsequent calls + +## Data Generation Approaches + +Choose your approach based on scale and where you need to write data: + +### Approach 1: Spark + Faker (Recommended for most cases) + +**Best for:** Any dataset size, especially >100K rows, writing to Unity Catalog + +Generate data with Pandas + Faker locally, convert to Spark DataFrame for saving to Databricks. + +**Key features:** +- Full access to 100+ Faker providers (names, addresses, companies, etc.) +- Use Pandas UDFs for parallelism with large datasets +- Flexible custom logic for complex patterns +- Direct integration with Unity Catalog via Spark + +**Example:** +```python +from pyspark.sql import functions as F +from pyspark.sql.functions import pandas_udf +from pyspark.sql.types import StringType +import pandas as pd +from faker import Faker +from databricks.connect import DatabricksSession + +spark = DatabricksSession.builder.serverless(True).getOrCreate() + +# Define Pandas UDFs for Faker data (batch processing) +@pandas_udf(StringType()) +def fake_name(ids: pd.Series) -> pd.Series: + fake = Faker() + return pd.Series([fake.name() for _ in range(len(ids))]) + +@pandas_udf(StringType()) +def fake_email(ids: pd.Series) -> pd.Series: + fake = Faker() + return pd.Series([fake.email() for _ in range(len(ids))]) + +# Generate with Spark + Pandas UDFs +customers_df = ( + spark.range(0, N_CUSTOMERS, numPartitions=8) + .select( + F.concat(F.lit("CUST-"), F.lpad(F.col("id").cast("string"), 5, "0")).alias("customer_id"), + fake_name(F.col("id")).alias("name"), + fake_email(F.col("id")).alias("email"), + F.when(F.rand() < 0.6, "Free") + .when(F.rand() < 0.9, "Pro") + .otherwise("Enterprise").alias("tier"), + ) +) +customers_df.write.mode("overwrite").parquet(f"{VOLUME_PATH}/customers") +``` + +**Scaling with Pandas UDFs (for large datasets):** +```python +from pyspark.sql import functions as F +from pyspark.sql.functions import pandas_udf +from pyspark.sql.types import StringType +import pandas as pd +from faker import Faker + +@pandas_udf(StringType()) +def generate_company_batch(ids: pd.Series) -> pd.Series: + """Batch generate company names - more efficient than row-by-row UDF.""" + fake = Faker() + return pd.Series([fake.company() for _ in range(len(ids))]) + +# Generate with Spark parallelism + batch processing +customers_df = ( + spark.range(0, 1_000_000, numPartitions=32) + .withColumn("name", generate_company_batch(F.col("id"))) +) +``` + +### Approach 2: Polars (For local development) -### Handling Failures +**Best for:** Quick prototyping, datasets <100K rows, no Spark dependency needed -When execution fails: -1. Read the error from the result -2. **Edit the local Python file** to fix the issue -3. Re-execute using the same `cluster_id` and `context_id` (faster, keeps installed libraries) -4. If the context is corrupted, omit `context_id` to create a fresh one +Generate entirely with Polars + Faker locally, export to parquet files. -### Installing Libraries +**Key features:** +- Fast local generation (no Spark overhead) +- Simple, clean API +- Perfect for testing and prototyping +- Can upload resulting parquet to Databricks volumes -Databricks provides Spark, pandas, numpy, and common data libraries by default. **Only install a library if you get an import error.** +**Example:** +```python +import polars as pl +from faker import Faker +import numpy as np + +fake = Faker() + +# Generate with Polars +customers = pl.DataFrame({ + "customer_id": [f"CUST-{i:05d}" for i in range(N_CUSTOMERS)], + "name": [fake.name() for _ in range(N_CUSTOMERS)], + "email": [fake.email() for _ in range(N_CUSTOMERS)], + "tier": np.random.choice(["Free", "Pro", "Enterprise"], N_CUSTOMERS, p=[0.6, 0.3, 0.1]), +}) + +# Save locally +customers.write_parquet("./output/customers.parquet") +``` + +### Decision Guide + +| Need | Recommended Approach | +|------|---------------------| +| Write to Unity Catalog | **Spark + Faker** | +| Scale to millions of rows | **Spark + Faker** with Pandas UDFs | +| Quick local prototype | **Polars** | +| Realistic text (names/addresses) | **Either** (both use Faker) | +| No Spark dependency | **Polars** | + +### Approach 3: Faker with Spark UDFs + +**Best for:** Realistic text data (names, addresses, companies), complex custom patterns + +Faker provides 100+ data providers for realistic text. Wrap it in Spark UDFs for parallelism. + +**Key features:** +- Access to 100+ Faker providers (names, addresses, companies, phone numbers, etc.) +- Custom UDFs for complex conditional logic +- Row-level coherence where attributes correlate logically +- Flexibility for domain-specific patterns + +**Example:** +```python +from pyspark.sql import functions as F +from pyspark.sql.types import StringType, DoubleType +from faker import Faker +import numpy as np + +# Define Faker UDFs for realistic text +@F.udf(returnType=StringType()) +def generate_company(): + return Faker().company() + +@F.udf(returnType=StringType()) +def generate_address(): + return Faker().address().replace('\n', ', ') + +@F.udf(returnType=DoubleType()) +def generate_lognormal_amount(tier): + """Generate amount based on tier using log-normal distribution.""" + np.random.seed(hash(tier) % (2**32)) + if tier == "Enterprise": + return float(np.random.lognormal(mean=10, sigma=0.8)) + elif tier == "Pro": + return float(np.random.lognormal(mean=8, sigma=0.7)) + else: + return float(np.random.lognormal(mean=5, sigma=0.6)) + +# Generate with Spark parallelism +customers_df = ( + spark.range(0, 1_000_000, numPartitions=32) + .select( + F.concat(F.lit("CUST-"), F.lpad(F.col("id").cast("string"), 5, "0")).alias("customer_id"), + generate_company().alias("name"), + generate_address().alias("address"), + F.when(F.rand(42) < 0.6, "Free") + .when(F.rand(42) < 0.9, "Pro") + .otherwise("Enterprise").alias("tier") + ) +) + +# Add tier-based amounts +customers_df = customers_df.withColumn("arr", generate_lognormal_amount(F.col("tier"))) +``` + +### When to Use Each Approach + +| Scenario | Recommended Approach | +|----------|---------------------| +| Generating 1M+ rows | **Spark + Faker with Pandas UDFs** | +| Need realistic names/addresses/emails | **Faker** (Spark or Polars) | +| Writing to Unity Catalog | **Spark + Faker** | +| Complex conditional row logic | **Spark + Faker UDFs** | +| Foreign key with complex weighting | **Spark + Faker** | +| Quick prototyping (small data) | **Polars** | +| No Spark dependency needed | **Polars** | + +## Workflow + +### Primary: Databricks Connect with Serverless + +The recommended workflow for development and interactive data generation: + +1. **Configure Databricks Connect** (one-time setup): + - Install: `pip install "databricks-connect>=17.3,<18" faker polars numpy pandas holidays` + - Configure `~/.databrickscfg` with `serverless_compute_id = auto` + +2. **Write Python script locally** (e.g., `scripts/generate_data.py`): + ```python + from databricks.connect import DatabricksSession + spark = DatabricksSession.builder.serverless(True).getOrCreate() + # Your data generation code here + ``` + +3. **Run locally** - Spark operations execute on serverless compute: + ```bash + python scripts/generate_data.py + ``` + +4. **Iterate quickly**: Edit file, re-run immediately. No cluster spin-up time. + +### Production: Serverless Job + +For scheduled or production workloads: + +1. **Write Python script locally** + +2. **Upload to workspace** using `upload_file` MCP tool: + - `local_path`: "scripts/generate_data.py" + - `workspace_path`: "/Workspace/Users/{username}/datagen/{project_name}/generate_data.py" + +3. **Create serverless job** using `create_job` MCP tool: + - `name`: "generate_synthetic_data" + - `tasks`: [{ + "task_key": "generate", + "spark_python_task": { + "python_file": "/Workspace/Users/{username}/datagen/{project_name}/generate_data.py" + }, + "environment_key": "datagen_env" + }] + - `environments`: [{ + "environment_key": "datagen_env", + "spec": { + "client": "4", + "dependencies": ["faker", "polars", "numpy", "pandas", "holidays"] + } + }] + +4. **Run job** using `run_job_now` MCP tool + +5. **Monitor** using `get_run` or `wait_for_run` MCP tools + +### Production: DABs Bundle Deployment + +For scheduled runs with version control and CI/CD: + +```yaml +# databricks.yml +bundle: + name: synthetic-data-gen + +resources: + jobs: + generate_daily_data: + name: "Generate Daily Data" + schedule: + quartz_cron_expression: "0 0 6 * * ?" + tasks: + - task_key: generate + spark_python_task: + python_file: ./src/generate_data.py + environment_key: default + +environments: + default: + spec: + client: "4" + dependencies: + - faker + - polars + - numpy + - pandas + - holidays +``` + +**Note**: The `environments` block with `client: "4"` enables serverless compute. Dependencies are installed automatically. + +### Fallback: Classic Cluster + +Only use if serverless is unavailable: -Use `execute_databricks_command` tool: -- `code`: "%pip install faker" -- `cluster_id`: "" -- `context_id`: "" +1. **Install dependencies** using `execute_databricks_command` tool: + - `code`: "%pip install faker polars numpy pandas holidays" + - Save returned `cluster_id` and `context_id` + +2. **Execute script** using `run_python_file_on_databricks` tool: + - `file_path`: "scripts/generate_data.py" + - `cluster_id`: "" + - `context_id`: "" -The library is immediately available in the same context. +3. **Iterate**: Edit local file, re-execute with same context (faster, keeps installed libraries) -**Note:** Keeping the same `context_id` means installed libraries persist across calls. +**Note:** Classic clusters take 3-8 minutes to start. Prefer serverless for faster iteration. ## Storage Destination @@ -98,19 +473,78 @@ spark.sql(f"CREATE SCHEMA IF NOT EXISTS {CATALOG}.{SCHEMA}") spark.sql(f"CREATE VOLUME IF NOT EXISTS {CATALOG}.{SCHEMA}.raw_data") ``` -### Save to Volume as Raw Data (Never Tables) +### Output Formats -**Always save data to a Volume as parquet files, never directly to tables** (unless the user explicitly requests tables). This is the input for the downstream Spark Declarative Pipeline (SDP) that will handle bronze/silver/gold layers. +Choose your output format based on downstream needs: + +#### Parquet to Volumes (Default) + +Standard format for SDP pipeline input. Best compression and query performance. +Files may not use a file extension or might end with .parquet. ```python VOLUME_PATH = f"/Volumes/{CATALOG}/{SCHEMA}/raw_data" -# Save as parquet files (raw data) -spark.createDataFrame(customers_pdf).write.mode("overwrite").parquet(f"{VOLUME_PATH}/customers") -spark.createDataFrame(orders_pdf).write.mode("overwrite").parquet(f"{VOLUME_PATH}/orders") -spark.createDataFrame(tickets_pdf).write.mode("overwrite").parquet(f"{VOLUME_PATH}/tickets") +# Save as parquet files +customers_df.write.mode("overwrite").parquet(f"{VOLUME_PATH}/customers") +orders_df.write.mode("overwrite").parquet(f"{VOLUME_PATH}/orders") +tickets_df.write.mode("overwrite").parquet(f"{VOLUME_PATH}/tickets") +``` + +#### JSON to Volumes + +A common pattern user may request for simulate SDP ingestion from external data feeds such as logs. +File extension should be .json + +```python +VOLUME_PATH = f"/Volumes/{CATALOG}/{SCHEMA}/raw_data" + +# Save as JSON files +customers_df.write.mode("overwrite").json(f"{VOLUME_PATH}/customers_json") +orders_df.write.mode("overwrite").json(f"{VOLUME_PATH}/orders_json") ``` +#### CSV to Volumes + +A common pattern user may request for simulate SDP ingestion from external data feeds such as logs. +File extension should be .csv. + +```python +VOLUME_PATH = f"/Volumes/{CATALOG}/{SCHEMA}/raw_data" + +# Save as CSV with headers +customers_df.write.mode("overwrite").option("header", "true").csv(f"{VOLUME_PATH}/customers_csv") +orders_df.write.mode("overwrite").option("header", "true").csv(f"{VOLUME_PATH}/orders_csv") +``` + +#### Delta Table (Unity Catalog) + +When data is ready for direct analytics consumption (skip SDP pipeline). + +```python +# Ensure schema exists +spark.sql(f"CREATE SCHEMA IF NOT EXISTS {CATALOG}.{SCHEMA}") + +# Save as managed Delta tables +customers_df.write.mode("overwrite").saveAsTable(f"{CATALOG}.{SCHEMA}.customers") +orders_df.write.mode("overwrite").saveAsTable(f"{CATALOG}.{SCHEMA}.orders") + +# With additional options +customers_df.write \ + .mode("overwrite") \ + .option("overwriteSchema", "true") \ + .saveAsTable(f"{CATALOG}.{SCHEMA}.customers") +``` + +#### When to Use Each Format + +| Format | Use Case | +|--------|----------| +| **Parquet to Volumes** | Default - input for SDP bronze/silver/gold pipelines | +| **JSON to Volumes** | User request - a common pattern in real Databricks ingestion workloads | +| **CSV to Volumes** | User request - a common pattern in real Databricks ingestion workloads | +| **Delta Table** | Direct analytics - user may not want to build the ingestion and have data ready to query in notebooks or with SQL | + ## Raw Data Only - No Pre-Aggregated Fields (Unless Instructed Otherwise) **By default, generate raw, transactional data only.** Do not create fields that represent sums, totals, averages, or counts. @@ -159,7 +593,7 @@ orders_data.append({ ### Date Range: Last 6 Months from Today -**Always generate data for the last ~6 months ending at the current date.** This ensures: +**Always generate data for the last ~6 months ending at the current date, unless prompted with specific timeframe.** This ensures: - Data feels current and relevant for demos - Recent patterns are visible in dashboards - Downstream aggregations (daily/weekly/monthly) have enough history @@ -246,26 +680,112 @@ spark = SparkSession.builder.getOrCreate() # ... rest of script ``` +## Business Integrity Requirements + +Generated data MUST reflect business reality. Data should be realistic and tell a coherent story. + +| Pattern | Example | Implementation | +|---------|---------|----------------| +| **Value coherence** | Houses worth $200K-$2M, pens $1-$50 | Domain-appropriate ranges | +| **Tier behavior** | Premium users have more orders | Weighted sampling by tier | +| **Temporal patterns** | More orders on weekends, holidays | Time-based distributions | +| **Geographic patterns** | Regional pricing differences | Location-correlated values | +| **Multi-table integrity** | Orders reference valid customers | Foreign key validation | + +**Anti-pattern**: Flat/linear distributions (every customer has ~same # orders) + +**Correct**: Skewed distributions (80/20 rule - 20% of customers generate 80% of orders) + +### Bad Data Injection (Optional) + +When user requests bad data for testing data quality rules: + +```python +# Bad data configuration +BAD_DATA_CONFIG = { + "null_rate": 0.02, # 2% nulls in required fields + "outlier_rate": 0.01, # 1% impossible values + "duplicate_pk_rate": 0.005, # 0.5% duplicate primary keys + "orphan_fk_rate": 0.01, # 1% orphan foreign keys +} + +# Inject after generation +if INJECT_BAD_DATA: + # Nulls in required fields + null_mask = np.random.random(len(orders_pdf)) < BAD_DATA_CONFIG["null_rate"] + orders_pdf.loc[null_mask, "customer_id"] = None + + # Outliers (impossible values) + outlier_mask = np.random.random(len(orders_pdf)) < BAD_DATA_CONFIG["outlier_rate"] + orders_pdf.loc[outlier_mask, "amount"] = -999.99 # Negative amount + + # Orphan foreign keys + orphan_mask = np.random.random(len(orders_pdf)) < BAD_DATA_CONFIG["orphan_fk_rate"] + orders_pdf.loc[orphan_mask, "customer_id"] = "CUST-NONEXISTENT" +``` + +## Domain-Specific Guidance + +When generating data for specific domains, consider these realistic patterns: + +### Retail/E-commerce +- **Tables**: customers → orders → order_items → products +- **Patterns**: + - Seasonal spikes (holiday shopping) + - Cart abandonment (~70% of carts) + - Loyalty tier progression + - Regional pricing + +### Support/CRM +- **Tables**: accounts → contacts → tickets → interactions +- **Patterns**: + - Incident spikes during outages + - Resolution time varies by priority + - Enterprise accounts have more contacts + - CSAT correlates with resolution speed + +### Manufacturing/IoT +- **Tables**: equipment → sensors → readings → maintenance_orders +- **Patterns**: + - Sensor readings follow equipment lifecycle + - Anomalies precede maintenance events + - Seasonal production variations + - Equipment age affects failure rates + +### Financial Services +- **Tables**: accounts → transactions → payments → fraud_flags +- **Patterns**: + - Transaction amounts follow power law + - Fraud patterns (unusual times, amounts, locations) + - Account balance consistency + - Regulatory compliance (no negative balances) + +**Note**: These are guidance, not rigid schemas. Adapt to user's specific needs. + ## Key Principles -### 1. Use Pandas for Generation, Spark for Saving +### 1. Use Polars for Generation, Spark for Saving -Generate data with pandas (faster, easier), convert to Spark for saving: +Generate data with Polars (faster than Pandas), convert to Spark for saving: ```python -import pandas as pd +import polars as pl +import numpy as np +from faker import Faker -# Generate with pandas -customers_pdf = pd.DataFrame({ +fake = Faker() + +# Generate with Polars (faster than Pandas) +customers_pl = pl.DataFrame({ "customer_id": [f"CUST-{i:05d}" for i in range(N_CUSTOMERS)], "name": [fake.company() for _ in range(N_CUSTOMERS)], - "tier": np.random.choice(['Free', 'Pro', 'Enterprise'], N_CUSTOMERS, p=[0.6, 0.3, 0.1]), - "region": np.random.choice(['North', 'South', 'East', 'West'], N_CUSTOMERS, p=[0.4, 0.25, 0.2, 0.15]), + "tier": np.random.choice(['Free', 'Pro', 'Enterprise'], N_CUSTOMERS, p=[0.6, 0.3, 0.1]).tolist(), + "region": np.random.choice(['North', 'South', 'East', 'West'], N_CUSTOMERS, p=[0.4, 0.25, 0.2, 0.15]).tolist(), "created_at": [fake.date_between(start_date='-2y', end_date='-6m') for _ in range(N_CUSTOMERS)], }) # Convert to Spark and save -customers_df = spark.createDataFrame(customers_pdf) +customers_df = spark.createDataFrame(customers_pl.to_pandas()) customers_df.write.mode("overwrite").parquet(f"{VOLUME_PATH}/customers") ``` @@ -428,199 +948,152 @@ def generate_ticket(customer_id, tier, date): } ``` -## Complete Example +## Complete Examples + +### Example 1: E-commerce Data (Spark + Faker + Pandas) + +Generate e-commerce data with customers and orders tables, with referential integrity and tier-based distributions. + +**Full implementation:** See `scripts/generate_ecommerce_data.py` in this skill folder. -Save as `scripts/generate_data.py`: +**Features:** +- Serverless-first with fallback to classic cluster +- Configurable bad data injection for testing +- Incremental mode for scheduled jobs +- Weighted tier distribution with realistic amounts + +**Key configuration options:** ```python -"""Generate synthetic customer, order, and ticket data.""" -import numpy as np -import pandas as pd -from datetime import datetime, timedelta +USE_SERVERLESS = True # Recommended +WRITE_MODE = "overwrite" # or "append" for incremental +INJECT_BAD_DATA = False # Set True for data quality testing +``` + +**Usage:** Copy to your scripts folder, update CATALOG/SCHEMA, run with `python generate_ecommerce_data.py` + +### Example 2: Local Development with Polars + +Generate synthetic data locally without Spark dependency, then upload to Databricks. + +**Full implementation:** See `scripts/example_polars.py` in this skill folder. + +**Features:** +- Fast local generation (no Spark overhead) +- Perfect for prototyping and testing +- Outputs parquet files to local directory +- Upload to volumes with `databricks fs cp` + +**Key pattern:** + +```python +import polars as pl from faker import Faker -import holidays -from pyspark.sql import SparkSession -# ============================================================================= -# CONFIGURATION -# ============================================================================= -CATALOG = "my_catalog" -SCHEMA = "my_schema" -VOLUME_PATH = f"/Volumes/{CATALOG}/{SCHEMA}/raw_data" +# Generate with Polars +customers = pl.DataFrame({ + "customer_id": [f"CUST-{i:05d}" for i in range(N_CUSTOMERS)], + "name": [fake.name() for _ in range(N_CUSTOMERS)], + "tier": np.random.choice(["Free", "Pro", "Enterprise"], N_CUSTOMERS, p=[0.6, 0.3, 0.1]), +}) -N_CUSTOMERS = 2500 -N_ORDERS = 25000 -N_TICKETS = 8000 +# Save locally +customers.write_parquet("./output/customers.parquet") +``` -# Date range - last 6 months from today -END_DATE = datetime.now().replace(hour=0, minute=0, second=0, microsecond=0) -START_DATE = END_DATE - timedelta(days=180) +**Usage:** Run locally, then upload: `databricks fs cp -r ./output dbfs:/Volumes/{catalog}/{schema}/raw_data/` -# Special events (within the date range) -INCIDENT_END = END_DATE - timedelta(days=21) -INCIDENT_START = INCIDENT_END - timedelta(days=10) +### Example 3: Large-Scale with Faker UDFs -# Holiday calendar -US_HOLIDAYS = holidays.US(years=[START_DATE.year, END_DATE.year]) +Use Faker with Spark UDFs for realistic text data with parallelism. Best for datasets 100K+ rows. -SEED = 42 +**Full implementation:** See `scripts/example_faker_udf.py` in this skill folder. -# ============================================================================= -# SETUP -# ============================================================================= -np.random.seed(SEED) -Faker.seed(SEED) -fake = Faker() -spark = SparkSession.builder.getOrCreate() +**Features:** +- Serverless-first with fallback to classic cluster +- Parallel execution using Spark UDFs +- Realistic text data (company names, addresses, emails) +- Tier-based amount generation -# ============================================================================= -# CREATE INFRASTRUCTURE -# ============================================================================= -print(f"Creating catalog/schema/volume if needed...") -spark.sql(f"CREATE CATALOG IF NOT EXISTS {CATALOG}") -spark.sql(f"CREATE SCHEMA IF NOT EXISTS {CATALOG}.{SCHEMA}") -spark.sql(f"CREATE VOLUME IF NOT EXISTS {CATALOG}.{SCHEMA}.raw_data") +**Key pattern - Faker UDFs for realistic data:** + +```python +@F.udf(returnType=StringType()) +def generate_company(): + return Faker().company() + +@F.udf(returnType=DoubleType()) +def generate_lognormal_amount(tier): + np.random.seed(hash(str(tier)) % (2**32)) + if tier == "Enterprise": + return float(np.random.lognormal(mean=9, sigma=0.8)) + elif tier == "Pro": + return float(np.random.lognormal(mean=7, sigma=0.7)) + else: + return float(np.random.lognormal(mean=5, sigma=0.6)) + +# Use UDFs in Spark operations +customers_df = ( + spark.range(0, N_CUSTOMERS, numPartitions=PARTITIONS) + .select( + generate_company().alias("name"), + # ... other columns + ) + .withColumn("arr", generate_lognormal_amount(F.col("tier"))) +) +``` -print(f"Generating: {N_CUSTOMERS:,} customers, {N_ORDERS:,} orders, {N_TICKETS:,} tickets") +**Usage:** Copy `example_faker_udf.py` to your scripts folder and customize the UDFs and configuration. -# ============================================================================= -# 1. CUSTOMERS (Master Table) -# ============================================================================= -print("Generating customers...") +### Example 4: Legacy Approach (Faker + Pandas) + +For smaller datasets or when you need complex time-based patterns with row-level logic. +This approach uses Pandas for generation (single-threaded) and Spark for saving. + +**Note:** For datasets over 100K rows, prefer Faker UDFs for better performance. + +**Full implementation:** See `scripts/example_pandas.py` in this skill folder. +**Key pattern - Pandas generation with referential integrity:** + +```python +# Generate master table customers_pdf = pd.DataFrame({ "customer_id": [f"CUST-{i:05d}" for i in range(N_CUSTOMERS)], - "name": [fake.company() for _ in range(N_CUSTOMERS)], "tier": np.random.choice(['Free', 'Pro', 'Enterprise'], N_CUSTOMERS, p=[0.6, 0.3, 0.1]), - "region": np.random.choice(['North', 'South', 'East', 'West'], N_CUSTOMERS, p=[0.4, 0.25, 0.2, 0.15]), }) -# ARR correlates with tier -customers_pdf["arr"] = customers_pdf["tier"].apply( - lambda t: round(np.random.lognormal(11, 0.5), 2) if t == 'Enterprise' - else round(np.random.lognormal(8, 0.6), 2) if t == 'Pro' else 0 -) - -# Lookups for foreign keys +# Create lookups for foreign keys customer_ids = customers_pdf["customer_id"].tolist() customer_tier_map = dict(zip(customers_pdf["customer_id"], customers_pdf["tier"])) tier_weights = customers_pdf["tier"].map({'Enterprise': 5.0, 'Pro': 2.0, 'Free': 1.0}) customer_weights = (tier_weights / tier_weights.sum()).tolist() -print(f" Created {len(customers_pdf):,} customers") - -# ============================================================================= -# 2. ORDERS (References Customers) -# ============================================================================= -print("Generating orders...") - +# Generate related table with weighted sampling orders_data = [] for i in range(N_ORDERS): cid = np.random.choice(customer_ids, p=customer_weights) tier = customer_tier_map[cid] - amount = np.random.lognormal(7 if tier == 'Enterprise' else 5 if tier == 'Pro' else 3.5, 0.7) + orders_data.append({"order_id": f"ORD-{i:06d}", "customer_id": cid, ...}) - orders_data.append({ - "order_id": f"ORD-{i:06d}", - "customer_id": cid, - "amount": round(amount, 2), - "status": np.random.choice(['completed', 'pending', 'cancelled'], p=[0.85, 0.10, 0.05]), - "order_date": fake.date_between(start_date=START_DATE, end_date=END_DATE), - }) - -orders_pdf = pd.DataFrame(orders_data) -print(f" Created {len(orders_pdf):,} orders") - -# ============================================================================= -# 3. TICKETS (References Customers, with incident spike) -# ============================================================================= -print("Generating tickets...") - -def get_daily_volume(date, base=25): - vol = base * (0.6 if date.weekday() >= 5 else 1.0) - if date in US_HOLIDAYS: - vol *= 0.3 # Even lower on holidays - if INCIDENT_START <= date <= INCIDENT_END: - vol *= 3.0 - return int(vol * np.random.normal(1, 0.15)) - -# Distribute tickets across dates -tickets_data = [] -ticket_idx = 0 -for day in pd.date_range(START_DATE, END_DATE): - daily_count = get_daily_volume(day.to_pydatetime()) - is_incident = INCIDENT_START <= day.to_pydatetime() <= INCIDENT_END - - for _ in range(daily_count): - if ticket_idx >= N_TICKETS: - break - - cid = np.random.choice(customer_ids, p=customer_weights) - tier = customer_tier_map[cid] - - # Category - Auth dominates during incident - if is_incident: - category = np.random.choice(['Auth', 'Network', 'Billing', 'Account'], p=[0.65, 0.15, 0.1, 0.1]) - else: - category = np.random.choice(['Auth', 'Network', 'Billing', 'Account'], p=[0.25, 0.30, 0.25, 0.20]) - - # Priority correlates with tier - priority = np.random.choice(['Critical', 'High', 'Medium'], p=[0.3, 0.5, 0.2]) if tier == 'Enterprise' \ - else np.random.choice(['Critical', 'High', 'Medium', 'Low'], p=[0.05, 0.2, 0.45, 0.3]) - - # Resolution time correlates with priority - res_scale = {'Critical': 4, 'High': 12, 'Medium': 36, 'Low': 72} - resolution = np.random.exponential(scale=res_scale[priority]) - - # CSAT degrades during incident for Auth - if is_incident and category == 'Auth': - csat = np.random.choice([1, 2, 3, 4, 5], p=[0.15, 0.25, 0.35, 0.2, 0.05]) - else: - csat = 5 if resolution < 4 else (4 if resolution < 12 else np.random.choice([2, 3, 4], p=[0.2, 0.5, 0.3])) - - tickets_data.append({ - "ticket_id": f"TKT-{ticket_idx:06d}", - "customer_id": cid, - "category": category, - "priority": priority, - "resolution_hours": round(resolution, 1), - "csat_score": csat, - "created_at": day.strftime("%Y-%m-%d"), - }) - ticket_idx += 1 - - if ticket_idx >= N_TICKETS: - break - -tickets_pdf = pd.DataFrame(tickets_data) -print(f" Created {len(tickets_pdf):,} tickets") +# Convert to Spark for saving +spark.createDataFrame(customers_pdf).write.mode("overwrite").parquet(f"{VOLUME_PATH}/customers") +``` -# ============================================================================= -# 4. SAVE TO VOLUME -# ============================================================================= -print(f"\nSaving to {VOLUME_PATH}...") +**Usage:** Copy `example_pandas.py` to your scripts folder and customize the configuration and patterns. -spark.createDataFrame(customers_pdf).write.mode("overwrite").parquet(f"{VOLUME_PATH}/customers") -spark.createDataFrame(orders_pdf).write.mode("overwrite").parquet(f"{VOLUME_PATH}/orders") -spark.createDataFrame(tickets_pdf).write.mode("overwrite").parquet(f"{VOLUME_PATH}/tickets") +**To use any example script:** -print("Done!") +1. Copy the example file to your scripts directory +2. Update the CONFIGURATION section with your catalog/schema +3. Execute using one of the methods below -# ============================================================================= -# 5. VALIDATION -# ============================================================================= -print("\n=== VALIDATION ===") -print(f"Tier distribution: {customers_pdf['tier'].value_counts(normalize=True).to_dict()}") -print(f"Avg order by tier: {orders_pdf.merge(customers_pdf[['customer_id', 'tier']]).groupby('tier')['amount'].mean().to_dict()}") - -incident_tickets = tickets_pdf[tickets_pdf['created_at'].between( - INCIDENT_START.strftime("%Y-%m-%d"), INCIDENT_END.strftime("%Y-%m-%d") -)] -print(f"Incident period tickets: {len(incident_tickets):,} ({len(incident_tickets)/len(tickets_pdf)*100:.1f}%)") -print(f"Incident Auth %: {(incident_tickets['category'] == 'Auth').mean()*100:.1f}%") +**Execute with Databricks Connect:** +```bash +python scripts/generate_data.py ``` -Execute using `run_python_file_on_databricks` tool: +**Execute with classic cluster** using `run_python_file_on_databricks` tool: - `file_path`: "scripts/generate_data.py" If it fails, edit the file and re-run with the same `cluster_id` and `context_id`. @@ -636,19 +1109,43 @@ This returns schema, row counts, and column statistics to confirm the data was w ## Best Practices -1. **Ask for schema**: Default to `ai_dev_kit` catalog, ask user for schema name -2. **Create infrastructure**: Use `CREATE CATALOG/SCHEMA/VOLUME IF NOT EXISTS` -3. **Raw data only**: No `total_x`, `sum_x`, `avg_x` fields - SDP pipeline computes those -4. **Save to Volume, not tables**: Write parquet to `/Volumes/{catalog}/{schema}/raw_data/` -5. **Configuration at top**: All sizes, dates, and paths as variables -6. **Dynamic dates**: Use `datetime.now() - timedelta(days=180)` for last 6 months -7. **Pandas for generation**: Faster and easier than Spark for row-by-row logic +### Execution +1. **Use Databricks Connect with serverless** for development - instant start, local debugging +2. **Use serverless jobs** for production - automatic dependency management, scheduling +3. **Prefer serverless over classic** - avoid 3-8 minute cluster spin-up times +4. **Ask for schema**: Default to `ai_dev_kit` catalog, ask user for schema name +5. **Present plan before generating**: Show table spec with assumptions, get user approval + +### Data Generation +6. **Use Faker with Pandas UDFs for scale** (1M+ rows) - Spark parallelism +7. **Use Polars for quick prototyping** - fast local generation 8. **Master tables first**: Generate customers, then orders reference customer_ids 9. **Weighted sampling**: Enterprise customers generate more activity 10. **Distributions**: Log-normal for values, exponential for times, weighted categorical 11. **Time patterns**: Weekday/weekend, holidays, seasonality, event spikes 12. **Row coherence**: Priority affects resolution time affects CSAT 13. **Volume for aggregation**: 10K-50K rows minimum so patterns survive GROUP BY -14. **Always use files**: Write to local file, execute, edit if error, re-execute -15. **Context reuse**: Pass `cluster_id` and `context_id` for faster iterations -16. **Libraries**: Install `faker` and `holidays` first; most others are pre-installed + +### Output +14. **Create infrastructure in script**: Use `CREATE SCHEMA/VOLUME IF NOT EXISTS` +15. **Raw data only**: No `total_x`, `sum_x`, `avg_x` fields - SDP pipeline computes those +16. **Choose output format** based on downstream needs (Parquet/JSON/CSV/Delta) +17. **Configuration at top**: All sizes, dates, and paths as variables +18. **Dynamic dates**: Use `datetime.now() - timedelta(days=180)` for last 6 months + +## Common Issues + +| Issue | Solution | +|-------|----------| +| **"Either base environment or version must be provided"** | Add `"client": "4"` to `spec` in job environments (auto-injected by MCP tool) | +| **"ModuleNotFoundError: No module named 'faker'"** | Add `faker` to dependencies or install locally: `pip install faker` | +| **"ModuleNotFoundError: No module named 'polars'"** | Add `polars` to dependencies or install locally: `pip install polars` | +| **Serverless job fails to start** | Verify workspace has serverless compute enabled; check Unity Catalog permissions | +| **Faker UDF is slow** | Use `pandas_udf` for batched operations; adjust `numPartitions` | +| **Classic cluster startup is slow (3-8 min)** | Switch to Databricks Connect with serverless for instant start | +| **Out of memory with large data** | Increase `partitions` parameter in `spark.range()` | +| **Foreign keys don't match across tables** | Use same random seed across all generators | +| **Delta table write fails** | Ensure `CREATE SCHEMA IF NOT EXISTS` runs before `saveAsTable()` | +| **databricks-connect serverless issues** | Use version 17.3.x: `pip install "databricks-connect>=17.3,<18"` | +| **Databricks Connect connection fails** | Verify `~/.databrickscfg` has correct host and `serverless_compute_id = auto` | +| **Context corrupted on classic cluster** | Omit `context_id` to create fresh context, reinstall libraries | diff --git a/databricks-skills/synthetic-data-generation/scripts/example_faker_udf.py b/databricks-skills/synthetic-data-generation/scripts/example_faker_udf.py new file mode 100644 index 00000000..a5cb2727 --- /dev/null +++ b/databricks-skills/synthetic-data-generation/scripts/example_faker_udf.py @@ -0,0 +1,167 @@ +"""Generate synthetic data using Faker with Spark UDFs for parallelism. + +This approach is best for: +- Large datasets (100K+ rows) that need Spark parallelism +- Generating realistic text data with Faker providers +- Writing directly to Unity Catalog volumes +- Complex conditional logic in data generation +""" +from pyspark.sql import functions as F +from pyspark.sql.window import Window +from pyspark.sql.types import StringType, DoubleType, DateType +from faker import Faker +import numpy as np +from datetime import datetime, timedelta +from databricks.connect import DatabricksSession + +# ============================================================================= +# CONFIGURATION +# ============================================================================= +# Compute - Serverless recommended +USE_SERVERLESS = True # Set to False and provide CLUSTER_ID for classic compute +CLUSTER_ID = None # Only used if USE_SERVERLESS=False + +# Storage +CATALOG = "ai_dev_kit" # Change to your catalog +SCHEMA = "synthetic_data" # Change to your schema +VOLUME_PATH = f"/Volumes/{CATALOG}/{SCHEMA}/raw_data" + +# Data sizes - this example is designed for larger datasets +N_CUSTOMERS = 100_000 +N_ORDERS = 500_000 +PARTITIONS = 16 # Adjust based on data size + +# Date range - last 6 months from today +END_DATE = datetime.now().replace(hour=0, minute=0, second=0, microsecond=0) +START_DATE = END_DATE - timedelta(days=180) + +# Reproducibility +SEED = 42 + +# ============================================================================= +# SETUP +# ============================================================================= +print("Connecting to Databricks...") +if USE_SERVERLESS: + spark = DatabricksSession.builder.serverless(True).getOrCreate() + print("Connected to serverless compute!") +else: + if not CLUSTER_ID: + raise ValueError("CLUSTER_ID must be set when USE_SERVERLESS=False") + spark = DatabricksSession.builder.clusterId(CLUSTER_ID).getOrCreate() + print(f"Connected to cluster {CLUSTER_ID}!") + +# ============================================================================= +# DEFINE FAKER UDFs +# ============================================================================= +@F.udf(returnType=StringType()) +def generate_company(): + """Generate realistic company name.""" + return Faker().company() + +@F.udf(returnType=StringType()) +def generate_address(): + """Generate realistic address.""" + return Faker().address().replace('\n', ', ') + +@F.udf(returnType=StringType()) +def generate_email(company_name): + """Generate email based on company name.""" + if company_name: + domain = company_name.lower().replace(" ", "").replace(",", "")[:15] + return f"contact@{domain}.com" + return "unknown@example.com" + +@F.udf(returnType=DoubleType()) +def generate_lognormal_amount(tier): + """Generate amount based on tier using log-normal distribution.""" + np.random.seed(hash(str(tier)) % (2**32)) + if tier == "Enterprise": + return float(np.random.lognormal(mean=9, sigma=0.8)) + elif tier == "Pro": + return float(np.random.lognormal(mean=7, sigma=0.7)) + else: + return float(np.random.lognormal(mean=5, sigma=0.6)) + +# ============================================================================= +# CREATE INFRASTRUCTURE +# ============================================================================= +print("Creating infrastructure...") +spark.sql(f"CREATE SCHEMA IF NOT EXISTS {CATALOG}.{SCHEMA}") +spark.sql(f"CREATE VOLUME IF NOT EXISTS {CATALOG}.{SCHEMA}.raw_data") +print(f"Infrastructure ready: {VOLUME_PATH}") + +# ============================================================================= +# GENERATE CUSTOMERS +# ============================================================================= +print(f"Generating {N_CUSTOMERS:,} customers...") + +customers_df = ( + spark.range(0, N_CUSTOMERS, numPartitions=PARTITIONS) + .select( + F.concat(F.lit("CUST-"), F.lpad(F.col("id").cast("string"), 5, "0")).alias("customer_id"), + generate_company().alias("name"), + generate_address().alias("address"), + F.when(F.rand(SEED) < 0.6, "Free") + .when(F.rand(SEED) < 0.9, "Pro") + .otherwise("Enterprise").alias("tier"), + F.when(F.rand(SEED) < 0.4, "North") + .when(F.rand(SEED) < 0.65, "South") + .when(F.rand(SEED) < 0.85, "East") + .otherwise("West").alias("region") + ) +) + +# Add tier-based ARR and email +customers_df = ( + customers_df + .withColumn("arr", generate_lognormal_amount(F.col("tier"))) + .withColumn("email", generate_email(F.col("name"))) +) + +customers_df.write.mode("overwrite").parquet(f"{VOLUME_PATH}/customers") +print(f" Saved customers to {VOLUME_PATH}/customers") + +# ============================================================================= +# GENERATE ORDERS +# ============================================================================= +print(f"Generating {N_ORDERS:,} orders...") + +# Get customer IDs for foreign key +customer_lookup = customers_df.select("customer_id", "tier").cache() + +orders_df = ( + spark.range(0, N_ORDERS, numPartitions=PARTITIONS) + .select( + F.concat(F.lit("ORD-"), F.lpad(F.col("id").cast("string"), 6, "0")).alias("order_id"), + # Generate customer_idx for FK join (random selection from customer range) + (F.abs(F.hash(F.col("id"), F.lit(SEED))) % N_CUSTOMERS).alias("customer_idx"), + F.when(F.rand(SEED) < 0.85, "completed") + .when(F.rand(SEED) < 0.95, "pending") + .otherwise("cancelled").alias("status"), + F.date_add(F.lit(START_DATE.date()), + (F.rand(SEED) * 180).cast("int")).alias("order_date") + ) +) + +# Add customer_idx to lookup for join +customer_lookup_with_idx = customer_lookup.withColumn( + "customer_idx", + (F.row_number().over(Window.orderBy(F.monotonically_increasing_id())) - 1).cast("int") +) + +# Join to get customer_id and tier as foreign key +orders_with_fk = ( + orders_df + .join(customer_lookup_with_idx, on="customer_idx", how="left") + .drop("customer_idx") +) + +# Add tier-based amount +orders_with_fk = orders_with_fk.withColumn("amount", generate_lognormal_amount(F.col("tier"))) + +orders_with_fk.drop("tier").write.mode("overwrite").parquet(f"{VOLUME_PATH}/orders") +print(f" Saved orders to {VOLUME_PATH}/orders") + +customer_lookup.unpersist() +print("Done!") diff --git a/databricks-skills/synthetic-data-generation/scripts/example_pandas.py b/databricks-skills/synthetic-data-generation/scripts/example_pandas.py new file mode 100644 index 00000000..78b050ae --- /dev/null +++ b/databricks-skills/synthetic-data-generation/scripts/example_pandas.py @@ -0,0 +1,94 @@ +"""Generate synthetic data using Faker + Pandas (legacy approach for complex patterns).""" +import numpy as np +import pandas as pd +from datetime import datetime, timedelta +from faker import Faker +import holidays +from pyspark.sql import SparkSession + +# For Databricks Connect, replace with: +# from databricks.connect import DatabricksSession +# spark = DatabricksSession.builder.getOrCreate() + +spark = SparkSession.builder.getOrCreate() + +# ============================================================================= +# CONFIGURATION +# ============================================================================= +CATALOG = "my_catalog" +SCHEMA = "my_schema" +VOLUME_PATH = f"/Volumes/{CATALOG}/{SCHEMA}/raw_data" + +N_CUSTOMERS = 2500 +N_ORDERS = 25000 +N_TICKETS = 8000 + +END_DATE = datetime.now().replace(hour=0, minute=0, second=0, microsecond=0) +START_DATE = END_DATE - timedelta(days=180) +INCIDENT_END = END_DATE - timedelta(days=21) +INCIDENT_START = INCIDENT_END - timedelta(days=10) +US_HOLIDAYS = holidays.US(years=[START_DATE.year, END_DATE.year]) + +SEED = 42 +np.random.seed(SEED) +Faker.seed(SEED) +fake = Faker() + +# ============================================================================= +# CREATE INFRASTRUCTURE +# ============================================================================= +spark.sql(f"CREATE CATALOG IF NOT EXISTS {CATALOG}") +spark.sql(f"CREATE SCHEMA IF NOT EXISTS {CATALOG}.{SCHEMA}") +spark.sql(f"CREATE VOLUME IF NOT EXISTS {CATALOG}.{SCHEMA}.raw_data") + +# ============================================================================= +# GENERATE CUSTOMERS +# ============================================================================= +print(f"Generating {N_CUSTOMERS:,} customers...") + +customers_pdf = pd.DataFrame({ + "customer_id": [f"CUST-{i:05d}" for i in range(N_CUSTOMERS)], + "name": [fake.company() for _ in range(N_CUSTOMERS)], + "tier": np.random.choice(['Free', 'Pro', 'Enterprise'], N_CUSTOMERS, p=[0.6, 0.3, 0.1]), + "region": np.random.choice(['North', 'South', 'East', 'West'], N_CUSTOMERS, p=[0.4, 0.25, 0.2, 0.15]), +}) + +customers_pdf["arr"] = customers_pdf["tier"].apply( + lambda t: round(np.random.lognormal(11, 0.5), 2) if t == 'Enterprise' + else round(np.random.lognormal(8, 0.6), 2) if t == 'Pro' else 0 +) + +# Lookups for foreign keys +customer_ids = customers_pdf["customer_id"].tolist() +customer_tier_map = dict(zip(customers_pdf["customer_id"], customers_pdf["tier"])) +tier_weights = customers_pdf["tier"].map({'Enterprise': 5.0, 'Pro': 2.0, 'Free': 1.0}) +customer_weights = (tier_weights / tier_weights.sum()).tolist() + +# ============================================================================= +# GENERATE ORDERS +# ============================================================================= +print(f"Generating {N_ORDERS:,} orders...") + +orders_data = [] +for i in range(N_ORDERS): + cid = np.random.choice(customer_ids, p=customer_weights) + tier = customer_tier_map[cid] + amount = np.random.lognormal(7 if tier == 'Enterprise' else 5 if tier == 'Pro' else 3.5, 0.7) + orders_data.append({ + "order_id": f"ORD-{i:06d}", + "customer_id": cid, + "amount": round(amount, 2), + "status": np.random.choice(['completed', 'pending', 'cancelled'], p=[0.85, 0.10, 0.05]), + "order_date": fake.date_between(start_date=START_DATE, end_date=END_DATE), + }) +orders_pdf = pd.DataFrame(orders_data) + +# ============================================================================= +# SAVE TO VOLUME +# ============================================================================= +print(f"Saving to {VOLUME_PATH}...") + +spark.createDataFrame(customers_pdf).write.mode("overwrite").parquet(f"{VOLUME_PATH}/customers") +spark.createDataFrame(orders_pdf).write.mode("overwrite").parquet(f"{VOLUME_PATH}/orders") + +print("Done!") diff --git a/databricks-skills/synthetic-data-generation/scripts/example_polars.py b/databricks-skills/synthetic-data-generation/scripts/example_polars.py new file mode 100644 index 00000000..8fb1fa8d --- /dev/null +++ b/databricks-skills/synthetic-data-generation/scripts/example_polars.py @@ -0,0 +1,157 @@ +"""Generate synthetic data with Polars (local, no Spark dependency). + +This approach is best for: +- Quick prototyping and testing +- Datasets under 100K rows +- Local development without Databricks connection +- Generating parquet files to upload to volumes later +""" +import polars as pl +from faker import Faker +from datetime import datetime, timedelta +import numpy as np +import os + +# ============================================================================= +# CONFIGURATION +# ============================================================================= +# Output +OUTPUT_PATH = "./output" # Local directory for parquet files + +# Data sizes +N_CUSTOMERS = 5000 +N_ORDERS = 15000 + +# Date range - last 6 months from today +END_DATE = datetime.now().replace(hour=0, minute=0, second=0, microsecond=0) +START_DATE = END_DATE - timedelta(days=180) + +# Reproducibility +SEED = 42 + +# Tier distribution: Free 60%, Pro 30%, Enterprise 10% +TIER_VALUES = ["Free", "Pro", "Enterprise"] +TIER_WEIGHTS = [0.6, 0.3, 0.1] + +# Region distribution +REGION_VALUES = ["North", "South", "East", "West"] +REGION_WEIGHTS = [0.4, 0.25, 0.2, 0.15] + +# Order status distribution +STATUS_VALUES = ["pending", "processing", "shipped", "delivered", "cancelled"] +STATUS_WEIGHTS = [0.05, 0.10, 0.15, 0.65, 0.05] + +# Weighted order generation by tier (Enterprise generates more orders) +TIER_ORDER_WEIGHTS = {"Enterprise": 5.0, "Pro": 2.0, "Free": 1.0} + +# Log-normal parameters for order amounts by tier +TIER_AMOUNT_PARAMS = { + "Enterprise": {"mean": 7.5, "sigma": 0.8}, + "Pro": {"mean": 5.5, "sigma": 0.7}, + "Free": {"mean": 4.0, "sigma": 0.6}, +} + +# ============================================================================= +# SETUP +# ============================================================================= +np.random.seed(SEED) +Faker.seed(SEED) +fake = Faker() + +# Create output directory +os.makedirs(OUTPUT_PATH, exist_ok=True) + +# ============================================================================= +# GENERATE CUSTOMERS TABLE +# ============================================================================= +print(f"Generating {N_CUSTOMERS:,} customers...") + +customers = pl.DataFrame({ + "customer_id": [f"CUST-{i:05d}" for i in range(N_CUSTOMERS)], + "name": [fake.name() for _ in range(N_CUSTOMERS)], + "email": [fake.email() for _ in range(N_CUSTOMERS)], + "tier": np.random.choice(TIER_VALUES, N_CUSTOMERS, p=TIER_WEIGHTS).tolist(), + "region": np.random.choice(REGION_VALUES, N_CUSTOMERS, p=REGION_WEIGHTS).tolist(), + "created_at": [fake.date_between(start_date='-2y', end_date=START_DATE) for _ in range(N_CUSTOMERS)], +}) + +# Show tier distribution +print("Tier distribution:") +tier_counts = customers.group_by("tier").len().sort("tier") +for row in tier_counts.iter_rows(named=True): + pct = row["len"] / N_CUSTOMERS * 100 + print(f" {row['tier']}: {row['len']:,} ({pct:.1f}%)") + +# ============================================================================= +# GENERATE ORDERS TABLE WITH REFERENTIAL INTEGRITY +# ============================================================================= +print(f"\nGenerating {N_ORDERS:,} orders with weighted sampling by tier...") + +# Create lookups for foreign key generation +customer_ids = customers["customer_id"].to_list() +customer_tier_map = dict(zip(customers["customer_id"], customers["tier"])) + +# Weight by tier - Enterprise customers generate more orders +tier_weights_list = [TIER_ORDER_WEIGHTS[t] for t in customers["tier"].to_list()] +total_weight = sum(tier_weights_list) +customer_weights = [w / total_weight for w in tier_weights_list] + +# Generate orders with weighted sampling +orders_data = { + "order_id": [], + "customer_id": [], + "amount": [], + "order_date": [], + "status": [], +} + +for i in range(N_ORDERS): + cid = np.random.choice(customer_ids, p=customer_weights) + tier = customer_tier_map[cid] + + # Amount based on tier using log-normal distribution + params = TIER_AMOUNT_PARAMS[tier] + amount = np.random.lognormal(mean=params["mean"], sigma=params["sigma"]) + + orders_data["order_id"].append(f"ORD-{i:06d}") + orders_data["customer_id"].append(cid) + orders_data["amount"].append(round(amount, 2)) + orders_data["order_date"].append(fake.date_between(start_date=START_DATE, end_date=END_DATE)) + orders_data["status"].append(np.random.choice(STATUS_VALUES, p=STATUS_WEIGHTS)) + +orders = pl.DataFrame(orders_data) + +# Show order distribution by customer tier +orders_with_tier = orders.join( + customers.select(["customer_id", "tier"]), + on="customer_id" +) +orders_by_tier = orders_with_tier.group_by("tier").len().sort("tier") +print("\nOrders by customer tier:") +for row in orders_by_tier.iter_rows(named=True): + pct = row["len"] / N_ORDERS * 100 + print(f" {row['tier']}: {row['len']:,} ({pct:.1f}%)") + +# Show amount statistics by tier +print("\nAmount statistics by tier:") +for tier in TIER_VALUES: + tier_amounts = orders_with_tier.filter(pl.col("tier") == tier)["amount"] + if len(tier_amounts) > 0: + print(f" {tier}: avg=${tier_amounts.mean():,.2f}, median=${tier_amounts.median():,.2f}, " + f"min=${tier_amounts.min():,.2f}, max=${tier_amounts.max():,.2f}") + +# ============================================================================= +# SAVE TO PARQUET (LOCAL) +# ============================================================================= +print(f"\nSaving to Parquet files in {OUTPUT_PATH}...") + +customers.write_parquet(f"{OUTPUT_PATH}/customers.parquet") +print(f" Saved: {OUTPUT_PATH}/customers.parquet ({N_CUSTOMERS:,} rows)") + +orders.write_parquet(f"{OUTPUT_PATH}/orders.parquet") +print(f" Saved: {OUTPUT_PATH}/orders.parquet ({N_ORDERS:,} rows)") + +print(f"\nDone! Data saved locally to {OUTPUT_PATH}") +print(f" - customers.parquet: {N_CUSTOMERS:,} rows") +print(f" - orders.parquet: {N_ORDERS:,} rows") +print("\nTo upload to Databricks, use: databricks fs cp -r ./output dbfs:/Volumes/...") diff --git a/databricks-skills/synthetic-data-generation/scripts/generate_ecommerce_data.py b/databricks-skills/synthetic-data-generation/scripts/generate_ecommerce_data.py new file mode 100644 index 00000000..2580af56 --- /dev/null +++ b/databricks-skills/synthetic-data-generation/scripts/generate_ecommerce_data.py @@ -0,0 +1,217 @@ +"""Generate synthetic e-commerce data with customers and orders tables.""" +import numpy as np +import pandas as pd +from datetime import datetime, timedelta +from faker import Faker +from databricks.connect import DatabricksSession + +# ============================================================================= +# CONFIGURATION +# ============================================================================= +# Compute - Serverless recommended +USE_SERVERLESS = True # Set to False and provide CLUSTER_ID for classic compute +CLUSTER_ID = None # Only used if USE_SERVERLESS=False + +# Storage +CATALOG = "ai_dev_kit" # Change to your catalog +SCHEMA = "synthetic_data" # Change to your schema +VOLUME_PATH = f"/Volumes/{CATALOG}/{SCHEMA}/raw_data" + +# Data sizes +N_CUSTOMERS = 5000 +N_ORDERS = 15000 + +# Date range - last 6 months from today +END_DATE = datetime.now().replace(hour=0, minute=0, second=0, microsecond=0) +START_DATE = END_DATE - timedelta(days=180) + +# Write mode - "overwrite" for one-time, "append" for incremental/scheduled jobs +WRITE_MODE = "overwrite" + +# Bad data injection for testing data quality rules +INJECT_BAD_DATA = False # Set to True to inject bad data +BAD_DATA_CONFIG = { + "null_rate": 0.02, # 2% nulls in required fields + "outlier_rate": 0.01, # 1% impossible values + "duplicate_pk_rate": 0.005, # 0.5% duplicate primary keys + "orphan_fk_rate": 0.01, # 1% orphan foreign keys +} + +# Reproducibility +SEED = 42 + +# Tier distribution: Free 60%, Pro 30%, Enterprise 10% +TIER_VALUES = ["Free", "Pro", "Enterprise"] +TIER_WEIGHTS = [0.6, 0.3, 0.1] + +# Region distribution +REGION_VALUES = ["North", "South", "East", "West"] +REGION_WEIGHTS = [0.4, 0.25, 0.2, 0.15] + +# Order status distribution +STATUS_VALUES = ["pending", "processing", "shipped", "delivered", "cancelled"] +STATUS_WEIGHTS = [0.05, 0.10, 0.15, 0.65, 0.05] + +# Weighted order generation by tier (Enterprise generates more orders) +TIER_ORDER_WEIGHTS = {"Enterprise": 5.0, "Pro": 2.0, "Free": 1.0} + +# Log-normal parameters for order amounts by tier +TIER_AMOUNT_PARAMS = { + "Enterprise": {"mean": 7.5, "sigma": 0.8}, # ~$1800 avg, range $500-$8000+ + "Pro": {"mean": 5.5, "sigma": 0.7}, # ~$245 avg, range $50-$1000 + "Free": {"mean": 4.0, "sigma": 0.6}, # ~$55 avg, range $15-$200 +} + +# ============================================================================= +# SETUP +# ============================================================================= +np.random.seed(SEED) +Faker.seed(SEED) +fake = Faker() + +print("Connecting to Databricks...") +if USE_SERVERLESS: + spark = DatabricksSession.builder.serverless(True).getOrCreate() + print("Connected to serverless compute!") +else: + if not CLUSTER_ID: + raise ValueError("CLUSTER_ID must be set when USE_SERVERLESS=False") + spark = DatabricksSession.builder.clusterId(CLUSTER_ID).getOrCreate() + print(f"Connected to cluster {CLUSTER_ID}!") + +# ============================================================================= +# CREATE INFRASTRUCTURE +# ============================================================================= +print(f"\nCreating schema and volume...") +spark.sql(f"CREATE SCHEMA IF NOT EXISTS {CATALOG}.{SCHEMA}") +spark.sql(f"CREATE VOLUME IF NOT EXISTS {CATALOG}.{SCHEMA}.raw_data") +print(f"Infrastructure ready: {VOLUME_PATH}") + +# ============================================================================= +# GENERATE CUSTOMERS TABLE +# ============================================================================= +print(f"\nGenerating {N_CUSTOMERS:,} customers...") + +customers_pdf = pd.DataFrame({ + "customer_id": [f"CUST-{i:05d}" for i in range(N_CUSTOMERS)], + "name": [fake.name() for _ in range(N_CUSTOMERS)], + "email": [fake.email() for _ in range(N_CUSTOMERS)], + "tier": np.random.choice(TIER_VALUES, N_CUSTOMERS, p=TIER_WEIGHTS), + "region": np.random.choice(REGION_VALUES, N_CUSTOMERS, p=REGION_WEIGHTS), + "created_at": [fake.date_between(start_date='-2y', end_date=START_DATE) for _ in range(N_CUSTOMERS)], +}) + +# Show tier distribution +tier_counts = customers_pdf["tier"].value_counts() +print(f"Tier distribution:") +for tier in TIER_VALUES: + count = tier_counts.get(tier, 0) + pct = count / N_CUSTOMERS * 100 + print(f" {tier}: {count:,} ({pct:.1f}%)") + +# ============================================================================= +# GENERATE ORDERS TABLE WITH REFERENTIAL INTEGRITY +# ============================================================================= +print(f"\nGenerating {N_ORDERS:,} orders with weighted sampling by tier...") + +# Create lookups for foreign key generation +customer_ids = customers_pdf["customer_id"].tolist() +customer_tier_map = dict(zip(customers_pdf["customer_id"], customers_pdf["tier"])) + +# Weight by tier - Enterprise customers generate more orders +tier_weights_series = customers_pdf["tier"].map(TIER_ORDER_WEIGHTS) +customer_weights = (tier_weights_series / tier_weights_series.sum()).tolist() + +# Generate orders with weighted sampling +orders_data = [] +for i in range(N_ORDERS): + cid = np.random.choice(customer_ids, p=customer_weights) + tier = customer_tier_map[cid] + + # Amount based on tier using log-normal distribution + params = TIER_AMOUNT_PARAMS[tier] + amount = np.random.lognormal(mean=params["mean"], sigma=params["sigma"]) + + orders_data.append({ + "order_id": f"ORD-{i:06d}", + "customer_id": cid, + "amount": round(amount, 2), + "order_date": fake.date_between(start_date=START_DATE, end_date=END_DATE), + "status": np.random.choice(STATUS_VALUES, p=STATUS_WEIGHTS), + }) + +orders_pdf = pd.DataFrame(orders_data) + +# ============================================================================= +# INJECT BAD DATA (OPTIONAL) +# ============================================================================= +if INJECT_BAD_DATA: + print(f"\nInjecting bad data for quality testing...") + + # Nulls in required fields + null_count = int(len(orders_pdf) * BAD_DATA_CONFIG["null_rate"]) + null_indices = np.random.choice(orders_pdf.index, null_count, replace=False) + orders_pdf.loc[null_indices, "customer_id"] = None + print(f" Injected {null_count} null customer_ids") + + # Outliers (impossible values - negative amounts) + outlier_count = int(len(orders_pdf) * BAD_DATA_CONFIG["outlier_rate"]) + outlier_indices = np.random.choice(orders_pdf.index, outlier_count, replace=False) + orders_pdf.loc[outlier_indices, "amount"] = -999.99 + print(f" Injected {outlier_count} negative amounts") + + # Orphan foreign keys + orphan_count = int(len(orders_pdf) * BAD_DATA_CONFIG["orphan_fk_rate"]) + orphan_indices = np.random.choice(orders_pdf.index, orphan_count, replace=False) + orders_pdf.loc[orphan_indices, "customer_id"] = "CUST-NONEXISTENT" + print(f" Injected {orphan_count} orphan foreign keys") + + # Duplicate primary keys + dup_count = int(len(orders_pdf) * BAD_DATA_CONFIG["duplicate_pk_rate"]) + dup_indices = np.random.choice(orders_pdf.index[:-dup_count], dup_count, replace=False) + for i, idx in enumerate(dup_indices): + orders_pdf.loc[orders_pdf.index[-i-1], "order_id"] = orders_pdf.loc[idx, "order_id"] + print(f" Injected {dup_count} duplicate order_ids") + +# Show order distribution by customer tier +orders_by_tier = orders_pdf.merge( + customers_pdf[["customer_id", "tier"]], on="customer_id", how="left" +)["tier"].value_counts() +print(f"\nOrders by customer tier:") +for tier in TIER_VALUES: + count = orders_by_tier.get(tier, 0) + pct = count / N_ORDERS * 100 + print(f" {tier}: {count:,} ({pct:.1f}%)") + +# Show amount statistics by tier +print(f"\nAmount statistics by tier:") +for tier in TIER_VALUES: + tier_orders = orders_pdf.merge( + customers_pdf[["customer_id", "tier"]], on="customer_id", how="left" + ) + tier_amounts = tier_orders[tier_orders["tier"] == tier]["amount"] + if len(tier_amounts) > 0: + print(f" {tier}: avg=${tier_amounts.mean():,.2f}, median=${tier_amounts.median():,.2f}, " + f"min=${tier_amounts.min():,.2f}, max=${tier_amounts.max():,.2f}") + +# ============================================================================= +# SAVE TO PARQUET +# ============================================================================= +print(f"\nSaving to Parquet files in {VOLUME_PATH} (mode={WRITE_MODE})...") + +# Convert to Spark DataFrames +customers_df = spark.createDataFrame(customers_pdf) +orders_df = spark.createDataFrame(orders_pdf) + +# Save as Parquet +customers_df.write.mode(WRITE_MODE).parquet(f"{VOLUME_PATH}/customers") +print(f" Saved: {VOLUME_PATH}/customers ({N_CUSTOMERS:,} rows)") + +orders_df.write.mode(WRITE_MODE).parquet(f"{VOLUME_PATH}/orders") +print(f" Saved: {VOLUME_PATH}/orders ({N_ORDERS:,} rows)") + +print(f"\nDone! Data saved to {VOLUME_PATH}") +print(f" - customers: {N_CUSTOMERS:,} rows") +print(f" - orders: {N_ORDERS:,} rows") +if INJECT_BAD_DATA: + print(f" - Bad data injected: nulls, outliers, orphan FKs, duplicate PKs") From e73c86d8e530b9df36a180cc04499315de48d56f Mon Sep 17 00:00:00 2001 From: dustinvannoy-db Date: Mon, 16 Feb 2026 12:50:11 -0800 Subject: [PATCH 02/24] Fix databricks-connect version requirements for Python compatibility The serverless() method requires databricks-connect 15.1.0+, but version 17.x only supports Python 3.12. Updated documentation to specify: - Python 3.10/3.11: use >=15.1,<16.2 - Python 3.12: use >=16.2 Co-Authored-By: Claude Opus 4.5 --- .../synthetic-data-generation/SKILL.md | 14 +++-- .../data_gen_update_summary.md | 56 +++++++++++++++++++ 2 files changed, 65 insertions(+), 5 deletions(-) create mode 100644 databricks-skills/synthetic-data-generation/data_gen_update_summary.md diff --git a/databricks-skills/synthetic-data-generation/SKILL.md b/databricks-skills/synthetic-data-generation/SKILL.md index 72c74869..410a1b23 100644 --- a/databricks-skills/synthetic-data-generation/SKILL.md +++ b/databricks-skills/synthetic-data-generation/SKILL.md @@ -77,8 +77,12 @@ Run code locally while Spark operations execute on serverless compute. Best for **Setup:** ```bash -# Install locally - IMPORTANT: Use version 17.3.x (NOT 18.x which has serverless issues) -pip install "databricks-connect>=17.3,<18" faker polars numpy pandas +# Install locally - version depends on your Python version +# For Python 3.10 or 3.11: +pip install "databricks-connect>=15.1,<16.2" faker polars numpy pandas + +# For Python 3.12: +pip install "databricks-connect>=16.2" faker polars numpy pandas # Configure ~/.databrickscfg [DEFAULT] @@ -149,7 +153,7 @@ These libraries are useful for generating realistic synthetic data: - **numpy/pandas**: Statistical distributions and data manipulation - **holidays**: Provides country-specific holiday calendars for realistic date patterns -**For Databricks Connect:** Install locally with `pip install "databricks-connect>=17.3,<18" faker polars numpy pandas holidays` +**For Databricks Connect:** Install locally with `pip install "databricks-connect>=15.1,<16.2" faker polars numpy pandas holidays` (Python 3.10/3.11) or `pip install "databricks-connect>=16.2" faker polars numpy pandas holidays` (Python 3.12) **For Serverless Jobs:** Include in `environments.spec.dependencies`: `["faker", "polars", "numpy", "pandas", "holidays"]` @@ -348,7 +352,7 @@ customers_df = customers_df.withColumn("arr", generate_lognormal_amount(F.col("t The recommended workflow for development and interactive data generation: 1. **Configure Databricks Connect** (one-time setup): - - Install: `pip install "databricks-connect>=17.3,<18" faker polars numpy pandas holidays` + - Install: `pip install "databricks-connect>=15.1,<16.2" faker polars numpy pandas holidays` (Python 3.10/3.11) or `pip install "databricks-connect>=16.2" faker polars numpy pandas holidays` (Python 3.12) - Configure `~/.databrickscfg` with `serverless_compute_id = auto` 2. **Write Python script locally** (e.g., `scripts/generate_data.py`): @@ -1146,6 +1150,6 @@ This returns schema, row counts, and column statistics to confirm the data was w | **Out of memory with large data** | Increase `partitions` parameter in `spark.range()` | | **Foreign keys don't match across tables** | Use same random seed across all generators | | **Delta table write fails** | Ensure `CREATE SCHEMA IF NOT EXISTS` runs before `saveAsTable()` | -| **databricks-connect serverless issues** | Use version 17.3.x: `pip install "databricks-connect>=17.3,<18"` | +| **databricks-connect serverless issues** | Use `pip install "databricks-connect>=15.1,<16.2"` (Python 3.10/3.11) or `pip install "databricks-connect>=16.2"` (Python 3.12) | | **Databricks Connect connection fails** | Verify `~/.databrickscfg` has correct host and `serverless_compute_id = auto` | | **Context corrupted on classic cluster** | Omit `context_id` to create fresh context, reinstall libraries | diff --git a/databricks-skills/synthetic-data-generation/data_gen_update_summary.md b/databricks-skills/synthetic-data-generation/data_gen_update_summary.md new file mode 100644 index 00000000..1533c1de --- /dev/null +++ b/databricks-skills/synthetic-data-generation/data_gen_update_summary.md @@ -0,0 +1,56 @@ +# Synthetic Data Generation Skill Update Summary + +## Summary of Changes + +### SKILL.md Updates + +1. **Added Generation Planning Workflow** (new section at top) + - 3-step process: gather requirements, present table spec with assumptions, ask about data features + - Pre-generation checklist for user approval + - "Surprise Me" fallback option + +2. **Updated Compute Selection** + - Serverless-first with confirmation prompt + - databricks-connect version guidance (>=15.1,<16.2 for Python 3.10/3.11; >=16.2 for Python 3.12) + +3. **Replaced Data Generation Approaches** + - **Option 1: Spark + Faker** (recommended for most cases, writing to UC) + - **Option 2: Polars** (for local development, quick prototyping) + - Removed dbldatagen as primary approach + +4. **Added Deployment Options** + - Ephemeral script run (default) + - DABs bundle deployment with `client: "4"` for serverless + +5. **Added Business Integrity Requirements** + - Value coherence, tier behavior, temporal patterns, geographic patterns + - Bad data injection patterns (nulls, outliers, duplicates, orphan FKs) + +6. **Added Domain-Specific Guidance** + - Retail/E-commerce, Support/CRM, Manufacturing/IoT, Financial Services + +7. **Updated Complete Examples** + - Example 1: E-commerce Data (Spark + Faker + Pandas) + - Example 2: Local Development with Polars + - Example 3: Large-Scale with Faker UDFs + - Example 4: Legacy Approach (Faker + Pandas) + +### Script Updates + +| File | Action | +|------|--------| +| `scripts/generate_ecommerce_data.py` | Updated: serverless-first, bad data injection, incremental mode | +| `scripts/example_dbldatagen.py` | Deleted | +| `scripts/example_polars.py` | Created: local generation with Polars | +| `scripts/example_faker_udf.py` | Updated: serverless-first configuration | + +## Key Decisions Implemented + +1. **Serverless Default**: Yes, with user confirmation prompt +2. **databricks-connect Version**: >=15.1,<16.2 (Python 3.10/3.11) or >=16.2 (Python 3.12) +3. **Planning Phase**: Show table spec with assumptions before generating +4. **Templates**: Domain guidance (retail, manufacturing) without rigid schemas +5. **Incremental Mode**: Support `.mode("append")` for scheduled jobs +6. **Data Quality Features**: Bad data injection (nulls, outliers, duplicates, orphan FKs) +7. **NO dbldatagen by default**: Use Spark + Faker or Polars +8. **Deployment**: Both ephemeral scripts AND DABs bundles as options From 805dbb6e9bd67d2b4409a867f52230cbd16145dd Mon Sep 17 00:00:00 2001 From: dustinvannoy-db Date: Thu, 19 Feb 2026 15:51:11 -0800 Subject: [PATCH 03/24] Improve synthetic-data-generation skill with Spark preference and catalog management - Strongly recommend Spark + Faker for all data generation (default approach) - Only use Polars for <10K rows if user explicitly prefers local generation - Add volume upload instructions using databricks fs commands - Remove CREATE CATALOG statements - assume catalogs already exist - Update decision guides and examples to reflect Spark-first approach - Consolidate and simplify execution options and installation instructions - Update best practices and common issues sections Co-Authored-By: Claude Sonnet 4.5 --- .../synthetic-data-generation/SKILL.md | 389 +++++++----------- 1 file changed, 153 insertions(+), 236 deletions(-) diff --git a/databricks-skills/synthetic-data-generation/SKILL.md b/databricks-skills/synthetic-data-generation/SKILL.md index 5ca25d6d..fd53176e 100644 --- a/databricks-skills/synthetic-data-generation/SKILL.md +++ b/databricks-skills/synthetic-data-generation/SKILL.md @@ -1,11 +1,12 @@ --- name: synthetic-data-generation -description: "Generate realistic synthetic data using Spark + Faker or Polars. Supports serverless execution, multiple output formats (Parquet/JSON/CSV/Delta), and scales from thousands to millions of rows. Use for test data, demo datasets, or synthetic tables." +description: "Generate realistic synthetic data using Spark + Faker (strongly recommended). Supports serverless execution, multiple output formats (Parquet/JSON/CSV/Delta), and scales from thousands to millions of rows. For small datasets (<10K rows), can optionally generate locally and upload to volumes. Use for test data, demo datasets, or synthetic tables." --- # Synthetic Data Generation -Generate realistic, story-driven synthetic data for Databricks using Spark + Faker or Polars. +Generate realistic, story-driven synthetic data for Databricks using Spark + Faker (strongly recommended). +For small datasets (<10K rows), can optionally generate locally with Polars and upload to volumes. Always present a generation plan with assumptions before generating code. ## Generation Planning Workflow @@ -35,6 +36,10 @@ Show a clear specification with **YOUR ASSUMPTIONS surfaced**: - Date range: last 6 months from today - Status distribution: 65% delivered, 15% shipped, 10% processing, 5% pending, 5% cancelled +**Generation Approach:** +- **Total rows < 10K**: Notify user you'll generate data using **Spark** (strongly recommended for all use cases) +- **Total rows < 10K with user preference**: Only if user explicitly prefers local generation, generate with Polars locally and upload to volume using `databricks fs cp` + **Ask user**: "Does this look correct? Any adjustments needed?" ### Step 3: Ask About Data Features @@ -54,6 +59,8 @@ Prompt user with options (enabled by default unless otherwise noted): Before writing any generation code, verify: +- [ ] Generation approach determined: **Spark (strongly recommended)** or local generation with upload (only for <10K rows if user prefers) +- [ ] If using local generation: User notified and prefers this approach - [ ] User confirmed compute preference (serverless vs cluster) - [ ] Table specification shown and approved - [ ] Assumptions about distributions surfaced and confirmed @@ -63,26 +70,25 @@ Before writing any generation code, verify: **Do NOT proceed to code generation until user approves the plan.** -## Execution Options - -Choose your execution mode based on your needs: +## Execution Options & Installation -### Option 1: Databricks Connect with Serverless (Recommended) - -Run code locally while Spark operations execute on serverless compute. Best for development and interactive work. +Choose your execution mode based on your needs. **Serverless is strongly recommended** for all use cases. **When user requests data generation:** 1. Confirm serverless is acceptable: "I'll use serverless compute. Is that OK?" 2. If they request classic cluster: "Serverless is recommended for cost efficiency. Are you sure you need a classic cluster?" -**Setup:** +### Option 1: Databricks Connect with Serverless (Recommended) + +Run code locally while Spark operations execute on serverless compute. Best for development and interactive work. + +**Install locally (one-time setup):** ```bash -# Install locally - version depends on your Python version -# For Python 3.10 or 3.11: -pip install "databricks-connect>=15.1,<16.2" faker polars numpy pandas +# Python 3.10 or 3.11: +pip install "databricks-connect>=15.1,<16.2" faker polars numpy pandas holidays -# For Python 3.12: -pip install "databricks-connect>=16.2" faker polars numpy pandas +# Python 3.12: +pip install "databricks-connect>=16.2,<18.0" faker polars numpy pandas holidays # Configure ~/.databrickscfg [DEFAULT] @@ -96,80 +102,68 @@ auth_type = databricks-cli from databricks.connect import DatabricksSession spark = DatabricksSession.builder.serverless(True).getOrCreate() -# Now Spark operations execute on serverless compute +# Spark operations now execute on serverless compute ``` -**Benefits:** -- Instant start (no cluster spin-up) -- Local debugging with IDE integration -- Dependencies installed locally via pip -- Iterate quickly: edit file, re-run immediately +**Benefits:** Instant start, local debugging, fast iteration (edit file, re-run immediately) ### Option 2: Serverless Job (Production/Scheduled) -Submit jobs to serverless compute with dependencies managed via the `environments` parameter. Best for production workloads and scheduled jobs. +Submit jobs to serverless compute with automatic dependency management. Best for production and scheduled workloads. -**Use `create_job` MCP tool with environments:** -- `name`: "generate_synthetic_data" -- `tasks`: [{ task with `environment_key` reference }] -- `environments`: [{ +**Dependencies managed via `environments` parameter:** +```python +# Use create_job MCP tool with: +{ + "name": "generate_synthetic_data", + "tasks": [{ "environment_key": "datagen_env", ... }], + "environments": [{ "environment_key": "datagen_env", "spec": { "client": "4", "dependencies": ["faker", "polars", "numpy", "pandas", "holidays"] } }] +} +``` -**Benefits:** -- No local environment needed -- Automatic dependency management -- Scheduled execution support -- Production-ready scaling - -### Option 3: Classic Cluster (Fallback) +**Benefits:** No local setup, automatic dependency management, production-ready scaling -Execute on a classic all-purpose cluster. Use only if serverless is unavailable or you need specific cluster features. +### Option 3: Classic Cluster (Fallback Only) -**Warning:** Classic clusters take 3-8 minutes to start if not already running. Prefer serverless for faster iteration. +Use only if serverless unavailable or you need specific cluster features (GPUs, custom init scripts). -**Workflow:** -1. Install dependencies using `execute_databricks_command` tool: - - `code`: "%pip install faker polars numpy pandas holidays" - - Save returned `cluster_id` and `context_id` +**Warning:** Classic clusters take 3-8 minutes to start. Prefer serverless. -2. Execute script using `run_python_file_on_databricks` tool: - - `file_path`: "scripts/generate_data.py" - - `cluster_id`: "" - - `context_id`: "" +**Install dependencies in cluster:** +```python +# Using execute_databricks_command tool: +code = "%pip install faker polars numpy pandas holidays" +# Save returned cluster_id and context_id for subsequent calls +``` -**When to use:** Only when serverless is not available, or you need specific cluster configurations (GPUs, custom init scripts, etc.) +**When to use:** Only when serverless not available or specific cluster configurations required -## Common Libraries +## Required Libraries -These libraries are useful for generating realistic synthetic data: +Standard libraries for generating realistic synthetic data: -- **faker**: Generates realistic names, addresses, emails, companies, dates, etc. (100+ providers) -- **polars**: Fast local DataFrame library for small/medium datasets +- **faker**: Realistic names, addresses, emails, companies, dates (100+ providers) - **numpy/pandas**: Statistical distributions and data manipulation -- **holidays**: Provides country-specific holiday calendars for realistic date patterns - -**For Databricks Connect:** Install locally with `pip install "databricks-connect>=15.1,<16.2" faker polars numpy pandas holidays` (Python 3.10/3.11) or `pip install "databricks-connect>=16.2" faker polars numpy pandas holidays` (Python 3.12) - -**For Serverless Jobs:** Include in `environments.spec.dependencies`: `["faker", "polars", "numpy", "pandas", "holidays"]` +- **holidays**: Country-specific holiday calendars for realistic date patterns +- **polars**: Fast local DataFrame library (optional, only for local generation) -**For Classic Clusters:** Install using `execute_databricks_command` tool: -- `code`: "%pip install faker polars numpy pandas holidays" -- Save the returned `cluster_id` and `context_id` for subsequent calls +See **Execution Options & Installation** above for installation instructions per execution mode. ## Data Generation Approaches Choose your approach based on scale and where you need to write data: -### Approach 1: Spark + Faker (Recommended for most cases) +### Approach 1: Spark + Faker with Pandas UDFs (Recommended for most cases) **Best for:** Any dataset size, especially >100K rows, writing to Unity Catalog -Generate data with Pandas + Faker locally, convert to Spark DataFrame for saving to Databricks. +Generate data with Spark + Faker with Pandas UDFs, save to Databricks. **Key features:** - Full access to 100+ Faker providers (names, addresses, companies, etc.) @@ -235,17 +229,19 @@ customers_df = ( ) ``` -### Approach 2: Polars (For local development) +### Approach 2: Polars (For local development - Use only if Spark not suitable) -**Best for:** Quick prototyping, datasets <100K rows, no Spark dependency needed +**Important:** Spark is strongly recommended for all data generation. Only use this approach for datasets <10K rows if user explicitly prefers local generation. -Generate entirely with Polars + Faker locally, export to parquet files. +**Best for:** Quick prototyping when Spark is not needed, datasets <10K rows + +Generate entirely with Polars + Faker locally, export to parquet files, then upload to Databricks volumes. **Key features:** - Fast local generation (no Spark overhead) - Simple, clean API -- Perfect for testing and prototyping -- Can upload resulting parquet to Databricks volumes +- Perfect for quick prototyping with very small datasets +- Requires manual upload to Databricks volumes **Example:** ```python @@ -267,142 +263,57 @@ customers = pl.DataFrame({ customers.write_parquet("./output/customers.parquet") ``` -### Decision Guide +**Upload to Databricks Volume:** +After generating data locally, upload to a Databricks volume: -| Need | Recommended Approach | -|------|---------------------| -| Write to Unity Catalog | **Spark + Faker** | -| Scale to millions of rows | **Spark + Faker** with Pandas UDFs | -| Quick local prototype | **Polars** | -| Realistic text (names/addresses) | **Either** (both use Faker) | -| No Spark dependency | **Polars** | - -### Approach 3: Faker with Spark UDFs - -**Best for:** Realistic text data (names, addresses, companies), complex custom patterns - -Faker provides 100+ data providers for realistic text. Wrap it in Spark UDFs for parallelism. - -**Key features:** -- Access to 100+ Faker providers (names, addresses, companies, phone numbers, etc.) -- Custom UDFs for complex conditional logic -- Row-level coherence where attributes correlate logically -- Flexibility for domain-specific patterns - -**Example:** -```python -from pyspark.sql import functions as F -from pyspark.sql.types import StringType, DoubleType -from faker import Faker -import numpy as np +```bash +# Create directory in volume if needed +databricks fs mkdirs dbfs:/Volumes////source_data/ -# Define Faker UDFs for realistic text -@F.udf(returnType=StringType()) -def generate_company(): - return Faker().company() +# Upload local data to volume +databricks fs cp -r ./output/customers.parquet dbfs:/Volumes////source_data/ +databricks fs cp -r ./output/orders.parquet dbfs:/Volumes////source_data/ +``` -@F.udf(returnType=StringType()) -def generate_address(): - return Faker().address().replace('\n', ', ') +### Decision Guide -@F.udf(returnType=DoubleType()) -def generate_lognormal_amount(tier): - """Generate amount based on tier using log-normal distribution.""" - np.random.seed(hash(tier) % (2**32)) - if tier == "Enterprise": - return float(np.random.lognormal(mean=10, sigma=0.8)) - elif tier == "Pro": - return float(np.random.lognormal(mean=8, sigma=0.7)) - else: - return float(np.random.lognormal(mean=5, sigma=0.6)) +| Need | Recommended Approach | +|------|---------------------| +| **Any data generation (default)** | **Spark + Faker** with Pandas UDFs (strongly recommended) | +| Quick local prototype (<10K rows, user prefers local) | **Polars** (then upload with `databricks fs cp`) | -# Generate with Spark parallelism -customers_df = ( - spark.range(0, 1_000_000, numPartitions=32) - .select( - F.concat(F.lit("CUST-"), F.lpad(F.col("id").cast("string"), 5, "0")).alias("customer_id"), - generate_company().alias("name"), - generate_address().alias("address"), - F.when(F.rand(42) < 0.6, "Free") - .when(F.rand(42) < 0.9, "Pro") - .otherwise("Enterprise").alias("tier") - ) -) +**Default: Always use Spark + Faker unless user explicitly requests local generation for small datasets (<10K rows).** -# Add tier-based amounts -customers_df = customers_df.withColumn("arr", generate_lognormal_amount(F.col("tier"))) -``` ### When to Use Each Approach | Scenario | Recommended Approach | |----------|---------------------| +| **Default - any data generation** | **Spark + Faker with Pandas UDFs** (strongly recommended) | | Generating 1M+ rows | **Spark + Faker with Pandas UDFs** | -| Need realistic names/addresses/emails | **Faker** (Spark or Polars) | -| Writing to Unity Catalog | **Spark + Faker** | -| Complex conditional row logic | **Spark + Faker UDFs** | -| Foreign key with complex weighting | **Spark + Faker** | -| Quick prototyping (small data) | **Polars** | -| No Spark dependency needed | **Polars** | - -## Workflow - -### Primary: Databricks Connect with Serverless - -The recommended workflow for development and interactive data generation: - -1. **Configure Databricks Connect** (one-time setup): - - Install: `pip install "databricks-connect>=15.1,<16.2" faker polars numpy pandas holidays` (Python 3.10/3.11) or `pip install "databricks-connect>=16.2" faker polars numpy pandas holidays` (Python 3.12) - - Configure `~/.databrickscfg` with `serverless_compute_id = auto` - -2. **Write Python script locally** (e.g., `scripts/generate_data.py`): - ```python - from databricks.connect import DatabricksSession - spark = DatabricksSession.builder.serverless(True).getOrCreate() - # Your data generation code here - ``` - -3. **Run locally** - Spark operations execute on serverless compute: - ```bash - python scripts/generate_data.py - ``` - -4. **Iterate quickly**: Edit file, re-run immediately. No cluster spin-up time. - -### Production: Serverless Job +| Quick prototyping (<10K rows, user explicitly prefers local) | **Polars** (then upload with `databricks fs cp`) | -For scheduled or production workloads: +**Important:** Default to Spark + Faker for all cases. Only use Polars if dataset is <10K rows AND user explicitly requests local generation. -1. **Write Python script locally** +## Workflow -2. **Upload to workspace** using `upload_file` MCP tool: - - `local_path`: "scripts/generate_data.py" - - `workspace_path`: "/Workspace/Users/{username}/datagen/{project_name}/generate_data.py" +### Development (Databricks Connect) -3. **Create serverless job** using `create_job` MCP tool: - - `name`: "generate_synthetic_data" - - `tasks`: [{ - "task_key": "generate", - "spark_python_task": { - "python_file": "/Workspace/Users/{username}/datagen/{project_name}/generate_data.py" - }, - "environment_key": "datagen_env" - }] - - `environments`: [{ - "environment_key": "datagen_env", - "spec": { - "client": "4", - "dependencies": ["faker", "polars", "numpy", "pandas", "holidays"] - } - }] +1. **One-time setup**: Install dependencies locally (see **Execution Options & Installation** above) +2. **Write script**: Create `scripts/generate_data.py` with `DatabricksSession.builder.serverless(True)` +3. **Run locally**: `python scripts/generate_data.py` (Spark ops execute on serverless) +4. **Iterate**: Edit file, re-run immediately -4. **Run job** using `run_job_now` MCP tool +### Production (Serverless Job) -5. **Monitor** using `get_run` or `wait_for_run` MCP tools +1. **Write script locally** +2. **Upload** using `upload_file` MCP tool to `/Workspace/Users/{username}/datagen/{project}/` +3. **Create job** using `create_job` MCP tool with `environments` parameter (see Option 2 above) +4. **Run & monitor** using `run_job_now` and `wait_for_run` MCP tools -### Production: DABs Bundle Deployment +### Production (DABs Bundle) -For scheduled runs with version control and CI/CD: +For version control and CI/CD: ```yaml # databricks.yml @@ -433,25 +344,6 @@ environments: - holidays ``` -**Note**: The `environments` block with `client: "4"` enables serverless compute. Dependencies are installed automatically. - -### Fallback: Classic Cluster - -Only use if serverless is unavailable: - -1. **Install dependencies** using `execute_databricks_command` tool: - - `code`: "%pip install faker polars numpy pandas holidays" - - Save returned `cluster_id` and `context_id` - -2. **Execute script** using `run_python_file_on_databricks` tool: - - `file_path`: "scripts/generate_data.py" - - `cluster_id`: "" - - `context_id`: "" - -3. **Iterate**: Edit local file, re-execute with same context (faster, keeps installed libraries) - -**Note:** Classic clusters take 3-8 minutes to start. Prefer serverless for faster iteration. - ## Storage Destination ### Ask for Schema Name @@ -464,7 +356,9 @@ If the user provides just a schema name, use `ai_dev_kit.{schema}`. If they prov ### Create Infrastructure in the Script -Always create the catalog, schema, and volume **inside the Python script** using `spark.sql()`. Do NOT make separate MCP SQL calls - it's much slower. +Always create the schema and volume **inside the Python script** using `spark.sql()`. Do NOT make separate MCP SQL calls - it's much slower. + +**Important:** Do NOT create catalogs - assume they already exist. Only create schema and volume. The `spark` variable is available by default on Databricks clusters. @@ -472,7 +366,7 @@ The `spark` variable is available by default on Databricks clusters. # ============================================================================= # CREATE INFRASTRUCTURE (inside the Python script) # ============================================================================= -spark.sql(f"CREATE CATALOG IF NOT EXISTS {CATALOG}") +# Note: Assume catalog exists - do NOT create it spark.sql(f"CREATE SCHEMA IF NOT EXISTS {CATALOG}.{SCHEMA}") spark.sql(f"CREATE VOLUME IF NOT EXISTS {CATALOG}.{SCHEMA}.raw_data") ``` @@ -768,29 +662,53 @@ When generating data for specific domains, consider these realistic patterns: ## Key Principles -### 1. Use Polars for Generation, Spark for Saving +### 1. Use Spark + Faker for All Data Generation (Strongly Recommended) -Generate data with Polars (faster than Pandas), convert to Spark for saving: +**Default:** Generate data with Spark + Faker for all use cases. This provides scalability, parallelism, and direct integration with Unity Catalog. + +```python +from pyspark.sql import functions as F +from pyspark.sql.functions import pandas_udf +from pyspark.sql.types import StringType +import pandas as pd +from faker import Faker + +@pandas_udf(StringType()) +def fake_company(ids: pd.Series) -> pd.Series: + fake = Faker() + return pd.Series([fake.company() for _ in range(len(ids))]) + +# Generate with Spark + Pandas UDFs +customers_df = ( + spark.range(0, N_CUSTOMERS, numPartitions=8) + .select( + F.concat(F.lit("CUST-"), F.lpad(F.col("id").cast("string"), 5, "0")).alias("customer_id"), + fake_company(F.col("id")).alias("name"), + F.when(F.rand() < 0.6, "Free") + .when(F.rand() < 0.9, "Pro") + .otherwise("Enterprise").alias("tier"), + ) +) +customers_df.write.mode("overwrite").parquet(f"{VOLUME_PATH}/customers") +``` + +**Alternative (only for <10K rows if user prefers):** Generate with Polars locally and upload: ```python import polars as pl -import numpy as np from faker import Faker fake = Faker() -# Generate with Polars (faster than Pandas) +# Generate with Polars customers_pl = pl.DataFrame({ "customer_id": [f"CUST-{i:05d}" for i in range(N_CUSTOMERS)], "name": [fake.company() for _ in range(N_CUSTOMERS)], "tier": np.random.choice(['Free', 'Pro', 'Enterprise'], N_CUSTOMERS, p=[0.6, 0.3, 0.1]).tolist(), - "region": np.random.choice(['North', 'South', 'East', 'West'], N_CUSTOMERS, p=[0.4, 0.25, 0.2, 0.15]).tolist(), - "created_at": [fake.date_between(start_date='-2y', end_date='-6m') for _ in range(N_CUSTOMERS)], }) -# Convert to Spark and save -customers_df = spark.createDataFrame(customers_pl.to_pandas()) -customers_df.write.mode("overwrite").parquet(f"{VOLUME_PATH}/customers") +# Save locally then upload with: databricks fs cp -r ./output dbfs:/Volumes/{catalog}/{schema}/raw_data/ +customers_pl.write_parquet("./output/customers.parquet") ``` ### 2. Iterate on DataFrames for Referential Integrity @@ -976,7 +894,9 @@ INJECT_BAD_DATA = False # Set True for data quality testing **Usage:** Copy to your scripts folder, update CATALOG/SCHEMA, run with `python generate_ecommerce_data.py` -### Example 2: Local Development with Polars +### Example 2: Local Development with Polars (Only for <10K rows if user prefers) + +**Note:** Spark is strongly recommended. Only use this approach for datasets <10K rows if user explicitly prefers local generation. Generate synthetic data locally without Spark dependency, then upload to Databricks. @@ -984,9 +904,9 @@ Generate synthetic data locally without Spark dependency, then upload to Databri **Features:** - Fast local generation (no Spark overhead) -- Perfect for prototyping and testing +- For very small datasets (<10K rows) - Outputs parquet files to local directory -- Upload to volumes with `databricks fs cp` +- Requires manual upload to volumes with `databricks fs cp` **Key pattern:** @@ -1114,24 +1034,22 @@ This returns schema, row counts, and column statistics to confirm the data was w ## Best Practices ### Execution -1. **Use Databricks Connect with serverless** for development - instant start, local debugging -2. **Use serverless jobs** for production - automatic dependency management, scheduling -3. **Prefer serverless over classic** - avoid 3-8 minute cluster spin-up times -4. **Ask for schema**: Default to `ai_dev_kit` catalog, ask user for schema name -5. **Present plan before generating**: Show table spec with assumptions, get user approval +1. **Use serverless** (Databricks Connect for dev, jobs for production) - instant start, no cluster wait +2. **Ask for schema**: Default to `ai_dev_kit` catalog, ask user for schema name +3. **Present plan before generating**: Show table spec with assumptions, get user approval ### Data Generation -6. **Use Faker with Pandas UDFs for scale** (1M+ rows) - Spark parallelism -7. **Use Polars for quick prototyping** - fast local generation -8. **Master tables first**: Generate customers, then orders reference customer_ids -9. **Weighted sampling**: Enterprise customers generate more activity -10. **Distributions**: Log-normal for values, exponential for times, weighted categorical -11. **Time patterns**: Weekday/weekend, holidays, seasonality, event spikes -12. **Row coherence**: Priority affects resolution time affects CSAT -13. **Volume for aggregation**: 10K-50K rows minimum so patterns survive GROUP BY -14. **Always use files**: Write to local file, execute, edit if error, re-execute -15. **Context reuse**: Pass `cluster_id` and `context_id` for faster iterations -16. **Libraries**: Install `faker` and `holidays` first; most others are pre-installed +6. **Default to Spark + Faker** for all data generation - scalable, parallel, direct Unity Catalog integration +7. **Use Pandas UDFs for scale** (1M+ rows) - Spark parallelism with Faker +8. **Only use local generation** (<10K rows) if user explicitly prefers it - then upload with `databricks fs cp` +9. **Master tables first**: Generate customers, then orders reference customer_ids +10. **Weighted sampling**: Enterprise customers generate more activity +11. **Distributions**: Log-normal for values, exponential for times, weighted categorical +12. **Time patterns**: Weekday/weekend, holidays, seasonality, event spikes +13. **Row coherence**: Priority affects resolution time affects CSAT +14. **Volume for aggregation**: 10K-50K rows minimum so patterns survive GROUP BY +15. **Always use files**: Write to local file, execute, edit if error, re-execute +16. **Context reuse**: Pass `cluster_id` and `context_id` for faster iterations (classic cluster only) ## Related Skills @@ -1140,25 +1058,24 @@ This returns schema, row counts, and column statistics to confirm the data was w - **[databricks-unity-catalog](../databricks-unity-catalog/SKILL.md)** - for managing catalogs, schemas, and volumes where data is stored ### Output -14. **Create infrastructure in script**: Use `CREATE SCHEMA/VOLUME IF NOT EXISTS` -15. **Raw data only**: No `total_x`, `sum_x`, `avg_x` fields - SDP pipeline computes those -16. **Choose output format** based on downstream needs (Parquet/JSON/CSV/Delta) -17. **Configuration at top**: All sizes, dates, and paths as variables -18. **Dynamic dates**: Use `datetime.now() - timedelta(days=180)` for last 6 months +14. **Create infrastructure in script**: Use `CREATE SCHEMA/VOLUME IF NOT EXISTS` - do NOT create catalogs +15. **Assume catalogs exist**: Never auto-create catalogs, only create schema and volume +16. **Raw data only**: No `total_x`, `sum_x`, `avg_x` fields - SDP pipeline computes those +17. **Choose output format** based on downstream needs (Parquet/JSON/CSV/Delta) +18. **Configuration at top**: All sizes, dates, and paths as variables +19. **Dynamic dates**: Use `datetime.now() - timedelta(days=180)` for last 6 months ## Common Issues | Issue | Solution | |-------|----------| | **"Either base environment or version must be provided"** | Add `"client": "4"` to `spec` in job environments (auto-injected by MCP tool) | -| **"ModuleNotFoundError: No module named 'faker'"** | Add `faker` to dependencies or install locally: `pip install faker` | -| **"ModuleNotFoundError: No module named 'polars'"** | Add `polars` to dependencies or install locally: `pip install polars` | +| **"ModuleNotFoundError"** for faker/polars/etc. | See **Execution Options & Installation** section for dependency setup per execution mode | | **Serverless job fails to start** | Verify workspace has serverless compute enabled; check Unity Catalog permissions | | **Faker UDF is slow** | Use `pandas_udf` for batched operations; adjust `numPartitions` | | **Classic cluster startup is slow (3-8 min)** | Switch to Databricks Connect with serverless for instant start | | **Out of memory with large data** | Increase `partitions` parameter in `spark.range()` | | **Foreign keys don't match across tables** | Use same random seed across all generators | | **Delta table write fails** | Ensure `CREATE SCHEMA IF NOT EXISTS` runs before `saveAsTable()` | -| **databricks-connect serverless issues** | Use `pip install "databricks-connect>=15.1,<16.2"` (Python 3.10/3.11) or `pip install "databricks-connect>=16.2"` (Python 3.12) | -| **Databricks Connect connection fails** | Verify `~/.databrickscfg` has correct host and `serverless_compute_id = auto` | +| **Databricks Connect issues** | Verify correct version for your Python (see **Execution Options & Installation**), check `~/.databrickscfg` has `serverless_compute_id = auto` | | **Context corrupted on classic cluster** | Omit `context_id` to create fresh context, reinstall libraries | From fccf5756bd318d02530a77861e019ec1bc348d75 Mon Sep 17 00:00:00 2001 From: dustinvannoy-db Date: Thu, 19 Feb 2026 16:07:21 -0800 Subject: [PATCH 04/24] Cleanup data gen skill --- .../synthetic-data-generation/.gitignore | 7 -- .../synthetic-data-generation/SKILL.md | 102 ++++-------------- .../data_gen_update_summary.md | 56 ---------- .../scripts/example_pandas.py | 94 ---------------- 4 files changed, 20 insertions(+), 239 deletions(-) delete mode 100644 databricks-skills/synthetic-data-generation/.gitignore delete mode 100644 databricks-skills/synthetic-data-generation/data_gen_update_summary.md delete mode 100644 databricks-skills/synthetic-data-generation/scripts/example_pandas.py diff --git a/databricks-skills/synthetic-data-generation/.gitignore b/databricks-skills/synthetic-data-generation/.gitignore deleted file mode 100644 index 96dceadf..00000000 --- a/databricks-skills/synthetic-data-generation/.gitignore +++ /dev/null @@ -1,7 +0,0 @@ -# Generated data -output/ - -# Python -__pycache__/ -*.pyc -.venv/ diff --git a/databricks-skills/synthetic-data-generation/SKILL.md b/databricks-skills/synthetic-data-generation/SKILL.md index fd53176e..48def889 100644 --- a/databricks-skills/synthetic-data-generation/SKILL.md +++ b/databricks-skills/synthetic-data-generation/SKILL.md @@ -33,12 +33,11 @@ Show a clear specification with **YOUR ASSUMPTIONS surfaced**: **Assumptions I'm making:** - Amount distribution: log-normal by tier (Enterprise avg ~$1800, Pro ~$245, Free ~$55) -- Date range: last 6 months from today - Status distribution: 65% delivered, 15% shipped, 10% processing, 5% pending, 5% cancelled **Generation Approach:** -- **Total rows < 10K**: Notify user you'll generate data using **Spark** (strongly recommended for all use cases) -- **Total rows < 10K with user preference**: Only if user explicitly prefers local generation, generate with Polars locally and upload to volume using `databricks fs cp` +- **Default**: Generate data using **Spark** (recommended for all use cases) +- **Alternative for <10K rows**: Only if user explicitly prefers local generation, use Polars and upload to volume using `databricks fs cp` **Ask user**: "Does this look correct? Any adjustments needed?" @@ -182,24 +181,25 @@ from databricks.connect import DatabricksSession spark = DatabricksSession.builder.serverless(True).getOrCreate() -# Define Pandas UDFs for Faker data (batch processing) +# Define Pandas UDFs for Faker data (batch processing for parallelism) @pandas_udf(StringType()) def fake_name(ids: pd.Series) -> pd.Series: fake = Faker() return pd.Series([fake.name() for _ in range(len(ids))]) @pandas_udf(StringType()) -def fake_email(ids: pd.Series) -> pd.Series: +def fake_company(ids: pd.Series) -> pd.Series: fake = Faker() - return pd.Series([fake.email() for _ in range(len(ids))]) + return pd.Series([fake.company() for _ in range(len(ids))]) # Generate with Spark + Pandas UDFs +# Adjust numPartitions based on scale: 8 for <100K, 32 for 1M+ customers_df = ( spark.range(0, N_CUSTOMERS, numPartitions=8) .select( F.concat(F.lit("CUST-"), F.lpad(F.col("id").cast("string"), 5, "0")).alias("customer_id"), fake_name(F.col("id")).alias("name"), - fake_email(F.col("id")).alias("email"), + fake_company(F.col("id")).alias("company"), F.when(F.rand() < 0.6, "Free") .when(F.rand() < 0.9, "Pro") .otherwise("Enterprise").alias("tier"), @@ -208,30 +208,9 @@ customers_df = ( customers_df.write.mode("overwrite").parquet(f"{VOLUME_PATH}/customers") ``` -**Scaling with Pandas UDFs (for large datasets):** -```python -from pyspark.sql import functions as F -from pyspark.sql.functions import pandas_udf -from pyspark.sql.types import StringType -import pandas as pd -from faker import Faker - -@pandas_udf(StringType()) -def generate_company_batch(ids: pd.Series) -> pd.Series: - """Batch generate company names - more efficient than row-by-row UDF.""" - fake = Faker() - return pd.Series([fake.company() for _ in range(len(ids))]) - -# Generate with Spark parallelism + batch processing -customers_df = ( - spark.range(0, 1_000_000, numPartitions=32) - .withColumn("name", generate_company_batch(F.col("id"))) -) -``` - ### Approach 2: Polars (For local development - Use only if Spark not suitable) -**Important:** Spark is strongly recommended for all data generation. Only use this approach for datasets <10K rows if user explicitly prefers local generation. +**Important:** Only use this approach for datasets <10K rows if user explicitly prefers local generation. **Best for:** Quick prototyping when Spark is not needed, datasets <10K rows @@ -275,25 +254,15 @@ databricks fs cp -r ./output/customers.parquet dbfs:/Volumes/// databricks fs cp -r ./output/orders.parquet dbfs:/Volumes////source_data/ ``` -### Decision Guide - -| Need | Recommended Approach | -|------|---------------------| -| **Any data generation (default)** | **Spark + Faker** with Pandas UDFs (strongly recommended) | -| Quick local prototype (<10K rows, user prefers local) | **Polars** (then upload with `databricks fs cp`) | - -**Default: Always use Spark + Faker unless user explicitly requests local generation for small datasets (<10K rows).** - - ### When to Use Each Approach | Scenario | Recommended Approach | |----------|---------------------| -| **Default - any data generation** | **Spark + Faker with Pandas UDFs** (strongly recommended) | +| **Default - any data generation** | **Spark + Faker with Pandas UDFs** | | Generating 1M+ rows | **Spark + Faker with Pandas UDFs** | | Quick prototyping (<10K rows, user explicitly prefers local) | **Polars** (then upload with `databricks fs cp`) | -**Important:** Default to Spark + Faker for all cases. Only use Polars if dataset is <10K rows AND user explicitly requests local generation. +**Default:** Use Spark + Faker for all cases. Only use Polars if dataset is <10K rows AND user explicitly requests local generation. ## Workflow @@ -662,9 +631,9 @@ When generating data for specific domains, consider these realistic patterns: ## Key Principles -### 1. Use Spark + Faker for All Data Generation (Strongly Recommended) +### 1. Use Spark + Faker for All Data Generation -**Default:** Generate data with Spark + Faker for all use cases. This provides scalability, parallelism, and direct integration with Unity Catalog. +Generate data with Spark + Faker for all use cases. This provides scalability, parallelism, and direct integration with Unity Catalog. ```python from pyspark.sql import functions as F @@ -896,9 +865,7 @@ INJECT_BAD_DATA = False # Set True for data quality testing ### Example 2: Local Development with Polars (Only for <10K rows if user prefers) -**Note:** Spark is strongly recommended. Only use this approach for datasets <10K rows if user explicitly prefers local generation. - -Generate synthetic data locally without Spark dependency, then upload to Databricks. +Generate synthetic data locally without Spark dependency, then upload to Databricks. Only use for datasets <10K rows if user explicitly prefers local generation. **Full implementation:** See `scripts/example_polars.py` in this skill folder. @@ -978,31 +945,7 @@ This approach uses Pandas for generation (single-threaded) and Spark for saving. **Full implementation:** See `scripts/example_pandas.py` in this skill folder. -**Key pattern - Pandas generation with referential integrity:** - -```python -# Generate master table -customers_pdf = pd.DataFrame({ - "customer_id": [f"CUST-{i:05d}" for i in range(N_CUSTOMERS)], - "tier": np.random.choice(['Free', 'Pro', 'Enterprise'], N_CUSTOMERS, p=[0.6, 0.3, 0.1]), -}) - -# Create lookups for foreign keys -customer_ids = customers_pdf["customer_id"].tolist() -customer_tier_map = dict(zip(customers_pdf["customer_id"], customers_pdf["tier"])) -tier_weights = customers_pdf["tier"].map({'Enterprise': 5.0, 'Pro': 2.0, 'Free': 1.0}) -customer_weights = (tier_weights / tier_weights.sum()).tolist() - -# Generate related table with weighted sampling -orders_data = [] -for i in range(N_ORDERS): - cid = np.random.choice(customer_ids, p=customer_weights) - tier = customer_tier_map[cid] - orders_data.append({"order_id": f"ORD-{i:06d}", "customer_id": cid, ...}) - -# Convert to Spark for saving -spark.createDataFrame(customers_pdf).write.mode("overwrite").parquet(f"{VOLUME_PATH}/customers") -``` +**Key pattern:** Generate master table (customers) first, create lookups for foreign keys, then generate related tables (orders) with weighted sampling. See **Key Principles → Iterate on DataFrames for Referential Integrity** for detailed pattern. **Usage:** Copy `example_pandas.py` to your scripts folder and customize the configuration and patterns. @@ -1035,12 +978,12 @@ This returns schema, row counts, and column statistics to confirm the data was w ### Execution 1. **Use serverless** (Databricks Connect for dev, jobs for production) - instant start, no cluster wait -2. **Ask for schema**: Default to `ai_dev_kit` catalog, ask user for schema name +2. **Ask for catalog and schema**: Ask for catalog (default to `ai_dev_kit`), ask user for schema name 3. **Present plan before generating**: Show table spec with assumptions, get user approval ### Data Generation 6. **Default to Spark + Faker** for all data generation - scalable, parallel, direct Unity Catalog integration -7. **Use Pandas UDFs for scale** (1M+ rows) - Spark parallelism with Faker +7. **Use Pandas UDFs for scale** (10k+ rows) - Spark parallelism with Faker 8. **Only use local generation** (<10K rows) if user explicitly prefers it - then upload with `databricks fs cp` 9. **Master tables first**: Generate customers, then orders reference customer_ids 10. **Weighted sampling**: Enterprise customers generate more activity @@ -1052,18 +995,13 @@ This returns schema, row counts, and column statistics to confirm the data was w 16. **Context reuse**: Pass `cluster_id` and `context_id` for faster iterations (classic cluster only) ## Related Skills - -- **[spark-declarative-pipelines](../spark-declarative-pipelines/SKILL.md)** - for building bronze/silver/gold pipelines on top of generated data -- **[databricks-aibi-dashboards](../databricks-aibi-dashboards/SKILL.md)** - for visualizing the generated data in dashboards - **[databricks-unity-catalog](../databricks-unity-catalog/SKILL.md)** - for managing catalogs, schemas, and volumes where data is stored ### Output -14. **Create infrastructure in script**: Use `CREATE SCHEMA/VOLUME IF NOT EXISTS` - do NOT create catalogs -15. **Assume catalogs exist**: Never auto-create catalogs, only create schema and volume -16. **Raw data only**: No `total_x`, `sum_x`, `avg_x` fields - SDP pipeline computes those -17. **Choose output format** based on downstream needs (Parquet/JSON/CSV/Delta) -18. **Configuration at top**: All sizes, dates, and paths as variables -19. **Dynamic dates**: Use `datetime.now() - timedelta(days=180)` for last 6 months +17. **Create infrastructure in script**: Use `CREATE SCHEMA/VOLUME IF NOT EXISTS` - do NOT create catalogs +18. **Assume catalogs exist**: Never auto-create catalogs, only create schema and volume +19. **Choose output format** based on downstream needs (Parquet/JSON/CSV/Delta) +20. **Configuration at top**: All sizes, dates, and paths as variables ## Common Issues diff --git a/databricks-skills/synthetic-data-generation/data_gen_update_summary.md b/databricks-skills/synthetic-data-generation/data_gen_update_summary.md deleted file mode 100644 index 1533c1de..00000000 --- a/databricks-skills/synthetic-data-generation/data_gen_update_summary.md +++ /dev/null @@ -1,56 +0,0 @@ -# Synthetic Data Generation Skill Update Summary - -## Summary of Changes - -### SKILL.md Updates - -1. **Added Generation Planning Workflow** (new section at top) - - 3-step process: gather requirements, present table spec with assumptions, ask about data features - - Pre-generation checklist for user approval - - "Surprise Me" fallback option - -2. **Updated Compute Selection** - - Serverless-first with confirmation prompt - - databricks-connect version guidance (>=15.1,<16.2 for Python 3.10/3.11; >=16.2 for Python 3.12) - -3. **Replaced Data Generation Approaches** - - **Option 1: Spark + Faker** (recommended for most cases, writing to UC) - - **Option 2: Polars** (for local development, quick prototyping) - - Removed dbldatagen as primary approach - -4. **Added Deployment Options** - - Ephemeral script run (default) - - DABs bundle deployment with `client: "4"` for serverless - -5. **Added Business Integrity Requirements** - - Value coherence, tier behavior, temporal patterns, geographic patterns - - Bad data injection patterns (nulls, outliers, duplicates, orphan FKs) - -6. **Added Domain-Specific Guidance** - - Retail/E-commerce, Support/CRM, Manufacturing/IoT, Financial Services - -7. **Updated Complete Examples** - - Example 1: E-commerce Data (Spark + Faker + Pandas) - - Example 2: Local Development with Polars - - Example 3: Large-Scale with Faker UDFs - - Example 4: Legacy Approach (Faker + Pandas) - -### Script Updates - -| File | Action | -|------|--------| -| `scripts/generate_ecommerce_data.py` | Updated: serverless-first, bad data injection, incremental mode | -| `scripts/example_dbldatagen.py` | Deleted | -| `scripts/example_polars.py` | Created: local generation with Polars | -| `scripts/example_faker_udf.py` | Updated: serverless-first configuration | - -## Key Decisions Implemented - -1. **Serverless Default**: Yes, with user confirmation prompt -2. **databricks-connect Version**: >=15.1,<16.2 (Python 3.10/3.11) or >=16.2 (Python 3.12) -3. **Planning Phase**: Show table spec with assumptions before generating -4. **Templates**: Domain guidance (retail, manufacturing) without rigid schemas -5. **Incremental Mode**: Support `.mode("append")` for scheduled jobs -6. **Data Quality Features**: Bad data injection (nulls, outliers, duplicates, orphan FKs) -7. **NO dbldatagen by default**: Use Spark + Faker or Polars -8. **Deployment**: Both ephemeral scripts AND DABs bundles as options diff --git a/databricks-skills/synthetic-data-generation/scripts/example_pandas.py b/databricks-skills/synthetic-data-generation/scripts/example_pandas.py deleted file mode 100644 index 78b050ae..00000000 --- a/databricks-skills/synthetic-data-generation/scripts/example_pandas.py +++ /dev/null @@ -1,94 +0,0 @@ -"""Generate synthetic data using Faker + Pandas (legacy approach for complex patterns).""" -import numpy as np -import pandas as pd -from datetime import datetime, timedelta -from faker import Faker -import holidays -from pyspark.sql import SparkSession - -# For Databricks Connect, replace with: -# from databricks.connect import DatabricksSession -# spark = DatabricksSession.builder.getOrCreate() - -spark = SparkSession.builder.getOrCreate() - -# ============================================================================= -# CONFIGURATION -# ============================================================================= -CATALOG = "my_catalog" -SCHEMA = "my_schema" -VOLUME_PATH = f"/Volumes/{CATALOG}/{SCHEMA}/raw_data" - -N_CUSTOMERS = 2500 -N_ORDERS = 25000 -N_TICKETS = 8000 - -END_DATE = datetime.now().replace(hour=0, minute=0, second=0, microsecond=0) -START_DATE = END_DATE - timedelta(days=180) -INCIDENT_END = END_DATE - timedelta(days=21) -INCIDENT_START = INCIDENT_END - timedelta(days=10) -US_HOLIDAYS = holidays.US(years=[START_DATE.year, END_DATE.year]) - -SEED = 42 -np.random.seed(SEED) -Faker.seed(SEED) -fake = Faker() - -# ============================================================================= -# CREATE INFRASTRUCTURE -# ============================================================================= -spark.sql(f"CREATE CATALOG IF NOT EXISTS {CATALOG}") -spark.sql(f"CREATE SCHEMA IF NOT EXISTS {CATALOG}.{SCHEMA}") -spark.sql(f"CREATE VOLUME IF NOT EXISTS {CATALOG}.{SCHEMA}.raw_data") - -# ============================================================================= -# GENERATE CUSTOMERS -# ============================================================================= -print(f"Generating {N_CUSTOMERS:,} customers...") - -customers_pdf = pd.DataFrame({ - "customer_id": [f"CUST-{i:05d}" for i in range(N_CUSTOMERS)], - "name": [fake.company() for _ in range(N_CUSTOMERS)], - "tier": np.random.choice(['Free', 'Pro', 'Enterprise'], N_CUSTOMERS, p=[0.6, 0.3, 0.1]), - "region": np.random.choice(['North', 'South', 'East', 'West'], N_CUSTOMERS, p=[0.4, 0.25, 0.2, 0.15]), -}) - -customers_pdf["arr"] = customers_pdf["tier"].apply( - lambda t: round(np.random.lognormal(11, 0.5), 2) if t == 'Enterprise' - else round(np.random.lognormal(8, 0.6), 2) if t == 'Pro' else 0 -) - -# Lookups for foreign keys -customer_ids = customers_pdf["customer_id"].tolist() -customer_tier_map = dict(zip(customers_pdf["customer_id"], customers_pdf["tier"])) -tier_weights = customers_pdf["tier"].map({'Enterprise': 5.0, 'Pro': 2.0, 'Free': 1.0}) -customer_weights = (tier_weights / tier_weights.sum()).tolist() - -# ============================================================================= -# GENERATE ORDERS -# ============================================================================= -print(f"Generating {N_ORDERS:,} orders...") - -orders_data = [] -for i in range(N_ORDERS): - cid = np.random.choice(customer_ids, p=customer_weights) - tier = customer_tier_map[cid] - amount = np.random.lognormal(7 if tier == 'Enterprise' else 5 if tier == 'Pro' else 3.5, 0.7) - orders_data.append({ - "order_id": f"ORD-{i:06d}", - "customer_id": cid, - "amount": round(amount, 2), - "status": np.random.choice(['completed', 'pending', 'cancelled'], p=[0.85, 0.10, 0.05]), - "order_date": fake.date_between(start_date=START_DATE, end_date=END_DATE), - }) -orders_pdf = pd.DataFrame(orders_data) - -# ============================================================================= -# SAVE TO VOLUME -# ============================================================================= -print(f"Saving to {VOLUME_PATH}...") - -spark.createDataFrame(customers_pdf).write.mode("overwrite").parquet(f"{VOLUME_PATH}/customers") -spark.createDataFrame(orders_pdf).write.mode("overwrite").parquet(f"{VOLUME_PATH}/orders") - -print("Done!") From eb82b2179871bf85a6927eec3187fcacc2d628d4 Mon Sep 17 00:00:00 2001 From: dustinvannoy-db Date: Thu, 19 Feb 2026 17:15:13 -0800 Subject: [PATCH 05/24] Add stronger guidance to use Databricks Connect --- .../synthetic-data-generation/SKILL.md | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/databricks-skills/synthetic-data-generation/SKILL.md b/databricks-skills/synthetic-data-generation/SKILL.md index 48def889..0ed1531a 100644 --- a/databricks-skills/synthetic-data-generation/SKILL.md +++ b/databricks-skills/synthetic-data-generation/SKILL.md @@ -81,6 +81,14 @@ Choose your execution mode based on your needs. **Serverless is strongly recomme Run code locally while Spark operations execute on serverless compute. Best for development and interactive work. +# ❌ WRONG - DO NOT USE +from pyspark.sql import SparkSession +spark = SparkSession.builder.getOrCreate() # Will fail with RuntimeError + +# ✅ CORRECT - ALWAYS USE THIS +from databricks.connect import DatabricksSession +spark = DatabricksSession.builder.serverless(True).getOrCreate() + **Install locally (one-time setup):** ```bash # Python 3.10 or 3.11: @@ -508,7 +516,7 @@ import pandas as pd from datetime import datetime, timedelta from faker import Faker import holidays -from pyspark.sql import SparkSession +from databricks.connect import DatabricksSession # ============================================================================= # CONFIGURATION - Edit these values @@ -539,10 +547,12 @@ SEED = 42 # ============================================================================= # SETUP # ============================================================================= +# IMPORTANT: Always use DatabricksSession, NOT SparkSession! + np.random.seed(SEED) Faker.seed(SEED) fake = Faker() -spark = SparkSession.builder.getOrCreate() +spark = DatabricksSession.builder.serverless(True).getOrCreate() # ... rest of script ``` From c9ec6835e1076b1fcce57efd68139590565b1ecb Mon Sep 17 00:00:00 2001 From: dustinvannoy-db Date: Mon, 23 Feb 2026 22:34:44 -0800 Subject: [PATCH 06/24] Update data gen for different run modes --- .gitignore | 2 +- .../synthetic-data-generation/SKILL.md | 171 ++------ .../scripts/example_faker_udf.py | 128 +++++- .../scripts/generate_ecommerce_data.py | 8 + scripts/generate_support_tickets.py | 387 ++++++++++++++++++ 5 files changed, 539 insertions(+), 157 deletions(-) create mode 100644 scripts/generate_support_tickets.py diff --git a/.gitignore b/.gitignore index 385994fa..a170605d 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,7 @@ # Databricks AI Dev Kit .ai-dev-kit/ .claude/ - +.local # Python __pycache__/ diff --git a/databricks-skills/synthetic-data-generation/SKILL.md b/databricks-skills/synthetic-data-generation/SKILL.md index 0ed1531a..83c4d6d3 100644 --- a/databricks-skills/synthetic-data-generation/SKILL.md +++ b/databricks-skills/synthetic-data-generation/SKILL.md @@ -94,8 +94,8 @@ spark = DatabricksSession.builder.serverless(True).getOrCreate() # Python 3.10 or 3.11: pip install "databricks-connect>=15.1,<16.2" faker polars numpy pandas holidays -# Python 3.12: -pip install "databricks-connect>=16.2,<18.0" faker polars numpy pandas holidays +# Python 3.12+: +pip install "databricks-connect>=16.4,<18.0" faker polars numpy pandas holidays # Configure ~/.databrickscfg [DEFAULT] @@ -104,14 +104,28 @@ serverless_compute_id = auto auth_type = databricks-cli ``` -**In your script:** +**In your script (version-dependent):** + +**For Python 3.12+ with databricks-connect >= 16.4:** ```python -from databricks.connect import DatabricksSession +from databricks.connect import DatabricksSession, DatabricksEnv -spark = DatabricksSession.builder.serverless(True).getOrCreate() -# Spark operations now execute on serverless compute +env = DatabricksEnv().withAutoDependencies(upload_local=True, use_index=True) +spark = DatabricksSession.builder.withEnvironment(env).serverless(True).getOrCreate() + +# Spark operations now execute on serverless compute with auto-managed dependencies ``` +**For Python < 3.12 or databricks-connect < 16.4:** + +`DatabricksEnv()` and `withEnvironment()` are NOT available in older versions. You must use one of these alternatives: + +**Create a job with environment settings** + +Create a Databricks job with environment settings on the task. See **Option 2: Serverless Job** section below. + +**Note:** If you're using Polars for local generation (not Spark with Faker UDFs), these workarounds are NOT needed since dependencies run locally. + **Benefits:** Instant start, local debugging, fast iteration (edit file, re-run immediately) ### Option 2: Serverless Job (Production/Scheduled) @@ -180,15 +194,6 @@ Generate data with Spark + Faker with Pandas UDFs, save to Databricks. **Example:** ```python -from pyspark.sql import functions as F -from pyspark.sql.functions import pandas_udf -from pyspark.sql.types import StringType -import pandas as pd -from faker import Faker -from databricks.connect import DatabricksSession - -spark = DatabricksSession.builder.serverless(True).getOrCreate() - # Define Pandas UDFs for Faker data (batch processing for parallelism) @pandas_udf(StringType()) def fake_name(ids: pd.Series) -> pd.Series: @@ -505,58 +510,6 @@ N_CUSTOMERS = 2500 # Each has ~3 tickets on average N_ORDERS = 25000 # ~10 orders per customer average ``` -## Script Structure - -Always structure scripts with configuration variables at the top: - -```python -"""Generate synthetic data for [use case].""" -import numpy as np -import pandas as pd -from datetime import datetime, timedelta -from faker import Faker -import holidays -from databricks.connect import DatabricksSession - -# ============================================================================= -# CONFIGURATION - Edit these values -# ============================================================================= -CATALOG = "my_catalog" -SCHEMA = "my_schema" -VOLUME_PATH = f"/Volumes/{CATALOG}/{SCHEMA}/raw_data" - -# Data sizes - enough for aggregation patterns to survive -N_CUSTOMERS = 2500 -N_ORDERS = 25000 -N_TICKETS = 8000 - -# Date range - last 6 months from today -END_DATE = datetime.now().replace(hour=0, minute=0, second=0, microsecond=0) -START_DATE = END_DATE - timedelta(days=180) - -# Special events (within the date range) -INCIDENT_END = END_DATE - timedelta(days=21) -INCIDENT_START = INCIDENT_END - timedelta(days=10) - -# Holiday calendar for realistic patterns -US_HOLIDAYS = holidays.US(years=[START_DATE.year, END_DATE.year]) - -# Reproducibility -SEED = 42 - -# ============================================================================= -# SETUP -# ============================================================================= -# IMPORTANT: Always use DatabricksSession, NOT SparkSession! - -np.random.seed(SEED) -Faker.seed(SEED) -fake = Faker() -spark = DatabricksSession.builder.serverless(True).getOrCreate() - -# ... rest of script -``` - ## Business Integrity Requirements Generated data MUST reflect business reality. Data should be realistic and tell a coherent story. @@ -646,12 +599,6 @@ When generating data for specific domains, consider these realistic patterns: Generate data with Spark + Faker for all use cases. This provides scalability, parallelism, and direct integration with Unity Catalog. ```python -from pyspark.sql import functions as F -from pyspark.sql.functions import pandas_udf -from pyspark.sql.types import StringType -import pandas as pd -from faker import Faker - @pandas_udf(StringType()) def fake_company(ids: pd.Series) -> pd.Series: fake = Faker() @@ -885,22 +832,6 @@ Generate synthetic data locally without Spark dependency, then upload to Databri - Outputs parquet files to local directory - Requires manual upload to volumes with `databricks fs cp` -**Key pattern:** - -```python -import polars as pl -from faker import Faker - -# Generate with Polars -customers = pl.DataFrame({ - "customer_id": [f"CUST-{i:05d}" for i in range(N_CUSTOMERS)], - "name": [fake.name() for _ in range(N_CUSTOMERS)], - "tier": np.random.choice(["Free", "Pro", "Enterprise"], N_CUSTOMERS, p=[0.6, 0.3, 0.1]), -}) - -# Save locally -customers.write_parquet("./output/customers.parquet") -``` **Usage:** Run locally, then upload: `databricks fs cp -r ./output dbfs:/Volumes/{catalog}/{schema}/raw_data/` @@ -916,54 +847,9 @@ Use Faker with Spark UDFs for realistic text data with parallelism. Best for dat - Realistic text data (company names, addresses, emails) - Tier-based amount generation -**Key pattern - Faker UDFs for realistic data:** - -```python -@F.udf(returnType=StringType()) -def generate_company(): - return Faker().company() - -@F.udf(returnType=DoubleType()) -def generate_lognormal_amount(tier): - np.random.seed(hash(str(tier)) % (2**32)) - if tier == "Enterprise": - return float(np.random.lognormal(mean=9, sigma=0.8)) - elif tier == "Pro": - return float(np.random.lognormal(mean=7, sigma=0.7)) - else: - return float(np.random.lognormal(mean=5, sigma=0.6)) - -# Use UDFs in Spark operations -customers_df = ( - spark.range(0, N_CUSTOMERS, numPartitions=PARTITIONS) - .select( - generate_company().alias("name"), - # ... other columns - ) - .withColumn("arr", generate_lognormal_amount(F.col("tier"))) -) -``` **Usage:** Copy `example_faker_udf.py` to your scripts folder and customize the UDFs and configuration. -### Example 4: Legacy Approach (Faker + Pandas) - -For smaller datasets or when you need complex time-based patterns with row-level logic. -This approach uses Pandas for generation (single-threaded) and Spark for saving. - -**Note:** For datasets over 100K rows, prefer Faker UDFs for better performance. - -**Full implementation:** See `scripts/example_pandas.py` in this skill folder. - -**Key pattern:** Generate master table (customers) first, create lookups for foreign keys, then generate related tables (orders) with weighted sampling. See **Key Principles → Iterate on DataFrames for Referential Integrity** for detailed pattern. - -**Usage:** Copy `example_pandas.py` to your scripts folder and customize the configuration and patterns. - -**To use any example script:** - -1. Copy the example file to your scripts directory -2. Update the CONFIGURATION section with your catalog/schema -3. Execute using one of the methods below **Execute with Databricks Connect:** ```bash @@ -1001,17 +887,17 @@ This returns schema, row counts, and column statistics to confirm the data was w 12. **Time patterns**: Weekday/weekend, holidays, seasonality, event spikes 13. **Row coherence**: Priority affects resolution time affects CSAT 14. **Volume for aggregation**: 10K-50K rows minimum so patterns survive GROUP BY -15. **Always use files**: Write to local file, execute, edit if error, re-execute -16. **Context reuse**: Pass `cluster_id` and `context_id` for faster iterations (classic cluster only) + +15. **Context reuse**: Pass `cluster_id` and `context_id` for faster iterations (classic cluster only) ## Related Skills - **[databricks-unity-catalog](../databricks-unity-catalog/SKILL.md)** - for managing catalogs, schemas, and volumes where data is stored ### Output -17. **Create infrastructure in script**: Use `CREATE SCHEMA/VOLUME IF NOT EXISTS` - do NOT create catalogs -18. **Assume catalogs exist**: Never auto-create catalogs, only create schema and volume -19. **Choose output format** based on downstream needs (Parquet/JSON/CSV/Delta) -20. **Configuration at top**: All sizes, dates, and paths as variables +16. **Create infrastructure in script**: Use `CREATE SCHEMA/VOLUME IF NOT EXISTS` - do NOT create catalogs +17. **Assume catalogs exist**: Never auto-create catalogs, only create schema and volume +18. **Choose output format** based on downstream needs (Parquet/JSON/CSV/Delta) +19. **Configuration at top**: All sizes, dates, and paths as variables ## Common Issues @@ -1021,9 +907,6 @@ This returns schema, row counts, and column statistics to confirm the data was w | **"ModuleNotFoundError"** for faker/polars/etc. | See **Execution Options & Installation** section for dependency setup per execution mode | | **Serverless job fails to start** | Verify workspace has serverless compute enabled; check Unity Catalog permissions | | **Faker UDF is slow** | Use `pandas_udf` for batched operations; adjust `numPartitions` | -| **Classic cluster startup is slow (3-8 min)** | Switch to Databricks Connect with serverless for instant start | +| **Classic cluster startup is slow (3-8 min)** | Prompt user to check if cluster is running and suggest a replacement. | | **Out of memory with large data** | Increase `partitions` parameter in `spark.range()` | -| **Foreign keys don't match across tables** | Use same random seed across all generators | -| **Delta table write fails** | Ensure `CREATE SCHEMA IF NOT EXISTS` runs before `saveAsTable()` | -| **Databricks Connect issues** | Verify correct version for your Python (see **Execution Options & Installation**), check `~/.databrickscfg` has `serverless_compute_id = auto` | | **Context corrupted on classic cluster** | Omit `context_id` to create fresh context, reinstall libraries | diff --git a/databricks-skills/synthetic-data-generation/scripts/example_faker_udf.py b/databricks-skills/synthetic-data-generation/scripts/example_faker_udf.py index a5cb2727..020587d9 100644 --- a/databricks-skills/synthetic-data-generation/scripts/example_faker_udf.py +++ b/databricks-skills/synthetic-data-generation/scripts/example_faker_udf.py @@ -5,14 +5,18 @@ - Generating realistic text data with Faker providers - Writing directly to Unity Catalog volumes - Complex conditional logic in data generation + +This script automatically detects the environment and uses: +- DatabricksEnv with auto-dependencies if databricks-connect >= 16.4 and running locally +- Standard session creation if running on Databricks Runtime or older databricks-connect """ +import sys +import os from pyspark.sql import functions as F from pyspark.sql.window import Window -from pyspark.sql.types import StringType, DoubleType, DateType -from faker import Faker +from pyspark.sql.types import StringType, DoubleType import numpy as np from datetime import datetime, timedelta -from databricks.connect import DatabricksSession # ============================================================================= # CONFIGURATION @@ -39,17 +43,117 @@ SEED = 42 # ============================================================================= -# SETUP +# SETUP - Environment Detection and Session Creation # ============================================================================= -print("Connecting to Databricks...") -if USE_SERVERLESS: - spark = DatabricksSession.builder.serverless(True).getOrCreate() - print("Connected to serverless compute!") + +# Detect if running on Databricks Runtime vs locally with Databricks Connect +def is_databricks_runtime(): + """Check if running on Databricks Runtime (notebook/job) vs locally.""" + return "DATABRICKS_RUNTIME_VERSION" in os.environ + +# Get databricks-connect version if available +def get_databricks_connect_version(): + """Get databricks-connect version as (major, minor) tuple or None.""" + try: + import databricks.connect + version_str = databricks.connect.__version__ + parts = version_str.split('.') + return (int(parts[0]), int(parts[1])) + except (ImportError, AttributeError, ValueError, IndexError): + return None + +# Determine session creation strategy +on_runtime = is_databricks_runtime() +db_version = get_databricks_connect_version() + +print("=" * 80) +print("ENVIRONMENT DETECTION") +print("=" * 80) +print(f"Running on Databricks Runtime: {on_runtime}") +if db_version: + print(f"databricks-connect version: {db_version[0]}.{db_version[1]}") +else: + print("databricks-connect: not available") + +# Use DatabricksEnv with auto-dependencies if: +# - Running locally (not on Databricks Runtime) +# - databricks-connect >= 16.4 +use_auto_dependencies = (not on_runtime) and db_version and db_version >= (16, 4) + +if use_auto_dependencies: + print("✓ Using DatabricksEnv with auto-dependencies") + print("=" * 80) + from databricks.connect import DatabricksSession, DatabricksEnv + + env = DatabricksEnv().withAutoDependencies(upload_local=True, use_index=True) + + if USE_SERVERLESS: + spark = DatabricksSession.builder.withEnvironment(env).serverless(True).getOrCreate() + print("✓ Connected to serverless compute with auto-dependencies!") + else: + if not CLUSTER_ID: + raise ValueError("CLUSTER_ID must be set when USE_SERVERLESS=False") + spark = DatabricksSession.builder.withEnvironment(env).clusterId(CLUSTER_ID).getOrCreate() + print(f"✓ Connected to cluster {CLUSTER_ID} with auto-dependencies!") else: - if not CLUSTER_ID: - raise ValueError("CLUSTER_ID must be set when USE_SERVERLESS=False") - spark = DatabricksSession.builder.clusterId(CLUSTER_ID).getOrCreate() - print(f"Connected to cluster {CLUSTER_ID}!") + print("⚠ Using standard session (dependencies must be pre-installed)") + print("=" * 80) + + # Try to import libraries that will be used in UDFs + print("\nChecking UDF dependencies...") + missing_deps = [] + + try: + from faker import Faker + print(" ✓ faker") + except ImportError: + missing_deps.append("faker") + print(" ✗ faker - NOT INSTALLED") + + try: + import pandas as pd + print(" ✓ pandas") + except ImportError: + missing_deps.append("pandas") + print(" ✗ pandas - NOT INSTALLED") + + if missing_deps: + print("\n" + "=" * 80) + print("⚠ WARNING: Missing dependencies for UDFs") + print("=" * 80) + print(f"Missing libraries: {', '.join(missing_deps)}") + print("\nThese libraries are required in UDFs and must be installed:") + + if on_runtime: + print("\n→ SOLUTION: Install on the cluster or job:") + print(" - For interactive cluster: Run %pip install faker pandas numpy holidays") + print(" - For job: Add to job libraries or use init script") + else: + print("\n→ SOLUTION: Use one of these approaches:") + print(" 1. Upgrade databricks-connect to >= 16.4 (enables auto-dependencies)") + print(" 2. Create a job with environment settings in the task definition") + print(" 3. Use a classic cluster with libraries pre-installed") + + print("=" * 80) + sys.exit(1) + + print("\n✓ All UDF dependencies available") + print("=" * 80) + + # Create standard session + from databricks.connect import DatabricksSession + + if USE_SERVERLESS: + spark = DatabricksSession.builder.serverless(True).getOrCreate() + print("✓ Connected to serverless compute") + else: + if not CLUSTER_ID: + raise ValueError("CLUSTER_ID must be set when USE_SERVERLESS=False") + spark = DatabricksSession.builder.clusterId(CLUSTER_ID).getOrCreate() + print(f"✓ Connected to cluster {CLUSTER_ID}") + +# Import Faker for UDF definitions (already checked above) +from faker import Faker # ============================================================================= # DEFINE FAKER UDFs diff --git a/databricks-skills/synthetic-data-generation/scripts/generate_ecommerce_data.py b/databricks-skills/synthetic-data-generation/scripts/generate_ecommerce_data.py index 2580af56..cb3223c0 100644 --- a/databricks-skills/synthetic-data-generation/scripts/generate_ecommerce_data.py +++ b/databricks-skills/synthetic-data-generation/scripts/generate_ecommerce_data.py @@ -70,6 +70,14 @@ fake = Faker() print("Connecting to Databricks...") + +# NOTE: This script uses Faker locally with Pandas (not in Spark UDFs), so it +# does NOT require DatabricksEnv or auto-dependencies. It works with all versions: +# - Python 3.10, 3.11, 3.12+ +# - databricks-connect 15.1+ (any version) +# +# If you need to use Faker in Spark UDFs, see example_faker_udf.py instead. + if USE_SERVERLESS: spark = DatabricksSession.builder.serverless(True).getOrCreate() print("Connected to serverless compute!") diff --git a/scripts/generate_support_tickets.py b/scripts/generate_support_tickets.py new file mode 100644 index 00000000..31382b00 --- /dev/null +++ b/scripts/generate_support_tickets.py @@ -0,0 +1,387 @@ +"""Generate large-scale synthetic support ticket data. + +This script automatically detects the environment and uses: +- DatabricksEnv with auto-dependencies if databricks-connect >= 16.4 and running locally +- Standard session creation if running on Databricks Runtime or older databricks-connect +""" +import sys +import os +import numpy as np +import pandas as pd +from datetime import datetime, timedelta +from pyspark.sql import functions as F +from pyspark.sql.functions import pandas_udf +from pyspark.sql.types import StringType, DoubleType, IntegerType + +# ============================================================================= +# CONFIGURATION - Edit these values +# ============================================================================= +CATALOG = "dustin_vannoy_catalog" +SCHEMA = "sdg_test_large_delta" + +# Data sizes +N_CUSTOMERS = 100000 +N_TICKETS = 500000 + +# Date ranges +END_DATE = datetime.now().replace(hour=0, minute=0, second=0, microsecond=0) +CUSTOMER_START_DATE = END_DATE - timedelta(days=1095) # Last 3 years +TICKET_START_DATE = END_DATE - timedelta(days=180) # Last 6 months + +# Reproducibility +SEED = 42 + +# Spark partitions for parallelism (adjust based on scale) +CUSTOMER_PARTITIONS = 32 +TICKET_PARTITIONS = 64 + +# ============================================================================= +# SETUP - Environment Detection and Session Creation +# ============================================================================= +np.random.seed(SEED) + +# Detect if running on Databricks Runtime vs locally with Databricks Connect +def is_databricks_runtime(): + """Check if running on Databricks Runtime (notebook/job) vs locally.""" + return "DATABRICKS_RUNTIME_VERSION" in os.environ + +# Get databricks-connect version if available +def get_databricks_connect_version(): + """Get databricks-connect version as (major, minor) tuple or None.""" + try: + import databricks.connect + version_str = databricks.connect.__version__ + parts = version_str.split('.') + return (int(parts[0]), int(parts[1])) + except (ImportError, AttributeError, ValueError, IndexError): + return None + +print("=" * 80) +print("SYNTHETIC DATA GENERATION - SUPPORT TICKETS") +print("=" * 80) +print(f"Catalog: {CATALOG}") +print(f"Schema: {SCHEMA}") +print(f"Customers: {N_CUSTOMERS:,}") +print(f"Tickets: {N_TICKETS:,}") +print(f"Customer partitions: {CUSTOMER_PARTITIONS}") +print(f"Ticket partitions: {TICKET_PARTITIONS}") +print("=" * 80) + +# Determine session creation strategy +on_runtime = is_databricks_runtime() +db_version = get_databricks_connect_version() + +print("\nENVIRONMENT DETECTION") +print("=" * 80) +print(f"Running on Databricks Runtime: {on_runtime}") +if db_version: + print(f"databricks-connect version: {db_version[0]}.{db_version[1]}") +else: + print("databricks-connect: not available") + +# Use DatabricksEnv with auto-dependencies if: +# - Running locally (not on Databricks Runtime) +# - databricks-connect >= 16.4 +use_auto_dependencies = (not on_runtime) and db_version and db_version >= (16, 4) + +if use_auto_dependencies: + print("✓ Using DatabricksEnv with auto-dependencies") + print("=" * 80) + from databricks.connect import DatabricksSession, DatabricksEnv + + env = DatabricksEnv().withAutoDependencies(upload_local=True, use_index=True) + spark = ( + DatabricksSession.builder + .withEnvironment(env) + .config("spark.databricks.sql.externalUDF.env.enabled", "true") + .config("spark.databricks.sql.udf.routineEnvironmentSettings.enabled", "true") + .serverless(True) + .getOrCreate() + ) + print("✓ Connected to serverless compute with auto-dependencies!") +else: + print("⚠ Using standard session (dependencies must be pre-installed)") + print("=" * 80) + + # Try to import libraries that will be used in UDFs + print("\nChecking UDF dependencies...") + missing_deps = [] + + try: + from faker import Faker + print(" ✓ faker") + except ImportError: + missing_deps.append("faker") + print(" ✗ faker - NOT INSTALLED") + + try: + import pandas as pd + print(" ✓ pandas") + except ImportError: + missing_deps.append("pandas") + print(" ✗ pandas - NOT INSTALLED") + + if missing_deps: + print("\n" + "=" * 80) + print("⚠ WARNING: Missing dependencies for UDFs") + print("=" * 80) + print(f"Missing libraries: {', '.join(missing_deps)}") + print("\nThese libraries are required in UDFs and must be installed:") + + if on_runtime: + print("\n→ SOLUTION: Install on the cluster or job:") + print(" - For interactive cluster: Run %pip install faker pandas numpy holidays") + print(" - For job: Add to job libraries or use init script") + else: + print("\n→ SOLUTION: Use one of these approaches:") + print(" 1. Upgrade databricks-connect to >= 16.4 (enables auto-dependencies)") + print(" 2. Create a job with environment settings in the task definition") + print(" 3. Use a classic cluster with libraries pre-installed") + + print("=" * 80) + sys.exit(1) + + print("\n✓ All UDF dependencies available") + print("=" * 80) + + # Create standard session + from databricks.connect import DatabricksSession + + spark = ( + DatabricksSession.builder + .config("spark.databricks.sql.externalUDF.env.enabled", "true") + .config("spark.databricks.sql.udf.routineEnvironmentSettings.enabled", "true") + .serverless(True) + .getOrCreate() + ) + print("✓ Connected to serverless compute") + +# Import Faker for later use (already checked above) +from faker import Faker +Faker.seed(SEED) +fake = Faker() + +# ============================================================================= +# CREATE INFRASTRUCTURE +# ============================================================================= +print("\n[1/4] Creating infrastructure...") +# Note: Assume catalog exists - do NOT create it +spark.sql(f"CREATE SCHEMA IF NOT EXISTS {CATALOG}.{SCHEMA}") +print(f"✓ Schema created/verified") + +# ============================================================================= +# DEFINE PANDAS UDFs FOR FAKER DATA +# ============================================================================= +print("\n[2/4] Defining data generation UDFs...") + +@pandas_udf(StringType()) +def fake_company(ids: pd.Series) -> pd.Series: + """Generate realistic company names.""" + fake = Faker() + Faker.seed(SEED) + return pd.Series([fake.company() for _ in range(len(ids))]) + +@pandas_udf(DoubleType()) +def generate_arr(tiers: pd.Series) -> pd.Series: + """Generate ARR based on tier using log-normal distribution.""" + np.random.seed(SEED) + result = [] + for tier in tiers: + if tier == "Enterprise": + # Mean ~$500K + arr = np.random.lognormal(mean=13, sigma=0.8) + elif tier == "Pro": + # Mean ~$50K + arr = np.random.lognormal(mean=11, sigma=0.7) + else: # Free + arr = 0.0 + result.append(round(arr, 2)) + return pd.Series(result) + +@pandas_udf(StringType()) +def generate_priority(tiers: pd.Series) -> pd.Series: + """Generate priority based on tier.""" + np.random.seed(SEED) + result = [] + for tier in tiers: + if tier == "Enterprise": + priority = np.random.choice(['Critical', 'High', 'Medium'], p=[0.3, 0.5, 0.2]) + elif tier == "Pro": + priority = np.random.choice(['Critical', 'High', 'Medium', 'Low'], p=[0.1, 0.3, 0.45, 0.15]) + else: # Free + priority = np.random.choice(['Critical', 'High', 'Medium', 'Low'], p=[0.02, 0.15, 0.40, 0.43]) + result.append(priority) + return pd.Series(result) + +@pandas_udf(DoubleType()) +def generate_resolution_hours(priorities: pd.Series) -> pd.Series: + """Generate resolution hours based on priority using exponential distribution.""" + np.random.seed(SEED) + result = [] + scale_map = {'Critical': 4, 'High': 12, 'Medium': 36, 'Low': 72} + for priority in priorities: + scale = scale_map.get(priority, 24) + hours = np.random.exponential(scale=scale) + result.append(round(hours, 2)) + return pd.Series(result) + +@pandas_udf(IntegerType()) +def generate_csat(resolution_hours: pd.Series) -> pd.Series: + """Generate CSAT score based on resolution time.""" + np.random.seed(SEED) + result = [] + for hours in resolution_hours: + if hours < 4: + csat = np.random.choice([4, 5], p=[0.3, 0.7]) + elif hours < 24: + csat = np.random.choice([3, 4, 5], p=[0.2, 0.5, 0.3]) + elif hours < 72: + csat = np.random.choice([2, 3, 4], p=[0.3, 0.5, 0.2]) + else: + csat = np.random.choice([1, 2, 3], p=[0.4, 0.4, 0.2]) + result.append(int(csat)) + return pd.Series(result) + +print("✓ UDFs defined") + +# ============================================================================= +# GENERATE CUSTOMERS TABLE +# ============================================================================= +print(f"\n[3/4] Generating {N_CUSTOMERS:,} customers...") + +# Generate base customer data with Spark +customers_df = ( + spark.range(0, N_CUSTOMERS, numPartitions=CUSTOMER_PARTITIONS) + .select( + # customer_id: CUST-00001 format + F.concat(F.lit("CUST-"), F.lpad(F.col("id").cast("string"), 6, "0")).alias("customer_id"), + + # tier: Enterprise 10%, Pro 30%, Free 60% + F.when(F.rand(SEED) < 0.10, "Enterprise") + .when(F.rand(SEED + 1) < 0.40, "Pro") # 0.10 + 0.30 = 0.40 + .otherwise("Free").alias("tier"), + + # region: North 35%, South 25%, East 25%, West 15% + F.when(F.rand(SEED + 2) < 0.35, "North") + .when(F.rand(SEED + 3) < 0.60, "South") # 0.35 + 0.25 = 0.60 + .when(F.rand(SEED + 4) < 0.85, "East") # 0.60 + 0.25 = 0.85 + .otherwise("West").alias("region"), + + # signup_date: random date in last 3 years + (F.lit(CUSTOMER_START_DATE.timestamp()) + + (F.rand(SEED + 5) * (END_DATE.timestamp() - CUSTOMER_START_DATE.timestamp())) + ).cast("timestamp").cast("date").alias("signup_date"), + ) +) + +# Add company_name and arr using UDFs +customers_df = ( + customers_df + .withColumn("company_name", fake_company(F.col("customer_id"))) + .withColumn("arr", generate_arr(F.col("tier"))) +) + +# Reorder columns +customers_df = customers_df.select( + "customer_id", "company_name", "tier", "arr", "region", "signup_date" +) + +# Save to Delta table +print(f"Writing customers to {CATALOG}.{SCHEMA}.customers...") +customers_df.write.mode("overwrite").option("overwriteSchema", "true").saveAsTable(f"{CATALOG}.{SCHEMA}.customers") + +# Get customer count +customer_count = spark.table(f"{CATALOG}.{SCHEMA}.customers").count() +print(f"✓ Created customers table with {customer_count:,} rows") + +# ============================================================================= +# GENERATE TICKETS TABLE +# ============================================================================= +print(f"\n[4/4] Generating {N_TICKETS:,} tickets...") + +# Create a broadcast map of customer_id -> tier for weighted sampling +customers_sample = spark.table(f"{CATALOG}.{SCHEMA}.customers").select("customer_id", "tier").collect() +customer_ids = [row.customer_id for row in customers_sample] +customer_tiers = {row.customer_id: row.tier for row in customers_sample} + +# Create weights: Enterprise 5x, Pro 2x, Free 1x +tier_weights = {"Enterprise": 5.0, "Pro": 2.0, "Free": 1.0} +weights = [tier_weights[customer_tiers[cid]] for cid in customer_ids] +weights = np.array(weights) / np.sum(weights) + +# Sample customer_ids with replacement based on weights +np.random.seed(SEED) +sampled_customer_ids = np.random.choice(customer_ids, size=N_TICKETS, replace=True, p=weights) + +# Create tickets DataFrame from sampled customer_ids +tickets_pdf = pd.DataFrame({ + "ticket_id": [f"TKT-{i:07d}" for i in range(N_TICKETS)], + "customer_id": sampled_customer_ids, +}) + +# Convert to Spark DataFrame +tickets_df = spark.createDataFrame(tickets_pdf, schema="ticket_id STRING, customer_id STRING") + +# Repartition for better parallelism +tickets_df = tickets_df.repartition(TICKET_PARTITIONS) + +# Join with customers to get tier for priority generation +tickets_df = tickets_df.join( + spark.table(f"{CATALOG}.{SCHEMA}.customers").select("customer_id", "tier"), + on="customer_id", + how="left" +) + +# Add priority, resolution_hours, csat_score, created_at +tickets_df = ( + tickets_df + .withColumn("priority", generate_priority(F.col("tier"))) + .withColumn("resolution_hours", generate_resolution_hours(F.col("priority"))) + .withColumn("csat_score", generate_csat(F.col("resolution_hours"))) + .withColumn( + "created_at", + (F.lit(TICKET_START_DATE.timestamp()) + + (F.rand(SEED + 10) * (END_DATE.timestamp() - TICKET_START_DATE.timestamp())) + ).cast("timestamp") + ) +) + +# Drop the tier column (only needed for generation) +tickets_df = tickets_df.select( + "ticket_id", "customer_id", "priority", "resolution_hours", "csat_score", "created_at" +) + +# Save to Delta table +print(f"Writing tickets to {CATALOG}.{SCHEMA}.tickets...") +tickets_df.write.mode("overwrite").option("overwriteSchema", "true").saveAsTable(f"{CATALOG}.{SCHEMA}.tickets") + +# Get ticket count +ticket_count = spark.table(f"{CATALOG}.{SCHEMA}.tickets").count() +print(f"✓ Created tickets table with {ticket_count:,} rows") + +# ============================================================================= +# VALIDATION +# ============================================================================= +print("\n" + "=" * 80) +print("GENERATION COMPLETE") +print("=" * 80) + +# Show sample data +print("\nCustomers sample:") +spark.table(f"{CATALOG}.{SCHEMA}.customers").show(5, truncate=False) + +print("\nTickets sample:") +spark.table(f"{CATALOG}.{SCHEMA}.tickets").show(5, truncate=False) + +# Show statistics +print("\nCustomer tier distribution:") +spark.table(f"{CATALOG}.{SCHEMA}.customers").groupBy("tier").count().orderBy("tier").show() + +print("\nTicket priority distribution:") +spark.table(f"{CATALOG}.{SCHEMA}.tickets").groupBy("priority").count().orderBy("priority").show() + +print("\n" + "=" * 80) +print(f"✓ Tables created:") +print(f" - {CATALOG}.{SCHEMA}.customers ({customer_count:,} rows)") +print(f" - {CATALOG}.{SCHEMA}.tickets ({ticket_count:,} rows)") +print("=" * 80) From 728e454892e0529b59721e1a94c610a7d8269801 Mon Sep 17 00:00:00 2001 From: dustinvannoy-db Date: Tue, 24 Feb 2026 13:17:07 -0800 Subject: [PATCH 07/24] Small updates to databricks-connect and environments --- .../synthetic-data-generation/SKILL.md | 38 ++++++++++++++++--- .../scripts/example_faker_udf.py | 9 +++-- 2 files changed, 38 insertions(+), 9 deletions(-) diff --git a/databricks-skills/synthetic-data-generation/SKILL.md b/databricks-skills/synthetic-data-generation/SKILL.md index 83c4d6d3..f16ecf0f 100644 --- a/databricks-skills/synthetic-data-generation/SKILL.md +++ b/databricks-skills/synthetic-data-generation/SKILL.md @@ -92,10 +92,11 @@ spark = DatabricksSession.builder.serverless(True).getOrCreate() **Install locally (one-time setup):** ```bash # Python 3.10 or 3.11: -pip install "databricks-connect>=15.1,<16.2" faker polars numpy pandas holidays +pip install "databricks-connect>=15.1,<16.2" faker numpy pandas holidays # Python 3.12+: -pip install "databricks-connect>=16.4,<18.0" faker polars numpy pandas holidays +# IMPORTANT: Use 16.4.x for stable withDependencies API (17.x has breaking changes) +pip install "databricks-connect>=16.4,<17.0" faker numpy pandas holidays # Configure ~/.databrickscfg [DEFAULT] @@ -110,10 +111,37 @@ auth_type = databricks-cli ```python from databricks.connect import DatabricksSession, DatabricksEnv -env = DatabricksEnv().withAutoDependencies(upload_local=True, use_index=True) -spark = DatabricksSession.builder.withEnvironment(env).serverless(True).getOrCreate() +# Pass dependencies as simple package name strings +env = DatabricksEnv().withDependencies("faker", "pandas", "numpy", "holidays") -# Spark operations now execute on serverless compute with auto-managed dependencies +# Create session +spark = ( + DatabricksSession.builder + .withEnvironment(env) + .serverless(True) + .getOrCreate() +) + +# Spark operations now execute on serverless compute with managed dependencies +``` + +**Version Detection (if needed in your script):** +```python +import importlib.metadata + +def get_databricks_connect_version(): + """Get databricks-connect version as (major, minor) tuple.""" + try: + version_str = importlib.metadata.version('databricks-connect') + parts = version_str.split('.') + return (int(parts[0]), int(parts[1])) + except Exception: + return None + +db_version = get_databricks_connect_version() +if db_version and db_version >= (16, 4): + # Use DatabricksEnv with withDependencies + pass ``` **For Python < 3.12 or databricks-connect < 16.4:** diff --git a/databricks-skills/synthetic-data-generation/scripts/example_faker_udf.py b/databricks-skills/synthetic-data-generation/scripts/example_faker_udf.py index 020587d9..9deefb54 100644 --- a/databricks-skills/synthetic-data-generation/scripts/example_faker_udf.py +++ b/databricks-skills/synthetic-data-generation/scripts/example_faker_udf.py @@ -81,20 +81,21 @@ def get_databricks_connect_version(): use_auto_dependencies = (not on_runtime) and db_version and db_version >= (16, 4) if use_auto_dependencies: - print("✓ Using DatabricksEnv with auto-dependencies") + print("✓ Using DatabricksEnv with managed dependencies") print("=" * 80) from databricks.connect import DatabricksSession, DatabricksEnv - env = DatabricksEnv().withAutoDependencies(upload_local=True, use_index=True) + # Pass dependencies as simple package name strings + env = DatabricksEnv().withDependencies("faker", "pandas", "numpy", "holidays") if USE_SERVERLESS: spark = DatabricksSession.builder.withEnvironment(env).serverless(True).getOrCreate() - print("✓ Connected to serverless compute with auto-dependencies!") + print("✓ Connected to serverless compute with managed dependencies!") else: if not CLUSTER_ID: raise ValueError("CLUSTER_ID must be set when USE_SERVERLESS=False") spark = DatabricksSession.builder.withEnvironment(env).clusterId(CLUSTER_ID).getOrCreate() - print(f"✓ Connected to cluster {CLUSTER_ID} with auto-dependencies!") + print(f"✓ Connected to cluster {CLUSTER_ID} with managed dependencies!") else: print("⚠ Using standard session (dependencies must be pre-installed)") print("=" * 80) From 3f2c9e032b28e747db2cfd810be628cfde94477d Mon Sep 17 00:00:00 2001 From: dustinvannoy-db Date: Tue, 24 Feb 2026 14:45:42 -0800 Subject: [PATCH 08/24] Updates to improve serverless dbconnect and polars local for data gen --- .../synthetic-data-generation/SKILL.md | 964 +++--------------- .../references/1-setup-and-execution.md | 201 ++++ .../references/2-generation-approaches.md | 205 ++++ .../references/3-data-patterns.md | 284 ++++++ .../references/4-domain-guidance.md | 256 +++++ .../references/5-output-formats.md | 188 ++++ .../references/6-troubleshooting.md | 261 +++++ .../scripts/example_faker_udf.py | 272 ----- .../scripts/example_polars.py | 157 --- .../scripts/generate_ecommerce_data.py | 225 ---- .../scripts/generate_synthetic_data.py | 387 +++++++ 11 files changed, 1902 insertions(+), 1498 deletions(-) create mode 100644 databricks-skills/synthetic-data-generation/references/1-setup-and-execution.md create mode 100644 databricks-skills/synthetic-data-generation/references/2-generation-approaches.md create mode 100644 databricks-skills/synthetic-data-generation/references/3-data-patterns.md create mode 100644 databricks-skills/synthetic-data-generation/references/4-domain-guidance.md create mode 100644 databricks-skills/synthetic-data-generation/references/5-output-formats.md create mode 100644 databricks-skills/synthetic-data-generation/references/6-troubleshooting.md delete mode 100644 databricks-skills/synthetic-data-generation/scripts/example_faker_udf.py delete mode 100644 databricks-skills/synthetic-data-generation/scripts/example_polars.py delete mode 100644 databricks-skills/synthetic-data-generation/scripts/generate_ecommerce_data.py create mode 100644 databricks-skills/synthetic-data-generation/scripts/generate_synthetic_data.py diff --git a/databricks-skills/synthetic-data-generation/SKILL.md b/databricks-skills/synthetic-data-generation/SKILL.md index f16ecf0f..2468a53e 100644 --- a/databricks-skills/synthetic-data-generation/SKILL.md +++ b/databricks-skills/synthetic-data-generation/SKILL.md @@ -1,17 +1,36 @@ --- name: synthetic-data-generation -description: "Generate realistic synthetic data using Spark + Faker (strongly recommended). Supports serverless execution, multiple output formats (Parquet/JSON/CSV/Delta), and scales from thousands to millions of rows. For small datasets (<10K rows), can optionally generate locally and upload to volumes. Use for test data, demo datasets, or synthetic tables." +description: "Generate realistic synthetic data using Spark + Faker (strongly recommended). Supports serverless execution, multiple output formats (Parquet/JSON/CSV/Delta), and scales from thousands to millions of rows. For small datasets (<10K rows), can optionally generate locally and upload to volumes. Use when user mentions 'synthetic data', 'test data', 'generate data', 'demo dataset', 'Faker', or 'sample data'." --- # Synthetic Data Generation -Generate realistic, story-driven synthetic data for Databricks using Spark + Faker (strongly recommended). -For small datasets (<10K rows), can optionally generate locally with Polars and upload to volumes. -Always present a generation plan with assumptions before generating code. +Generate realistic, story-driven synthetic data for Databricks using **Spark + Faker + Pandas UDFs** (strongly recommended). + +## Quick Reference + +| Topic | Guide | When to Use | +|-------|-------|-------------| +| **Setup & Execution** | [references/1-setup-and-execution.md](references/1-setup-and-execution.md) | Setting up environment, choosing compute, installing dependencies | +| **Generation Approaches** | [references/2-generation-approaches.md](references/2-generation-approaches.md) | Choosing Spark UDFs vs Polars local, writing generation code | +| **Data Patterns** | [references/3-data-patterns.md](references/3-data-patterns.md) | Creating realistic distributions, referential integrity, time patterns | +| **Domain Guidance** | [references/4-domain-guidance.md](references/4-domain-guidance.md) | E-commerce, IoT, financial, support/CRM domain patterns | +| **Output Formats** | [references/5-output-formats.md](references/5-output-formats.md) | Choosing output format, saving to volumes/tables | +| **Troubleshooting** | [references/6-troubleshooting.md](references/6-troubleshooting.md) | Fixing errors, debugging issues | +| **Example Script** | [scripts/generate_synthetic_data.py](scripts/generate_synthetic_data.py) | Complete Spark + Pandas UDF example | + +## Critical Rules + +1. **Always use Spark + Faker + Pandas UDFs** for data generation (scalable, parallel) +2. **Present a plan for user approval** before generating any code +3. **Ask for catalog/schema** - default to `ai_dev_kit.` +4. **Use serverless compute** unless user explicitly requests classic cluster +5. **Generate raw data only** - no pre-aggregated fields (unless user requests) +6. **Create master tables first** - then generate related tables with valid FKs ## Generation Planning Workflow -**Before generating any code, you MUST present a plan for user approval.** Give them a "Surprise Me" option if they don't want to specify details. +**Before generating any code, you MUST present a plan for user approval.** ### Step 1: Gather Requirements @@ -20,7 +39,6 @@ Ask the user about: - How many tables? What relationships between them? - Approximate row counts per table? - Output format preference? (Parquet to Volume is default) -- One-time generation or scheduled job? ### Step 2: Present Table Specification @@ -28,913 +46,171 @@ Show a clear specification with **YOUR ASSUMPTIONS surfaced**: | Table | Columns | Rows | Key Assumptions | |-------|---------|------|-----------------| -| customers | customer_id, name, email, tier, region, created_at | 5,000 | Tier weighted: Free 60%, Pro 30%, Enterprise 10% | -| orders | order_id, customer_id (FK), amount, order_date, status | 15,000 | Enterprise customers generate 5x more orders than Free | +| customers | customer_id, name, email, tier, region | 5,000 | Tier: Free 60%, Pro 30%, Enterprise 10% | +| orders | order_id, customer_id (FK), amount, status | 15,000 | Enterprise customers generate 5x more orders | **Assumptions I'm making:** -- Amount distribution: log-normal by tier (Enterprise avg ~$1800, Pro ~$245, Free ~$55) -- Status distribution: 65% delivered, 15% shipped, 10% processing, 5% pending, 5% cancelled - -**Generation Approach:** -- **Default**: Generate data using **Spark** (recommended for all use cases) -- **Alternative for <10K rows**: Only if user explicitly prefers local generation, use Polars and upload to volume using `databricks fs cp` +- Amount distribution: log-normal by tier (Enterprise ~$1800, Pro ~$245, Free ~$55) +- Status: 65% delivered, 15% shipped, 10% processing, 5% pending, 5% cancelled **Ask user**: "Does this look correct? Any adjustments needed?" ### Step 3: Ask About Data Features -Prompt user with options (enabled by default unless otherwise noted): - [x] Skew (non-uniform distributions) - **Enabled by default** -- [x] Joins (referential integrity between tables) - **Enabled by default** +- [x] Joins (referential integrity) - **Enabled by default** - [ ] Bad data injection (for data quality testing) - - Nulls in required fields - - Outliers/impossible values (house price $1, age 500) - - Duplicate primary keys - - Orphan foreign keys (referencing non-existent parents) -- [ ] Multi-language text (non-English names/addresses) -- [ ] Incremental mode (append vs overwrite) - for scheduled jobs +- [ ] Multi-language text +- [ ] Incremental mode (append vs overwrite) ### Pre-Generation Checklist -Before writing any generation code, verify: - -- [ ] Generation approach determined: **Spark (strongly recommended)** or local generation with upload (only for <10K rows if user prefers) -- [ ] If using local generation: User notified and prefers this approach -- [ ] User confirmed compute preference (serverless vs cluster) +- [ ] User confirmed compute preference (serverless recommended) - [ ] Table specification shown and approved -- [ ] Assumptions about distributions surfaced and confirmed +- [ ] Assumptions about distributions confirmed - [ ] Output location confirmed (catalog.schema) -- [ ] Data features selected (skew, joins, bad data, etc.) -- [ ] Row counts appropriate for use case +- [ ] Data features selected **Do NOT proceed to code generation until user approves the plan.** -## Execution Options & Installation - -Choose your execution mode based on your needs. **Serverless is strongly recommended** for all use cases. - -**When user requests data generation:** -1. Confirm serverless is acceptable: "I'll use serverless compute. Is that OK?" -2. If they request classic cluster: "Serverless is recommended for cost efficiency. Are you sure you need a classic cluster?" +## Quick Start: Spark + Faker + Pandas UDFs -### Option 1: Databricks Connect with Serverless (Recommended) - -Run code locally while Spark operations execute on serverless compute. Best for development and interactive work. - -# ❌ WRONG - DO NOT USE -from pyspark.sql import SparkSession -spark = SparkSession.builder.getOrCreate() # Will fail with RuntimeError - -# ✅ CORRECT - ALWAYS USE THIS -from databricks.connect import DatabricksSession -spark = DatabricksSession.builder.serverless(True).getOrCreate() - -**Install locally (one-time setup):** -```bash -# Python 3.10 or 3.11: -pip install "databricks-connect>=15.1,<16.2" faker numpy pandas holidays - -# Python 3.12+: -# IMPORTANT: Use 16.4.x for stable withDependencies API (17.x has breaking changes) -pip install "databricks-connect>=16.4,<17.0" faker numpy pandas holidays - -# Configure ~/.databrickscfg -[DEFAULT] -host = https://your-workspace.cloud.databricks.com/ -serverless_compute_id = auto -auth_type = databricks-cli -``` - -**In your script (version-dependent):** - -**For Python 3.12+ with databricks-connect >= 16.4:** ```python from databricks.connect import DatabricksSession, DatabricksEnv +from pyspark.sql import functions as F +from pyspark.sql.types import StringType, DoubleType +import pandas as pd +import numpy as np -# Pass dependencies as simple package name strings -env = DatabricksEnv().withDependencies("faker", "pandas", "numpy", "holidays") - -# Create session -spark = ( - DatabricksSession.builder - .withEnvironment(env) - .serverless(True) - .getOrCreate() -) - -# Spark operations now execute on serverless compute with managed dependencies -``` - -**Version Detection (if needed in your script):** -```python -import importlib.metadata - -def get_databricks_connect_version(): - """Get databricks-connect version as (major, minor) tuple.""" - try: - version_str = importlib.metadata.version('databricks-connect') - parts = version_str.split('.') - return (int(parts[0]), int(parts[1])) - except Exception: - return None - -db_version = get_databricks_connect_version() -if db_version and db_version >= (16, 4): - # Use DatabricksEnv with withDependencies - pass -``` - -**For Python < 3.12 or databricks-connect < 16.4:** - -`DatabricksEnv()` and `withEnvironment()` are NOT available in older versions. You must use one of these alternatives: - -**Create a job with environment settings** - -Create a Databricks job with environment settings on the task. See **Option 2: Serverless Job** section below. - -**Note:** If you're using Polars for local generation (not Spark with Faker UDFs), these workarounds are NOT needed since dependencies run locally. - -**Benefits:** Instant start, local debugging, fast iteration (edit file, re-run immediately) - -### Option 2: Serverless Job (Production/Scheduled) - -Submit jobs to serverless compute with automatic dependency management. Best for production and scheduled workloads. - -**Dependencies managed via `environments` parameter:** -```python -# Use create_job MCP tool with: -{ - "name": "generate_synthetic_data", - "tasks": [{ "environment_key": "datagen_env", ... }], - "environments": [{ - "environment_key": "datagen_env", - "spec": { - "client": "4", - "dependencies": ["faker", "polars", "numpy", "pandas", "holidays"] - } - }] -} -``` - -**Benefits:** No local setup, automatic dependency management, production-ready scaling - -### Option 3: Classic Cluster (Fallback Only) - -Use only if serverless unavailable or you need specific cluster features (GPUs, custom init scripts). - -**Warning:** Classic clusters take 3-8 minutes to start. Prefer serverless. - -**Install dependencies in cluster:** -```python -# Using execute_databricks_command tool: -code = "%pip install faker polars numpy pandas holidays" -# Save returned cluster_id and context_id for subsequent calls -``` - -**When to use:** Only when serverless not available or specific cluster configurations required - -## Required Libraries - -Standard libraries for generating realistic synthetic data: - -- **faker**: Realistic names, addresses, emails, companies, dates (100+ providers) -- **numpy/pandas**: Statistical distributions and data manipulation -- **holidays**: Country-specific holiday calendars for realistic date patterns -- **polars**: Fast local DataFrame library (optional, only for local generation) - -See **Execution Options & Installation** above for installation instructions per execution mode. - -## Data Generation Approaches - -Choose your approach based on scale and where you need to write data: - -### Approach 1: Spark + Faker with Pandas UDFs (Recommended for most cases) - -**Best for:** Any dataset size, especially >100K rows, writing to Unity Catalog - -Generate data with Spark + Faker with Pandas UDFs, save to Databricks. - -**Key features:** -- Full access to 100+ Faker providers (names, addresses, companies, etc.) -- Use Pandas UDFs for parallelism with large datasets -- Flexible custom logic for complex patterns -- Direct integration with Unity Catalog via Spark +# Setup with managed dependencies (databricks-connect 16.4+) +env = DatabricksEnv().withDependencies("faker", "pandas", "numpy") +spark = DatabricksSession.builder.withEnvironment(env).serverless(True).getOrCreate() -**Example:** -```python -# Define Pandas UDFs for Faker data (batch processing for parallelism) -@pandas_udf(StringType()) +# Define Pandas UDFs +@F.pandas_udf(StringType()) def fake_name(ids: pd.Series) -> pd.Series: + from faker import Faker fake = Faker() return pd.Series([fake.name() for _ in range(len(ids))]) -@pandas_udf(StringType()) -def fake_company(ids: pd.Series) -> pd.Series: - fake = Faker() - return pd.Series([fake.company() for _ in range(len(ids))]) - -# Generate with Spark + Pandas UDFs -# Adjust numPartitions based on scale: 8 for <100K, 32 for 1M+ +@F.pandas_udf(DoubleType()) +def generate_amount(tiers: pd.Series) -> pd.Series: + amounts = [] + for tier in tiers: + if tier == "Enterprise": + amounts.append(float(np.random.lognormal(7.5, 0.8))) + elif tier == "Pro": + amounts.append(float(np.random.lognormal(5.5, 0.7))) + else: + amounts.append(float(np.random.lognormal(4.0, 0.6))) + return pd.Series(amounts) + +# Generate customers customers_df = ( - spark.range(0, N_CUSTOMERS, numPartitions=8) + spark.range(0, 10000, numPartitions=16) .select( F.concat(F.lit("CUST-"), F.lpad(F.col("id").cast("string"), 5, "0")).alias("customer_id"), fake_name(F.col("id")).alias("name"), - fake_company(F.col("id")).alias("company"), F.when(F.rand() < 0.6, "Free") .when(F.rand() < 0.9, "Pro") .otherwise("Enterprise").alias("tier"), ) + .withColumn("arr", generate_amount(F.col("tier"))) ) -customers_df.write.mode("overwrite").parquet(f"{VOLUME_PATH}/customers") -``` -### Approach 2: Polars (For local development - Use only if Spark not suitable) - -**Important:** Only use this approach for datasets <10K rows if user explicitly prefers local generation. - -**Best for:** Quick prototyping when Spark is not needed, datasets <10K rows - -Generate entirely with Polars + Faker locally, export to parquet files, then upload to Databricks volumes. - -**Key features:** -- Fast local generation (no Spark overhead) -- Simple, clean API -- Perfect for quick prototyping with very small datasets -- Requires manual upload to Databricks volumes - -**Example:** -```python -import polars as pl -from faker import Faker -import numpy as np - -fake = Faker() - -# Generate with Polars -customers = pl.DataFrame({ - "customer_id": [f"CUST-{i:05d}" for i in range(N_CUSTOMERS)], - "name": [fake.name() for _ in range(N_CUSTOMERS)], - "email": [fake.email() for _ in range(N_CUSTOMERS)], - "tier": np.random.choice(["Free", "Pro", "Enterprise"], N_CUSTOMERS, p=[0.6, 0.3, 0.1]), -}) - -# Save locally -customers.write_parquet("./output/customers.parquet") -``` - -**Upload to Databricks Volume:** -After generating data locally, upload to a Databricks volume: - -```bash -# Create directory in volume if needed -databricks fs mkdirs dbfs:/Volumes////source_data/ - -# Upload local data to volume -databricks fs cp -r ./output/customers.parquet dbfs:/Volumes////source_data/ -databricks fs cp -r ./output/orders.parquet dbfs:/Volumes////source_data/ -``` - -### When to Use Each Approach - -| Scenario | Recommended Approach | -|----------|---------------------| -| **Default - any data generation** | **Spark + Faker with Pandas UDFs** | -| Generating 1M+ rows | **Spark + Faker with Pandas UDFs** | -| Quick prototyping (<10K rows, user explicitly prefers local) | **Polars** (then upload with `databricks fs cp`) | - -**Default:** Use Spark + Faker for all cases. Only use Polars if dataset is <10K rows AND user explicitly requests local generation. - -## Workflow - -### Development (Databricks Connect) - -1. **One-time setup**: Install dependencies locally (see **Execution Options & Installation** above) -2. **Write script**: Create `scripts/generate_data.py` with `DatabricksSession.builder.serverless(True)` -3. **Run locally**: `python scripts/generate_data.py` (Spark ops execute on serverless) -4. **Iterate**: Edit file, re-run immediately - -### Production (Serverless Job) - -1. **Write script locally** -2. **Upload** using `upload_file` MCP tool to `/Workspace/Users/{username}/datagen/{project}/` -3. **Create job** using `create_job` MCP tool with `environments` parameter (see Option 2 above) -4. **Run & monitor** using `run_job_now` and `wait_for_run` MCP tools - -### Production (DABs Bundle) - -For version control and CI/CD: - -```yaml -# databricks.yml -bundle: - name: synthetic-data-gen - -resources: - jobs: - generate_daily_data: - name: "Generate Daily Data" - schedule: - quartz_cron_expression: "0 0 6 * * ?" - tasks: - - task_key: generate - spark_python_task: - python_file: ./src/generate_data.py - environment_key: default - -environments: - default: - spec: - client: "4" - dependencies: - - faker - - polars - - numpy - - pandas - - holidays -``` - -## Storage Destination - -### Ask for Schema Name - -By default, use the `ai_dev_kit` catalog. Ask the user which schema to use: - -> "I'll save the data to `ai_dev_kit.`. What schema name would you like to use? (You can also specify a different catalog if needed.)" - -If the user provides just a schema name, use `ai_dev_kit.{schema}`. If they provide `catalog.schema`, use that instead. - -### Create Infrastructure in the Script - -Always create the schema and volume **inside the Python script** using `spark.sql()`. Do NOT make separate MCP SQL calls - it's much slower. - -**Important:** Do NOT create catalogs - assume they already exist. Only create schema and volume. - -The `spark` variable is available by default on Databricks clusters. - -```python -# ============================================================================= -# CREATE INFRASTRUCTURE (inside the Python script) -# ============================================================================= -# Note: Assume catalog exists - do NOT create it -spark.sql(f"CREATE SCHEMA IF NOT EXISTS {CATALOG}.{SCHEMA}") -spark.sql(f"CREATE VOLUME IF NOT EXISTS {CATALOG}.{SCHEMA}.raw_data") +# Save to Unity Catalog +spark.sql("CREATE SCHEMA IF NOT EXISTS ai_dev_kit.my_schema") +spark.sql("CREATE VOLUME IF NOT EXISTS ai_dev_kit.my_schema.raw_data") +customers_df.write.mode("overwrite").parquet("/Volumes/ai_dev_kit/my_schema/raw_data/customers") ``` -### Output Formats - -Choose your output format based on downstream needs: - -#### Parquet to Volumes (Default) - -Standard format for SDP pipeline input. Best compression and query performance. -Files may not use a file extension or might end with .parquet. +## Common Patterns +### Weighted Tier Distribution ```python -VOLUME_PATH = f"/Volumes/{CATALOG}/{SCHEMA}/raw_data" - -# Save as parquet files -customers_df.write.mode("overwrite").parquet(f"{VOLUME_PATH}/customers") -orders_df.write.mode("overwrite").parquet(f"{VOLUME_PATH}/orders") -tickets_df.write.mode("overwrite").parquet(f"{VOLUME_PATH}/tickets") +F.when(F.rand() < 0.6, "Free") + .when(F.rand() < 0.9, "Pro") + .otherwise("Enterprise").alias("tier") ``` -#### JSON to Volumes - -A common pattern user may request for simulate SDP ingestion from external data feeds such as logs. -File extension should be .json - +### Log-Normal Amounts (Realistic Pricing) ```python -VOLUME_PATH = f"/Volumes/{CATALOG}/{SCHEMA}/raw_data" - -# Save as JSON files -customers_df.write.mode("overwrite").json(f"{VOLUME_PATH}/customers_json") -orders_df.write.mode("overwrite").json(f"{VOLUME_PATH}/orders_json") +@F.pandas_udf(DoubleType()) +def generate_amount(tiers: pd.Series) -> pd.Series: + return pd.Series([ + float(np.random.lognormal({"Enterprise": 7.5, "Pro": 5.5, "Free": 4.0}[t], 0.7)) + for t in tiers + ]) ``` -#### CSV to Volumes - -A common pattern user may request for simulate SDP ingestion from external data feeds such as logs. -File extension should be .csv. - -```python -VOLUME_PATH = f"/Volumes/{CATALOG}/{SCHEMA}/raw_data" - -# Save as CSV with headers -customers_df.write.mode("overwrite").option("header", "true").csv(f"{VOLUME_PATH}/customers_csv") -orders_df.write.mode("overwrite").option("header", "true").csv(f"{VOLUME_PATH}/orders_csv") -``` - -#### Delta Table (Unity Catalog) - -When data is ready for direct analytics consumption (skip SDP pipeline). - -```python -# Ensure schema exists -spark.sql(f"CREATE SCHEMA IF NOT EXISTS {CATALOG}.{SCHEMA}") - -# Save as managed Delta tables -customers_df.write.mode("overwrite").saveAsTable(f"{CATALOG}.{SCHEMA}.customers") -orders_df.write.mode("overwrite").saveAsTable(f"{CATALOG}.{SCHEMA}.orders") - -# With additional options -customers_df.write \ - .mode("overwrite") \ - .option("overwriteSchema", "true") \ - .saveAsTable(f"{CATALOG}.{SCHEMA}.customers") -``` - -#### When to Use Each Format - -| Format | Use Case | -|--------|----------| -| **Parquet to Volumes** | Default - input for SDP bronze/silver/gold pipelines | -| **JSON to Volumes** | User request - a common pattern in real Databricks ingestion workloads | -| **CSV to Volumes** | User request - a common pattern in real Databricks ingestion workloads | -| **Delta Table** | Direct analytics - user may not want to build the ingestion and have data ready to query in notebooks or with SQL | - -## Raw Data Only - No Pre-Aggregated Fields (Unless Instructed Otherwise) - -**By default, generate raw, transactional data only.** Do not create fields that represent sums, totals, averages, or counts. - -- One row = one event/transaction/record -- No columns like `total_orders`, `sum_revenue`, `avg_csat`, `order_count` -- Each row has its own individual values, not rollups - -**Why?** A Spark Declarative Pipeline (SDP) will typically be built after data generation to: -- Ingest raw data (bronze layer) -- Clean and validate (silver layer) -- Aggregate and compute metrics (gold layer) - -The synthetic data is the **source** for this pipeline. Aggregations happen downstream. - -**Note:** If the user specifically requests aggregated fields or summary tables, follow their instructions. - -```python -# GOOD - Raw transactional data -# Customer table: one row per customer, no aggregated fields -customers_data.append({ - "customer_id": cid, - "name": fake.company(), - "tier": "Enterprise", - "region": "North", -}) - -# Order table: one row per order -orders_data.append({ - "order_id": f"ORD-{i:06d}", - "customer_id": cid, - "amount": 150.00, # This order's amount - "order_date": "2024-10-15", -}) - -# BAD - Don't add pre-aggregated fields -# customers_data.append({ -# "customer_id": cid, -# "total_orders": 47, # NO - this is an aggregation -# "total_revenue": 12500.00, # NO - this is a sum -# "avg_order_value": 265.95, # NO - this is an average -# }) -``` - -## Temporality and Data Volume - -### Date Range: Last 6 Months from Today - -**Always generate data for the last ~6 months ending at the current date, unless prompted with specific timeframe.** This ensures: -- Data feels current and relevant for demos -- Recent patterns are visible in dashboards -- Downstream aggregations (daily/weekly/monthly) have enough history - +### Date Range (Last 6 Months) ```python from datetime import datetime, timedelta - -# Dynamic date range - last 6 months from today -END_DATE = datetime.now().replace(hour=0, minute=0, second=0, microsecond=0) +END_DATE = datetime.now() START_DATE = END_DATE - timedelta(days=180) -# Place special events within this range (e.g., incident 3 weeks ago) -INCIDENT_END = END_DATE - timedelta(days=21) -INCIDENT_START = INCIDENT_END - timedelta(days=10) -``` - -### Data Volume for Aggregation - -Generate enough data so patterns remain visible after downstream aggregation (SDP pipelines often aggregate by day/week/region/category). Rules of thumb: - -| Grain | Minimum Records | Rationale | -|-------|-----------------|-----------| -| Daily time series | 50-100/day | See trends after weekly rollup | -| Per category | 500+ per category | Statistical significance | -| Per customer | 5-20 events/customer | Enough for customer-level analysis | -| Total rows | 10K-50K minimum | Patterns survive GROUP BY | - -```python -# Example: 8000 tickets over 180 days = ~44/day average -# After weekly aggregation: ~310 records per week per category -# After monthly by region: still enough to see patterns -N_TICKETS = 8000 -N_CUSTOMERS = 2500 # Each has ~3 tickets on average -N_ORDERS = 25000 # ~10 orders per customer average -``` - -## Business Integrity Requirements - -Generated data MUST reflect business reality. Data should be realistic and tell a coherent story. - -| Pattern | Example | Implementation | -|---------|---------|----------------| -| **Value coherence** | Houses worth $200K-$2M, pens $1-$50 | Domain-appropriate ranges | -| **Tier behavior** | Premium users have more orders | Weighted sampling by tier | -| **Temporal patterns** | More orders on weekends, holidays | Time-based distributions | -| **Geographic patterns** | Regional pricing differences | Location-correlated values | -| **Multi-table integrity** | Orders reference valid customers | Foreign key validation | - -**Anti-pattern**: Flat/linear distributions (every customer has ~same # orders) - -**Correct**: Skewed distributions (80/20 rule - 20% of customers generate 80% of orders) - -### Bad Data Injection (Optional) - -When user requests bad data for testing data quality rules: - -```python -# Bad data configuration -BAD_DATA_CONFIG = { - "null_rate": 0.02, # 2% nulls in required fields - "outlier_rate": 0.01, # 1% impossible values - "duplicate_pk_rate": 0.005, # 0.5% duplicate primary keys - "orphan_fk_rate": 0.01, # 1% orphan foreign keys -} - -# Inject after generation -if INJECT_BAD_DATA: - # Nulls in required fields - null_mask = np.random.random(len(orders_pdf)) < BAD_DATA_CONFIG["null_rate"] - orders_pdf.loc[null_mask, "customer_id"] = None - - # Outliers (impossible values) - outlier_mask = np.random.random(len(orders_pdf)) < BAD_DATA_CONFIG["outlier_rate"] - orders_pdf.loc[outlier_mask, "amount"] = -999.99 # Negative amount - - # Orphan foreign keys - orphan_mask = np.random.random(len(orders_pdf)) < BAD_DATA_CONFIG["orphan_fk_rate"] - orders_pdf.loc[orphan_mask, "customer_id"] = "CUST-NONEXISTENT" -``` - -## Domain-Specific Guidance - -When generating data for specific domains, consider these realistic patterns: - -### Retail/E-commerce -- **Tables**: customers → orders → order_items → products -- **Patterns**: - - Seasonal spikes (holiday shopping) - - Cart abandonment (~70% of carts) - - Loyalty tier progression - - Regional pricing - -### Support/CRM -- **Tables**: accounts → contacts → tickets → interactions -- **Patterns**: - - Incident spikes during outages - - Resolution time varies by priority - - Enterprise accounts have more contacts - - CSAT correlates with resolution speed - -### Manufacturing/IoT -- **Tables**: equipment → sensors → readings → maintenance_orders -- **Patterns**: - - Sensor readings follow equipment lifecycle - - Anomalies precede maintenance events - - Seasonal production variations - - Equipment age affects failure rates - -### Financial Services -- **Tables**: accounts → transactions → payments → fraud_flags -- **Patterns**: - - Transaction amounts follow power law - - Fraud patterns (unusual times, amounts, locations) - - Account balance consistency - - Regulatory compliance (no negative balances) - -**Note**: These are guidance, not rigid schemas. Adapt to user's specific needs. - -## Key Principles - -### 1. Use Spark + Faker for All Data Generation - -Generate data with Spark + Faker for all use cases. This provides scalability, parallelism, and direct integration with Unity Catalog. - -```python -@pandas_udf(StringType()) -def fake_company(ids: pd.Series) -> pd.Series: - fake = Faker() - return pd.Series([fake.company() for _ in range(len(ids))]) - -# Generate with Spark + Pandas UDFs -customers_df = ( - spark.range(0, N_CUSTOMERS, numPartitions=8) - .select( - F.concat(F.lit("CUST-"), F.lpad(F.col("id").cast("string"), 5, "0")).alias("customer_id"), - fake_company(F.col("id")).alias("name"), - F.when(F.rand() < 0.6, "Free") - .when(F.rand() < 0.9, "Pro") - .otherwise("Enterprise").alias("tier"), - ) -) -customers_df.write.mode("overwrite").parquet(f"{VOLUME_PATH}/customers") -``` - -**Alternative (only for <10K rows if user prefers):** Generate with Polars locally and upload: - -```python -import polars as pl -from faker import Faker - -fake = Faker() - -# Generate with Polars -customers_pl = pl.DataFrame({ - "customer_id": [f"CUST-{i:05d}" for i in range(N_CUSTOMERS)], - "name": [fake.company() for _ in range(N_CUSTOMERS)], - "tier": np.random.choice(['Free', 'Pro', 'Enterprise'], N_CUSTOMERS, p=[0.6, 0.3, 0.1]).tolist(), -}) - -# Save locally then upload with: databricks fs cp -r ./output dbfs:/Volumes/{catalog}/{schema}/raw_data/ -customers_pl.write_parquet("./output/customers.parquet") +F.date_add(F.lit(START_DATE.date()), (F.rand() * 180).cast("int")).alias("order_date") ``` -### 2. Iterate on DataFrames for Referential Integrity - -Generate master tables first, then iterate on them to create related tables with matching IDs: - +### Infrastructure Creation ```python -# 1. Generate customers (master table) -customers_pdf = pd.DataFrame({ - "customer_id": [f"CUST-{i:05d}" for i in range(N_CUSTOMERS)], - "tier": np.random.choice(['Free', 'Pro', 'Enterprise'], N_CUSTOMERS, p=[0.6, 0.3, 0.1]), - # ... -}) - -# 2. Create lookup for foreign key generation -customer_ids = customers_pdf["customer_id"].tolist() -customer_tier_map = dict(zip(customers_pdf["customer_id"], customers_pdf["tier"])) - -# Weight by tier - Enterprise customers generate more orders -tier_weights = customers_pdf["tier"].map({'Enterprise': 5.0, 'Pro': 2.0, 'Free': 1.0}) -customer_weights = (tier_weights / tier_weights.sum()).tolist() - -# 3. Generate orders with valid foreign keys and tier-based logic -orders_data = [] -for i in range(N_ORDERS): - cid = np.random.choice(customer_ids, p=customer_weights) - tier = customer_tier_map[cid] - - # Amount depends on tier - if tier == 'Enterprise': - amount = np.random.lognormal(7, 0.8) - elif tier == 'Pro': - amount = np.random.lognormal(5, 0.7) - else: - amount = np.random.lognormal(3.5, 0.6) - - orders_data.append({ - "order_id": f"ORD-{i:06d}", - "customer_id": cid, - "amount": round(amount, 2), - "order_date": fake.date_between(start_date=START_DATE, end_date=END_DATE), - }) - -orders_pdf = pd.DataFrame(orders_data) - -# 4. Generate tickets that reference both customers and orders -order_ids = orders_pdf["order_id"].tolist() -tickets_data = [] -for i in range(N_TICKETS): - cid = np.random.choice(customer_ids, p=customer_weights) - oid = np.random.choice(order_ids) # Or None for general inquiry - - tickets_data.append({ - "ticket_id": f"TKT-{i:06d}", - "customer_id": cid, - "order_id": oid if np.random.random() > 0.3 else None, - # ... - }) - -tickets_pdf = pd.DataFrame(tickets_data) -``` - -### 3. Non-Linear Distributions - -**Never use uniform distributions** - real data is rarely uniform: - -```python -# BAD - Uniform (unrealistic) -prices = np.random.uniform(10, 1000, size=N_ORDERS) - -# GOOD - Log-normal (realistic for prices, salaries, order amounts) -prices = np.random.lognormal(mean=4.5, sigma=0.8, size=N_ORDERS) - -# GOOD - Pareto/power law (popularity, wealth, page views) -popularity = (np.random.pareto(a=2.5, size=N_PRODUCTS) + 1) * 10 - -# GOOD - Exponential (time between events, resolution time) -resolution_hours = np.random.exponential(scale=24, size=N_TICKETS) - -# GOOD - Weighted categorical -regions = np.random.choice( - ['North', 'South', 'East', 'West'], - size=N_CUSTOMERS, - p=[0.40, 0.25, 0.20, 0.15] -) -``` - -### 4. Time-Based Patterns - -Add weekday/weekend effects, holidays, seasonality, and event spikes: - -```python -import holidays - -# Load holiday calendar -US_HOLIDAYS = holidays.US(years=[START_DATE.year, END_DATE.year]) - -def get_daily_multiplier(date): - """Calculate volume multiplier for a given date.""" - multiplier = 1.0 - - # Weekend drop - if date.weekday() >= 5: - multiplier *= 0.6 - - # Holiday drop (even lower than weekends) - if date in US_HOLIDAYS: - multiplier *= 0.3 - - # Q4 seasonality (higher in Oct-Dec) - multiplier *= 1 + 0.15 * (date.month - 6) / 6 - - # Incident spike - if INCIDENT_START <= date <= INCIDENT_END: - multiplier *= 3.0 - - # Random noise - multiplier *= np.random.normal(1, 0.1) - - return max(0.1, multiplier) - -# Distribute tickets across dates with realistic patterns -date_range = pd.date_range(START_DATE, END_DATE, freq='D') -daily_volumes = [int(BASE_DAILY_TICKETS * get_daily_multiplier(d)) for d in date_range] -``` - -### 5. Row Coherence - -Attributes within a row should correlate logically: - -```python -def generate_ticket(customer_id, tier, date): - """Generate a coherent ticket where attributes correlate.""" - - # Priority correlates with tier - if tier == 'Enterprise': - priority = np.random.choice(['Critical', 'High', 'Medium'], p=[0.3, 0.5, 0.2]) - else: - priority = np.random.choice(['Critical', 'High', 'Medium', 'Low'], p=[0.05, 0.2, 0.45, 0.3]) - - # Resolution time correlates with priority - resolution_scale = {'Critical': 4, 'High': 12, 'Medium': 36, 'Low': 72} - resolution_hours = np.random.exponential(scale=resolution_scale[priority]) - - # CSAT correlates with resolution time - if resolution_hours < 4: - csat = np.random.choice([4, 5], p=[0.3, 0.7]) - elif resolution_hours < 24: - csat = np.random.choice([3, 4, 5], p=[0.2, 0.5, 0.3]) - else: - csat = np.random.choice([1, 2, 3, 4], p=[0.1, 0.3, 0.4, 0.2]) - - return { - "customer_id": customer_id, - "priority": priority, - "resolution_hours": round(resolution_hours, 1), - "csat_score": csat, - "created_at": date, - } -``` - -## Complete Examples - -### Example 1: E-commerce Data (Spark + Faker + Pandas) - -Generate e-commerce data with customers and orders tables, with referential integrity and tier-based distributions. - -**Full implementation:** See `scripts/generate_ecommerce_data.py` in this skill folder. - -**Features:** -- Serverless-first with fallback to classic cluster -- Configurable bad data injection for testing -- Incremental mode for scheduled jobs -- Weighted tier distribution with realistic amounts - -**Key configuration options:** - -```python -USE_SERVERLESS = True # Recommended -WRITE_MODE = "overwrite" # or "append" for incremental -INJECT_BAD_DATA = False # Set True for data quality testing +# Always in script - assume catalog exists +spark.sql(f"CREATE SCHEMA IF NOT EXISTS {CATALOG}.{SCHEMA}") +spark.sql(f"CREATE VOLUME IF NOT EXISTS {CATALOG}.{SCHEMA}.raw_data") ``` -**Usage:** Copy to your scripts folder, update CATALOG/SCHEMA, run with `python generate_ecommerce_data.py` - -### Example 2: Local Development with Polars (Only for <10K rows if user prefers) - -Generate synthetic data locally without Spark dependency, then upload to Databricks. Only use for datasets <10K rows if user explicitly prefers local generation. - -**Full implementation:** See `scripts/example_polars.py` in this skill folder. - -**Features:** -- Fast local generation (no Spark overhead) -- For very small datasets (<10K rows) -- Outputs parquet files to local directory -- Requires manual upload to volumes with `databricks fs cp` - +## Execution Modes -**Usage:** Run locally, then upload: `databricks fs cp -r ./output dbfs:/Volumes/{catalog}/{schema}/raw_data/` +| Mode | Best For | Setup | +|------|----------|-------| +| **DB Connect 16.4+ Serverless** | Local dev, Python 3.12+ | `DatabricksEnv().withDependencies(...)` | +| **Serverless Job** | Production, scheduled | Job with `environments` parameter | +| **Classic Cluster** | Fallback only | Manual `%pip install` | -### Example 3: Large-Scale with Faker UDFs +See [references/1-setup-and-execution.md](references/1-setup-and-execution.md) for detailed setup instructions. -Use Faker with Spark UDFs for realistic text data with parallelism. Best for datasets 100K+ rows. +## Output Formats -**Full implementation:** See `scripts/example_faker_udf.py` in this skill folder. +| Format | Use Case | Code | +|--------|----------|------| +| **Parquet** (default) | SDP pipeline input | `df.write.parquet(path)` | +| **JSON** | Log-style ingestion | `df.write.json(path)` | +| **CSV** | Legacy systems | `df.write.option("header", "true").csv(path)` | +| **Delta Table** | Direct analytics | `df.write.saveAsTable("catalog.schema.table")` | -**Features:** -- Serverless-first with fallback to classic cluster -- Parallel execution using Spark UDFs -- Realistic text data (company names, addresses, emails) -- Tier-based amount generation +See [references/5-output-formats.md](references/5-output-formats.md) for detailed options. - -**Usage:** Copy `example_faker_udf.py` to your scripts folder and customize the UDFs and configuration. - - -**Execute with Databricks Connect:** -```bash -python scripts/generate_data.py -``` - -**Execute with classic cluster** using `run_python_file_on_databricks` tool: -- `file_path`: "scripts/generate_data.py" - -If it fails, edit the file and re-run with the same `cluster_id` and `context_id`. - -### Validate Generated Data - -After successful execution, use `get_volume_folder_details` tool to verify the generated data: -- `volume_path`: "my_catalog/my_schema/raw_data/customers" -- `format`: "parquet" -- `table_stat_level`: "SIMPLE" - -This returns schema, row counts, and column statistics to confirm the data was written correctly. - -## Best Practices +## Best Practices Summary ### Execution -1. **Use serverless** (Databricks Connect for dev, jobs for production) - instant start, no cluster wait -2. **Ask for catalog and schema**: Ask for catalog (default to `ai_dev_kit`), ask user for schema name -3. **Present plan before generating**: Show table spec with assumptions, get user approval +- Use serverless (instant start, no cluster wait) +- Ask for catalog/schema (default `ai_dev_kit`) +- Present plan before generating ### Data Generation -6. **Default to Spark + Faker** for all data generation - scalable, parallel, direct Unity Catalog integration -7. **Use Pandas UDFs for scale** (10k+ rows) - Spark parallelism with Faker -8. **Only use local generation** (<10K rows) if user explicitly prefers it - then upload with `databricks fs cp` -9. **Master tables first**: Generate customers, then orders reference customer_ids -10. **Weighted sampling**: Enterprise customers generate more activity -11. **Distributions**: Log-normal for values, exponential for times, weighted categorical -12. **Time patterns**: Weekday/weekend, holidays, seasonality, event spikes -13. **Row coherence**: Priority affects resolution time affects CSAT -14. **Volume for aggregation**: 10K-50K rows minimum so patterns survive GROUP BY - -15. **Context reuse**: Pass `cluster_id` and `context_id` for faster iterations (classic cluster only) +- **Spark + Faker + Pandas UDFs** for all cases +- Master tables first, then related tables with valid FKs +- Non-linear distributions (log-normal, Pareto, exponential) +- Time patterns (weekday/weekend, holidays, seasonality) +- Row coherence (correlated attributes) + +### Output +- Create infrastructure in script (`CREATE SCHEMA/VOLUME IF NOT EXISTS`) +- Do NOT create catalogs - assume they exist +- Parquet to volumes as default ## Related Skills -- **[databricks-unity-catalog](../databricks-unity-catalog/SKILL.md)** - for managing catalogs, schemas, and volumes where data is stored -### Output -16. **Create infrastructure in script**: Use `CREATE SCHEMA/VOLUME IF NOT EXISTS` - do NOT create catalogs -17. **Assume catalogs exist**: Never auto-create catalogs, only create schema and volume -18. **Choose output format** based on downstream needs (Parquet/JSON/CSV/Delta) -19. **Configuration at top**: All sizes, dates, and paths as variables +- **databricks-unity-catalog** - Managing catalogs, schemas, and volumes +- **databricks-asset-bundles** - DABs for production deployment ## Common Issues | Issue | Solution | |-------|----------| -| **"Either base environment or version must be provided"** | Add `"client": "4"` to `spec` in job environments (auto-injected by MCP tool) | -| **"ModuleNotFoundError"** for faker/polars/etc. | See **Execution Options & Installation** section for dependency setup per execution mode | -| **Serverless job fails to start** | Verify workspace has serverless compute enabled; check Unity Catalog permissions | -| **Faker UDF is slow** | Use `pandas_udf` for batched operations; adjust `numPartitions` | -| **Classic cluster startup is slow (3-8 min)** | Prompt user to check if cluster is running and suggest a replacement. | -| **Out of memory with large data** | Increase `partitions` parameter in `spark.range()` | -| **Context corrupted on classic cluster** | Omit `context_id` to create fresh context, reinstall libraries | +| `ModuleNotFoundError: faker` | See [references/1-setup-and-execution.md](references/1-setup-and-execution.md) | +| Faker UDF is slow | Use `pandas_udf` for batch processing | +| Out of memory | Increase `numPartitions` in `spark.range()` | +| Referential integrity errors | Generate master tables first, cache, then join | + +See [references/6-troubleshooting.md](references/6-troubleshooting.md) for full troubleshooting guide. diff --git a/databricks-skills/synthetic-data-generation/references/1-setup-and-execution.md b/databricks-skills/synthetic-data-generation/references/1-setup-and-execution.md new file mode 100644 index 00000000..a05eeadf --- /dev/null +++ b/databricks-skills/synthetic-data-generation/references/1-setup-and-execution.md @@ -0,0 +1,201 @@ +# Setup and Execution Guide + +This guide covers all execution modes for synthetic data generation, organized by Databricks Connect version and Python version. + +## Quick Decision Matrix + +| Your Environment | Recommended Approach | +|------------------|---------------------| +| Python 3.12+ with databricks-connect >= 16.4 | DatabricksEnv with withDependencies API | +| Python 3.10/3.11 with older databricks-connect | Serverless job with environments parameter | +| Running on Databricks Runtime (notebook/job) | Dependencies pre-installed or %pip install | +| Classic compute (fallback only) | Manual cluster setup | + +## Option 1: Databricks Connect 16.4+ with Serverless (Recommended) + +**Best for:** Python 3.12+, local development with serverless compute + +**Install locally:** +```bash +pip install "databricks-connect>=16.4,<17.0" faker numpy pandas holidays +``` + +**Configure ~/.databrickscfg:** +```ini +[DEFAULT] +host = https://your-workspace.cloud.databricks.com/ +serverless_compute_id = auto +auth_type = databricks-cli +``` + +**In your script:** +```python +from databricks.connect import DatabricksSession, DatabricksEnv + +# Pass dependencies as simple package name strings +env = DatabricksEnv().withDependencies("faker", "pandas", "numpy", "holidays") + +# Create session with managed dependencies +spark = ( + DatabricksSession.builder + .withEnvironment(env) + .serverless(True) + .getOrCreate() +) + +# Spark operations now execute on serverless compute with managed dependencies +``` + +**Version Detection (if needed):** +```python +import importlib.metadata + +def get_databricks_connect_version(): + """Get databricks-connect version as (major, minor) tuple.""" + try: + version_str = importlib.metadata.version('databricks-connect') + parts = version_str.split('.') + return (int(parts[0]), int(parts[1])) + except Exception: + return None + +db_version = get_databricks_connect_version() +if db_version and db_version >= (16, 4): + # Use DatabricksEnv with withDependencies + pass +``` + +**Benefits:** +- Instant start, no cluster wait +- Local debugging and fast iteration +- Automatic dependency management +- Edit file, re-run immediately + +## Option 2: Older Databricks Connect or Python < 3.12 + +**Best for:** Python 3.10/3.11, databricks-connect 15.1-16.3 + +`DatabricksEnv()` and `withEnvironment()` are NOT available in older versions. Use serverless jobs with environments parameter instead. + +**Install locally:** +```bash +pip install "databricks-connect>=15.1,<16.2" faker numpy pandas holidays +``` + +**Create a serverless job with environment settings:** +```python +# Use create_job MCP tool with: +{ + "name": "generate_synthetic_data", + "tasks": [{ "environment_key": "datagen_env", ... }], + "environments": [{ + "environment_key": "datagen_env", + "spec": { + "client": "4", + "dependencies": ["faker", "numpy", "pandas", "holidays"] + } + }] +} +``` + +**DABs bundle configuration:** +```yaml +# databricks.yml +bundle: + name: synthetic-data-gen + +resources: + jobs: + generate_data: + name: "Generate Synthetic Data" + tasks: + - task_key: generate + spark_python_task: + python_file: ./src/generate_data.py + environment_key: default + +environments: + default: + spec: + client: "4" + dependencies: + - faker + - numpy + - pandas + - holidays +``` + +## Option 3: Classic Cluster (Fallback Only) + +**Use only when:** Serverless unavailable, or specific cluster features needed (GPUs, custom init scripts) + +**Warning:** Classic clusters take 3-8 minutes to start. Always prefer serverless. + +**Install dependencies in cluster:** +```python +# In notebook or using execute_databricks_command tool: +%pip install faker numpy pandas holidays +``` + +**Connect from local script:** +```python +from databricks.connect import DatabricksSession + +spark = DatabricksSession.builder.clusterId("your-cluster-id").getOrCreate() +``` + +## Required Libraries + +Standard libraries for generating realistic synthetic data: + +| Library | Purpose | Required For | +|---------|---------|--------------| +| **faker** | Realistic names, addresses, emails, companies | Text data generation | +| **numpy** | Statistical distributions | Non-linear distributions | +| **pandas** | Data manipulation, Pandas UDFs | Spark UDF definitions | +| **holidays** | Country-specific holiday calendars | Time-based patterns | + +## Environment Detection Pattern + +Use this pattern to auto-detect environment and choose the right session creation: + +```python +import os + +def is_databricks_runtime(): + """Check if running on Databricks Runtime vs locally.""" + return "DATABRICKS_RUNTIME_VERSION" in os.environ + +def get_databricks_connect_version(): + """Get databricks-connect version as (major, minor) tuple or None.""" + try: + import databricks.connect + version_str = databricks.connect.__version__ + parts = version_str.split('.') + return (int(parts[0]), int(parts[1])) + except (ImportError, AttributeError, ValueError, IndexError): + return None + +on_runtime = is_databricks_runtime() +db_version = get_databricks_connect_version() + +# Use DatabricksEnv if: locally + databricks-connect >= 16.4 +use_auto_dependencies = (not on_runtime) and db_version and db_version >= (16, 4) + +if use_auto_dependencies: + from databricks.connect import DatabricksSession, DatabricksEnv + env = DatabricksEnv().withDependencies("faker", "pandas", "numpy", "holidays") + spark = DatabricksSession.builder.withEnvironment(env).serverless(True).getOrCreate() +else: + from databricks.connect import DatabricksSession + spark = DatabricksSession.builder.serverless(True).getOrCreate() +``` + +## Common Setup Issues + +| Issue | Solution | +|-------|----------| +| `ModuleNotFoundError: faker` | Install dependencies per execution mode above | +| `DatabricksEnv not found` | Upgrade to databricks-connect >= 16.4 or use job with environments | +| `serverless_compute_id` error | Add `serverless_compute_id = auto` to ~/.databrickscfg | +| Classic cluster startup slow | Use serverless instead (instant start) | diff --git a/databricks-skills/synthetic-data-generation/references/2-generation-approaches.md b/databricks-skills/synthetic-data-generation/references/2-generation-approaches.md new file mode 100644 index 00000000..d820cd10 --- /dev/null +++ b/databricks-skills/synthetic-data-generation/references/2-generation-approaches.md @@ -0,0 +1,205 @@ +# Data Generation Approaches + +Choose your approach based on scale and requirements. **Spark + Faker + Pandas UDFs is strongly preferred** for all cases. + +## Decision Table + +| Scenario | Recommended Approach | +|----------|---------------------| +| **Default - any data generation** | **Spark + Faker + Pandas UDFs** | +| Large datasets (100K+ rows) | **Spark + Faker + Pandas UDFs** | +| Medium datasets (10K-100K rows) | **Spark + Faker + Pandas UDFs** | +| Small datasets (<10K rows) | **Spark + Faker + Pandas UDFs** (or Polars if user prefers local) | + +**Rule:** Always use Spark + Faker + Pandas UDFs unless user explicitly requests local generation for <10K rows. + +--- + +## Approach 1: Spark + Faker + Pandas UDFs (Strongly Preferred) + +**Best for:** All dataset sizes, direct write to Unity Catalog + +**Why this approach:** +- Scales from thousands to millions of rows +- Parallel execution via Spark +- Direct integration with Unity Catalog +- No intermediate files or uploads needed +- Works with serverless and classic compute + +### Basic Pattern + +```python +from pyspark.sql import functions as F +from pyspark.sql.types import StringType, DoubleType +from faker import Faker +import pandas as pd +import numpy as np + +# Define Pandas UDFs for Faker data (batch processing for parallelism) +@F.pandas_udf(StringType()) +def fake_name(ids: pd.Series) -> pd.Series: + fake = Faker() + return pd.Series([fake.name() for _ in range(len(ids))]) + +@F.pandas_udf(StringType()) +def fake_company(ids: pd.Series) -> pd.Series: + fake = Faker() + return pd.Series([fake.company() for _ in range(len(ids))]) + +@F.pandas_udf(StringType()) +def fake_email(ids: pd.Series) -> pd.Series: + fake = Faker() + return pd.Series([fake.email() for _ in range(len(ids))]) + +@F.pandas_udf(DoubleType()) +def generate_lognormal_amount(tiers: pd.Series) -> pd.Series: + """Generate amount based on tier using log-normal distribution.""" + amounts = [] + for tier in tiers: + if tier == "Enterprise": + amounts.append(float(np.random.lognormal(mean=7.5, sigma=0.8))) + elif tier == "Pro": + amounts.append(float(np.random.lognormal(mean=5.5, sigma=0.7))) + else: + amounts.append(float(np.random.lognormal(mean=4.0, sigma=0.6))) + return pd.Series(amounts) +``` + +### Generate Data with Spark + Pandas UDFs + +```python +# Configuration +N_CUSTOMERS = 100_000 +PARTITIONS = 16 # Adjust based on data size: 8 for <100K, 32 for 1M+ +VOLUME_PATH = f"/Volumes/{CATALOG}/{SCHEMA}/raw_data" + +# Generate customers with Spark + Pandas UDFs +customers_df = ( + spark.range(0, N_CUSTOMERS, numPartitions=PARTITIONS) + .select( + F.concat(F.lit("CUST-"), F.lpad(F.col("id").cast("string"), 5, "0")).alias("customer_id"), + fake_name(F.col("id")).alias("name"), + fake_company(F.col("id")).alias("company"), + fake_email(F.col("id")).alias("email"), + F.when(F.rand() < 0.6, "Free") + .when(F.rand() < 0.9, "Pro") + .otherwise("Enterprise").alias("tier"), + F.when(F.rand() < 0.4, "North") + .when(F.rand() < 0.65, "South") + .when(F.rand() < 0.85, "East") + .otherwise("West").alias("region"), + ) +) + +# Add tier-based amount +customers_df = customers_df.withColumn("arr", generate_lognormal_amount(F.col("tier"))) + +# Write directly to Unity Catalog volume +customers_df.write.mode("overwrite").parquet(f"{VOLUME_PATH}/customers") +``` + +### Partitioning Strategy + +| Data Size | Recommended Partitions | +|-----------|----------------------| +| < 100K rows | 8 partitions | +| 100K - 500K rows | 16 partitions | +| 500K - 1M rows | 32 partitions | +| 1M+ rows | 64+ partitions | + +--- + +## Approach 2: Polars + Local Generation + Upload (Secondary Option) + +**Use only when:** Dataset <10K rows AND user explicitly prefers local generation + +**Why this approach exists:** +- No Spark overhead for tiny datasets +- Quick prototyping in local environment +- When Databricks Connect not available + +**Limitations:** +- Doesn't scale past ~100K rows +- Requires manual upload step +- No direct Unity Catalog integration + +### Install Local Dependencies + +```bash +# Preferred: use uv for fast, reliable installs +uv pip install polars faker numpy + +# Alternative if uv not available +pip install polars faker numpy +``` + +### Generate Locally with Polars + +```python +import polars as pl +from faker import Faker +import numpy as np + +fake = Faker() +N_CUSTOMERS = 5000 + +# Generate with Polars +customers = pl.DataFrame({ + "customer_id": [f"CUST-{i:05d}" for i in range(N_CUSTOMERS)], + "name": [fake.name() for _ in range(N_CUSTOMERS)], + "email": [fake.email() for _ in range(N_CUSTOMERS)], + "tier": np.random.choice(["Free", "Pro", "Enterprise"], N_CUSTOMERS, p=[0.6, 0.3, 0.1]).tolist(), + "region": np.random.choice(["North", "South", "East", "West"], N_CUSTOMERS, p=[0.4, 0.25, 0.2, 0.15]).tolist(), +}) + +# Save locally +customers.write_parquet("./output/customers.parquet") +``` + +### Upload to Databricks Volume + +After generating data locally, upload to a Databricks volume: + +```bash +# Create directory in volume if needed +databricks fs mkdirs dbfs:/Volumes////source_data/ + +# Upload local data to volume +databricks fs cp -r ./output/customers.parquet dbfs:/Volumes////source_data/ +databricks fs cp -r ./output/orders.parquet dbfs:/Volumes////source_data/ +``` + +### When to Actually Use Polars + +Only recommend Polars when ALL conditions are met: +1. Dataset is < 10K rows +2. User explicitly requests local generation +3. Quick prototyping without Databricks connection + +Otherwise, **always use Spark + Faker + Pandas UDFs**. + +--- + +## Storage Destinations + +### Ask for Catalog and Schema + +By default, use the `ai_dev_kit` catalog. Ask the user which schema to use: + +> "I'll save the data to `ai_dev_kit.`. What schema name would you like to use? (You can also specify a different catalog if needed.)" + +### Create Infrastructure in Script + +Always create the schema and volume **inside the Python script** using `spark.sql()`: + +```python +CATALOG = "ai_dev_kit" +SCHEMA = "synthetic_data" +VOLUME_PATH = f"/Volumes/{CATALOG}/{SCHEMA}/raw_data" + +# Note: Assume catalog exists - do NOT create it +spark.sql(f"CREATE SCHEMA IF NOT EXISTS {CATALOG}.{SCHEMA}") +spark.sql(f"CREATE VOLUME IF NOT EXISTS {CATALOG}.{SCHEMA}.raw_data") +``` + +**Important:** Do NOT create catalogs - assume they already exist. Only create schema and volume. diff --git a/databricks-skills/synthetic-data-generation/references/3-data-patterns.md b/databricks-skills/synthetic-data-generation/references/3-data-patterns.md new file mode 100644 index 00000000..ae9b0697 --- /dev/null +++ b/databricks-skills/synthetic-data-generation/references/3-data-patterns.md @@ -0,0 +1,284 @@ +# Data Patterns Guide + +Creating realistic, coherent synthetic data with Spark + Pandas UDFs. + +## 5 Key Principles + +1. **Use Spark + Faker + Pandas UDFs** for all generation +2. **Referential Integrity** - master tables first, weighted sampling +3. **Non-Linear Distributions** - log-normal, Pareto, exponential +4. **Time-Based Patterns** - weekday/weekend, holidays, seasonality +5. **Row Coherence** - correlated attributes within each row + +--- + +## Principle 1: Use Spark + Faker + Pandas UDFs + +Generate data with Spark + Faker for all use cases. Pandas UDFs provide efficient, distributed Faker calls that scale seamlessly from thousands to millions of rows. + +### Define Pandas UDFs + +```python +from pyspark.sql import functions as F +from pyspark.sql.types import StringType, DoubleType +from faker import Faker +import pandas as pd +import numpy as np + +@F.pandas_udf(StringType()) +def fake_company(ids: pd.Series) -> pd.Series: + fake = Faker() + return pd.Series([fake.company() for _ in range(len(ids))]) + +@F.pandas_udf(StringType()) +def fake_address(ids: pd.Series) -> pd.Series: + fake = Faker() + return pd.Series([fake.address().replace('\n', ', ') for _ in range(len(ids))]) + +@F.pandas_udf(DoubleType()) +def generate_lognormal_amount(tiers: pd.Series) -> pd.Series: + amounts = [] + for tier in tiers: + if tier == "Enterprise": + amounts.append(float(np.random.lognormal(mean=7.5, sigma=0.8))) + elif tier == "Pro": + amounts.append(float(np.random.lognormal(mean=5.5, sigma=0.7))) + else: + amounts.append(float(np.random.lognormal(mean=4.0, sigma=0.6))) + return pd.Series(amounts) +``` + +### Generate with Spark + +```python +# Adjust numPartitions based on scale: 8 for <100K, 32 for 1M+ +customers_df = ( + spark.range(0, N_CUSTOMERS, numPartitions=16) + .select( + F.concat(F.lit("CUST-"), F.lpad(F.col("id").cast("string"), 5, "0")).alias("customer_id"), + fake_company(F.col("id")).alias("name"), + F.when(F.rand() < 0.6, "Free") + .when(F.rand() < 0.9, "Pro") + .otherwise("Enterprise").alias("tier"), + ) +) +customers_df.write.mode("overwrite").parquet(f"{VOLUME_PATH}/customers") +``` + +--- + +## Principle 2: Referential Integrity + +Generate master tables first, then iterate on them to create related tables with matching IDs. + +### Pattern: Weighted Sampling by Tier + +```python +from pyspark.sql.window import Window + +# 1. Generate and cache customers (master table) +customers_df = ( + spark.range(0, N_CUSTOMERS, numPartitions=PARTITIONS) + .select( + F.concat(F.lit("CUST-"), F.lpad(F.col("id").cast("string"), 5, "0")).alias("customer_id"), + F.when(F.rand(SEED) < 0.6, "Free") + .when(F.rand(SEED) < 0.9, "Pro") + .otherwise("Enterprise").alias("tier"), + ) +) +customer_lookup = customers_df.select("customer_id", "tier").cache() + +# 2. Generate orders with valid foreign keys +orders_df = spark.range(0, N_ORDERS, numPartitions=PARTITIONS) + +# Map order to customer using hash-based distribution +orders_df = orders_df.select( + F.concat(F.lit("ORD-"), F.lpad(F.col("id").cast("string"), 6, "0")).alias("order_id"), + (F.abs(F.hash(F.col("id"), F.lit(SEED))) % N_CUSTOMERS).alias("customer_idx"), +) + +# Add customer_idx to lookup for join +customer_lookup_with_idx = customer_lookup.withColumn( + "customer_idx", + (F.row_number().over(Window.orderBy(F.monotonically_increasing_id())) - 1).cast("int") +) + +# Join to get valid foreign keys +orders_with_fk = orders_df.join(customer_lookup_with_idx, on="customer_idx", how="left") + +customer_lookup.unpersist() +``` + +### Anti-Pattern: Random FK Generation + +```python +# BAD - May generate non-existent customer IDs +orders_df = spark.range(0, N_ORDERS).select( + F.concat(F.lit("CUST-"), (F.rand() * 99999).cast("int")).alias("customer_id") # WRONG! +) +``` + +--- + +## Principle 3: Non-Linear Distributions + +**Never use uniform distributions** - real data is rarely uniform. + +### Distribution Types + +| Distribution | Use Case | Example | +|--------------|----------|---------| +| **Log-normal** | Prices, salaries, order amounts | `np.random.lognormal(mean=4.5, sigma=0.8)` | +| **Pareto/Power law** | Popularity, wealth, page views | `(np.random.pareto(a=2.5) + 1) * 10` | +| **Exponential** | Time between events, resolution time | `np.random.exponential(scale=24)` | +| **Weighted categorical** | Status, region, tier | `np.random.choice(vals, p=[0.4, 0.3, 0.2, 0.1])` | + +### Pandas UDF for Log-Normal Amounts + +```python +@F.pandas_udf(DoubleType()) +def generate_lognormal_amount(tiers: pd.Series) -> pd.Series: + """Generate amount based on tier using log-normal distribution.""" + amounts = [] + for tier in tiers: + if tier == "Enterprise": + amounts.append(float(np.random.lognormal(mean=7.5, sigma=0.8))) # ~$1800 avg + elif tier == "Pro": + amounts.append(float(np.random.lognormal(mean=5.5, sigma=0.7))) # ~$245 avg + else: + amounts.append(float(np.random.lognormal(mean=4.0, sigma=0.6))) # ~$55 avg + return pd.Series(amounts) +``` + +### Anti-Pattern: Uniform Distribution + +```python +# BAD - Uniform (unrealistic) +prices = np.random.uniform(10, 1000, size=N_ORDERS) + +# GOOD - Log-normal (realistic for prices) +prices = np.random.lognormal(mean=4.5, sigma=0.8, size=N_ORDERS) +``` + +--- + +## Principle 4: Time-Based Patterns + +Add weekday/weekend effects, holidays, seasonality, and event spikes. + +### Holiday and Weekday Multipliers + +```python +import holidays +from datetime import datetime, timedelta + +# Load holiday calendar +US_HOLIDAYS = holidays.US(years=[START_DATE.year, END_DATE.year]) + +def get_daily_multiplier(date): + """Calculate volume multiplier for a given date.""" + multiplier = 1.0 + + # Weekend drop + if date.weekday() >= 5: + multiplier *= 0.6 + + # Holiday drop (even lower than weekends) + if date in US_HOLIDAYS: + multiplier *= 0.3 + + # Q4 seasonality (higher in Oct-Dec) + multiplier *= 1 + 0.15 * (date.month - 6) / 6 + + # Incident spike (if applicable) + if INCIDENT_START <= date <= INCIDENT_END: + multiplier *= 3.0 + + # Random noise + multiplier *= np.random.normal(1, 0.1) + + return max(0.1, multiplier) +``` + +### Date Range: Last 6 Months + +Always generate data for the last ~6 months ending at the current date: + +```python +from datetime import datetime, timedelta + +END_DATE = datetime.now().replace(hour=0, minute=0, second=0, microsecond=0) +START_DATE = END_DATE - timedelta(days=180) +``` + +--- + +## Principle 5: Row Coherence + +Attributes within a row should correlate logically. + +### Coherent Ticket Generation + +```python +@F.pandas_udf("struct") +def generate_coherent_ticket(tiers: pd.Series) -> pd.DataFrame: + """Generate coherent ticket where attributes correlate.""" + results = [] + for tier in tiers: + # Priority correlates with tier + if tier == 'Enterprise': + priority = np.random.choice(['Critical', 'High', 'Medium'], p=[0.3, 0.5, 0.2]) + else: + priority = np.random.choice(['Critical', 'High', 'Medium', 'Low'], p=[0.05, 0.2, 0.45, 0.3]) + + # Resolution time correlates with priority + resolution_scale = {'Critical': 4, 'High': 12, 'Medium': 36, 'Low': 72} + resolution_hours = np.random.exponential(scale=resolution_scale[priority]) + + # CSAT correlates with resolution time + if resolution_hours < 4: + csat = np.random.choice([4, 5], p=[0.3, 0.7]) + elif resolution_hours < 24: + csat = np.random.choice([3, 4, 5], p=[0.2, 0.5, 0.3]) + else: + csat = np.random.choice([1, 2, 3, 4], p=[0.1, 0.3, 0.4, 0.2]) + + results.append({ + "priority": priority, + "resolution_hours": round(resolution_hours, 1), + "csat_score": int(csat), + }) + + return pd.DataFrame(results) +``` + +### Correlation Examples + +| Attribute A | Attribute B | Correlation | +|------------|-------------|-------------| +| Customer tier | Order amount | Enterprise = higher amounts | +| Ticket priority | Resolution time | Critical = faster resolution | +| Resolution time | CSAT score | Faster = higher satisfaction | +| Region | Product preference | Regional variations | +| Time of day | Transaction type | Business hours = B2B | + +--- + +## Data Volume for Aggregation + +Generate enough data so patterns remain visible after downstream aggregation: + +| Grain | Minimum Records | Rationale | +|-------|-----------------|-----------| +| Daily time series | 50-100/day | See trends after weekly rollup | +| Per category | 500+ per category | Statistical significance | +| Per customer | 5-20 events/customer | Customer-level analysis | +| Total rows | 10K-50K minimum | Patterns survive GROUP BY | + +```python +# Example: 8000 tickets over 180 days = ~44/day average +# After weekly aggregation: ~310 records per week +N_TICKETS = 8000 +N_CUSTOMERS = 2500 # Each has ~3 tickets on average +N_ORDERS = 25000 # ~10 orders per customer average +``` diff --git a/databricks-skills/synthetic-data-generation/references/4-domain-guidance.md b/databricks-skills/synthetic-data-generation/references/4-domain-guidance.md new file mode 100644 index 00000000..0519bcce --- /dev/null +++ b/databricks-skills/synthetic-data-generation/references/4-domain-guidance.md @@ -0,0 +1,256 @@ +# Domain-Specific Guidance + +Realistic patterns for common data domains. All examples use Spark + Faker + Pandas UDFs. + +--- + +## Retail/E-commerce + +### Tables +``` +customers → orders → order_items → products +``` + +### Key Patterns + +| Pattern | Implementation | +|---------|----------------| +| Seasonal spikes | Q4 holiday shopping (1.5-2x volume in Nov-Dec) | +| Cart abandonment | ~70% of carts never complete | +| Loyalty tier progression | Free → Pro → Enterprise over time | +| Regional pricing | 5-15% price variation by region | + +### Realistic Distributions + +```python +@F.pandas_udf(DoubleType()) +def generate_order_amount(tiers: pd.Series) -> pd.Series: + """E-commerce order amounts by tier.""" + amounts = [] + for tier in tiers: + if tier == "Premium": + amounts.append(float(np.random.lognormal(mean=5.5, sigma=0.9))) # ~$245 avg + elif tier == "Standard": + amounts.append(float(np.random.lognormal(mean=4.2, sigma=0.7))) # ~$67 avg + else: # Basic + amounts.append(float(np.random.lognormal(mean=3.5, sigma=0.6))) # ~$33 avg + return pd.Series(amounts) + +# Order status with cart abandonment +status_weights = [0.70, 0.08, 0.07, 0.10, 0.05] # abandoned, pending, processing, shipped, delivered +``` + +### Schema Example + +```python +# Products +products_df = spark.range(0, N_PRODUCTS).select( + F.concat(F.lit("PROD-"), F.lpad(F.col("id").cast("string"), 5, "0")).alias("product_id"), + fake_product_name(F.col("id")).alias("name"), + F.array(F.lit("Electronics"), F.lit("Clothing"), F.lit("Home"), F.lit("Sports"))[ + (F.rand() * 4).cast("int") + ].alias("category"), + generate_price(F.col("id")).alias("base_price"), +) +``` + +--- + +## Support/CRM + +### Tables +``` +accounts → contacts → tickets → interactions +``` + +### Key Patterns + +| Pattern | Implementation | +|---------|----------------| +| Incident spikes | 3-5x volume during outages | +| Resolution by priority | Critical: 4h avg, Low: 72h avg | +| Enterprise contacts | 5-10 contacts per account vs 1-2 for SMB | +| CSAT correlation | Faster resolution = higher satisfaction | + +### Realistic Distributions + +```python +@F.pandas_udf("struct") +def generate_ticket_metrics(tiers: pd.Series) -> pd.DataFrame: + """Support ticket metrics with correlated attributes.""" + results = [] + for tier in tiers: + # Priority correlates with tier + if tier == 'Enterprise': + priority = np.random.choice(['Critical', 'High', 'Medium'], p=[0.3, 0.5, 0.2]) + else: + priority = np.random.choice(['Critical', 'High', 'Medium', 'Low'], p=[0.05, 0.2, 0.45, 0.3]) + + # Resolution time by priority (exponential distribution) + resolution_scale = {'Critical': 4, 'High': 12, 'Medium': 36, 'Low': 72} + resolution_hours = np.random.exponential(scale=resolution_scale[priority]) + + # CSAT correlates with resolution time + if resolution_hours < 4: + csat = np.random.choice([4, 5], p=[0.3, 0.7]) + elif resolution_hours < 24: + csat = np.random.choice([3, 4, 5], p=[0.2, 0.5, 0.3]) + else: + csat = np.random.choice([1, 2, 3, 4], p=[0.1, 0.3, 0.4, 0.2]) + + results.append({"priority": priority, "resolution_hours": round(resolution_hours, 1), "csat": int(csat)}) + return pd.DataFrame(results) +``` + +### Schema Example + +```python +# Tickets with coherent attributes +tickets_df = ( + spark.range(0, N_TICKETS, numPartitions=PARTITIONS) + .select( + F.concat(F.lit("TKT-"), F.lpad(F.col("id").cast("string"), 6, "0")).alias("ticket_id"), + # FK to customer (weighted by tier) + ... + ) + .withColumn("metrics", generate_ticket_metrics(F.col("tier"))) + .select("*", "metrics.*") + .drop("metrics") +) +``` + +--- + +## Manufacturing/IoT + +### Tables +``` +equipment → sensors → readings → maintenance_orders +``` + +### Key Patterns + +| Pattern | Implementation | +|---------|----------------| +| Sensor lifecycle | Normal → degraded → failure progression | +| Anomaly precursors | Anomalies precede maintenance by 2-7 days | +| Seasonal production | Summer/winter production variations | +| Equipment age | Failure rate increases with age | + +### Realistic Distributions + +```python +@F.pandas_udf(DoubleType()) +def generate_sensor_reading(equipment_ages: pd.Series) -> pd.Series: + """Sensor readings with age-based degradation.""" + readings = [] + for age_days in equipment_ages: + # Base reading with age-based drift + base = 100.0 + drift = (age_days / 365) * 5 # 5 units drift per year + noise = np.random.normal(0, 2) + + # Occasional anomalies (more likely with age) + anomaly_prob = min(0.01 + (age_days / 365) * 0.02, 0.1) + if np.random.random() < anomaly_prob: + noise += np.random.choice([-1, 1]) * np.random.exponential(10) + + readings.append(base + drift + noise) + return pd.Series(readings) +``` + +### Schema Example + +```python +# Sensor readings time series +readings_df = ( + spark.range(0, N_READINGS, numPartitions=PARTITIONS) + .select( + F.concat(F.lit("READ-"), F.col("id").cast("string")).alias("reading_id"), + # FK to sensor + ((F.col("id") % N_SENSORS) + 1).alias("sensor_id"), + F.date_add(F.lit(START_DATE.date()), (F.col("id") / READINGS_PER_DAY).cast("int")).alias("timestamp"), + generate_sensor_reading(F.col("equipment_age")).alias("value"), + ) +) +``` + +--- + +## Financial Services + +### Tables +``` +accounts → transactions → payments → fraud_flags +``` + +### Key Patterns + +| Pattern | Implementation | +|---------|----------------| +| Transaction power law | 80% of volume from 20% of accounts | +| Fraud patterns | Unusual times, amounts, locations | +| Balance consistency | Transactions maintain positive balance | +| Regulatory compliance | No negative balances, valid amounts | + +### Realistic Distributions + +```python +@F.pandas_udf(DoubleType()) +def generate_transaction_amount(account_types: pd.Series) -> pd.Series: + """Transaction amounts following power law by account type.""" + amounts = [] + for acct_type in account_types: + if acct_type == "Corporate": + # Power law for corporate (few large transactions) + amount = (np.random.pareto(a=1.5) + 1) * 1000 + elif acct_type == "Premium": + amount = np.random.lognormal(mean=6, sigma=1.2) + else: # Standard + amount = np.random.lognormal(mean=4, sigma=0.8) + amounts.append(min(amount, 1_000_000)) # Cap at $1M + return pd.Series(amounts) + +@F.pandas_udf(BooleanType()) +def generate_fraud_flag(amounts: pd.Series, hours: pd.Series) -> pd.Series: + """Flag suspicious transactions based on amount and time.""" + flags = [] + for amount, hour in zip(amounts, hours): + # Higher fraud probability for: large amounts + unusual hours + base_prob = 0.001 + if amount > 5000: + base_prob *= 3 + if hour < 6 or hour > 22: + base_prob *= 2 + flags.append(np.random.random() < base_prob) + return pd.Series(flags) +``` + +### Schema Example + +```python +# Transactions with fraud indicators +transactions_df = ( + spark.range(0, N_TRANSACTIONS, numPartitions=PARTITIONS) + .select( + F.concat(F.lit("TXN-"), F.lpad(F.col("id").cast("string"), 10, "0")).alias("transaction_id"), + # FK to account + ... + generate_transaction_amount(F.col("account_type")).alias("amount"), + F.hour(F.col("timestamp")).alias("hour"), + ) + .withColumn("is_suspicious", generate_fraud_flag(F.col("amount"), F.col("hour"))) +) +``` + +--- + +## General Best Practices + +1. **Start with domain tables**: Define the core entities and relationships first +2. **Add domain-specific distributions**: Use realistic statistical patterns for your domain +3. **Include edge cases**: Every domain has edge cases (returns, cancellations, failures) +4. **Time-based patterns matter**: Most domains have daily/weekly/seasonal patterns +5. **Correlate attributes**: Attributes within a row should make business sense together + +**Note:** These are guidance patterns, not rigid schemas. Adapt to user's specific requirements. diff --git a/databricks-skills/synthetic-data-generation/references/5-output-formats.md b/databricks-skills/synthetic-data-generation/references/5-output-formats.md new file mode 100644 index 00000000..e018efcf --- /dev/null +++ b/databricks-skills/synthetic-data-generation/references/5-output-formats.md @@ -0,0 +1,188 @@ +# Output Formats Guide + +Where and how to save generated synthetic data. + +## Storage Destination + +### Ask for Catalog and Schema + +By default, use the `ai_dev_kit` catalog. Ask the user which schema to use: + +> "I'll save the data to `ai_dev_kit.`. What schema name would you like to use? (You can also specify a different catalog if needed.)" + +If the user provides just a schema name, use `ai_dev_kit.{schema}`. If they provide `catalog.schema`, use that instead. + +### Create Infrastructure in Script + +Always create the schema and volume **inside the Python script** using `spark.sql()`. Do NOT make separate MCP SQL calls - it's much slower. + +```python +CATALOG = "ai_dev_kit" +SCHEMA = "synthetic_data" +VOLUME_PATH = f"/Volumes/{CATALOG}/{SCHEMA}/raw_data" + +# Note: Assume catalog exists - do NOT create it +spark.sql(f"CREATE SCHEMA IF NOT EXISTS {CATALOG}.{SCHEMA}") +spark.sql(f"CREATE VOLUME IF NOT EXISTS {CATALOG}.{SCHEMA}.raw_data") +``` + +**Important:** Do NOT create catalogs - assume they already exist. Only create schema and volume. + +--- + +## Format Comparison + +| Format | Use Case | Extension | Best For | +|--------|----------|-----------|----------| +| **Parquet** | Default - SDP pipeline input | `.parquet` or none | Best compression, query performance | +| **JSON** | Log-style ingestion | `.json` | Simulating external data feeds | +| **CSV** | Legacy systems | `.csv` | Human-readable, spreadsheet import | +| **Delta Table** | Direct analytics | N/A | Skip SDP, query immediately | + +--- + +## Parquet to Volumes (Default) + +Standard format for SDP pipeline input. Best compression and query performance. + +```python +VOLUME_PATH = f"/Volumes/{CATALOG}/{SCHEMA}/raw_data" + +# Save as parquet files (directory format) +customers_df.write.mode("overwrite").parquet(f"{VOLUME_PATH}/customers") +orders_df.write.mode("overwrite").parquet(f"{VOLUME_PATH}/orders") +tickets_df.write.mode("overwrite").parquet(f"{VOLUME_PATH}/tickets") +``` + +**Notes:** +- Files may not use a file extension or might end with `.parquet` +- Spark writes as a directory with part files +- Use `mode("overwrite")` for one-time generation +- Use `mode("append")` for incremental/scheduled jobs + +--- + +## JSON to Volumes + +Common pattern for simulating SDP ingestion from external data feeds (logs, webhooks). + +```python +VOLUME_PATH = f"/Volumes/{CATALOG}/{SCHEMA}/raw_data" + +# Save as JSON files +customers_df.write.mode("overwrite").json(f"{VOLUME_PATH}/customers_json") +orders_df.write.mode("overwrite").json(f"{VOLUME_PATH}/orders_json") +``` + +**When to use:** +- Simulating log ingestion +- External API data feeds +- User explicitly requests JSON format + +--- + +## CSV to Volumes + +Common pattern for simulating data from legacy systems or spreadsheet exports. + +```python +VOLUME_PATH = f"/Volumes/{CATALOG}/{SCHEMA}/raw_data" + +# Save as CSV with headers +customers_df.write.mode("overwrite").option("header", "true").csv(f"{VOLUME_PATH}/customers_csv") +orders_df.write.mode("overwrite").option("header", "true").csv(f"{VOLUME_PATH}/orders_csv") +``` + +**Options:** +```python +# Full options for CSV +df.write \ + .mode("overwrite") \ + .option("header", "true") \ + .option("delimiter", ",") \ + .option("quote", '"') \ + .option("escape", "\\") \ + .csv(f"{VOLUME_PATH}/data_csv") +``` + +**When to use:** +- Legacy system integration +- Human-readable data +- Spreadsheet import testing + +--- + +## Delta Table (Unity Catalog) + +Write directly to managed Delta tables when data is ready for analytics consumption (skip SDP pipeline). + +```python +# Ensure schema exists +spark.sql(f"CREATE SCHEMA IF NOT EXISTS {CATALOG}.{SCHEMA}") + +# Save as managed Delta tables +customers_df.write.mode("overwrite").saveAsTable(f"{CATALOG}.{SCHEMA}.customers") +orders_df.write.mode("overwrite").saveAsTable(f"{CATALOG}.{SCHEMA}.orders") + +# With additional options +customers_df.write \ + .mode("overwrite") \ + .option("overwriteSchema", "true") \ + .saveAsTable(f"{CATALOG}.{SCHEMA}.customers") +``` + +**When to use:** +- User wants data ready to query immediately +- Skip the SDP bronze/silver/gold pipeline +- Direct notebook or SQL analytics + +--- + +## Write Modes + +| Mode | Behavior | Use Case | +|------|----------|----------| +| `overwrite` | Replace existing data | One-time generation, regeneration | +| `append` | Add to existing data | Incremental/scheduled jobs | +| `ignore` | Skip if exists | Idempotent generation | +| `error` | Fail if exists | Safety check | + +### Incremental Generation Pattern + +```python +WRITE_MODE = "append" # For scheduled jobs + +# Only generate new records since last run +from datetime import datetime, timedelta + +LAST_RUN = datetime.now() - timedelta(days=1) +END_DATE = datetime.now() + +# Generate only new data +new_orders_df = generate_orders(start_date=LAST_RUN, end_date=END_DATE) +new_orders_df.write.mode(WRITE_MODE).parquet(f"{VOLUME_PATH}/orders") +``` + +--- + +## Validation After Write + +After successful execution, validate the generated data: + +```python +# Read back and verify +customers_check = spark.read.parquet(f"{VOLUME_PATH}/customers") +orders_check = spark.read.parquet(f"{VOLUME_PATH}/orders") + +print(f"Customers: {customers_check.count():,} rows") +print(f"Orders: {orders_check.count():,} rows") + +# Verify distributions +customers_check.groupBy("tier").count().show() +orders_check.describe("amount").show() +``` + +Or use `get_volume_folder_details` MCP tool: +- `volume_path`: "my_catalog/my_schema/raw_data/customers" +- `format`: "parquet" +- `table_stat_level`: "SIMPLE" diff --git a/databricks-skills/synthetic-data-generation/references/6-troubleshooting.md b/databricks-skills/synthetic-data-generation/references/6-troubleshooting.md new file mode 100644 index 00000000..f0052878 --- /dev/null +++ b/databricks-skills/synthetic-data-generation/references/6-troubleshooting.md @@ -0,0 +1,261 @@ +# Troubleshooting Guide + +Common issues and solutions for synthetic data generation. + +## Environment Issues + +### ModuleNotFoundError: faker (or other library) + +**Problem:** Dependencies not available in execution environment. + +**Solutions by execution mode:** + +| Mode | Solution | +|------|----------| +| **DB Connect 16.4+** | Use `DatabricksEnv().withDependencies("faker", "pandas", ...)` | +| **Older DB Connect** | Create job with `environments` parameter | +| **Databricks Runtime** | Run `%pip install faker numpy pandas holidays` | +| **Classic cluster** | Add to cluster libraries or init script | + +```python +# For DB Connect 16.4+ +from databricks.connect import DatabricksSession, DatabricksEnv + +env = DatabricksEnv().withDependencies("faker", "pandas", "numpy", "holidays") +spark = DatabricksSession.builder.withEnvironment(env).serverless(True).getOrCreate() +``` + +### DatabricksEnv not found + +**Problem:** Using older databricks-connect version. + +**Solution:** Upgrade to 16.4+ or use job-based approach: + +```bash +# Upgrade +pip install "databricks-connect>=16.4,<17.0" + +# Or use job with environments parameter instead +``` + +### serverless_compute_id error + +**Problem:** Missing serverless configuration. + +**Solution:** Add to `~/.databrickscfg`: + +```ini +[DEFAULT] +host = https://your-workspace.cloud.databricks.com/ +serverless_compute_id = auto +auth_type = databricks-cli +``` + +--- + +## Execution Issues + +### Serverless job fails to start + +**Possible causes:** +1. Workspace doesn't have serverless enabled +2. Unity Catalog permissions missing +3. Invalid environment configuration + +**Solutions:** +```python +# Verify serverless is available +# Try creating a simple job first to test + +# Check Unity Catalog permissions +spark.sql("SELECT current_catalog(), current_schema()") +``` + +### Classic cluster startup slow (3-8 minutes) + +**Problem:** Clusters take time to start. + +**Solution:** Switch to serverless: + +```python +# Instead of: +# spark = DatabricksSession.builder.clusterId("xxx").getOrCreate() + +# Use: +spark = DatabricksSession.builder.serverless(True).getOrCreate() +``` + +### "Either base environment or version must be provided" + +**Problem:** Missing `client` in job environment spec. + +**Solution:** Add `"client": "4"` to the spec: + +```python +{ + "environments": [{ + "environment_key": "datagen_env", + "spec": { + "client": "4", # Required! + "dependencies": ["faker", "numpy", "pandas"] + } + }] +} +``` + +--- + +## Data Generation Issues + +### Faker UDF is slow + +**Problem:** Single-row UDFs don't parallelize well. + +**Solution:** Use `pandas_udf` for batch processing: + +```python +# SLOW - scalar UDF +@F.udf(returnType=StringType()) +def slow_fake_name(): + return Faker().name() + +# FAST - pandas UDF (batch processing) +@F.pandas_udf(StringType()) +def fast_fake_name(ids: pd.Series) -> pd.Series: + fake = Faker() + return pd.Series([fake.name() for _ in range(len(ids))]) +``` + +### Out of memory with large data + +**Problem:** Not enough partitions for data size. + +**Solution:** Increase partitions: + +```python +# For large datasets (1M+ rows) +customers_df = spark.range(0, N_CUSTOMERS, numPartitions=64) # Increase from default +``` + +| Data Size | Recommended Partitions | +|-----------|----------------------| +| < 100K | 8 | +| 100K - 500K | 16 | +| 500K - 1M | 32 | +| 1M+ | 64+ | + +### Context corrupted on classic cluster + +**Problem:** Stale execution context. + +**Solution:** Create fresh context (omit context_id), reinstall libraries: + +```python +# Don't reuse context_id if you see strange errors +# Let it create a new context +``` + +### Referential integrity violations + +**Problem:** Foreign keys reference non-existent parent records. + +**Solution:** Generate master tables first, cache, then join: + +```python +# 1. Generate and cache master table +customers_df = spark.range(0, N_CUSTOMERS)... +customer_lookup = customers_df.select("customer_id").cache() + +# 2. Generate child table with valid FKs +orders_df = spark.range(0, N_ORDERS).join( + customer_lookup, + on=, + how="left" +) + +# 3. Clean up +customer_lookup.unpersist() +``` + +--- + +## Data Quality Issues + +### Uniform distributions (unrealistic) + +**Problem:** All customers have similar order counts, amounts are evenly distributed. + +**Solution:** Use non-linear distributions: + +```python +# BAD - uniform +amounts = np.random.uniform(10, 1000, N) + +# GOOD - log-normal (realistic) +amounts = np.random.lognormal(mean=5, sigma=0.8, N) +``` + +### Missing time-based patterns + +**Problem:** Data doesn't reflect weekday/weekend or seasonal patterns. + +**Solution:** Add multipliers: + +```python +import holidays + +US_HOLIDAYS = holidays.US(years=[2024, 2025]) + +def get_multiplier(date): + mult = 1.0 + if date.weekday() >= 5: # Weekend + mult *= 0.6 + if date in US_HOLIDAYS: + mult *= 0.3 + return mult +``` + +### Incoherent row attributes + +**Problem:** Enterprise customer has low-value orders, critical ticket has slow resolution. + +**Solution:** Correlate attributes: + +```python +# Priority based on tier +if tier == 'Enterprise': + priority = np.random.choice(['Critical', 'High'], p=[0.4, 0.6]) +else: + priority = np.random.choice(['Medium', 'Low'], p=[0.6, 0.4]) + +# Resolution based on priority +resolution_scale = {'Critical': 4, 'High': 12, 'Medium': 36, 'Low': 72} +resolution_hours = np.random.exponential(scale=resolution_scale[priority]) +``` + +--- + +## Validation Steps + +After generation, verify your data: + +```python +# 1. Check row counts +print(f"Customers: {customers_df.count():,}") +print(f"Orders: {orders_df.count():,}") + +# 2. Verify distributions +customers_df.groupBy("tier").count().show() +orders_df.describe("amount").show() + +# 3. Check referential integrity +orphans = orders_df.join( + customers_df, + orders_df.customer_id == customers_df.customer_id, + "left_anti" +) +print(f"Orphan orders: {orphans.count()}") + +# 4. Verify date range +orders_df.select(F.min("order_date"), F.max("order_date")).show() +``` diff --git a/databricks-skills/synthetic-data-generation/scripts/example_faker_udf.py b/databricks-skills/synthetic-data-generation/scripts/example_faker_udf.py deleted file mode 100644 index 9deefb54..00000000 --- a/databricks-skills/synthetic-data-generation/scripts/example_faker_udf.py +++ /dev/null @@ -1,272 +0,0 @@ -"""Generate synthetic data using Faker with Spark UDFs for parallelism. - -This approach is best for: -- Large datasets (100K+ rows) that need Spark parallelism -- Generating realistic text data with Faker providers -- Writing directly to Unity Catalog volumes -- Complex conditional logic in data generation - -This script automatically detects the environment and uses: -- DatabricksEnv with auto-dependencies if databricks-connect >= 16.4 and running locally -- Standard session creation if running on Databricks Runtime or older databricks-connect -""" -import sys -import os -from pyspark.sql import functions as F -from pyspark.sql.window import Window -from pyspark.sql.types import StringType, DoubleType -import numpy as np -from datetime import datetime, timedelta - -# ============================================================================= -# CONFIGURATION -# ============================================================================= -# Compute - Serverless recommended -USE_SERVERLESS = True # Set to False and provide CLUSTER_ID for classic compute -CLUSTER_ID = None # Only used if USE_SERVERLESS=False - -# Storage -CATALOG = "ai_dev_kit" # Change to your catalog -SCHEMA = "synthetic_data" # Change to your schema -VOLUME_PATH = f"/Volumes/{CATALOG}/{SCHEMA}/raw_data" - -# Data sizes - this example is designed for larger datasets -N_CUSTOMERS = 100_000 -N_ORDERS = 500_000 -PARTITIONS = 16 # Adjust based on data size - -# Date range - last 6 months from today -END_DATE = datetime.now().replace(hour=0, minute=0, second=0, microsecond=0) -START_DATE = END_DATE - timedelta(days=180) - -# Reproducibility -SEED = 42 - -# ============================================================================= -# SETUP - Environment Detection and Session Creation -# ============================================================================= - -# Detect if running on Databricks Runtime vs locally with Databricks Connect -def is_databricks_runtime(): - """Check if running on Databricks Runtime (notebook/job) vs locally.""" - return "DATABRICKS_RUNTIME_VERSION" in os.environ - -# Get databricks-connect version if available -def get_databricks_connect_version(): - """Get databricks-connect version as (major, minor) tuple or None.""" - try: - import databricks.connect - version_str = databricks.connect.__version__ - parts = version_str.split('.') - return (int(parts[0]), int(parts[1])) - except (ImportError, AttributeError, ValueError, IndexError): - return None - -# Determine session creation strategy -on_runtime = is_databricks_runtime() -db_version = get_databricks_connect_version() - -print("=" * 80) -print("ENVIRONMENT DETECTION") -print("=" * 80) -print(f"Running on Databricks Runtime: {on_runtime}") -if db_version: - print(f"databricks-connect version: {db_version[0]}.{db_version[1]}") -else: - print("databricks-connect: not available") - -# Use DatabricksEnv with auto-dependencies if: -# - Running locally (not on Databricks Runtime) -# - databricks-connect >= 16.4 -use_auto_dependencies = (not on_runtime) and db_version and db_version >= (16, 4) - -if use_auto_dependencies: - print("✓ Using DatabricksEnv with managed dependencies") - print("=" * 80) - from databricks.connect import DatabricksSession, DatabricksEnv - - # Pass dependencies as simple package name strings - env = DatabricksEnv().withDependencies("faker", "pandas", "numpy", "holidays") - - if USE_SERVERLESS: - spark = DatabricksSession.builder.withEnvironment(env).serverless(True).getOrCreate() - print("✓ Connected to serverless compute with managed dependencies!") - else: - if not CLUSTER_ID: - raise ValueError("CLUSTER_ID must be set when USE_SERVERLESS=False") - spark = DatabricksSession.builder.withEnvironment(env).clusterId(CLUSTER_ID).getOrCreate() - print(f"✓ Connected to cluster {CLUSTER_ID} with managed dependencies!") -else: - print("⚠ Using standard session (dependencies must be pre-installed)") - print("=" * 80) - - # Try to import libraries that will be used in UDFs - print("\nChecking UDF dependencies...") - missing_deps = [] - - try: - from faker import Faker - print(" ✓ faker") - except ImportError: - missing_deps.append("faker") - print(" ✗ faker - NOT INSTALLED") - - try: - import pandas as pd - print(" ✓ pandas") - except ImportError: - missing_deps.append("pandas") - print(" ✗ pandas - NOT INSTALLED") - - if missing_deps: - print("\n" + "=" * 80) - print("⚠ WARNING: Missing dependencies for UDFs") - print("=" * 80) - print(f"Missing libraries: {', '.join(missing_deps)}") - print("\nThese libraries are required in UDFs and must be installed:") - - if on_runtime: - print("\n→ SOLUTION: Install on the cluster or job:") - print(" - For interactive cluster: Run %pip install faker pandas numpy holidays") - print(" - For job: Add to job libraries or use init script") - else: - print("\n→ SOLUTION: Use one of these approaches:") - print(" 1. Upgrade databricks-connect to >= 16.4 (enables auto-dependencies)") - print(" 2. Create a job with environment settings in the task definition") - print(" 3. Use a classic cluster with libraries pre-installed") - - print("=" * 80) - sys.exit(1) - - print("\n✓ All UDF dependencies available") - print("=" * 80) - - # Create standard session - from databricks.connect import DatabricksSession - - if USE_SERVERLESS: - spark = DatabricksSession.builder.serverless(True).getOrCreate() - print("✓ Connected to serverless compute") - else: - if not CLUSTER_ID: - raise ValueError("CLUSTER_ID must be set when USE_SERVERLESS=False") - spark = DatabricksSession.builder.clusterId(CLUSTER_ID).getOrCreate() - print(f"✓ Connected to cluster {CLUSTER_ID}") - -# Import Faker for UDF definitions (already checked above) -from faker import Faker - -# ============================================================================= -# DEFINE FAKER UDFs -# ============================================================================= -@F.udf(returnType=StringType()) -def generate_company(): - """Generate realistic company name.""" - return Faker().company() - -@F.udf(returnType=StringType()) -def generate_address(): - """Generate realistic address.""" - return Faker().address().replace('\n', ', ') - -@F.udf(returnType=StringType()) -def generate_email(company_name): - """Generate email based on company name.""" - if company_name: - domain = company_name.lower().replace(" ", "").replace(",", "")[:15] - return f"contact@{domain}.com" - return "unknown@example.com" - -@F.udf(returnType=DoubleType()) -def generate_lognormal_amount(tier): - """Generate amount based on tier using log-normal distribution.""" - np.random.seed(hash(str(tier)) % (2**32)) - if tier == "Enterprise": - return float(np.random.lognormal(mean=9, sigma=0.8)) - elif tier == "Pro": - return float(np.random.lognormal(mean=7, sigma=0.7)) - else: - return float(np.random.lognormal(mean=5, sigma=0.6)) - -# ============================================================================= -# CREATE INFRASTRUCTURE -# ============================================================================= -print("Creating infrastructure...") -spark.sql(f"CREATE SCHEMA IF NOT EXISTS {CATALOG}.{SCHEMA}") -spark.sql(f"CREATE VOLUME IF NOT EXISTS {CATALOG}.{SCHEMA}.raw_data") -print(f"Infrastructure ready: {VOLUME_PATH}") - -# ============================================================================= -# GENERATE CUSTOMERS -# ============================================================================= -print(f"Generating {N_CUSTOMERS:,} customers...") - -customers_df = ( - spark.range(0, N_CUSTOMERS, numPartitions=PARTITIONS) - .select( - F.concat(F.lit("CUST-"), F.lpad(F.col("id").cast("string"), 5, "0")).alias("customer_id"), - generate_company().alias("name"), - generate_address().alias("address"), - F.when(F.rand(SEED) < 0.6, "Free") - .when(F.rand(SEED) < 0.9, "Pro") - .otherwise("Enterprise").alias("tier"), - F.when(F.rand(SEED) < 0.4, "North") - .when(F.rand(SEED) < 0.65, "South") - .when(F.rand(SEED) < 0.85, "East") - .otherwise("West").alias("region") - ) -) - -# Add tier-based ARR and email -customers_df = ( - customers_df - .withColumn("arr", generate_lognormal_amount(F.col("tier"))) - .withColumn("email", generate_email(F.col("name"))) -) - -customers_df.write.mode("overwrite").parquet(f"{VOLUME_PATH}/customers") -print(f" Saved customers to {VOLUME_PATH}/customers") - -# ============================================================================= -# GENERATE ORDERS -# ============================================================================= -print(f"Generating {N_ORDERS:,} orders...") - -# Get customer IDs for foreign key -customer_lookup = customers_df.select("customer_id", "tier").cache() - -orders_df = ( - spark.range(0, N_ORDERS, numPartitions=PARTITIONS) - .select( - F.concat(F.lit("ORD-"), F.lpad(F.col("id").cast("string"), 6, "0")).alias("order_id"), - # Generate customer_idx for FK join (random selection from customer range) - (F.abs(F.hash(F.col("id"), F.lit(SEED))) % N_CUSTOMERS).alias("customer_idx"), - F.when(F.rand(SEED) < 0.85, "completed") - .when(F.rand(SEED) < 0.95, "pending") - .otherwise("cancelled").alias("status"), - F.date_add(F.lit(START_DATE.date()), - (F.rand(SEED) * 180).cast("int")).alias("order_date") - ) -) - -# Add customer_idx to lookup for join -customer_lookup_with_idx = customer_lookup.withColumn( - "customer_idx", - (F.row_number().over(Window.orderBy(F.monotonically_increasing_id())) - 1).cast("int") -) - -# Join to get customer_id and tier as foreign key -orders_with_fk = ( - orders_df - .join(customer_lookup_with_idx, on="customer_idx", how="left") - .drop("customer_idx") -) - -# Add tier-based amount -orders_with_fk = orders_with_fk.withColumn("amount", generate_lognormal_amount(F.col("tier"))) - -orders_with_fk.drop("tier").write.mode("overwrite").parquet(f"{VOLUME_PATH}/orders") -print(f" Saved orders to {VOLUME_PATH}/orders") - -customer_lookup.unpersist() -print("Done!") diff --git a/databricks-skills/synthetic-data-generation/scripts/example_polars.py b/databricks-skills/synthetic-data-generation/scripts/example_polars.py deleted file mode 100644 index 8fb1fa8d..00000000 --- a/databricks-skills/synthetic-data-generation/scripts/example_polars.py +++ /dev/null @@ -1,157 +0,0 @@ -"""Generate synthetic data with Polars (local, no Spark dependency). - -This approach is best for: -- Quick prototyping and testing -- Datasets under 100K rows -- Local development without Databricks connection -- Generating parquet files to upload to volumes later -""" -import polars as pl -from faker import Faker -from datetime import datetime, timedelta -import numpy as np -import os - -# ============================================================================= -# CONFIGURATION -# ============================================================================= -# Output -OUTPUT_PATH = "./output" # Local directory for parquet files - -# Data sizes -N_CUSTOMERS = 5000 -N_ORDERS = 15000 - -# Date range - last 6 months from today -END_DATE = datetime.now().replace(hour=0, minute=0, second=0, microsecond=0) -START_DATE = END_DATE - timedelta(days=180) - -# Reproducibility -SEED = 42 - -# Tier distribution: Free 60%, Pro 30%, Enterprise 10% -TIER_VALUES = ["Free", "Pro", "Enterprise"] -TIER_WEIGHTS = [0.6, 0.3, 0.1] - -# Region distribution -REGION_VALUES = ["North", "South", "East", "West"] -REGION_WEIGHTS = [0.4, 0.25, 0.2, 0.15] - -# Order status distribution -STATUS_VALUES = ["pending", "processing", "shipped", "delivered", "cancelled"] -STATUS_WEIGHTS = [0.05, 0.10, 0.15, 0.65, 0.05] - -# Weighted order generation by tier (Enterprise generates more orders) -TIER_ORDER_WEIGHTS = {"Enterprise": 5.0, "Pro": 2.0, "Free": 1.0} - -# Log-normal parameters for order amounts by tier -TIER_AMOUNT_PARAMS = { - "Enterprise": {"mean": 7.5, "sigma": 0.8}, - "Pro": {"mean": 5.5, "sigma": 0.7}, - "Free": {"mean": 4.0, "sigma": 0.6}, -} - -# ============================================================================= -# SETUP -# ============================================================================= -np.random.seed(SEED) -Faker.seed(SEED) -fake = Faker() - -# Create output directory -os.makedirs(OUTPUT_PATH, exist_ok=True) - -# ============================================================================= -# GENERATE CUSTOMERS TABLE -# ============================================================================= -print(f"Generating {N_CUSTOMERS:,} customers...") - -customers = pl.DataFrame({ - "customer_id": [f"CUST-{i:05d}" for i in range(N_CUSTOMERS)], - "name": [fake.name() for _ in range(N_CUSTOMERS)], - "email": [fake.email() for _ in range(N_CUSTOMERS)], - "tier": np.random.choice(TIER_VALUES, N_CUSTOMERS, p=TIER_WEIGHTS).tolist(), - "region": np.random.choice(REGION_VALUES, N_CUSTOMERS, p=REGION_WEIGHTS).tolist(), - "created_at": [fake.date_between(start_date='-2y', end_date=START_DATE) for _ in range(N_CUSTOMERS)], -}) - -# Show tier distribution -print("Tier distribution:") -tier_counts = customers.group_by("tier").len().sort("tier") -for row in tier_counts.iter_rows(named=True): - pct = row["len"] / N_CUSTOMERS * 100 - print(f" {row['tier']}: {row['len']:,} ({pct:.1f}%)") - -# ============================================================================= -# GENERATE ORDERS TABLE WITH REFERENTIAL INTEGRITY -# ============================================================================= -print(f"\nGenerating {N_ORDERS:,} orders with weighted sampling by tier...") - -# Create lookups for foreign key generation -customer_ids = customers["customer_id"].to_list() -customer_tier_map = dict(zip(customers["customer_id"], customers["tier"])) - -# Weight by tier - Enterprise customers generate more orders -tier_weights_list = [TIER_ORDER_WEIGHTS[t] for t in customers["tier"].to_list()] -total_weight = sum(tier_weights_list) -customer_weights = [w / total_weight for w in tier_weights_list] - -# Generate orders with weighted sampling -orders_data = { - "order_id": [], - "customer_id": [], - "amount": [], - "order_date": [], - "status": [], -} - -for i in range(N_ORDERS): - cid = np.random.choice(customer_ids, p=customer_weights) - tier = customer_tier_map[cid] - - # Amount based on tier using log-normal distribution - params = TIER_AMOUNT_PARAMS[tier] - amount = np.random.lognormal(mean=params["mean"], sigma=params["sigma"]) - - orders_data["order_id"].append(f"ORD-{i:06d}") - orders_data["customer_id"].append(cid) - orders_data["amount"].append(round(amount, 2)) - orders_data["order_date"].append(fake.date_between(start_date=START_DATE, end_date=END_DATE)) - orders_data["status"].append(np.random.choice(STATUS_VALUES, p=STATUS_WEIGHTS)) - -orders = pl.DataFrame(orders_data) - -# Show order distribution by customer tier -orders_with_tier = orders.join( - customers.select(["customer_id", "tier"]), - on="customer_id" -) -orders_by_tier = orders_with_tier.group_by("tier").len().sort("tier") -print("\nOrders by customer tier:") -for row in orders_by_tier.iter_rows(named=True): - pct = row["len"] / N_ORDERS * 100 - print(f" {row['tier']}: {row['len']:,} ({pct:.1f}%)") - -# Show amount statistics by tier -print("\nAmount statistics by tier:") -for tier in TIER_VALUES: - tier_amounts = orders_with_tier.filter(pl.col("tier") == tier)["amount"] - if len(tier_amounts) > 0: - print(f" {tier}: avg=${tier_amounts.mean():,.2f}, median=${tier_amounts.median():,.2f}, " - f"min=${tier_amounts.min():,.2f}, max=${tier_amounts.max():,.2f}") - -# ============================================================================= -# SAVE TO PARQUET (LOCAL) -# ============================================================================= -print(f"\nSaving to Parquet files in {OUTPUT_PATH}...") - -customers.write_parquet(f"{OUTPUT_PATH}/customers.parquet") -print(f" Saved: {OUTPUT_PATH}/customers.parquet ({N_CUSTOMERS:,} rows)") - -orders.write_parquet(f"{OUTPUT_PATH}/orders.parquet") -print(f" Saved: {OUTPUT_PATH}/orders.parquet ({N_ORDERS:,} rows)") - -print(f"\nDone! Data saved locally to {OUTPUT_PATH}") -print(f" - customers.parquet: {N_CUSTOMERS:,} rows") -print(f" - orders.parquet: {N_ORDERS:,} rows") -print("\nTo upload to Databricks, use: databricks fs cp -r ./output dbfs:/Volumes/...") diff --git a/databricks-skills/synthetic-data-generation/scripts/generate_ecommerce_data.py b/databricks-skills/synthetic-data-generation/scripts/generate_ecommerce_data.py deleted file mode 100644 index cb3223c0..00000000 --- a/databricks-skills/synthetic-data-generation/scripts/generate_ecommerce_data.py +++ /dev/null @@ -1,225 +0,0 @@ -"""Generate synthetic e-commerce data with customers and orders tables.""" -import numpy as np -import pandas as pd -from datetime import datetime, timedelta -from faker import Faker -from databricks.connect import DatabricksSession - -# ============================================================================= -# CONFIGURATION -# ============================================================================= -# Compute - Serverless recommended -USE_SERVERLESS = True # Set to False and provide CLUSTER_ID for classic compute -CLUSTER_ID = None # Only used if USE_SERVERLESS=False - -# Storage -CATALOG = "ai_dev_kit" # Change to your catalog -SCHEMA = "synthetic_data" # Change to your schema -VOLUME_PATH = f"/Volumes/{CATALOG}/{SCHEMA}/raw_data" - -# Data sizes -N_CUSTOMERS = 5000 -N_ORDERS = 15000 - -# Date range - last 6 months from today -END_DATE = datetime.now().replace(hour=0, minute=0, second=0, microsecond=0) -START_DATE = END_DATE - timedelta(days=180) - -# Write mode - "overwrite" for one-time, "append" for incremental/scheduled jobs -WRITE_MODE = "overwrite" - -# Bad data injection for testing data quality rules -INJECT_BAD_DATA = False # Set to True to inject bad data -BAD_DATA_CONFIG = { - "null_rate": 0.02, # 2% nulls in required fields - "outlier_rate": 0.01, # 1% impossible values - "duplicate_pk_rate": 0.005, # 0.5% duplicate primary keys - "orphan_fk_rate": 0.01, # 1% orphan foreign keys -} - -# Reproducibility -SEED = 42 - -# Tier distribution: Free 60%, Pro 30%, Enterprise 10% -TIER_VALUES = ["Free", "Pro", "Enterprise"] -TIER_WEIGHTS = [0.6, 0.3, 0.1] - -# Region distribution -REGION_VALUES = ["North", "South", "East", "West"] -REGION_WEIGHTS = [0.4, 0.25, 0.2, 0.15] - -# Order status distribution -STATUS_VALUES = ["pending", "processing", "shipped", "delivered", "cancelled"] -STATUS_WEIGHTS = [0.05, 0.10, 0.15, 0.65, 0.05] - -# Weighted order generation by tier (Enterprise generates more orders) -TIER_ORDER_WEIGHTS = {"Enterprise": 5.0, "Pro": 2.0, "Free": 1.0} - -# Log-normal parameters for order amounts by tier -TIER_AMOUNT_PARAMS = { - "Enterprise": {"mean": 7.5, "sigma": 0.8}, # ~$1800 avg, range $500-$8000+ - "Pro": {"mean": 5.5, "sigma": 0.7}, # ~$245 avg, range $50-$1000 - "Free": {"mean": 4.0, "sigma": 0.6}, # ~$55 avg, range $15-$200 -} - -# ============================================================================= -# SETUP -# ============================================================================= -np.random.seed(SEED) -Faker.seed(SEED) -fake = Faker() - -print("Connecting to Databricks...") - -# NOTE: This script uses Faker locally with Pandas (not in Spark UDFs), so it -# does NOT require DatabricksEnv or auto-dependencies. It works with all versions: -# - Python 3.10, 3.11, 3.12+ -# - databricks-connect 15.1+ (any version) -# -# If you need to use Faker in Spark UDFs, see example_faker_udf.py instead. - -if USE_SERVERLESS: - spark = DatabricksSession.builder.serverless(True).getOrCreate() - print("Connected to serverless compute!") -else: - if not CLUSTER_ID: - raise ValueError("CLUSTER_ID must be set when USE_SERVERLESS=False") - spark = DatabricksSession.builder.clusterId(CLUSTER_ID).getOrCreate() - print(f"Connected to cluster {CLUSTER_ID}!") - -# ============================================================================= -# CREATE INFRASTRUCTURE -# ============================================================================= -print(f"\nCreating schema and volume...") -spark.sql(f"CREATE SCHEMA IF NOT EXISTS {CATALOG}.{SCHEMA}") -spark.sql(f"CREATE VOLUME IF NOT EXISTS {CATALOG}.{SCHEMA}.raw_data") -print(f"Infrastructure ready: {VOLUME_PATH}") - -# ============================================================================= -# GENERATE CUSTOMERS TABLE -# ============================================================================= -print(f"\nGenerating {N_CUSTOMERS:,} customers...") - -customers_pdf = pd.DataFrame({ - "customer_id": [f"CUST-{i:05d}" for i in range(N_CUSTOMERS)], - "name": [fake.name() for _ in range(N_CUSTOMERS)], - "email": [fake.email() for _ in range(N_CUSTOMERS)], - "tier": np.random.choice(TIER_VALUES, N_CUSTOMERS, p=TIER_WEIGHTS), - "region": np.random.choice(REGION_VALUES, N_CUSTOMERS, p=REGION_WEIGHTS), - "created_at": [fake.date_between(start_date='-2y', end_date=START_DATE) for _ in range(N_CUSTOMERS)], -}) - -# Show tier distribution -tier_counts = customers_pdf["tier"].value_counts() -print(f"Tier distribution:") -for tier in TIER_VALUES: - count = tier_counts.get(tier, 0) - pct = count / N_CUSTOMERS * 100 - print(f" {tier}: {count:,} ({pct:.1f}%)") - -# ============================================================================= -# GENERATE ORDERS TABLE WITH REFERENTIAL INTEGRITY -# ============================================================================= -print(f"\nGenerating {N_ORDERS:,} orders with weighted sampling by tier...") - -# Create lookups for foreign key generation -customer_ids = customers_pdf["customer_id"].tolist() -customer_tier_map = dict(zip(customers_pdf["customer_id"], customers_pdf["tier"])) - -# Weight by tier - Enterprise customers generate more orders -tier_weights_series = customers_pdf["tier"].map(TIER_ORDER_WEIGHTS) -customer_weights = (tier_weights_series / tier_weights_series.sum()).tolist() - -# Generate orders with weighted sampling -orders_data = [] -for i in range(N_ORDERS): - cid = np.random.choice(customer_ids, p=customer_weights) - tier = customer_tier_map[cid] - - # Amount based on tier using log-normal distribution - params = TIER_AMOUNT_PARAMS[tier] - amount = np.random.lognormal(mean=params["mean"], sigma=params["sigma"]) - - orders_data.append({ - "order_id": f"ORD-{i:06d}", - "customer_id": cid, - "amount": round(amount, 2), - "order_date": fake.date_between(start_date=START_DATE, end_date=END_DATE), - "status": np.random.choice(STATUS_VALUES, p=STATUS_WEIGHTS), - }) - -orders_pdf = pd.DataFrame(orders_data) - -# ============================================================================= -# INJECT BAD DATA (OPTIONAL) -# ============================================================================= -if INJECT_BAD_DATA: - print(f"\nInjecting bad data for quality testing...") - - # Nulls in required fields - null_count = int(len(orders_pdf) * BAD_DATA_CONFIG["null_rate"]) - null_indices = np.random.choice(orders_pdf.index, null_count, replace=False) - orders_pdf.loc[null_indices, "customer_id"] = None - print(f" Injected {null_count} null customer_ids") - - # Outliers (impossible values - negative amounts) - outlier_count = int(len(orders_pdf) * BAD_DATA_CONFIG["outlier_rate"]) - outlier_indices = np.random.choice(orders_pdf.index, outlier_count, replace=False) - orders_pdf.loc[outlier_indices, "amount"] = -999.99 - print(f" Injected {outlier_count} negative amounts") - - # Orphan foreign keys - orphan_count = int(len(orders_pdf) * BAD_DATA_CONFIG["orphan_fk_rate"]) - orphan_indices = np.random.choice(orders_pdf.index, orphan_count, replace=False) - orders_pdf.loc[orphan_indices, "customer_id"] = "CUST-NONEXISTENT" - print(f" Injected {orphan_count} orphan foreign keys") - - # Duplicate primary keys - dup_count = int(len(orders_pdf) * BAD_DATA_CONFIG["duplicate_pk_rate"]) - dup_indices = np.random.choice(orders_pdf.index[:-dup_count], dup_count, replace=False) - for i, idx in enumerate(dup_indices): - orders_pdf.loc[orders_pdf.index[-i-1], "order_id"] = orders_pdf.loc[idx, "order_id"] - print(f" Injected {dup_count} duplicate order_ids") - -# Show order distribution by customer tier -orders_by_tier = orders_pdf.merge( - customers_pdf[["customer_id", "tier"]], on="customer_id", how="left" -)["tier"].value_counts() -print(f"\nOrders by customer tier:") -for tier in TIER_VALUES: - count = orders_by_tier.get(tier, 0) - pct = count / N_ORDERS * 100 - print(f" {tier}: {count:,} ({pct:.1f}%)") - -# Show amount statistics by tier -print(f"\nAmount statistics by tier:") -for tier in TIER_VALUES: - tier_orders = orders_pdf.merge( - customers_pdf[["customer_id", "tier"]], on="customer_id", how="left" - ) - tier_amounts = tier_orders[tier_orders["tier"] == tier]["amount"] - if len(tier_amounts) > 0: - print(f" {tier}: avg=${tier_amounts.mean():,.2f}, median=${tier_amounts.median():,.2f}, " - f"min=${tier_amounts.min():,.2f}, max=${tier_amounts.max():,.2f}") - -# ============================================================================= -# SAVE TO PARQUET -# ============================================================================= -print(f"\nSaving to Parquet files in {VOLUME_PATH} (mode={WRITE_MODE})...") - -# Convert to Spark DataFrames -customers_df = spark.createDataFrame(customers_pdf) -orders_df = spark.createDataFrame(orders_pdf) - -# Save as Parquet -customers_df.write.mode(WRITE_MODE).parquet(f"{VOLUME_PATH}/customers") -print(f" Saved: {VOLUME_PATH}/customers ({N_CUSTOMERS:,} rows)") - -orders_df.write.mode(WRITE_MODE).parquet(f"{VOLUME_PATH}/orders") -print(f" Saved: {VOLUME_PATH}/orders ({N_ORDERS:,} rows)") - -print(f"\nDone! Data saved to {VOLUME_PATH}") -print(f" - customers: {N_CUSTOMERS:,} rows") -print(f" - orders: {N_ORDERS:,} rows") -if INJECT_BAD_DATA: - print(f" - Bad data injected: nulls, outliers, orphan FKs, duplicate PKs") diff --git a/databricks-skills/synthetic-data-generation/scripts/generate_synthetic_data.py b/databricks-skills/synthetic-data-generation/scripts/generate_synthetic_data.py new file mode 100644 index 00000000..f1011a29 --- /dev/null +++ b/databricks-skills/synthetic-data-generation/scripts/generate_synthetic_data.py @@ -0,0 +1,387 @@ +"""Generate synthetic data using Spark + Faker + Pandas UDFs. + +This is the recommended approach for ALL data generation tasks: +- Scales from thousands to millions of rows +- Parallel execution via Spark +- Direct write to Unity Catalog +- Works with serverless and classic compute + +Auto-detects environment and uses: +- DatabricksEnv with managed dependencies if databricks-connect >= 16.4 (local) +- Standard session if running on Databricks Runtime or older databricks-connect +""" +import sys +import os +from pyspark.sql import functions as F +from pyspark.sql.window import Window +from pyspark.sql.types import StringType, DoubleType, StructType, StructField, IntegerType +import numpy as np +import pandas as pd +from datetime import datetime, timedelta + +# ============================================================================= +# CONFIGURATION +# ============================================================================= +# Compute - Serverless strongly recommended +USE_SERVERLESS = True # Set to False and provide CLUSTER_ID for classic compute +CLUSTER_ID = None # Only used if USE_SERVERLESS=False + +# Storage - Update these for your environment +CATALOG = "ai_dev_kit" # Change to your catalog +SCHEMA = "synthetic_data" # Change to your schema +VOLUME_PATH = f"/Volumes/{CATALOG}/{SCHEMA}/raw_data" + +# Data sizes +N_CUSTOMERS = 10_000 +N_ORDERS = 50_000 +PARTITIONS = 16 # Adjust: 8 for <100K, 32 for 1M+ + +# Date range - last 6 months from today +END_DATE = datetime.now().replace(hour=0, minute=0, second=0, microsecond=0) +START_DATE = END_DATE - timedelta(days=180) + +# Write mode - "overwrite" for one-time, "append" for incremental +WRITE_MODE = "overwrite" + +# Bad data injection for testing data quality rules +INJECT_BAD_DATA = False # Set to True to inject bad data +BAD_DATA_CONFIG = { + "null_rate": 0.02, # 2% nulls in required fields + "outlier_rate": 0.01, # 1% impossible values + "orphan_fk_rate": 0.01, # 1% orphan foreign keys +} + +# Reproducibility +SEED = 42 + +# Tier distribution: Free 60%, Pro 30%, Enterprise 10% +TIER_PROBS = [0.6, 0.3, 0.1] + +# Region distribution +REGION_PROBS = [0.4, 0.25, 0.2, 0.15] + +# ============================================================================= +# ENVIRONMENT DETECTION AND SESSION CREATION +# ============================================================================= + +def is_databricks_runtime(): + """Check if running on Databricks Runtime (notebook/job) vs locally.""" + return "DATABRICKS_RUNTIME_VERSION" in os.environ + +def get_databricks_connect_version(): + """Get databricks-connect version as (major, minor) tuple or None.""" + try: + import databricks.connect + version_str = databricks.connect.__version__ + parts = version_str.split('.') + return (int(parts[0]), int(parts[1])) + except (ImportError, AttributeError, ValueError, IndexError): + return None + +# Detect environment +on_runtime = is_databricks_runtime() +db_version = get_databricks_connect_version() + +print("=" * 80) +print("ENVIRONMENT DETECTION") +print("=" * 80) +print(f"Running on Databricks Runtime: {on_runtime}") +if db_version: + print(f"databricks-connect version: {db_version[0]}.{db_version[1]}") +else: + print("databricks-connect: not available") + +# Use DatabricksEnv with managed dependencies if: +# - Running locally (not on Databricks Runtime) +# - databricks-connect >= 16.4 +use_managed_deps = (not on_runtime) and db_version and db_version >= (16, 4) + +if use_managed_deps: + print("Using DatabricksEnv with managed dependencies") + print("=" * 80) + from databricks.connect import DatabricksSession, DatabricksEnv + + env = DatabricksEnv().withDependencies("faker", "pandas", "numpy", "holidays") + + if USE_SERVERLESS: + spark = DatabricksSession.builder.withEnvironment(env).serverless(True).getOrCreate() + print("Connected to serverless compute with managed dependencies!") + else: + if not CLUSTER_ID: + raise ValueError("CLUSTER_ID must be set when USE_SERVERLESS=False") + spark = DatabricksSession.builder.withEnvironment(env).clusterId(CLUSTER_ID).getOrCreate() + print(f"Connected to cluster {CLUSTER_ID} with managed dependencies!") +else: + print("Using standard session (dependencies must be pre-installed)") + print("=" * 80) + + # Check that UDF dependencies are available + print("\nChecking UDF dependencies...") + missing_deps = [] + + try: + from faker import Faker + print(" faker: OK") + except ImportError: + missing_deps.append("faker") + print(" faker: MISSING") + + try: + import pandas as pd + print(" pandas: OK") + except ImportError: + missing_deps.append("pandas") + print(" pandas: MISSING") + + if missing_deps: + print("\n" + "=" * 80) + print("ERROR: Missing dependencies for UDFs") + print("=" * 80) + print(f"Missing: {', '.join(missing_deps)}") + if on_runtime: + print("\nSolution: Run %pip install faker pandas numpy holidays") + else: + print("\nSolution: Upgrade to databricks-connect >= 16.4 for managed deps") + print(" Or create a job with environment settings") + print("=" * 80) + sys.exit(1) + + print("\nAll dependencies available") + print("=" * 80) + + from databricks.connect import DatabricksSession + + if USE_SERVERLESS: + spark = DatabricksSession.builder.serverless(True).getOrCreate() + print("Connected to serverless compute") + else: + if not CLUSTER_ID: + raise ValueError("CLUSTER_ID must be set when USE_SERVERLESS=False") + spark = DatabricksSession.builder.clusterId(CLUSTER_ID).getOrCreate() + print(f"Connected to cluster {CLUSTER_ID}") + +# Import Faker for UDF definitions +from faker import Faker + +# ============================================================================= +# DEFINE PANDAS UDFs FOR FAKER DATA +# ============================================================================= + +@F.pandas_udf(StringType()) +def fake_name(ids: pd.Series) -> pd.Series: + """Generate realistic person names.""" + fake = Faker() + Faker.seed(SEED) + return pd.Series([fake.name() for _ in range(len(ids))]) + +@F.pandas_udf(StringType()) +def fake_company(ids: pd.Series) -> pd.Series: + """Generate realistic company names.""" + fake = Faker() + Faker.seed(SEED) + return pd.Series([fake.company() for _ in range(len(ids))]) + +@F.pandas_udf(StringType()) +def fake_address(ids: pd.Series) -> pd.Series: + """Generate realistic addresses.""" + fake = Faker() + Faker.seed(SEED) + return pd.Series([fake.address().replace('\n', ', ') for _ in range(len(ids))]) + +@F.pandas_udf(StringType()) +def fake_email(names: pd.Series) -> pd.Series: + """Generate email based on name.""" + emails = [] + for name in names: + if name: + domain = name.lower().replace(" ", ".").replace(",", "")[:20] + emails.append(f"{domain}@example.com") + else: + emails.append("unknown@example.com") + return pd.Series(emails) + +@F.pandas_udf(DoubleType()) +def generate_lognormal_amount(tiers: pd.Series) -> pd.Series: + """Generate amount based on tier using log-normal distribution.""" + np.random.seed(SEED) + amounts = [] + for tier in tiers: + if tier == "Enterprise": + amounts.append(float(np.random.lognormal(mean=7.5, sigma=0.8))) # ~$1800 avg + elif tier == "Pro": + amounts.append(float(np.random.lognormal(mean=5.5, sigma=0.7))) # ~$245 avg + else: + amounts.append(float(np.random.lognormal(mean=4.0, sigma=0.6))) # ~$55 avg + return pd.Series(amounts) + +# ============================================================================= +# CREATE INFRASTRUCTURE +# ============================================================================= +print("\nCreating infrastructure...") +spark.sql(f"CREATE SCHEMA IF NOT EXISTS {CATALOG}.{SCHEMA}") +spark.sql(f"CREATE VOLUME IF NOT EXISTS {CATALOG}.{SCHEMA}.raw_data") +print(f"Infrastructure ready: {VOLUME_PATH}") + +# ============================================================================= +# GENERATE CUSTOMERS (Master Table) +# ============================================================================= +print(f"\nGenerating {N_CUSTOMERS:,} customers...") + +customers_df = ( + spark.range(0, N_CUSTOMERS, numPartitions=PARTITIONS) + .select( + F.concat(F.lit("CUST-"), F.lpad(F.col("id").cast("string"), 5, "0")).alias("customer_id"), + fake_name(F.col("id")).alias("name"), + fake_company(F.col("id")).alias("company"), + fake_address(F.col("id")).alias("address"), + # Tier distribution: Free 60%, Pro 30%, Enterprise 10% + F.when(F.rand(SEED) < TIER_PROBS[0], "Free") + .when(F.rand(SEED) < TIER_PROBS[0] + TIER_PROBS[1], "Pro") + .otherwise("Enterprise").alias("tier"), + # Region distribution + F.when(F.rand(SEED) < REGION_PROBS[0], "North") + .when(F.rand(SEED) < REGION_PROBS[0] + REGION_PROBS[1], "South") + .when(F.rand(SEED) < REGION_PROBS[0] + REGION_PROBS[1] + REGION_PROBS[2], "East") + .otherwise("West").alias("region"), + # Created date (within last 2 years before start date) + F.date_sub(F.lit(START_DATE.date()), (F.rand(SEED) * 730).cast("int")).alias("created_at"), + ) +) + +# Add tier-based ARR and email +customers_df = ( + customers_df + .withColumn("arr", F.round(generate_lognormal_amount(F.col("tier")), 2)) + .withColumn("email", fake_email(F.col("name"))) +) + +# Save customers +customers_df.write.mode(WRITE_MODE).parquet(f"{VOLUME_PATH}/customers") +print(f" Saved customers to {VOLUME_PATH}/customers") + +# Show tier distribution +print("\n Tier distribution:") +customers_df.groupBy("tier").count().orderBy("tier").show() + +# ============================================================================= +# GENERATE ORDERS (Child Table with Referential Integrity) +# ============================================================================= +print(f"\nGenerating {N_ORDERS:,} orders with referential integrity...") + +# Cache customer lookup for FK generation +customer_lookup = customers_df.select("customer_id", "tier").cache() + +# Generate orders base +orders_df = ( + spark.range(0, N_ORDERS, numPartitions=PARTITIONS) + .select( + F.concat(F.lit("ORD-"), F.lpad(F.col("id").cast("string"), 6, "0")).alias("order_id"), + # Generate customer_idx for FK join (hash-based distribution) + (F.abs(F.hash(F.col("id"), F.lit(SEED))) % N_CUSTOMERS).alias("customer_idx"), + # Order status + F.when(F.rand(SEED) < 0.65, "delivered") + .when(F.rand(SEED) < 0.80, "shipped") + .when(F.rand(SEED) < 0.90, "processing") + .when(F.rand(SEED) < 0.95, "pending") + .otherwise("cancelled").alias("status"), + # Order date within date range + F.date_add(F.lit(START_DATE.date()), (F.rand(SEED) * 180).cast("int")).alias("order_date"), + ) +) + +# Add customer_idx to lookup for join +customer_lookup_with_idx = customer_lookup.withColumn( + "customer_idx", + (F.row_number().over(Window.orderBy(F.monotonically_increasing_id())) - 1).cast("int") +) + +# Join to get customer_id and tier as foreign key +orders_with_fk = ( + orders_df + .join(customer_lookup_with_idx, on="customer_idx", how="left") + .drop("customer_idx") +) + +# Add tier-based amount +orders_with_fk = orders_with_fk.withColumn( + "amount", + F.round(generate_lognormal_amount(F.col("tier")), 2) +) + +# ============================================================================= +# INJECT BAD DATA (OPTIONAL) +# ============================================================================= +if INJECT_BAD_DATA: + print("\nInjecting bad data for quality testing...") + + # Calculate counts + null_count = int(N_ORDERS * BAD_DATA_CONFIG["null_rate"]) + outlier_count = int(N_ORDERS * BAD_DATA_CONFIG["outlier_rate"]) + orphan_count = int(N_ORDERS * BAD_DATA_CONFIG["orphan_fk_rate"]) + + # Add bad data flags + orders_with_fk = orders_with_fk.withColumn( + "row_num", + F.row_number().over(Window.orderBy(F.monotonically_increasing_id())) + ) + + # Inject nulls in customer_id for first null_count rows + orders_with_fk = orders_with_fk.withColumn( + "customer_id", + F.when(F.col("row_num") <= null_count, None).otherwise(F.col("customer_id")) + ) + + # Inject negative amounts for next outlier_count rows + orders_with_fk = orders_with_fk.withColumn( + "amount", + F.when( + (F.col("row_num") > null_count) & (F.col("row_num") <= null_count + outlier_count), + F.lit(-999.99) + ).otherwise(F.col("amount")) + ) + + # Inject orphan FKs for next orphan_count rows + orders_with_fk = orders_with_fk.withColumn( + "customer_id", + F.when( + (F.col("row_num") > null_count + outlier_count) & + (F.col("row_num") <= null_count + outlier_count + orphan_count), + F.lit("CUST-NONEXISTENT") + ).otherwise(F.col("customer_id")) + ) + + orders_with_fk = orders_with_fk.drop("row_num") + + print(f" Injected {null_count} null customer_ids") + print(f" Injected {outlier_count} negative amounts") + print(f" Injected {orphan_count} orphan foreign keys") + +# Drop tier column (not needed in final output) +orders_final = orders_with_fk.drop("tier") + +# Save orders +orders_final.write.mode(WRITE_MODE).parquet(f"{VOLUME_PATH}/orders") +print(f" Saved orders to {VOLUME_PATH}/orders") + +# Show status distribution +print("\n Status distribution:") +orders_final.groupBy("status").count().orderBy("status").show() + +# ============================================================================= +# CLEANUP AND SUMMARY +# ============================================================================= +customer_lookup.unpersist() + +print("\n" + "=" * 80) +print("GENERATION COMPLETE") +print("=" * 80) +print(f"Catalog: {CATALOG}") +print(f"Schema: {SCHEMA}") +print(f"Volume: {VOLUME_PATH}") +print(f"\nGenerated data:") +print(f" - customers: {N_CUSTOMERS:,} rows") +print(f" - orders: {N_ORDERS:,} rows") +if INJECT_BAD_DATA: + print(f" - Bad data injected: nulls, outliers, orphan FKs") +print(f"\nDate range: {START_DATE.date()} to {END_DATE.date()}") +print("=" * 80) From c15572f27c6543f071c3357259f65df00b05e81d Mon Sep 17 00:00:00 2001 From: dustinvannoy-db Date: Tue, 24 Feb 2026 15:56:44 -0800 Subject: [PATCH 09/24] Add guidance on cache with serverless --- .../synthetic-data-generation/SKILL.md | 4 +- .../references/3-data-patterns.md | 26 +- .../references/6-troubleshooting.md | 47 ++- scripts/generate_support_tickets.py | 387 ------------------ 4 files changed, 57 insertions(+), 407 deletions(-) delete mode 100644 scripts/generate_support_tickets.py diff --git a/databricks-skills/synthetic-data-generation/SKILL.md b/databricks-skills/synthetic-data-generation/SKILL.md index 2468a53e..34d82d87 100644 --- a/databricks-skills/synthetic-data-generation/SKILL.md +++ b/databricks-skills/synthetic-data-generation/SKILL.md @@ -27,6 +27,7 @@ Generate realistic, story-driven synthetic data for Databricks using **Spark + F 4. **Use serverless compute** unless user explicitly requests classic cluster 5. **Generate raw data only** - no pre-aggregated fields (unless user requests) 6. **Create master tables first** - then generate related tables with valid FKs +7. **NEVER use `.cache()` or `.persist()` with serverless compute** - these operations are NOT supported and will fail with `AnalysisException: PERSIST TABLE is not supported on serverless compute`. Instead, write master tables to Delta first, then read them back for FK joins. ## Generation Planning Workflow @@ -211,6 +212,7 @@ See [references/5-output-formats.md](references/5-output-formats.md) for detaile | `ModuleNotFoundError: faker` | See [references/1-setup-and-execution.md](references/1-setup-and-execution.md) | | Faker UDF is slow | Use `pandas_udf` for batch processing | | Out of memory | Increase `numPartitions` in `spark.range()` | -| Referential integrity errors | Generate master tables first, cache, then join | +| Referential integrity errors | Write master table to Delta first, read back for FK joins | +| `PERSIST TABLE is not supported on serverless` | **NEVER use `.cache()` or `.persist()` with serverless** - write to Delta table first, then read back | See [references/6-troubleshooting.md](references/6-troubleshooting.md) for full troubleshooting guide. diff --git a/databricks-skills/synthetic-data-generation/references/3-data-patterns.md b/databricks-skills/synthetic-data-generation/references/3-data-patterns.md index ae9b0697..351f1bd7 100644 --- a/databricks-skills/synthetic-data-generation/references/3-data-patterns.md +++ b/databricks-skills/synthetic-data-generation/references/3-data-patterns.md @@ -71,24 +71,34 @@ customers_df.write.mode("overwrite").parquet(f"{VOLUME_PATH}/customers") Generate master tables first, then iterate on them to create related tables with matching IDs. +> **CRITICAL:** Do NOT use `.cache()` or `.persist()` with serverless compute - these operations are not supported and will fail. Instead, write master tables to Delta first, then read them back for FK joins. + ### Pattern: Weighted Sampling by Tier ```python from pyspark.sql.window import Window -# 1. Generate and cache customers (master table) +# 1. Generate customers (master table) with index for FK mapping customers_df = ( spark.range(0, N_CUSTOMERS, numPartitions=PARTITIONS) .select( + F.col("id").alias("customer_idx"), # Keep index for FK joins F.concat(F.lit("CUST-"), F.lpad(F.col("id").cast("string"), 5, "0")).alias("customer_id"), F.when(F.rand(SEED) < 0.6, "Free") .when(F.rand(SEED) < 0.9, "Pro") .otherwise("Enterprise").alias("tier"), ) ) -customer_lookup = customers_df.select("customer_id", "tier").cache() -# 2. Generate orders with valid foreign keys +# 2. Write to Delta table (do NOT use cache with serverless!) +customers_df.write.mode("overwrite").saveAsTable(f"{CATALOG}.{SCHEMA}.customers") + +# 3. Read back for FK lookups +customer_lookup = spark.table(f"{CATALOG}.{SCHEMA}.customers").select( + "customer_idx", "customer_id", "tier" +) + +# 4. Generate orders with valid foreign keys orders_df = spark.range(0, N_ORDERS, numPartitions=PARTITIONS) # Map order to customer using hash-based distribution @@ -97,16 +107,8 @@ orders_df = orders_df.select( (F.abs(F.hash(F.col("id"), F.lit(SEED))) % N_CUSTOMERS).alias("customer_idx"), ) -# Add customer_idx to lookup for join -customer_lookup_with_idx = customer_lookup.withColumn( - "customer_idx", - (F.row_number().over(Window.orderBy(F.monotonically_increasing_id())) - 1).cast("int") -) - # Join to get valid foreign keys -orders_with_fk = orders_df.join(customer_lookup_with_idx, on="customer_idx", how="left") - -customer_lookup.unpersist() +orders_with_fk = orders_df.join(customer_lookup, on="customer_idx", how="left") ``` ### Anti-Pattern: Random FK Generation diff --git a/databricks-skills/synthetic-data-generation/references/6-troubleshooting.md b/databricks-skills/synthetic-data-generation/references/6-troubleshooting.md index f0052878..a40efc38 100644 --- a/databricks-skills/synthetic-data-generation/references/6-troubleshooting.md +++ b/databricks-skills/synthetic-data-generation/references/6-troubleshooting.md @@ -55,6 +55,37 @@ auth_type = databricks-cli ## Execution Issues +### CRITICAL: cache() and persist() NOT supported on serverless + +**Problem:** Using `.cache()` or `.persist()` on serverless compute fails with: +``` +AnalysisException: [NOT_SUPPORTED_WITH_SERVERLESS] PERSIST TABLE is not supported on serverless compute. +``` + +**Why this happens:** Serverless compute does not support caching DataFrames in memory. This is a fundamental limitation of the serverless architecture. + +**Solution:** Write master tables to Delta first, then read them back for FK joins: + +```python +# BAD - will fail on serverless +customers_df = spark.range(0, N_CUSTOMERS)... +customers_df.cache() # ❌ FAILS: "PERSIST TABLE is not supported on serverless compute" + +# GOOD - write to Delta, then read back +customers_df = spark.range(0, N_CUSTOMERS)... +customers_df.write.mode("overwrite").saveAsTable(f"{CATALOG}.{SCHEMA}.customers") +customer_lookup = spark.table(f"{CATALOG}.{SCHEMA}.customers") # ✓ Read from Delta +``` + +**Best practice for referential integrity:** +1. Generate master table (e.g., customers) +2. Write to Delta table +3. Read back for FK lookup joins +4. Generate child tables (e.g., orders, tickets) with valid FKs +5. Write child tables to Delta + +--- + ### Serverless job fails to start **Possible causes:** @@ -159,24 +190,26 @@ customers_df = spark.range(0, N_CUSTOMERS, numPartitions=64) # Increase from de **Problem:** Foreign keys reference non-existent parent records. -**Solution:** Generate master tables first, cache, then join: +**Solution:** Write master table to Delta first, then read back for FK joins: ```python -# 1. Generate and cache master table +# 1. Generate and WRITE master table (do NOT use cache with serverless!) customers_df = spark.range(0, N_CUSTOMERS)... -customer_lookup = customers_df.select("customer_id").cache() +customers_df.write.mode("overwrite").saveAsTable(f"{CATALOG}.{SCHEMA}.customers") + +# 2. Read back for FK lookups +customer_lookup = spark.table(f"{CATALOG}.{SCHEMA}.customers").select("customer_id", "tier") -# 2. Generate child table with valid FKs +# 3. Generate child table with valid FKs orders_df = spark.range(0, N_ORDERS).join( customer_lookup, on=, how="left" ) - -# 3. Clean up -customer_lookup.unpersist() ``` +> **WARNING:** Do NOT use `.cache()` or `.persist()` with serverless compute. See the dedicated section above. + --- ## Data Quality Issues diff --git a/scripts/generate_support_tickets.py b/scripts/generate_support_tickets.py deleted file mode 100644 index 31382b00..00000000 --- a/scripts/generate_support_tickets.py +++ /dev/null @@ -1,387 +0,0 @@ -"""Generate large-scale synthetic support ticket data. - -This script automatically detects the environment and uses: -- DatabricksEnv with auto-dependencies if databricks-connect >= 16.4 and running locally -- Standard session creation if running on Databricks Runtime or older databricks-connect -""" -import sys -import os -import numpy as np -import pandas as pd -from datetime import datetime, timedelta -from pyspark.sql import functions as F -from pyspark.sql.functions import pandas_udf -from pyspark.sql.types import StringType, DoubleType, IntegerType - -# ============================================================================= -# CONFIGURATION - Edit these values -# ============================================================================= -CATALOG = "dustin_vannoy_catalog" -SCHEMA = "sdg_test_large_delta" - -# Data sizes -N_CUSTOMERS = 100000 -N_TICKETS = 500000 - -# Date ranges -END_DATE = datetime.now().replace(hour=0, minute=0, second=0, microsecond=0) -CUSTOMER_START_DATE = END_DATE - timedelta(days=1095) # Last 3 years -TICKET_START_DATE = END_DATE - timedelta(days=180) # Last 6 months - -# Reproducibility -SEED = 42 - -# Spark partitions for parallelism (adjust based on scale) -CUSTOMER_PARTITIONS = 32 -TICKET_PARTITIONS = 64 - -# ============================================================================= -# SETUP - Environment Detection and Session Creation -# ============================================================================= -np.random.seed(SEED) - -# Detect if running on Databricks Runtime vs locally with Databricks Connect -def is_databricks_runtime(): - """Check if running on Databricks Runtime (notebook/job) vs locally.""" - return "DATABRICKS_RUNTIME_VERSION" in os.environ - -# Get databricks-connect version if available -def get_databricks_connect_version(): - """Get databricks-connect version as (major, minor) tuple or None.""" - try: - import databricks.connect - version_str = databricks.connect.__version__ - parts = version_str.split('.') - return (int(parts[0]), int(parts[1])) - except (ImportError, AttributeError, ValueError, IndexError): - return None - -print("=" * 80) -print("SYNTHETIC DATA GENERATION - SUPPORT TICKETS") -print("=" * 80) -print(f"Catalog: {CATALOG}") -print(f"Schema: {SCHEMA}") -print(f"Customers: {N_CUSTOMERS:,}") -print(f"Tickets: {N_TICKETS:,}") -print(f"Customer partitions: {CUSTOMER_PARTITIONS}") -print(f"Ticket partitions: {TICKET_PARTITIONS}") -print("=" * 80) - -# Determine session creation strategy -on_runtime = is_databricks_runtime() -db_version = get_databricks_connect_version() - -print("\nENVIRONMENT DETECTION") -print("=" * 80) -print(f"Running on Databricks Runtime: {on_runtime}") -if db_version: - print(f"databricks-connect version: {db_version[0]}.{db_version[1]}") -else: - print("databricks-connect: not available") - -# Use DatabricksEnv with auto-dependencies if: -# - Running locally (not on Databricks Runtime) -# - databricks-connect >= 16.4 -use_auto_dependencies = (not on_runtime) and db_version and db_version >= (16, 4) - -if use_auto_dependencies: - print("✓ Using DatabricksEnv with auto-dependencies") - print("=" * 80) - from databricks.connect import DatabricksSession, DatabricksEnv - - env = DatabricksEnv().withAutoDependencies(upload_local=True, use_index=True) - spark = ( - DatabricksSession.builder - .withEnvironment(env) - .config("spark.databricks.sql.externalUDF.env.enabled", "true") - .config("spark.databricks.sql.udf.routineEnvironmentSettings.enabled", "true") - .serverless(True) - .getOrCreate() - ) - print("✓ Connected to serverless compute with auto-dependencies!") -else: - print("⚠ Using standard session (dependencies must be pre-installed)") - print("=" * 80) - - # Try to import libraries that will be used in UDFs - print("\nChecking UDF dependencies...") - missing_deps = [] - - try: - from faker import Faker - print(" ✓ faker") - except ImportError: - missing_deps.append("faker") - print(" ✗ faker - NOT INSTALLED") - - try: - import pandas as pd - print(" ✓ pandas") - except ImportError: - missing_deps.append("pandas") - print(" ✗ pandas - NOT INSTALLED") - - if missing_deps: - print("\n" + "=" * 80) - print("⚠ WARNING: Missing dependencies for UDFs") - print("=" * 80) - print(f"Missing libraries: {', '.join(missing_deps)}") - print("\nThese libraries are required in UDFs and must be installed:") - - if on_runtime: - print("\n→ SOLUTION: Install on the cluster or job:") - print(" - For interactive cluster: Run %pip install faker pandas numpy holidays") - print(" - For job: Add to job libraries or use init script") - else: - print("\n→ SOLUTION: Use one of these approaches:") - print(" 1. Upgrade databricks-connect to >= 16.4 (enables auto-dependencies)") - print(" 2. Create a job with environment settings in the task definition") - print(" 3. Use a classic cluster with libraries pre-installed") - - print("=" * 80) - sys.exit(1) - - print("\n✓ All UDF dependencies available") - print("=" * 80) - - # Create standard session - from databricks.connect import DatabricksSession - - spark = ( - DatabricksSession.builder - .config("spark.databricks.sql.externalUDF.env.enabled", "true") - .config("spark.databricks.sql.udf.routineEnvironmentSettings.enabled", "true") - .serverless(True) - .getOrCreate() - ) - print("✓ Connected to serverless compute") - -# Import Faker for later use (already checked above) -from faker import Faker -Faker.seed(SEED) -fake = Faker() - -# ============================================================================= -# CREATE INFRASTRUCTURE -# ============================================================================= -print("\n[1/4] Creating infrastructure...") -# Note: Assume catalog exists - do NOT create it -spark.sql(f"CREATE SCHEMA IF NOT EXISTS {CATALOG}.{SCHEMA}") -print(f"✓ Schema created/verified") - -# ============================================================================= -# DEFINE PANDAS UDFs FOR FAKER DATA -# ============================================================================= -print("\n[2/4] Defining data generation UDFs...") - -@pandas_udf(StringType()) -def fake_company(ids: pd.Series) -> pd.Series: - """Generate realistic company names.""" - fake = Faker() - Faker.seed(SEED) - return pd.Series([fake.company() for _ in range(len(ids))]) - -@pandas_udf(DoubleType()) -def generate_arr(tiers: pd.Series) -> pd.Series: - """Generate ARR based on tier using log-normal distribution.""" - np.random.seed(SEED) - result = [] - for tier in tiers: - if tier == "Enterprise": - # Mean ~$500K - arr = np.random.lognormal(mean=13, sigma=0.8) - elif tier == "Pro": - # Mean ~$50K - arr = np.random.lognormal(mean=11, sigma=0.7) - else: # Free - arr = 0.0 - result.append(round(arr, 2)) - return pd.Series(result) - -@pandas_udf(StringType()) -def generate_priority(tiers: pd.Series) -> pd.Series: - """Generate priority based on tier.""" - np.random.seed(SEED) - result = [] - for tier in tiers: - if tier == "Enterprise": - priority = np.random.choice(['Critical', 'High', 'Medium'], p=[0.3, 0.5, 0.2]) - elif tier == "Pro": - priority = np.random.choice(['Critical', 'High', 'Medium', 'Low'], p=[0.1, 0.3, 0.45, 0.15]) - else: # Free - priority = np.random.choice(['Critical', 'High', 'Medium', 'Low'], p=[0.02, 0.15, 0.40, 0.43]) - result.append(priority) - return pd.Series(result) - -@pandas_udf(DoubleType()) -def generate_resolution_hours(priorities: pd.Series) -> pd.Series: - """Generate resolution hours based on priority using exponential distribution.""" - np.random.seed(SEED) - result = [] - scale_map = {'Critical': 4, 'High': 12, 'Medium': 36, 'Low': 72} - for priority in priorities: - scale = scale_map.get(priority, 24) - hours = np.random.exponential(scale=scale) - result.append(round(hours, 2)) - return pd.Series(result) - -@pandas_udf(IntegerType()) -def generate_csat(resolution_hours: pd.Series) -> pd.Series: - """Generate CSAT score based on resolution time.""" - np.random.seed(SEED) - result = [] - for hours in resolution_hours: - if hours < 4: - csat = np.random.choice([4, 5], p=[0.3, 0.7]) - elif hours < 24: - csat = np.random.choice([3, 4, 5], p=[0.2, 0.5, 0.3]) - elif hours < 72: - csat = np.random.choice([2, 3, 4], p=[0.3, 0.5, 0.2]) - else: - csat = np.random.choice([1, 2, 3], p=[0.4, 0.4, 0.2]) - result.append(int(csat)) - return pd.Series(result) - -print("✓ UDFs defined") - -# ============================================================================= -# GENERATE CUSTOMERS TABLE -# ============================================================================= -print(f"\n[3/4] Generating {N_CUSTOMERS:,} customers...") - -# Generate base customer data with Spark -customers_df = ( - spark.range(0, N_CUSTOMERS, numPartitions=CUSTOMER_PARTITIONS) - .select( - # customer_id: CUST-00001 format - F.concat(F.lit("CUST-"), F.lpad(F.col("id").cast("string"), 6, "0")).alias("customer_id"), - - # tier: Enterprise 10%, Pro 30%, Free 60% - F.when(F.rand(SEED) < 0.10, "Enterprise") - .when(F.rand(SEED + 1) < 0.40, "Pro") # 0.10 + 0.30 = 0.40 - .otherwise("Free").alias("tier"), - - # region: North 35%, South 25%, East 25%, West 15% - F.when(F.rand(SEED + 2) < 0.35, "North") - .when(F.rand(SEED + 3) < 0.60, "South") # 0.35 + 0.25 = 0.60 - .when(F.rand(SEED + 4) < 0.85, "East") # 0.60 + 0.25 = 0.85 - .otherwise("West").alias("region"), - - # signup_date: random date in last 3 years - (F.lit(CUSTOMER_START_DATE.timestamp()) + - (F.rand(SEED + 5) * (END_DATE.timestamp() - CUSTOMER_START_DATE.timestamp())) - ).cast("timestamp").cast("date").alias("signup_date"), - ) -) - -# Add company_name and arr using UDFs -customers_df = ( - customers_df - .withColumn("company_name", fake_company(F.col("customer_id"))) - .withColumn("arr", generate_arr(F.col("tier"))) -) - -# Reorder columns -customers_df = customers_df.select( - "customer_id", "company_name", "tier", "arr", "region", "signup_date" -) - -# Save to Delta table -print(f"Writing customers to {CATALOG}.{SCHEMA}.customers...") -customers_df.write.mode("overwrite").option("overwriteSchema", "true").saveAsTable(f"{CATALOG}.{SCHEMA}.customers") - -# Get customer count -customer_count = spark.table(f"{CATALOG}.{SCHEMA}.customers").count() -print(f"✓ Created customers table with {customer_count:,} rows") - -# ============================================================================= -# GENERATE TICKETS TABLE -# ============================================================================= -print(f"\n[4/4] Generating {N_TICKETS:,} tickets...") - -# Create a broadcast map of customer_id -> tier for weighted sampling -customers_sample = spark.table(f"{CATALOG}.{SCHEMA}.customers").select("customer_id", "tier").collect() -customer_ids = [row.customer_id for row in customers_sample] -customer_tiers = {row.customer_id: row.tier for row in customers_sample} - -# Create weights: Enterprise 5x, Pro 2x, Free 1x -tier_weights = {"Enterprise": 5.0, "Pro": 2.0, "Free": 1.0} -weights = [tier_weights[customer_tiers[cid]] for cid in customer_ids] -weights = np.array(weights) / np.sum(weights) - -# Sample customer_ids with replacement based on weights -np.random.seed(SEED) -sampled_customer_ids = np.random.choice(customer_ids, size=N_TICKETS, replace=True, p=weights) - -# Create tickets DataFrame from sampled customer_ids -tickets_pdf = pd.DataFrame({ - "ticket_id": [f"TKT-{i:07d}" for i in range(N_TICKETS)], - "customer_id": sampled_customer_ids, -}) - -# Convert to Spark DataFrame -tickets_df = spark.createDataFrame(tickets_pdf, schema="ticket_id STRING, customer_id STRING") - -# Repartition for better parallelism -tickets_df = tickets_df.repartition(TICKET_PARTITIONS) - -# Join with customers to get tier for priority generation -tickets_df = tickets_df.join( - spark.table(f"{CATALOG}.{SCHEMA}.customers").select("customer_id", "tier"), - on="customer_id", - how="left" -) - -# Add priority, resolution_hours, csat_score, created_at -tickets_df = ( - tickets_df - .withColumn("priority", generate_priority(F.col("tier"))) - .withColumn("resolution_hours", generate_resolution_hours(F.col("priority"))) - .withColumn("csat_score", generate_csat(F.col("resolution_hours"))) - .withColumn( - "created_at", - (F.lit(TICKET_START_DATE.timestamp()) + - (F.rand(SEED + 10) * (END_DATE.timestamp() - TICKET_START_DATE.timestamp())) - ).cast("timestamp") - ) -) - -# Drop the tier column (only needed for generation) -tickets_df = tickets_df.select( - "ticket_id", "customer_id", "priority", "resolution_hours", "csat_score", "created_at" -) - -# Save to Delta table -print(f"Writing tickets to {CATALOG}.{SCHEMA}.tickets...") -tickets_df.write.mode("overwrite").option("overwriteSchema", "true").saveAsTable(f"{CATALOG}.{SCHEMA}.tickets") - -# Get ticket count -ticket_count = spark.table(f"{CATALOG}.{SCHEMA}.tickets").count() -print(f"✓ Created tickets table with {ticket_count:,} rows") - -# ============================================================================= -# VALIDATION -# ============================================================================= -print("\n" + "=" * 80) -print("GENERATION COMPLETE") -print("=" * 80) - -# Show sample data -print("\nCustomers sample:") -spark.table(f"{CATALOG}.{SCHEMA}.customers").show(5, truncate=False) - -print("\nTickets sample:") -spark.table(f"{CATALOG}.{SCHEMA}.tickets").show(5, truncate=False) - -# Show statistics -print("\nCustomer tier distribution:") -spark.table(f"{CATALOG}.{SCHEMA}.customers").groupBy("tier").count().orderBy("tier").show() - -print("\nTicket priority distribution:") -spark.table(f"{CATALOG}.{SCHEMA}.tickets").groupBy("priority").count().orderBy("priority").show() - -print("\n" + "=" * 80) -print(f"✓ Tables created:") -print(f" - {CATALOG}.{SCHEMA}.customers ({customer_count:,} rows)") -print(f" - {CATALOG}.{SCHEMA}.tickets ({ticket_count:,} rows)") -print("=" * 80) From bdb3ab6ffcacd88c687f59aa36054eff3e0091e5 Mon Sep 17 00:00:00 2001 From: dustinvannoy-db Date: Wed, 25 Feb 2026 09:41:56 -0800 Subject: [PATCH 10/24] Update data gen for better cluster/job guidance --- .../synthetic-data-generation/SKILL.md | 34 ++++- .../references/1-setup-and-execution.md | 120 +++++++++++++++--- 2 files changed, 129 insertions(+), 25 deletions(-) diff --git a/databricks-skills/synthetic-data-generation/SKILL.md b/databricks-skills/synthetic-data-generation/SKILL.md index 34d82d87..4b71e4f6 100644 --- a/databricks-skills/synthetic-data-generation/SKILL.md +++ b/databricks-skills/synthetic-data-generation/SKILL.md @@ -33,9 +33,25 @@ Generate realistic, story-driven synthetic data for Databricks using **Spark + F **Before generating any code, you MUST present a plan for user approval.** +### ⚠️ MUST DO: Confirm Catalog Before Proceeding + +**You MUST explicitly ask the user which catalog to use.** Do not assume or proceed without confirmation. + +Example prompt to user: +> "Which Unity Catalog should I use for this data? Default is `ai_dev_kit` but you can specify any catalog you have access to." + +When presenting your plan, always show the selected catalog prominently: +``` +📍 Output Location: catalog_name.schema_name + Volume: /Volumes/catalog_name/schema_name/raw_data/ +``` + +This makes it easy for the user to spot and correct if needed. + ### Step 1: Gather Requirements Ask the user about: +- **Catalog/Schema** - Which catalog to use? (default: `ai_dev_kit.`) - What domain/scenario? (e-commerce, support tickets, IoT sensors, etc.) - How many tables? What relationships between them? - Approximate row counts per table? @@ -43,7 +59,14 @@ Ask the user about: ### Step 2: Present Table Specification -Show a clear specification with **YOUR ASSUMPTIONS surfaced**: +Show a clear specification with **YOUR ASSUMPTIONS surfaced**. Always start with the output location: + +``` +📍 Output Location: ai_dev_kit.ecommerce_demo + Volume: /Volumes/ai_dev_kit/ecommerce_demo/raw_data/ + + ⬆️ Change this? Just let me know which catalog.schema to use instead. +``` | Table | Columns | Rows | Key Assumptions | |-------|---------|------|-----------------| @@ -54,7 +77,7 @@ Show a clear specification with **YOUR ASSUMPTIONS surfaced**: - Amount distribution: log-normal by tier (Enterprise ~$1800, Pro ~$245, Free ~$55) - Status: 65% delivered, 15% shipped, 10% processing, 5% pending, 5% cancelled -**Ask user**: "Does this look correct? Any adjustments needed?" +**Ask user**: "Does this look correct? Any adjustments to the catalog, tables, or distributions?" ### Step 3: Ask About Data Features @@ -66,13 +89,14 @@ Show a clear specification with **YOUR ASSUMPTIONS surfaced**: ### Pre-Generation Checklist -- [ ] User confirmed compute preference (serverless recommended) +- [ ] **Catalog confirmed** - User explicitly approved which catalog to use +- [ ] Output location shown prominently in plan (easy to spot/change) - [ ] Table specification shown and approved - [ ] Assumptions about distributions confirmed -- [ ] Output location confirmed (catalog.schema) +- [ ] User confirmed compute preference (serverless recommended) - [ ] Data features selected -**Do NOT proceed to code generation until user approves the plan.** +**Do NOT proceed to code generation until user approves the plan, including the catalog.** ## Quick Start: Spark + Faker + Pandas UDFs diff --git a/databricks-skills/synthetic-data-generation/references/1-setup-and-execution.md b/databricks-skills/synthetic-data-generation/references/1-setup-and-execution.md index a05eeadf..97c8387e 100644 --- a/databricks-skills/synthetic-data-generation/references/1-setup-and-execution.md +++ b/databricks-skills/synthetic-data-generation/references/1-setup-and-execution.md @@ -77,27 +77,49 @@ if db_version and db_version >= (16, 4): `DatabricksEnv()` and `withEnvironment()` are NOT available in older versions. Use serverless jobs with environments parameter instead. -**Install locally:** -```bash -pip install "databricks-connect>=15.1,<16.2" faker numpy pandas holidays -``` +### Serverless Job Configuration Requirements -**Create a serverless job with environment settings:** -```python -# Use create_job MCP tool with: +**MUST use `"client": "4"` in the Environment Spec:** + +```json { - "name": "generate_synthetic_data", - "tasks": [{ "environment_key": "datagen_env", ... }], "environments": [{ "environment_key": "datagen_env", "spec": { "client": "4", - "dependencies": ["faker", "numpy", "pandas", "holidays"] + "dependencies": ["faker", "numpy", "pandas"] } }] } ``` +> **Note:** Using `"client": "1"` will fail with environment configuration errors. + +### Script Deployment: Must Be FILE Type (Not Notebook) + +When deploying Python scripts for serverless jobs, the script MUST be imported as a FILE type. Using `--language PYTHON` creates a NOTEBOOK, which fails with `spark_python_task`. + +```bash +# Correct - imports as FILE type +databricks workspace import /Users/@databricks.com/scripts/my_script.py \ + --file ./my_script.py --format AUTO + +# Verify it's FILE type (not NOTEBOOK) +databricks workspace list /Users/@databricks.com/scripts/ +# Should show: FILE (not NOTEBOOK) +``` + +**Job config must reference the workspace path:** + +```json +{ + "spark_python_task": { + "python_file": "/Users/@databricks.com/scripts/my_script.py" + }, + "environment_key": "datagen_env" +} +``` + **DABs bundle configuration:** ```yaml # databricks.yml @@ -125,23 +147,81 @@ environments: - holidays ``` -## Option 3: Classic Cluster (Fallback Only) +## Option 3: Classic Cluster -**Use only when:** Serverless unavailable, or specific cluster features needed (GPUs, custom init scripts) +**Use when:** Serverless unavailable, or specific cluster features needed (GPUs, custom init scripts) -**Warning:** Classic clusters take 3-8 minutes to start. Always prefer serverless. +### Step 1: Check Python Version Compatibility -**Install dependencies in cluster:** -```python -# In notebook or using execute_databricks_command tool: -%pip install faker numpy pandas holidays +Pandas UDFs require matching Python minor versions between local and cluster. + +```bash +# Check local Python +python --version + +# Check cluster DBR version → Python version +# DBR 17.x = Python 3.12 +# DBR 15.4 LTS = Python 3.11 +# DBR 14.3 LTS = Python 3.10 +databricks clusters get | grep spark_version +``` + +### Step 2a: If Versions Match → Use Databricks Connect + +```bash +# Install matching databricks-connect version (must match DBR major.minor) +uv pip install "databricks-connect==17.3.*" faker numpy pandas holidays +``` + +```bash +# Install libraries on cluster +databricks libraries install --json '{ + "cluster_id": "", + "libraries": [{"pypi": {"package": "faker"}}] +}' + +# Wait for INSTALLED status +databricks libraries cluster-status ``` -**Connect from local script:** ```python +# Run locally via Databricks Connect from databricks.connect import DatabricksSession -spark = DatabricksSession.builder.clusterId("your-cluster-id").getOrCreate() +spark = DatabricksSession.builder.clusterId("").getOrCreate() +# Your Spark code runs on the cluster +``` + +### Step 2b: If Versions Don't Match → Submit as Job + +**Ask user for approval before submitting.** Example prompt: +> "Your local Python (3.11) doesn't match the cluster (3.12). Pandas UDFs require matching versions. Should I submit this as a job to run directly on the cluster instead?" + +```bash +# Import notebook to workspace +databricks workspace import /Users/you@company.com/my_notebook \ + --file script.py --language PYTHON --overwrite + +# Submit job to run on cluster +databricks jobs submit --json '{ + "run_name": "Generate Data", + "tasks": [{ + "task_key": "generate", + "existing_cluster_id": "", + "notebook_task": { + "notebook_path": "/Users/you@company.com/my_notebook" + } + }] +}' +``` + +### Classic Cluster Decision Flow + +``` +Local Python == Cluster Python? + ├─ YES → Install libs on cluster, run via Databricks Connect + └─ NO → Ask user: "Submit as job instead?" + └─ Import notebook + submit job ``` ## Required Libraries @@ -198,4 +278,4 @@ else: | `ModuleNotFoundError: faker` | Install dependencies per execution mode above | | `DatabricksEnv not found` | Upgrade to databricks-connect >= 16.4 or use job with environments | | `serverless_compute_id` error | Add `serverless_compute_id = auto` to ~/.databrickscfg | -| Classic cluster startup slow | Use serverless instead (instant start) | +| Classic cluster startup slow | Use serverless instead (instant start) | \ No newline at end of file From 0b9c9b306a28c074c559b08fd7a6ff0c41342cd1 Mon Sep 17 00:00:00 2001 From: dustinvannoy-db Date: Wed, 25 Feb 2026 10:31:50 -0800 Subject: [PATCH 11/24] Update classic library install --- databricks-skills/synthetic-data-generation/SKILL.md | 2 +- .../references/1-setup-and-execution.md | 6 +----- .../references/6-troubleshooting.md | 6 +++--- .../scripts/generate_synthetic_data.py | 6 +++--- 4 files changed, 8 insertions(+), 12 deletions(-) diff --git a/databricks-skills/synthetic-data-generation/SKILL.md b/databricks-skills/synthetic-data-generation/SKILL.md index 4b71e4f6..704b65c0 100644 --- a/databricks-skills/synthetic-data-generation/SKILL.md +++ b/databricks-skills/synthetic-data-generation/SKILL.md @@ -190,7 +190,7 @@ spark.sql(f"CREATE VOLUME IF NOT EXISTS {CATALOG}.{SCHEMA}.raw_data") |------|----------|-------| | **DB Connect 16.4+ Serverless** | Local dev, Python 3.12+ | `DatabricksEnv().withDependencies(...)` | | **Serverless Job** | Production, scheduled | Job with `environments` parameter | -| **Classic Cluster** | Fallback only | Manual `%pip install` | +| **Classic Cluster** | Fallback only | Use Databricks CLI to install libraries. `databricks libraries install --json '{"cluster_id": "", "libraries": [{"pypi": {"package": "faker"}}, {"pypi": {"package": "holidays"}}]}'` | See [references/1-setup-and-execution.md](references/1-setup-and-execution.md) for detailed setup instructions. diff --git a/databricks-skills/synthetic-data-generation/references/1-setup-and-execution.md b/databricks-skills/synthetic-data-generation/references/1-setup-and-execution.md index 97c8387e..6aeed5bb 100644 --- a/databricks-skills/synthetic-data-generation/references/1-setup-and-execution.md +++ b/databricks-skills/synthetic-data-generation/references/1-setup-and-execution.md @@ -8,7 +8,6 @@ This guide covers all execution modes for synthetic data generation, organized b |------------------|---------------------| | Python 3.12+ with databricks-connect >= 16.4 | DatabricksEnv with withDependencies API | | Python 3.10/3.11 with older databricks-connect | Serverless job with environments parameter | -| Running on Databricks Runtime (notebook/job) | Dependencies pre-installed or %pip install | | Classic compute (fallback only) | Manual cluster setup | ## Option 1: Databricks Connect 16.4+ with Serverless (Recommended) @@ -175,10 +174,7 @@ uv pip install "databricks-connect==17.3.*" faker numpy pandas holidays ```bash # Install libraries on cluster -databricks libraries install --json '{ - "cluster_id": "", - "libraries": [{"pypi": {"package": "faker"}}] -}' +`databricks libraries install --json '{"cluster_id": "", "libraries": [{"pypi": {"package": "faker"}}, {"pypi": {"package": "holidays"}}]}'` # Wait for INSTALLED status databricks libraries cluster-status diff --git a/databricks-skills/synthetic-data-generation/references/6-troubleshooting.md b/databricks-skills/synthetic-data-generation/references/6-troubleshooting.md index a40efc38..92d86a6d 100644 --- a/databricks-skills/synthetic-data-generation/references/6-troubleshooting.md +++ b/databricks-skills/synthetic-data-generation/references/6-troubleshooting.md @@ -13,9 +13,9 @@ Common issues and solutions for synthetic data generation. | Mode | Solution | |------|----------| | **DB Connect 16.4+** | Use `DatabricksEnv().withDependencies("faker", "pandas", ...)` | -| **Older DB Connect** | Create job with `environments` parameter | -| **Databricks Runtime** | Run `%pip install faker numpy pandas holidays` | -| **Classic cluster** | Add to cluster libraries or init script | +| **Older DB Connect with Serverless** | Create job with `environments` parameter | +| **Databricks Runtime** | Use Databricks CLI to install `faker holidays` | +| **Classic cluster** | Use Databricks CLI to install libraries. `databricks libraries install --json '{"cluster_id": "", "libraries": [{"pypi": {"package": "faker"}}, {"pypi": {"package": "holidays"}}]}'` | ```python # For DB Connect 16.4+ diff --git a/databricks-skills/synthetic-data-generation/scripts/generate_synthetic_data.py b/databricks-skills/synthetic-data-generation/scripts/generate_synthetic_data.py index f1011a29..102b268a 100644 --- a/databricks-skills/synthetic-data-generation/scripts/generate_synthetic_data.py +++ b/databricks-skills/synthetic-data-generation/scripts/generate_synthetic_data.py @@ -110,7 +110,7 @@ def get_databricks_connect_version(): if not CLUSTER_ID: raise ValueError("CLUSTER_ID must be set when USE_SERVERLESS=False") spark = DatabricksSession.builder.withEnvironment(env).clusterId(CLUSTER_ID).getOrCreate() - print(f"Connected to cluster {CLUSTER_ID} with managed dependencies!") + print(f"Connected to cluster with managed dependencies!") else: print("Using standard session (dependencies must be pre-installed)") print("=" * 80) @@ -139,7 +139,7 @@ def get_databricks_connect_version(): print("=" * 80) print(f"Missing: {', '.join(missing_deps)}") if on_runtime: - print("\nSolution: Run %pip install faker pandas numpy holidays") + print("\nSolution: Run install using Databricks CLI: Use Databricks CLI to install libraries: databricks libraries install --json '{"cluster_id": "", "libraries": [{"pypi": {"package": "faker"}}, {"pypi": {"package": "holidays"}}]}'") else: print("\nSolution: Upgrade to databricks-connect >= 16.4 for managed deps") print(" Or create a job with environment settings") @@ -158,7 +158,7 @@ def get_databricks_connect_version(): if not CLUSTER_ID: raise ValueError("CLUSTER_ID must be set when USE_SERVERLESS=False") spark = DatabricksSession.builder.clusterId(CLUSTER_ID).getOrCreate() - print(f"Connected to cluster {CLUSTER_ID}") + print(f"Connected to cluster ") # Import Faker for UDF definitions from faker import Faker From d177f62c01b99bce9870e771d64b6f0886c319cd Mon Sep 17 00:00:00 2001 From: dustinvannoy-db Date: Wed, 25 Feb 2026 11:11:52 -0800 Subject: [PATCH 12/24] Suggest uv and improve python task job payload --- .../synthetic-data-generation/SKILL.md | 14 +++++++++++ .../references/1-setup-and-execution.md | 25 ++++++++++--------- .../references/5-output-formats.md | 2 +- .../references/6-troubleshooting.md | 5 ++-- .../scripts/generate_synthetic_data.py | 2 +- 5 files changed, 32 insertions(+), 16 deletions(-) diff --git a/databricks-skills/synthetic-data-generation/SKILL.md b/databricks-skills/synthetic-data-generation/SKILL.md index 704b65c0..e246458f 100644 --- a/databricks-skills/synthetic-data-generation/SKILL.md +++ b/databricks-skills/synthetic-data-generation/SKILL.md @@ -19,6 +19,20 @@ Generate realistic, story-driven synthetic data for Databricks using **Spark + F | **Troubleshooting** | [references/6-troubleshooting.md](references/6-troubleshooting.md) | Fixing errors, debugging issues | | **Example Script** | [scripts/generate_synthetic_data.py](scripts/generate_synthetic_data.py) | Complete Spark + Pandas UDF example | +## Package Manager + +Prefer `uv` for all Python operations. Fall back to `pip` only if `uv` is not available. + +```bash +# Preferred +uv pip install "databricks-connect>=16.4,<17.0" faker numpy pandas holidays +uv run python generate_data.py + +# Fallback if uv not available +pip install "databricks-connect>=16.4,<17.0" faker numpy pandas holidays +python generate_data.py +``` + ## Critical Rules 1. **Always use Spark + Faker + Pandas UDFs** for data generation (scalable, parallel) diff --git a/databricks-skills/synthetic-data-generation/references/1-setup-and-execution.md b/databricks-skills/synthetic-data-generation/references/1-setup-and-execution.md index 6aeed5bb..f63062f9 100644 --- a/databricks-skills/synthetic-data-generation/references/1-setup-and-execution.md +++ b/databricks-skills/synthetic-data-generation/references/1-setup-and-execution.md @@ -16,6 +16,10 @@ This guide covers all execution modes for synthetic data generation, organized b **Install locally:** ```bash +# Preferred +uv pip install "databricks-connect>=16.4,<17.0" faker numpy pandas holidays + +# Fallback if uv not available pip install "databricks-connect>=16.4,<17.0" faker numpy pandas holidays ``` @@ -94,18 +98,15 @@ if db_version and db_version >= (16, 4): > **Note:** Using `"client": "1"` will fail with environment configuration errors. -### Script Deployment: Must Be FILE Type (Not Notebook) +### Script Deployment -When deploying Python scripts for serverless jobs, the script MUST be imported as a FILE type. Using `--language PYTHON` creates a NOTEBOOK, which fails with `spark_python_task`. +Deploy Python files (.py) to the workspace for serverless jobs: ```bash -# Correct - imports as FILE type databricks workspace import /Users/@databricks.com/scripts/my_script.py \ --file ./my_script.py --format AUTO -# Verify it's FILE type (not NOTEBOOK) databricks workspace list /Users/@databricks.com/scripts/ -# Should show: FILE (not NOTEBOOK) ``` **Job config must reference the workspace path:** @@ -156,7 +157,7 @@ Pandas UDFs require matching Python minor versions between local and cluster. ```bash # Check local Python -python --version +uv run python --version # or: python --version # Check cluster DBR version → Python version # DBR 17.x = Python 3.12 @@ -194,9 +195,9 @@ spark = DatabricksSession.builder.clusterId("").getOrCreate() > "Your local Python (3.11) doesn't match the cluster (3.12). Pandas UDFs require matching versions. Should I submit this as a job to run directly on the cluster instead?" ```bash -# Import notebook to workspace -databricks workspace import /Users/you@company.com/my_notebook \ - --file script.py --language PYTHON --overwrite +# Upload script to workspace +databricks workspace import /Users/you@company.com/scripts/generate_data.py \ + --file generate_data.py --format AUTO --overwrite # Submit job to run on cluster databricks jobs submit --json '{ @@ -204,8 +205,8 @@ databricks jobs submit --json '{ "tasks": [{ "task_key": "generate", "existing_cluster_id": "", - "notebook_task": { - "notebook_path": "/Users/you@company.com/my_notebook" + "spark_python_task": { + "python_file": "/Users/you@company.com/scripts/generate_data.py" } }] }' @@ -217,7 +218,7 @@ databricks jobs submit --json '{ Local Python == Cluster Python? ├─ YES → Install libs on cluster, run via Databricks Connect └─ NO → Ask user: "Submit as job instead?" - └─ Import notebook + submit job + └─ Upload script + submit job ``` ## Required Libraries diff --git a/databricks-skills/synthetic-data-generation/references/5-output-formats.md b/databricks-skills/synthetic-data-generation/references/5-output-formats.md index e018efcf..49214e91 100644 --- a/databricks-skills/synthetic-data-generation/references/5-output-formats.md +++ b/databricks-skills/synthetic-data-generation/references/5-output-formats.md @@ -134,7 +134,7 @@ customers_df.write \ **When to use:** - User wants data ready to query immediately - Skip the SDP bronze/silver/gold pipeline -- Direct notebook or SQL analytics +- Direct SQL analytics --- diff --git a/databricks-skills/synthetic-data-generation/references/6-troubleshooting.md b/databricks-skills/synthetic-data-generation/references/6-troubleshooting.md index 92d86a6d..42961e1d 100644 --- a/databricks-skills/synthetic-data-generation/references/6-troubleshooting.md +++ b/databricks-skills/synthetic-data-generation/references/6-troubleshooting.md @@ -32,8 +32,9 @@ spark = DatabricksSession.builder.withEnvironment(env).serverless(True).getOrCre **Solution:** Upgrade to 16.4+ or use job-based approach: ```bash -# Upgrade -pip install "databricks-connect>=16.4,<17.0" +# Upgrade (prefer uv, fall back to pip) +uv pip install "databricks-connect>=16.4,<17.0" +# or: pip install "databricks-connect>=16.4,<17.0" # Or use job with environments parameter instead ``` diff --git a/databricks-skills/synthetic-data-generation/scripts/generate_synthetic_data.py b/databricks-skills/synthetic-data-generation/scripts/generate_synthetic_data.py index 102b268a..d500b15e 100644 --- a/databricks-skills/synthetic-data-generation/scripts/generate_synthetic_data.py +++ b/databricks-skills/synthetic-data-generation/scripts/generate_synthetic_data.py @@ -65,7 +65,7 @@ # ============================================================================= def is_databricks_runtime(): - """Check if running on Databricks Runtime (notebook/job) vs locally.""" + """Check if running on Databricks Runtime vs locally.""" return "DATABRICKS_RUNTIME_VERSION" in os.environ def get_databricks_connect_version(): From 84ae64f18aeb1c017fa5cbf39c496146403e28de Mon Sep 17 00:00:00 2001 From: dustinvannoy-db Date: Thu, 26 Feb 2026 16:19:30 -0800 Subject: [PATCH 13/24] Add new data gen tests (first 3) --- .../databricks-data-generation/baseline.yaml | 18 + .../candidates.yaml | 7 + .../ground_truth.yaml | 544 ++++++++++++++++++ .../databricks-data-generation/manifest.yaml | 45 ++ 4 files changed, 614 insertions(+) create mode 100644 .test/baselines/databricks-data-generation/baseline.yaml create mode 100644 .test/skills/databricks-data-generation/candidates.yaml create mode 100644 .test/skills/databricks-data-generation/ground_truth.yaml create mode 100644 .test/skills/databricks-data-generation/manifest.yaml diff --git a/.test/baselines/databricks-data-generation/baseline.yaml b/.test/baselines/databricks-data-generation/baseline.yaml new file mode 100644 index 00000000..9735cae9 --- /dev/null +++ b/.test/baselines/databricks-data-generation/baseline.yaml @@ -0,0 +1,18 @@ +run_id: '20260226_161521' +created_at: '2026-02-26T16:15:21.384113' +skill_name: databricks-data-generation +metrics: + pass_rate: 1.0 + total_tests: 3 + passed_tests: 3 + failed_tests: 0 +test_results: +- id: sdg_dbconnect_small_parquet_exec_001 + passed: true + execution_mode: local +- id: sdg_serverless_job_catalog_json_009 + passed: true + execution_mode: local +- id: sdg_dbconnect_large_delta_exec_002 + passed: true + execution_mode: local diff --git a/.test/skills/databricks-data-generation/candidates.yaml b/.test/skills/databricks-data-generation/candidates.yaml new file mode 100644 index 00000000..fda8a58b --- /dev/null +++ b/.test/skills/databricks-data-generation/candidates.yaml @@ -0,0 +1,7 @@ +# Candidates for databricks-data-generation skill +# Test cases pending review before promotion to ground_truth.yaml +# +# Use `/skill-test databricks-data-generation add` to create new candidates +# Use `/skill-test databricks-data-generation review` to promote candidates to ground truth + +candidates: [] diff --git a/.test/skills/databricks-data-generation/ground_truth.yaml b/.test/skills/databricks-data-generation/ground_truth.yaml new file mode 100644 index 00000000..42679262 --- /dev/null +++ b/.test/skills/databricks-data-generation/ground_truth.yaml @@ -0,0 +1,544 @@ +test_cases: + # Test 1: Databricks Connect + Small + Parquet + 2 tables with referential integrity + # - id: "sdg_dbconnect_small_parquet_001" + # inputs: + # prompt: | + # Generate synthetic e-commerce data using Databricks Connect with serverless compute. + # Create 2 related tables with referential integrity: + # - customers (5,000 rows): customer_id, name, email, tier (Free/Pro/Enterprise weighted 60/30/10), region, created_at + # - orders (15,000 rows): order_id, customer_id (FK to customers), amount (log-normal by tier), order_date, status + + # Save as Parquet files to a Unity Catalog volume. Use schema name 'sdg_test_small_parquet'. + # Enterprise customers should generate more orders than Free tier (weighted sampling). + # expectations: + # expected_facts: + # - "DatabricksSession" + # - "serverless" + # - "parquet" + # - "customer_id" + # - "referential integrity" + # - "weighted" + # - "log-normal" + # expected_patterns: + # - pattern: "DatabricksSession\\.builder.*serverless.*True" + # min_count: 1 + # description: "Databricks Connect serverless configuration" + # - pattern: "\\.write.*parquet" + # min_count: 1 + # description: "Parquet output format" + # - pattern: "customer_id" + # min_count: 3 + # description: "Foreign key reference in multiple tables" + # - pattern: "lognormal|log-normal|log_normal" + # min_count: 1 + # description: "Log-normal distribution for amounts" + # guidelines: + # - "Must use DatabricksSession.builder.serverless(True).getOrCreate()" + # - "Orders table customer_id must only contain IDs from customers table" + # - "Enterprise tier customers must have higher weight for order generation" + # - "Amount distribution must use log-normal, not uniform" + # metadata: + # category: "happy_path" + # difficulty: "easy" + # source: "manual" + # tags: ["databricks-connect", "small", "parquet", "referential-integrity"] + + # # Test 2: Databricks Connect + Large + Delta + 2 tables + # - id: "sdg_dbconnect_large_delta_002" + # inputs: + # prompt: | + # Generate large-scale synthetic support ticket data using Databricks Connect with serverless. + # Create 2 related tables with referential integrity: + # - customers (100,000 rows): customer_id, company_name, tier, arr (log-normal), region, signup_date + # - tickets (500,000 rows): ticket_id, customer_id (FK), priority (correlates with tier), resolution_hours (exponential), csat_score, created_at + + # Save as Delta tables registered in Unity Catalog. Use schema name 'sdg_test_large_delta'. + # Use Spark + Faker + Pandas UDFs with appropriate partitioning for this scale. + # Priority should correlate with tier (Enterprise gets more Critical/High priorities). + # expectations: + # expected_facts: + # - "pandas_udf" + # - "DatabricksSession" + # - "Delta" + # - "saveAsTable" + # - "customer_id" + # - "priority" + # - "exponential" + # expected_patterns: + # - pattern: "@F\\.pandas_udf|pandas_udf" + # min_count: 1 + # description: "Pandas UDF for Faker parallelism" + # - pattern: "saveAsTable" + # min_count: 2 + # description: "Delta table registration" + # - pattern: "numPartitions.*=.*\\d+" + # min_count: 1 + # description: "Partitioned generation for scale" + # - pattern: "exponential" + # min_count: 1 + # description: "Exponential distribution for resolution times" + # guidelines: + # - "Must use Spark + Faker + Pandas UDFs for scalable generation" + # - "Tickets must reference valid customer_ids from customers table" + # - "Priority distribution must vary by customer tier" + # - "Resolution hours must use exponential distribution" + # metadata: + # category: "happy_path" + # difficulty: "medium" + # source: "manual" + # tags: ["databricks-connect", "large", "delta", "pandas-udf", "referential-integrity"] + + # # Test 3: Serverless Job + Small + JSON + 2 tables + # - id: "sdg_serverless_job_small_json_003" + # inputs: + # prompt: | + # Generate synthetic product catalog data that will run as a serverless Databricks job. + # Create 2 related tables with referential integrity: + # - products (3,000 rows): product_id, name, category (weighted), price (log-normal), inventory_count + # - sales (10,000 rows): sale_id, product_id (FK to products), quantity, sale_date, discount_pct + + # Save as JSON files to a Unity Catalog volume. Use schema name 'sdg_test_small_json'. + # Create a job definition with environments for dependencies (faker). + # Popular product categories should have more sales (weighted sampling). + # expectations: + # expected_facts: + # - "serverless" + # - "environments" + # - "dependencies" + # - "json" + # - "product_id" + # - "weighted" + # expected_patterns: + # - pattern: "environments.*spec.*dependencies" + # min_count: 1 + # description: "Serverless job environment configuration" + # - pattern: "\\.write.*json" + # min_count: 1 + # description: "JSON output format" + # - pattern: "product_id" + # min_count: 3 + # description: "Foreign key reference in multiple tables" + # - pattern: "create_job|run_job_now" + # min_count: 1 + # description: "Job creation or execution" + # guidelines: + # - "Must create serverless job with environments parameter for dependencies" + # - "Sales table product_id must only reference valid products" + # - "Product categories must be weighted (not uniform)" + # - "Job spec must include spec.client and spec.dependencies" + # metadata: + # category: "happy_path" + # difficulty: "medium" + # source: "manual" + # tags: ["serverless-job", "small", "json", "referential-integrity"] + + # # Test 4: Serverless Job + Large + CSV + 2 tables + # - id: "sdg_serverless_job_large_csv_004" + # inputs: + # prompt: | + # Generate large-scale synthetic financial transaction data as a serverless Databricks job. + # Create 2 related tables with referential integrity: + # - users (200,000 rows): user_id, username, account_type (Basic/Premium/VIP weighted 70/25/5), country, created_at + # - transactions (1,000,000 rows): txn_id, user_id (FK to users), amount (log-normal varies by account_type), txn_type, timestamp + + # Save as CSV files with headers to a Unity Catalog volume. Use schema name 'sdg_test_large_csv'. + # Use Spark + Faker + Pandas UDFs with high partition count for this scale. + # VIP users should have larger transaction amounts. + # Create the job with proper environments configuration. + # expectations: + # expected_facts: + # - "pandas_udf" + # - "serverless" + # - "CSV" + # - "header" + # - "user_id" + # - "environments" + # - "log-normal" + # expected_patterns: + # - pattern: "@F\\.pandas_udf|pandas_udf" + # min_count: 1 + # description: "Pandas UDF for Faker parallelism" + # - pattern: "\\.csv.*header.*true|\\.option.*header.*true.*\\.csv" + # min_count: 1 + # description: "CSV with headers" + # - pattern: "environments.*dependencies" + # min_count: 1 + # description: "Job environment configuration" + # - pattern: "numPartitions.*=.*\\d{2,}" + # min_count: 1 + # description: "High partition count for 1M rows" + # guidelines: + # - "Must use Spark + Faker + Pandas UDFs for million-row generation" + # - "Transactions must reference valid user_ids" + # - "Transaction amounts must scale with account_type (VIP > Premium > Basic)" + # - "CSV output must include header row" + # metadata: + # category: "happy_path" + # difficulty: "hard" + # source: "manual" + # tags: ["serverless-job", "large", "csv", "pandas-udf", "referential-integrity"] + + # # Test 5: Classic Cluster + Small + Delta + 2 tables + # - id: "sdg_classic_small_delta_005" + # inputs: + # prompt: | + # Generate synthetic HR data to run on a classic Databricks cluster. + # Create 2 related tables with referential integrity: + # - employees (2,000 rows): employee_id, name, department (weighted), hire_date, salary (log-normal by dept) + # - projects (5,000 rows): project_id, employee_id (FK to employees), project_name, hours_logged, status + + # Save as Delta tables in Unity Catalog. Use schema name 'sdg_test_classic_delta'. + # First install dependencies with pip, then run the Python script. + # Engineering department should have higher salaries than other departments. + # expectations: + # expected_facts: + # - "classic cluster" + # - "pip install" + # - "Delta" + # - "saveAsTable" + # - "employee_id" + # - "context_id" + # expected_patterns: + # - pattern: "%pip install|pip install" + # min_count: 1 + # description: "Dependency installation on classic cluster" + # - pattern: "execute_databricks_command|run_python_file_on_databricks" + # min_count: 1 + # description: "Classic cluster execution tools" + # - pattern: "saveAsTable" + # min_count: 2 + # description: "Delta table output" + # - pattern: "context_id" + # min_count: 1 + # description: "Context reuse for pip + script execution" + # guidelines: + # - "Must use execute_databricks_command for pip install" + # - "Must reuse context_id between pip install and script execution" + # - "Projects must reference valid employee_ids" + # - "Salary must vary by department using log-normal distribution" + # metadata: + # category: "happy_path" + # difficulty: "medium" + # source: "manual" + # tags: ["classic-cluster", "small", "delta", "referential-integrity"] + + # # Test 6: Classic Cluster + Large + Parquet + 2 tables + # - id: "sdg_classic_large_parquet_006" + # inputs: + # prompt: | + # Generate large-scale synthetic supply chain data to run on a classic Databricks cluster. + # Create 2 related tables with referential integrity: + # - suppliers (50,000 rows): supplier_id, company_name, country, rating (1-5 weighted toward 3-4), active_since + # - inventory (300,000 rows): inventory_id, supplier_id (FK to suppliers), product_sku, quantity (Pareto), unit_cost, last_restock_date + + # Save as Parquet files to a Unity Catalog volume. Use schema name 'sdg_test_classic_parquet'. + # Use Spark + Faker + Pandas UDFs with appropriate partitioning. Higher-rated suppliers should have more inventory items. + # Install dependencies first, then execute the script with context reuse. + # expectations: + # expected_facts: + # - "classic cluster" + # - "pandas_udf" + # - "parquet" + # - "supplier_id" + # - "Pareto" + # - "pip install" + # expected_patterns: + # - pattern: "@F\\.pandas_udf|pandas_udf" + # min_count: 1 + # description: "Pandas UDF for Faker parallelism" + # - pattern: "\\.write.*parquet" + # min_count: 1 + # description: "Parquet output" + # - pattern: "supplier_id" + # min_count: 3 + # description: "Foreign key across tables" + # - pattern: "pareto|power.*law" + # min_count: 1 + # description: "Pareto distribution for quantities" + # guidelines: + # - "Must use Spark + Faker + Pandas UDFs for large-scale generation" + # - "Must install dependencies and reuse context" + # - "Inventory must reference valid supplier_ids" + # - "Quantity must use Pareto/power-law distribution" + # metadata: + # category: "happy_path" + # difficulty: "hard" + # source: "manual" + # tags: ["classic-cluster", "large", "parquet", "pandas-udf", "referential-integrity"] + + # # Test 7: Databricks Connect + Medium + CSV + 3 tables (complex referential integrity) + # - id: "sdg_dbconnect_medium_csv_3tables_007" + # inputs: + # prompt: | + # Generate synthetic retail order data using Databricks Connect with serverless. + # Create 3 related tables with full referential integrity: + # - customers (10,000 rows): customer_id, name, email, membership_level (Bronze/Silver/Gold/Platinum weighted 50/30/15/5), region + # - orders (50,000 rows): order_id, customer_id (FK to customers), order_date, total_amount, status + # - line_items (150,000 rows): line_item_id, order_id (FK to orders), product_name, quantity, unit_price + + # Save as CSV files with headers to Unity Catalog volume. Use schema name 'sdg_test_medium_csv'. + # Use Faker with Spark UDFs for realistic product names. + # Higher membership levels should have more orders. Order total_amount should equal sum of line_items. + # expectations: + # expected_facts: + # - "DatabricksSession" + # - "serverless" + # - "CSV" + # - "customer_id" + # - "order_id" + # - "line_item" + # - "Faker" + # - "UDF" + # expected_patterns: + # - pattern: "DatabricksSession.*serverless" + # min_count: 1 + # description: "Databricks Connect configuration" + # - pattern: "@F\\.udf|@udf" + # min_count: 1 + # description: "Spark UDF for Faker" + # - pattern: "customer_id" + # min_count: 4 + # description: "FK in customers and orders" + # - pattern: "order_id" + # min_count: 4 + # description: "FK in orders and line_items" + # - pattern: "\\.csv.*header" + # min_count: 1 + # description: "CSV with headers" + # guidelines: + # - "Must maintain referential integrity across all 3 tables" + # - "line_items.order_id must reference valid orders" + # - "orders.customer_id must reference valid customers" + # - "Should use Faker UDFs for realistic product names" + # - "Membership level should weight order distribution" + # metadata: + # category: "happy_path" + # difficulty: "hard" + # source: "manual" + # tags: ["databricks-connect", "medium", "csv", "faker-udf", "3-tables", "referential-integrity"] + + # # Test 8: Serverless Job + Medium + JSON + 3 tables (CRM data) + # - id: "sdg_serverless_job_medium_json_3tables_008" + # inputs: + # prompt: | + # Generate synthetic CRM data as a serverless Databricks job. + # Create 3 related tables with referential integrity: + # - accounts (8,000 rows): account_id, company_name, industry (weighted), annual_revenue (log-normal), tier (SMB/Mid-Market/Enterprise) + # - contacts (25,000 rows): contact_id, account_id (FK to accounts), first_name, last_name, email, title, is_primary + # - activities (80,000 rows): activity_id, contact_id (FK to contacts), activity_type (Call/Email/Meeting weighted), activity_date, duration_minutes (exponential), notes + + # Save as JSON files to Unity Catalog volume. Use schema name 'sdg_test_medium_json'. + # Create job with environments for faker and holidays dependencies. + # Enterprise accounts should have more contacts. Use realistic time patterns (weekday bias, business hours). + # expectations: + # expected_facts: + # - "serverless" + # - "environments" + # - "JSON" + # - "account_id" + # - "contact_id" + # - "activity" + # - "weekday" + # - "exponential" + # expected_patterns: + # - pattern: "environments.*dependencies" + # min_count: 1 + # description: "Serverless job environment" + # - pattern: "\\.write.*json" + # min_count: 1 + # description: "JSON output" + # - pattern: "account_id" + # min_count: 4 + # description: "FK across tables" + # - pattern: "contact_id" + # min_count: 4 + # description: "FK in contacts and activities" + # - pattern: "weekday|weekend|business.*hours" + # min_count: 1 + # description: "Time-based patterns" + # guidelines: + # - "Must create job with environments specifying faker and holidays" + # - "contacts.account_id must reference valid accounts" + # - "activities.contact_id must reference valid contacts" + # - "Activity dates should show weekday bias (more on Mon-Fri)" + # - "Duration should use exponential distribution" + # metadata: + # category: "happy_path" + # difficulty: "hard" + # source: "manual" + # tags: ["serverless-job", "medium", "json", "3-tables", "time-patterns", "referential-integrity"] + + # ============================================================================= + # EXECUTED TEST CASES (with verified outputs) + # ============================================================================= + + # Test 9: Databricks Connect + Small + Parquet + 2 tables (EXECUTED) + - id: "sdg_dbconnect_small_parquet_exec_001" + inputs: + prompt: | + Generate synthetic e-commerce data locally then save it to Unity Catalog. + Create 2 related tables with referential integrity: + - customers (5,000 rows): customer_id, name, email, tier (Free/Pro/Enterprise weighted 60/30/10), region, created_at + - orders (15,000 rows): order_id, customer_id (FK to customers), amount, order_date, status + + Save as Parquet files to a Unity Catalog volume. Use schema name 'devkit_gen1_test_small_parquet'. + Enterprise customers should generate more orders than Free tier. + expectations: + expected_facts: + - "DatabricksSession" + - "serverless" + - "parquet" + - "customer_id" + - "referential integrity" + - "weighted" + - "log-normal" + - "pandas_udf" + expected_patterns: + - pattern: "DatabricksSession.*serverless.*True" + min_count: 1 + description: "Databricks Connect serverless configuration" + - pattern: "\\.write.*parquet" + min_count: 1 + description: "Parquet output format" + - pattern: "customer_id" + min_count: 3 + description: "Foreign key reference in multiple tables" + - pattern: "lognormal" + min_count: 1 + description: "Log-normal distribution for amounts" + - pattern: "@F\\.pandas_udf|pandas_udf" + min_count: 1 + description: "Pandas UDF for Faker parallelism" + guidelines: + - "Must use DatabricksSession.builder.serverless(True).getOrCreate()" + - "Orders table customer_id must only contain IDs from customers table" + - "Enterprise tier customers must have higher weight for order generation" + - "Amount distribution must use log-normal, not uniform" + - "Must use Spark + Faker + Pandas UDFs approach" + metadata: + category: "happy_path" + difficulty: "easy" + source: "interactive_execution" + execution_date: "2025-02-25" + execution_verified: true + tags: ["databricks-connect", "small", "parquet", "referential-integrity", "pandas-udf", "executed"] + + # Test 10: Serverless Job + Small + JSON + 2 tables (EXECUTED) + - id: "sdg_serverless_job_catalog_json_009" + inputs: + prompt: | + Generate synthetic product catalog data that will run as a serverless Databricks job. + Create 2 related tables with referential integrity: + - products (3,000 rows): product_id, name, category (weighted), price (log-normal), inventory_count + - sales (10,000 rows): sale_id, product_id (FK to products), quantity, sale_date, discount_pct + + Save as JSON files to a Unity Catalog volume. Use schema name 'devkit_gen3_test_small_json'. + Create a job definition with environments for dependencies (faker). + Popular product categories should have more sales (weighted sampling). + expectations: + expected_facts: + - "serverless" + - "environments" + - "dependencies" + - "client" + - "json" + - "product_id" + - "weighted" + - "lognormal" + - "pandas_udf" + expected_patterns: + - pattern: "environments.*spec.*dependencies" + min_count: 1 + description: "Serverless job environment configuration" + - pattern: '"client":\\s*"4"' + min_count: 1 + description: "Correct client version for serverless" + - pattern: "\\.write.*json" + min_count: 1 + description: "JSON output format" + - pattern: "product_id" + min_count: 3 + description: "Foreign key reference in multiple places" + - pattern: "@F\\.pandas_udf|pandas_udf" + min_count: 1 + description: "Pandas UDF for Faker parallelism" + - pattern: "lognormal|log-normal|log_normal" + min_count: 1 + description: "Log-normal distribution for prices" + - pattern: "CREATE SCHEMA IF NOT EXISTS|CREATE VOLUME IF NOT EXISTS" + min_count: 1 + description: "Infrastructure creation in script" + guidelines: + - "Must create serverless job with environments parameter for dependencies" + - "Job spec must include client: 4 (not 1)" + - "Sales table product_id must only reference valid products (FK integrity)" + - "Product categories must be weighted (not uniform)" + - "Price distribution must use log-normal, not uniform" + - "Script must create schema and volume infrastructure" + - "Must NOT use .cache() or .persist() (serverless incompatible)" + - "Popular categories should have more sales (weighted sampling)" + metadata: + category: "happy_path" + difficulty: "medium" + source: "interactive_execution" + execution_date: "2026-02-26" + execution_verified: true + job_run_id: "560746964795126" + tags: ["serverless-job", "small", "json", "referential-integrity", "weighted-sampling", "executed"] + + # Test 11: Databricks Connect + Large + Delta + 2 tables (EXECUTED) + - id: "sdg_dbconnect_large_delta_exec_002" + inputs: + prompt: | + Generate large-scale support ticket data. + Create 2 related tables with referential integrity: + - customers (100,000 rows): customer_id, company_name, tier, arr (log-normal), region, signup_date + - tickets (500,000 rows): ticket_id, customer_id (FK), priority (correlates with tier), resolution_hours (exponential), csat_score, created_at + + Save as Delta tables registered in Unity Catalog. Use schema name 'devkit_gen2_test_large_delta'. + Priority should correlate with tier (Enterprise gets more Critical/High priorities). + expectations: + expected_facts: + - "pandas_udf" + - "DatabricksSession" + - "Delta" + - "saveAsTable" + - "customer_id" + - "priority" + - "exponential" + - "lognormal" + - "serverless" + expected_patterns: + - pattern: "@F\\.pandas_udf|pandas_udf" + min_count: 1 + description: "Pandas UDF for Faker parallelism" + - pattern: "saveAsTable" + min_count: 2 + description: "Delta table registration" + - pattern: "numPartitions.*=.*\\d+" + min_count: 1 + description: "Partitioned generation for scale" + - pattern: "exponential" + min_count: 1 + description: "Exponential distribution for resolution times" + - pattern: "lognormal" + min_count: 1 + description: "Log-normal distribution for ARR" + - pattern: "DatabricksSession.*serverless.*True" + min_count: 1 + description: "Databricks Connect serverless configuration" + guidelines: + - "Must use DatabricksSession.builder.serverless(True).getOrCreate()" + - "Must use Spark + Faker + Pandas UDFs for scalable generation" + - "Tickets must reference valid customer_ids from customers table" + - "Priority distribution must vary by customer tier" + - "Resolution hours must use exponential distribution" + - "ARR must use log-normal distribution" + - "Must use high partition count (64+) for large-scale generation" + metadata: + category: "happy_path" + difficulty: "medium" + source: "interactive_execution" + execution_date: "2026-02-26" + execution_verified: true + tags: ["databricks-connect", "large", "delta", "pandas-udf", "referential-integrity", "executed"] diff --git a/.test/skills/databricks-data-generation/manifest.yaml b/.test/skills/databricks-data-generation/manifest.yaml new file mode 100644 index 00000000..80b6c21f --- /dev/null +++ b/.test/skills/databricks-data-generation/manifest.yaml @@ -0,0 +1,45 @@ +skill_name: databricks-data-generation +description: Tests for Databricks synthetic data generation skill covering Spark + Faker + Pandas UDFs, execution methods, output formats, and referential integrity +version: 1.0.0 + +scorers: + enabled: + - python_syntax # Check Python code blocks for syntax errors + - no_hallucinated_apis # Detect deprecated/wrong APIs + - pattern_adherence # Regex match against expected patterns + - expected_facts_present # Check if required facts mentioned + + llm_scorers: + - Safety + - guidelines_from_expectations + + default_guidelines: + - "Response must generate complete, runnable Python code" + - "Code must use the execution method specified in the prompt" + - "Code must save data in the output format specified" + - "When generating multiple tables, foreign key columns must use consistent ID formats" + - "Must use non-uniform distributions (log-normal, exponential, weighted) for realistic data" + - "Must include configuration section at top of script with CATALOG, SCHEMA, and size variables" + - "Must create catalog, schema, and volume infrastructure within the Python script" + - "Child tables must reference valid IDs from parent tables for referential integrity" + +quality_gates: + syntax_valid: 1.0 # 100% - all Python syntax must be valid + pattern_adherence: 0.9 # 90% - follow expected patterns + execution_success: 0.8 # 80% - code execution success rate + no_hallucinations: 1.0 # 100% - no deprecated/invalid APIs + +trace_expectations: + tool_limits: + Bash: 10 + Read: 20 + Write: 15 + Edit: 15 + mcp__databricks__run_python_file_on_databricks: 5 + mcp__databricks__execute_databricks_command: 5 + mcp__databricks__create_job: 3 + mcp__databricks__run_job_now: 3 + token_budget: + max_total: 200000 + required_tools: [] + banned_tools: [] From ded1cf2a766a0abf9b4ed18448c4be01472e1442 Mon Sep 17 00:00:00 2001 From: dustinvannoy-db Date: Thu, 26 Feb 2026 22:05:12 -0800 Subject: [PATCH 14/24] Update data gen ground_truth and baseline --- .../databricks-data-generation/baseline.yaml | 32 +- .../ground_truth.yaml | 1036 ++++++++++------- 2 files changed, 662 insertions(+), 406 deletions(-) diff --git a/.test/baselines/databricks-data-generation/baseline.yaml b/.test/baselines/databricks-data-generation/baseline.yaml index 9735cae9..980a16fb 100644 --- a/.test/baselines/databricks-data-generation/baseline.yaml +++ b/.test/baselines/databricks-data-generation/baseline.yaml @@ -1,18 +1,36 @@ -run_id: '20260226_161521' -created_at: '2026-02-26T16:15:21.384113' +run_id: '20260226_220320' +created_at: '2026-02-26T22:03:20.645758' skill_name: databricks-data-generation metrics: pass_rate: 1.0 - total_tests: 3 - passed_tests: 3 + total_tests: 9 + passed_tests: 9 failed_tests: 0 test_results: -- id: sdg_dbconnect_small_parquet_exec_001 +- id: gen_dbconnect_small_parquet_exec_001 passed: true execution_mode: local -- id: sdg_serverless_job_catalog_json_009 +- id: gen_dbconnect_large_delta_exec_002 passed: true execution_mode: local -- id: sdg_dbconnect_large_delta_exec_002 +- id: gen_serverless_job_catalog_json_003 + passed: true + execution_mode: local +- id: gen_serverless_job_large_delta_financial_004 + passed: true + execution_mode: local +- id: gen_classic_small_delta_hr_005 + passed: true + execution_mode: local +- id: gen_classic_large_parquet_supply_chain_006 + passed: true + execution_mode: local +- id: gen_dbconnect_medium_csv_3tables_retail_007 + passed: true + execution_mode: local +- id: gen_serverless_job_medium_json_3tables_crm_008 + passed: true + execution_mode: local +- id: gen_dbconnect_incremental_delta_variant_009 passed: true execution_mode: local diff --git a/.test/skills/databricks-data-generation/ground_truth.yaml b/.test/skills/databricks-data-generation/ground_truth.yaml index 42679262..9094bf4a 100644 --- a/.test/skills/databricks-data-generation/ground_truth.yaml +++ b/.test/skills/databricks-data-generation/ground_truth.yaml @@ -1,379 +1,10 @@ test_cases: - # Test 1: Databricks Connect + Small + Parquet + 2 tables with referential integrity - # - id: "sdg_dbconnect_small_parquet_001" - # inputs: - # prompt: | - # Generate synthetic e-commerce data using Databricks Connect with serverless compute. - # Create 2 related tables with referential integrity: - # - customers (5,000 rows): customer_id, name, email, tier (Free/Pro/Enterprise weighted 60/30/10), region, created_at - # - orders (15,000 rows): order_id, customer_id (FK to customers), amount (log-normal by tier), order_date, status - - # Save as Parquet files to a Unity Catalog volume. Use schema name 'sdg_test_small_parquet'. - # Enterprise customers should generate more orders than Free tier (weighted sampling). - # expectations: - # expected_facts: - # - "DatabricksSession" - # - "serverless" - # - "parquet" - # - "customer_id" - # - "referential integrity" - # - "weighted" - # - "log-normal" - # expected_patterns: - # - pattern: "DatabricksSession\\.builder.*serverless.*True" - # min_count: 1 - # description: "Databricks Connect serverless configuration" - # - pattern: "\\.write.*parquet" - # min_count: 1 - # description: "Parquet output format" - # - pattern: "customer_id" - # min_count: 3 - # description: "Foreign key reference in multiple tables" - # - pattern: "lognormal|log-normal|log_normal" - # min_count: 1 - # description: "Log-normal distribution for amounts" - # guidelines: - # - "Must use DatabricksSession.builder.serverless(True).getOrCreate()" - # - "Orders table customer_id must only contain IDs from customers table" - # - "Enterprise tier customers must have higher weight for order generation" - # - "Amount distribution must use log-normal, not uniform" - # metadata: - # category: "happy_path" - # difficulty: "easy" - # source: "manual" - # tags: ["databricks-connect", "small", "parquet", "referential-integrity"] - - # # Test 2: Databricks Connect + Large + Delta + 2 tables - # - id: "sdg_dbconnect_large_delta_002" - # inputs: - # prompt: | - # Generate large-scale synthetic support ticket data using Databricks Connect with serverless. - # Create 2 related tables with referential integrity: - # - customers (100,000 rows): customer_id, company_name, tier, arr (log-normal), region, signup_date - # - tickets (500,000 rows): ticket_id, customer_id (FK), priority (correlates with tier), resolution_hours (exponential), csat_score, created_at - - # Save as Delta tables registered in Unity Catalog. Use schema name 'sdg_test_large_delta'. - # Use Spark + Faker + Pandas UDFs with appropriate partitioning for this scale. - # Priority should correlate with tier (Enterprise gets more Critical/High priorities). - # expectations: - # expected_facts: - # - "pandas_udf" - # - "DatabricksSession" - # - "Delta" - # - "saveAsTable" - # - "customer_id" - # - "priority" - # - "exponential" - # expected_patterns: - # - pattern: "@F\\.pandas_udf|pandas_udf" - # min_count: 1 - # description: "Pandas UDF for Faker parallelism" - # - pattern: "saveAsTable" - # min_count: 2 - # description: "Delta table registration" - # - pattern: "numPartitions.*=.*\\d+" - # min_count: 1 - # description: "Partitioned generation for scale" - # - pattern: "exponential" - # min_count: 1 - # description: "Exponential distribution for resolution times" - # guidelines: - # - "Must use Spark + Faker + Pandas UDFs for scalable generation" - # - "Tickets must reference valid customer_ids from customers table" - # - "Priority distribution must vary by customer tier" - # - "Resolution hours must use exponential distribution" - # metadata: - # category: "happy_path" - # difficulty: "medium" - # source: "manual" - # tags: ["databricks-connect", "large", "delta", "pandas-udf", "referential-integrity"] - - # # Test 3: Serverless Job + Small + JSON + 2 tables - # - id: "sdg_serverless_job_small_json_003" - # inputs: - # prompt: | - # Generate synthetic product catalog data that will run as a serverless Databricks job. - # Create 2 related tables with referential integrity: - # - products (3,000 rows): product_id, name, category (weighted), price (log-normal), inventory_count - # - sales (10,000 rows): sale_id, product_id (FK to products), quantity, sale_date, discount_pct - - # Save as JSON files to a Unity Catalog volume. Use schema name 'sdg_test_small_json'. - # Create a job definition with environments for dependencies (faker). - # Popular product categories should have more sales (weighted sampling). - # expectations: - # expected_facts: - # - "serverless" - # - "environments" - # - "dependencies" - # - "json" - # - "product_id" - # - "weighted" - # expected_patterns: - # - pattern: "environments.*spec.*dependencies" - # min_count: 1 - # description: "Serverless job environment configuration" - # - pattern: "\\.write.*json" - # min_count: 1 - # description: "JSON output format" - # - pattern: "product_id" - # min_count: 3 - # description: "Foreign key reference in multiple tables" - # - pattern: "create_job|run_job_now" - # min_count: 1 - # description: "Job creation or execution" - # guidelines: - # - "Must create serverless job with environments parameter for dependencies" - # - "Sales table product_id must only reference valid products" - # - "Product categories must be weighted (not uniform)" - # - "Job spec must include spec.client and spec.dependencies" - # metadata: - # category: "happy_path" - # difficulty: "medium" - # source: "manual" - # tags: ["serverless-job", "small", "json", "referential-integrity"] - - # # Test 4: Serverless Job + Large + CSV + 2 tables - # - id: "sdg_serverless_job_large_csv_004" - # inputs: - # prompt: | - # Generate large-scale synthetic financial transaction data as a serverless Databricks job. - # Create 2 related tables with referential integrity: - # - users (200,000 rows): user_id, username, account_type (Basic/Premium/VIP weighted 70/25/5), country, created_at - # - transactions (1,000,000 rows): txn_id, user_id (FK to users), amount (log-normal varies by account_type), txn_type, timestamp - - # Save as CSV files with headers to a Unity Catalog volume. Use schema name 'sdg_test_large_csv'. - # Use Spark + Faker + Pandas UDFs with high partition count for this scale. - # VIP users should have larger transaction amounts. - # Create the job with proper environments configuration. - # expectations: - # expected_facts: - # - "pandas_udf" - # - "serverless" - # - "CSV" - # - "header" - # - "user_id" - # - "environments" - # - "log-normal" - # expected_patterns: - # - pattern: "@F\\.pandas_udf|pandas_udf" - # min_count: 1 - # description: "Pandas UDF for Faker parallelism" - # - pattern: "\\.csv.*header.*true|\\.option.*header.*true.*\\.csv" - # min_count: 1 - # description: "CSV with headers" - # - pattern: "environments.*dependencies" - # min_count: 1 - # description: "Job environment configuration" - # - pattern: "numPartitions.*=.*\\d{2,}" - # min_count: 1 - # description: "High partition count for 1M rows" - # guidelines: - # - "Must use Spark + Faker + Pandas UDFs for million-row generation" - # - "Transactions must reference valid user_ids" - # - "Transaction amounts must scale with account_type (VIP > Premium > Basic)" - # - "CSV output must include header row" - # metadata: - # category: "happy_path" - # difficulty: "hard" - # source: "manual" - # tags: ["serverless-job", "large", "csv", "pandas-udf", "referential-integrity"] - - # # Test 5: Classic Cluster + Small + Delta + 2 tables - # - id: "sdg_classic_small_delta_005" - # inputs: - # prompt: | - # Generate synthetic HR data to run on a classic Databricks cluster. - # Create 2 related tables with referential integrity: - # - employees (2,000 rows): employee_id, name, department (weighted), hire_date, salary (log-normal by dept) - # - projects (5,000 rows): project_id, employee_id (FK to employees), project_name, hours_logged, status - - # Save as Delta tables in Unity Catalog. Use schema name 'sdg_test_classic_delta'. - # First install dependencies with pip, then run the Python script. - # Engineering department should have higher salaries than other departments. - # expectations: - # expected_facts: - # - "classic cluster" - # - "pip install" - # - "Delta" - # - "saveAsTable" - # - "employee_id" - # - "context_id" - # expected_patterns: - # - pattern: "%pip install|pip install" - # min_count: 1 - # description: "Dependency installation on classic cluster" - # - pattern: "execute_databricks_command|run_python_file_on_databricks" - # min_count: 1 - # description: "Classic cluster execution tools" - # - pattern: "saveAsTable" - # min_count: 2 - # description: "Delta table output" - # - pattern: "context_id" - # min_count: 1 - # description: "Context reuse for pip + script execution" - # guidelines: - # - "Must use execute_databricks_command for pip install" - # - "Must reuse context_id between pip install and script execution" - # - "Projects must reference valid employee_ids" - # - "Salary must vary by department using log-normal distribution" - # metadata: - # category: "happy_path" - # difficulty: "medium" - # source: "manual" - # tags: ["classic-cluster", "small", "delta", "referential-integrity"] - - # # Test 6: Classic Cluster + Large + Parquet + 2 tables - # - id: "sdg_classic_large_parquet_006" - # inputs: - # prompt: | - # Generate large-scale synthetic supply chain data to run on a classic Databricks cluster. - # Create 2 related tables with referential integrity: - # - suppliers (50,000 rows): supplier_id, company_name, country, rating (1-5 weighted toward 3-4), active_since - # - inventory (300,000 rows): inventory_id, supplier_id (FK to suppliers), product_sku, quantity (Pareto), unit_cost, last_restock_date - - # Save as Parquet files to a Unity Catalog volume. Use schema name 'sdg_test_classic_parquet'. - # Use Spark + Faker + Pandas UDFs with appropriate partitioning. Higher-rated suppliers should have more inventory items. - # Install dependencies first, then execute the script with context reuse. - # expectations: - # expected_facts: - # - "classic cluster" - # - "pandas_udf" - # - "parquet" - # - "supplier_id" - # - "Pareto" - # - "pip install" - # expected_patterns: - # - pattern: "@F\\.pandas_udf|pandas_udf" - # min_count: 1 - # description: "Pandas UDF for Faker parallelism" - # - pattern: "\\.write.*parquet" - # min_count: 1 - # description: "Parquet output" - # - pattern: "supplier_id" - # min_count: 3 - # description: "Foreign key across tables" - # - pattern: "pareto|power.*law" - # min_count: 1 - # description: "Pareto distribution for quantities" - # guidelines: - # - "Must use Spark + Faker + Pandas UDFs for large-scale generation" - # - "Must install dependencies and reuse context" - # - "Inventory must reference valid supplier_ids" - # - "Quantity must use Pareto/power-law distribution" - # metadata: - # category: "happy_path" - # difficulty: "hard" - # source: "manual" - # tags: ["classic-cluster", "large", "parquet", "pandas-udf", "referential-integrity"] - - # # Test 7: Databricks Connect + Medium + CSV + 3 tables (complex referential integrity) - # - id: "sdg_dbconnect_medium_csv_3tables_007" - # inputs: - # prompt: | - # Generate synthetic retail order data using Databricks Connect with serverless. - # Create 3 related tables with full referential integrity: - # - customers (10,000 rows): customer_id, name, email, membership_level (Bronze/Silver/Gold/Platinum weighted 50/30/15/5), region - # - orders (50,000 rows): order_id, customer_id (FK to customers), order_date, total_amount, status - # - line_items (150,000 rows): line_item_id, order_id (FK to orders), product_name, quantity, unit_price - - # Save as CSV files with headers to Unity Catalog volume. Use schema name 'sdg_test_medium_csv'. - # Use Faker with Spark UDFs for realistic product names. - # Higher membership levels should have more orders. Order total_amount should equal sum of line_items. - # expectations: - # expected_facts: - # - "DatabricksSession" - # - "serverless" - # - "CSV" - # - "customer_id" - # - "order_id" - # - "line_item" - # - "Faker" - # - "UDF" - # expected_patterns: - # - pattern: "DatabricksSession.*serverless" - # min_count: 1 - # description: "Databricks Connect configuration" - # - pattern: "@F\\.udf|@udf" - # min_count: 1 - # description: "Spark UDF for Faker" - # - pattern: "customer_id" - # min_count: 4 - # description: "FK in customers and orders" - # - pattern: "order_id" - # min_count: 4 - # description: "FK in orders and line_items" - # - pattern: "\\.csv.*header" - # min_count: 1 - # description: "CSV with headers" - # guidelines: - # - "Must maintain referential integrity across all 3 tables" - # - "line_items.order_id must reference valid orders" - # - "orders.customer_id must reference valid customers" - # - "Should use Faker UDFs for realistic product names" - # - "Membership level should weight order distribution" - # metadata: - # category: "happy_path" - # difficulty: "hard" - # source: "manual" - # tags: ["databricks-connect", "medium", "csv", "faker-udf", "3-tables", "referential-integrity"] - - # # Test 8: Serverless Job + Medium + JSON + 3 tables (CRM data) - # - id: "sdg_serverless_job_medium_json_3tables_008" - # inputs: - # prompt: | - # Generate synthetic CRM data as a serverless Databricks job. - # Create 3 related tables with referential integrity: - # - accounts (8,000 rows): account_id, company_name, industry (weighted), annual_revenue (log-normal), tier (SMB/Mid-Market/Enterprise) - # - contacts (25,000 rows): contact_id, account_id (FK to accounts), first_name, last_name, email, title, is_primary - # - activities (80,000 rows): activity_id, contact_id (FK to contacts), activity_type (Call/Email/Meeting weighted), activity_date, duration_minutes (exponential), notes - - # Save as JSON files to Unity Catalog volume. Use schema name 'sdg_test_medium_json'. - # Create job with environments for faker and holidays dependencies. - # Enterprise accounts should have more contacts. Use realistic time patterns (weekday bias, business hours). - # expectations: - # expected_facts: - # - "serverless" - # - "environments" - # - "JSON" - # - "account_id" - # - "contact_id" - # - "activity" - # - "weekday" - # - "exponential" - # expected_patterns: - # - pattern: "environments.*dependencies" - # min_count: 1 - # description: "Serverless job environment" - # - pattern: "\\.write.*json" - # min_count: 1 - # description: "JSON output" - # - pattern: "account_id" - # min_count: 4 - # description: "FK across tables" - # - pattern: "contact_id" - # min_count: 4 - # description: "FK in contacts and activities" - # - pattern: "weekday|weekend|business.*hours" - # min_count: 1 - # description: "Time-based patterns" - # guidelines: - # - "Must create job with environments specifying faker and holidays" - # - "contacts.account_id must reference valid accounts" - # - "activities.contact_id must reference valid contacts" - # - "Activity dates should show weekday bias (more on Mon-Fri)" - # - "Duration should use exponential distribution" - # metadata: - # category: "happy_path" - # difficulty: "hard" - # source: "manual" - # tags: ["serverless-job", "medium", "json", "3-tables", "time-patterns", "referential-integrity"] - # ============================================================================= # EXECUTED TEST CASES (with verified outputs) # ============================================================================= - # Test 9: Databricks Connect + Small + Parquet + 2 tables (EXECUTED) - - id: "sdg_dbconnect_small_parquet_exec_001" + # Test 1: Databricks Connect + Small + Parquet + 2 tables (EXECUTED) + - id: "gen_dbconnect_small_parquet_exec_001" inputs: prompt: | Generate synthetic e-commerce data locally then save it to Unity Catalog. @@ -423,8 +54,65 @@ test_cases: execution_verified: true tags: ["databricks-connect", "small", "parquet", "referential-integrity", "pandas-udf", "executed"] - # Test 10: Serverless Job + Small + JSON + 2 tables (EXECUTED) - - id: "sdg_serverless_job_catalog_json_009" + # Test 2: Databricks Connect + Large + Delta + 2 tables (EXECUTED) + - id: "gen_dbconnect_large_delta_exec_002" + inputs: + prompt: | + Generate large-scale support ticket data. + Create 2 related tables with referential integrity: + - customers (100,000 rows): customer_id, company_name, tier, arr (log-normal), region, signup_date + - tickets (500,000 rows): ticket_id, customer_id (FK), priority (correlates with tier), resolution_hours (exponential), csat_score, created_at + + Save as Delta tables registered in Unity Catalog. Use schema name 'devkit_gen2_test_large_delta'. + Priority should correlate with tier (Enterprise gets more Critical/High priorities). + expectations: + expected_facts: + - "pandas_udf" + - "DatabricksSession" + - "Delta" + - "saveAsTable" + - "customer_id" + - "priority" + - "exponential" + - "lognormal" + - "serverless" + expected_patterns: + - pattern: "@F\\.pandas_udf|pandas_udf" + min_count: 1 + description: "Pandas UDF for Faker parallelism" + - pattern: "saveAsTable" + min_count: 2 + description: "Delta table registration" + - pattern: "numPartitions.*=.*\\d+" + min_count: 1 + description: "Partitioned generation for scale" + - pattern: "exponential" + min_count: 1 + description: "Exponential distribution for resolution times" + - pattern: "lognormal" + min_count: 1 + description: "Log-normal distribution for ARR" + - pattern: "DatabricksSession.*serverless.*True" + min_count: 1 + description: "Databricks Connect serverless configuration" + guidelines: + - "Must use DatabricksSession.builder.serverless(True).getOrCreate()" + - "Must use Spark + Faker + Pandas UDFs for scalable generation" + - "Tickets must reference valid customer_ids from customers table" + - "Priority distribution must vary by customer tier" + - "Resolution hours must use exponential distribution" + - "ARR must use log-normal distribution" + - "Must use high partition count (64+) for large-scale generation" + metadata: + category: "happy_path" + difficulty: "medium" + source: "interactive_execution" + execution_date: "2026-02-26" + execution_verified: true + tags: ["databricks-connect", "large", "delta", "pandas-udf", "referential-integrity", "executed"] + + # Test 3: Serverless Job + Small + JSON + 2 tables (EXECUTED) + - id: "gen_serverless_job_catalog_json_003" inputs: prompt: | Generate synthetic product catalog data that will run as a serverless Databricks job. @@ -486,59 +174,609 @@ test_cases: job_run_id: "560746964795126" tags: ["serverless-job", "small", "json", "referential-integrity", "weighted-sampling", "executed"] - # Test 11: Databricks Connect + Large + Delta + 2 tables (EXECUTED) - - id: "sdg_dbconnect_large_delta_exec_002" + # Test 4: Serverless Job + Large + Delta + 2 tables (Financial Transactions) (EXECUTED) + - id: "gen_serverless_job_large_delta_financial_004" inputs: prompt: | - Generate large-scale support ticket data. + Generate large-scale synthetic financial transaction data as a serverless Databricks job. Create 2 related tables with referential integrity: - - customers (100,000 rows): customer_id, company_name, tier, arr (log-normal), region, signup_date - - tickets (500,000 rows): ticket_id, customer_id (FK), priority (correlates with tier), resolution_hours (exponential), csat_score, created_at + - users (200,000 rows): user_id, username, account_type (Basic/Premium/VIP weighted 70/25/5), country, created_at + - transactions (1,000,000 rows): txn_id, user_id (FK to users), amount (log-normal varies by account_type), txn_type, timestamp - Save as Delta tables registered in Unity Catalog. Use schema name 'devkit_gen2_test_large_delta'. - Priority should correlate with tier (Enterprise gets more Critical/High priorities). + Save as Delta tables to Unity Catalog. Use schema name 'devkit_gen4_test_large_delta'. + VIP users should have larger transaction amounts. + Create the job with proper environments configuration. expectations: expected_facts: + - "serverless" + - "environments" + - "dependencies" + - "client" + - "Delta" + - "saveAsTable" + - "user_id" + - "account_type" + - "lognormal" - "pandas_udf" - - "DatabricksSession" + - "VIP" + expected_patterns: + - pattern: "environments.*spec.*dependencies|environments.*dependencies" + min_count: 1 + description: "Serverless job environment configuration" + - pattern: '"client":\\s*"4"' + min_count: 1 + description: "Correct client version for serverless" + - pattern: "saveAsTable" + min_count: 2 + description: "Delta table registration for both tables" + - pattern: "user_id" + min_count: 3 + description: "Foreign key reference across tables" + - pattern: "@F\\.pandas_udf|pandas_udf" + min_count: 1 + description: "Pandas UDF for Faker parallelism at scale" + - pattern: "lognormal" + min_count: 1 + description: "Log-normal distribution for transaction amounts" + - pattern: "numPartitions.*=.*\\d{2,}" + min_count: 1 + description: "High partition count for 1M+ rows" + - pattern: "CREATE SCHEMA IF NOT EXISTS" + min_count: 1 + description: "Infrastructure creation in script" + - pattern: "VIP" + min_count: 2 + description: "VIP account type handling" + guidelines: + - "Must create serverless job with environments parameter for dependencies" + - "Job spec must include client: 4 (not 1)" + - "Transactions table user_id must only reference valid users (FK integrity)" + - "Account types must be weighted: Basic 70%, Premium 25%, VIP 5%" + - "Transaction amounts must vary by account_type (VIP > Premium > Basic)" + - "Must use Spark + Faker + Pandas UDFs for million-row generation" + - "Must NOT use .cache() or .persist() (serverless incompatible)" + - "Script must create schema infrastructure" + - "Must use high partition count (32+) for large-scale generation" + - "Must write users table to Delta first, then read back for FK joins" + metadata: + category: "happy_path" + difficulty: "hard" + source: "interactive_execution" + execution_date: "2026-02-26" + execution_verified: true + job_run_id: "849738704935095" + verified_output: + users_table: "dustin_vannoy_catalog.devkit_gen4_test_large_delta.users" + users_rows: 200000 + users_distribution: + Basic: 140330 + Premium: 56622 + VIP: 3048 + transactions_table: "dustin_vannoy_catalog.devkit_gen4_test_large_delta.transactions" + transactions_rows: 1000000 + amount_by_account_type: + Basic_avg: 39.62 + Premium_avg: 115.22 + VIP_avg: 549.87 + tags: ["serverless-job", "large", "delta", "pandas-udf", "referential-integrity", "financial", "executed"] + + # Test 5: Classic Cluster + Small + Delta + 2 tables (HR Data) + - id: "gen_classic_small_delta_hr_005" + inputs: + prompt: | + Generate synthetic HR data to run on a classic Databricks cluster. + Create 2 related tables with referential integrity: + - employees (2,000 rows): employee_id, name, department (weighted), hire_date, salary (log-normal by dept) + - projects (5,000 rows): project_id, employee_id (FK to employees), project_name, hours_logged, status + + Save as Delta tables in Unity Catalog. Use schema name 'devkit_gen5_test_small_delta'. + Engineering department should have higher salaries than other departments. + expectations: + expected_facts: + - "classic cluster" - "Delta" - "saveAsTable" + - "employee_id" + - "department" + - "salary" + - "lognormal" + - "pandas_udf" + - "weighted" + expected_patterns: + - pattern: "DatabricksSession\\.builder\\.remote\\(\\)|DatabricksSession\\.builder\\.clusterId" + min_count: 1 + description: "Classic cluster connection (not serverless)" + - pattern: "saveAsTable" + min_count: 2 + description: "Delta table output for both tables" + - pattern: "@F\\.pandas_udf|pandas_udf" + min_count: 1 + description: "Pandas UDF for Faker parallelism" + - pattern: "employee_id" + min_count: 3 + description: "Foreign key reference across tables" + - pattern: "lognormal" + min_count: 1 + description: "Log-normal distribution for salaries" + - pattern: "Engineering.*higher|Engineering.*\\d{6}" + min_count: 1 + description: "Engineering department has higher salary parameters" + - pattern: "CREATE SCHEMA IF NOT EXISTS" + min_count: 1 + description: "Infrastructure creation in script" + - pattern: "databricks libraries install|pip install" + min_count: 1 + description: "Library installation instructions for classic cluster" + guidelines: + - "Must use DatabricksSession.builder.remote() or clusterId() for classic cluster" + - "Must NOT use serverless(True) - this is classic cluster execution" + - "Must use Spark + Faker + Pandas UDFs approach" + - "Projects must reference valid employee_ids from employees table" + - "Salary must vary by department using log-normal distribution" + - "Engineering department must have highest salary parameters" + - "Must include instructions for installing faker/numpy on classic cluster" + - "Must write master table (employees) before generating child table (projects)" + metadata: + category: "happy_path" + difficulty: "medium" + source: "interactive_execution" + execution_date: "2026-02-26" + execution_verified: true + cluster_id: "0128-180718-qmk3usr4" + tables_created: + - "dustin_vannoy_catalog.devkit_gen5_test_small_delta.employees" + - "dustin_vannoy_catalog.devkit_gen5_test_small_delta.projects" + validation_results: + employees_count: 2000 + projects_count: 5000 + orphan_projects: 0 + engineering_avg_salary: 140349 + hr_avg_salary: 69956 + tags: ["classic-cluster", "small", "delta", "pandas-udf", "referential-integrity", "hr-data", "executed"] + + # Test 6: Classic Cluster + Large + Parquet + 2 tables (Supply Chain Data) + - id: "gen_classic_large_parquet_supply_chain_006" + inputs: + prompt: | + Generate large-scale synthetic supply chain data to run on a classic Databricks cluster. + Create 2 related tables with referential integrity: + - suppliers (50,000 rows): supplier_id, company_name, country, rating (1-5 weighted toward 3-4), active_since + - inventory (300,000 rows): inventory_id, supplier_id (FK to suppliers), product_sku, quantity (Pareto), unit_cost, last_restock_date + + Save as Parquet files to a Unity Catalog volume. Use schema name 'devkit_gen6_test_classic_parquet'. + Higher-rated suppliers should have more inventory items. + expectations: + expected_facts: + - "classic cluster" + - "clusterId" + - "pandas_udf" + - "parquet" + - "supplier_id" + - "inventory" + - "Pareto" + - "lognormal" + - "weighted" + - "rating" + expected_patterns: + - pattern: "DatabricksSession\\.builder\\.clusterId" + min_count: 1 + description: "Classic cluster connection (not serverless)" + - pattern: "@F\\.pandas_udf|pandas_udf" + min_count: 1 + description: "Pandas UDF for Faker parallelism" + - pattern: "\\.write.*parquet|write\\.mode.*parquet" + min_count: 2 + description: "Parquet output for both tables" + - pattern: "supplier_id" + min_count: 3 + description: "Foreign key reference across tables" + - pattern: "pareto|Pareto" + min_count: 1 + description: "Pareto distribution for quantities" + - pattern: "lognormal" + min_count: 1 + description: "Log-normal distribution for unit costs" + - pattern: "rating.*weighted|weights.*rating|\\[0\\.05.*0\\.15.*0\\.35.*0\\.35.*0\\.10\\]" + min_count: 1 + description: "Weighted rating distribution toward 3-4" + - pattern: "CREATE SCHEMA IF NOT EXISTS" + min_count: 1 + description: "Infrastructure creation in script" + - pattern: "CREATE VOLUME IF NOT EXISTS" + min_count: 1 + description: "Volume creation for Parquet output" + - pattern: "numPartitions.*=.*\\d{2}" + min_count: 1 + description: "High partition count for large-scale generation" + guidelines: + - "Must use DatabricksSession.builder.clusterId() for classic cluster (not serverless)" + - "Must NOT use serverless(True) - this is classic cluster execution" + - "Must use Spark + Faker + Pandas UDFs approach" + - "Inventory must reference valid supplier_ids from suppliers table (FK integrity)" + - "Quantity must use Pareto/power-law distribution (right-skewed)" + - "Unit cost must use log-normal distribution" + - "Rating distribution must be weighted toward 3-4 (approximately 5%/15%/35%/35%/10%)" + - "Higher-rated suppliers must have more inventory items on average" + - "Must write master table (suppliers) before generating child table (inventory)" + - "Must create schema and volume infrastructure within the script" + - "Must use high partition count (32+) for large-scale generation" + - "Must include instructions or code for installing faker/numpy on classic cluster" + metadata: + category: "happy_path" + difficulty: "hard" + source: "interactive_execution" + execution_date: "2026-02-26" + execution_verified: true + cluster_id: "0128-180718-qmk3usr4" + tables_created: + - "dustin_vannoy_catalog.devkit_gen6_test_classic_parquet.suppliers (Parquet)" + - "dustin_vannoy_catalog.devkit_gen6_test_classic_parquet.inventory (Parquet)" + validation_results: + suppliers_count: 50000 + inventory_count: 225159 + orphan_inventory: 0 + rating_distribution: "5%/15%/35%/35%/10% for ratings 1-5" + avg_items_rating_1: 2.18 + avg_items_rating_5: 6.07 + quantity_median: 13 + quantity_p95: 33 + quantity_max: 1460 + unit_cost_median: 49.42 + unit_cost_avg: 68.05 + tags: ["classic-cluster", "large", "parquet", "pandas-udf", "referential-integrity", "supply-chain", "pareto", "weighted-rating", "executed"] + + # Test 7: Databricks Connect + Medium + CSV + 3 tables (Retail Orders) + - id: "gen_dbconnect_medium_csv_3tables_retail_007" + inputs: + prompt: | + Generate synthetic retail order data using Databricks Connect with serverless. + Create 3 related tables with full referential integrity: + - customers (10,000 rows): customer_id, name, email, membership_level (Bronze/Silver/Gold/Platinum weighted 50/30/15/5), region + - orders (50,000 rows): order_id, customer_id (FK to customers), order_date, total_amount, status + - line_items (150,000 rows): line_item_id, order_id (FK to orders), product_name, quantity, unit_price + + Save as CSV files with headers to Unity Catalog volume. Use schema name 'devkit_gen7_test_medium_csv'. + Create realistic product names. + Higher membership levels should have more orders. + Order total_amount should equal sum of line_items. + expectations: + expected_facts: + - "DatabricksSession" + - "serverless" + - "CSV" + - "header" - "customer_id" - - "priority" - - "exponential" + - "order_id" + - "line_item" + - "Faker" + - "pandas_udf" + - "membership_level" + - "weighted" + - "total_amount" - "lognormal" + expected_patterns: + - pattern: "DatabricksSession.*serverless.*True" + min_count: 1 + description: "Databricks Connect serverless configuration" + - pattern: "DatabricksEnv.*withDependencies" + min_count: 1 + description: "Managed dependencies for serverless" + - pattern: "@F\\.pandas_udf|pandas_udf" + min_count: 1 + description: "Pandas UDF for Faker parallelism" + - pattern: "customer_id" + min_count: 5 + description: "FK in customers and orders (multiple references)" + - pattern: "order_id" + min_count: 5 + description: "FK in orders and line_items (multiple references)" + - pattern: "\\.option.*header.*true.*\\.csv|\\.csv.*header" + min_count: 1 + description: "CSV with headers" + - pattern: "Bronze|Silver|Gold|Platinum" + min_count: 4 + description: "All membership levels present" + - pattern: "lognormal" + min_count: 1 + description: "Log-normal distribution for pricing" + - pattern: "CREATE SCHEMA IF NOT EXISTS" + min_count: 1 + description: "Infrastructure creation in script" + - pattern: "CREATE VOLUME IF NOT EXISTS" + min_count: 1 + description: "Volume creation for CSV output" + - pattern: "saveAsTable.*_tmp|write.*saveAsTable" + min_count: 1 + description: "Temp Delta tables for FK integrity (no .cache() on serverless)" + - pattern: "total_amount.*sum|sum.*line_total|computed_total" + min_count: 1 + description: "Order total computed from line items" + guidelines: + - "Must use DatabricksSession.builder.serverless(True).getOrCreate()" + - "Must use DatabricksEnv().withDependencies() for managed dependencies" + - "Must use Spark + Faker + Pandas UDFs approach" + - "Must maintain referential integrity across all 3 tables" + - "orders.customer_id must reference valid customers" + - "line_items.order_id must reference valid orders" + - "Must NOT use .cache() or .persist() (serverless incompatible)" + - "Must write to temp Delta tables for FK integrity, not .cache()" + - "Membership level must be weighted: Bronze 50%, Silver 30%, Gold 15%, Platinum 5%" + - "Higher membership levels must generate more orders per customer" + - "Order total_amount must equal sum of (quantity * unit_price) from line_items" + - "Unit prices should use log-normal distribution for realistic pricing" + - "CSV output must include header row" + - "Must create schema and volume infrastructure within the script" + - "Should use Faker for realistic product names" + metadata: + category: "happy_path" + difficulty: "hard" + source: "interactive_execution" + execution_date: "2026-02-26" + execution_verified: true + verified_output: + customers_table: "dustin_vannoy_catalog.devkit_gen7_test_medium_csv (CSV)" + customers_rows: 10000 + membership_distribution: + Bronze: 4935 + Silver: 4051 + Gold: 965 + Platinum: 49 + orders_rows: 50000 + orders_per_customer_by_tier: + Bronze_avg: 3.5 + Silver_avg: 5.9 + Gold_avg: 8.3 + Platinum_avg: 11.4 + line_items_rows: 119704 + orphan_orders: 0 + orphan_line_items: 0 + total_amount_matches_line_items: true + tags: ["databricks-connect", "medium", "csv", "3-tables", "pandas-udf", "referential-integrity", "retail", "computed-totals", "executed"] + + # Test 8: Serverless Job + Medium + JSON + 3 tables (CRM Data) + - id: "gen_serverless_job_medium_json_3tables_crm_008" + inputs: + prompt: | + Generate synthetic CRM data as a serverless Databricks job. + Create 3 related tables with referential integrity: + - accounts (8,000 rows): account_id, company_name, industry (weighted), annual_revenue (log-normal), tier (SMB/Mid-Market/Enterprise) + - contacts (25,000 rows): contact_id, account_id (FK to accounts), first_name, last_name, email, title, is_primary + - activities (80,000 rows): activity_id, contact_id (FK to contacts), activity_type (Call/Email/Meeting weighted), activity_date, duration_minutes (exponential), notes + + Save as JSON files to Unity Catalog volume. Use schema name 'devkit_gen8_test_medium_json'. + Enterprise accounts should have more contacts. Use realistic time patterns (weekday bias, business hours). + expectations: + expected_facts: - "serverless" + - "environments" + - "dependencies" + - "client" + - "JSON" + - "account_id" + - "contact_id" + - "activity" + - "weekday" + - "exponential" + - "lognormal" + - "pandas_udf" + - "is_primary" + - "weighted" expected_patterns: + - pattern: "environments.*spec.*dependencies|environments.*dependencies" + min_count: 1 + description: "Serverless job environment configuration" + - pattern: '"client":\\s*"4"' + min_count: 1 + description: "Correct client version for serverless" + - pattern: "\\.write.*json" + min_count: 1 + description: "JSON output format" + - pattern: "account_id" + min_count: 4 + description: "FK across accounts and contacts tables" + - pattern: "contact_id" + min_count: 4 + description: "FK in contacts and activities tables" - pattern: "@F\\.pandas_udf|pandas_udf" min_count: 1 description: "Pandas UDF for Faker parallelism" - - pattern: "saveAsTable" - min_count: 2 - description: "Delta table registration" - - pattern: "numPartitions.*=.*\\d+" + - pattern: "lognormal" min_count: 1 - description: "Partitioned generation for scale" + description: "Log-normal distribution for annual_revenue" - pattern: "exponential" min_count: 1 - description: "Exponential distribution for resolution times" - - pattern: "lognormal" + description: "Exponential distribution for duration_minutes" + - pattern: "weekday|is_weekend|dayofweek" min_count: 1 - description: "Log-normal distribution for ARR" + description: "Weekday bias implementation" + - pattern: "CREATE SCHEMA IF NOT EXISTS" + min_count: 1 + description: "Infrastructure creation in script" + - pattern: "CREATE VOLUME IF NOT EXISTS" + min_count: 1 + description: "Volume creation for JSON output" + - pattern: "is_primary" + min_count: 2 + description: "Primary contact flag" + - pattern: "saveAsTable.*staging|_staging" + min_count: 2 + description: "Staging tables for FK integrity (no .cache() on serverless)" + guidelines: + - "Must create serverless job with environments parameter for dependencies" + - "Job spec must include client: 4 (not 1)" + - "Must use Spark + Faker + Pandas UDFs approach" + - "contacts.account_id must reference valid accounts (FK integrity)" + - "activities.contact_id must reference valid contacts (FK integrity)" + - "Industry must be weighted distribution (Tech, Finance, Healthcare, Retail, Manufacturing, Other)" + - "Tier must be weighted: SMB ~60%, Mid-Market ~30%, Enterprise ~10%" + - "Activity type must be weighted: Email ~50%, Call ~35%, Meeting ~15%" + - "Activity dates must show weekday bias (85%+ Mon-Fri)" + - "Activity dates must show business hours bias (70%+ 9am-5pm)" + - "Duration must use exponential distribution by activity type" + - "Annual revenue must use log-normal distribution by tier" + - "Enterprise accounts must have more contacts on average than SMB" + - "Must mark first contact per account as is_primary=True" + - "Must NOT use .cache() or .persist() (serverless incompatible)" + - "Must NOT use RDDs or broadcast variables (serverless incompatible)" + - "Must write to staging Delta tables for FK integrity, then export to JSON" + - "Must create schema and volume infrastructure within the script" + metadata: + category: "happy_path" + difficulty: "hard" + source: "interactive_execution" + execution_date: "2026-02-26" + execution_verified: true + job_id: 673682369587321 + job_run_id: "129954663948285" + verified_output: + accounts_table: "dustin_vannoy_catalog.devkit_gen8_test_medium_json.accounts_staging" + accounts_rows: 8000 + tier_distribution: + Enterprise: 304 + Mid-Market: 2945 + SMB: 4751 + industry_distribution: + Finance: 2750 + Technology: 2383 + Healthcare: 1803 + Retail: 860 + Manufacturing: 183 + Other: 21 + contacts_table: "dustin_vannoy_catalog.devkit_gen8_test_medium_json.contacts_staging" + contacts_rows: 25000 + contacts_per_tier_avg: + Enterprise: 6.375 + Mid-Market: 3.91 + SMB: 2.71 + activities_table: "dustin_vannoy_catalog.devkit_gen8_test_medium_json.activities_staging" + activities_rows: 80000 + activity_type_distribution: + Email: 39925 + Call: 34131 + Meeting: 5944 + weekday_percentage: 95.76 + weekend_percentage: 4.24 + orphan_contacts: 0 + orphan_activities: 0 + revenue_by_tier: + Enterprise_avg: 78721443.88 + Mid-Market_avg: 7354786.45 + SMB_avg: 684174.54 + duration_by_type_avg: + Email: 4.69 + Call: 14.66 + Meeting: 45.54 + json_output_path: "/Volumes/dustin_vannoy_catalog/devkit_gen8_test_medium_json/raw_data/" + tags: ["serverless-job", "medium", "json", "3-tables", "pandas-udf", "referential-integrity", "crm", "time-patterns", "weekday-bias", "executed"] + + # Test 9: Databricks Connect + Incremental + Delta + Variant Column (Event Log) + - id: "gen_dbconnect_incremental_delta_variant_009" + inputs: + prompt: | + Generate synthetic event log data capturing user activity in a chatbot application. This should include deeply nested json string that will convert to large variant column with very long strings which represent the conversation. + Create this as an incremental script which can append new data based on the max date that already exists. Each batch created should be based on a size and window set by variables. Start with 20,000 rows in the batch representing a 1 hour time window. + Save this to Delta table in UC (representing bronze/raw step in the process). + Save to catalog dustin_vannoy_catalog with schema name 'devkit_gen9_test_variant'. + expectations: + expected_facts: + - "DatabricksSession" + - "serverless" + - "Delta" + - "saveAsTable" + - "variant" + - "JSON" + - "nested" + - "incremental" + - "append" + - "max_timestamp" + - "batch" + - "conversation" + - "messages" + - "pandas_udf" + expected_patterns: - pattern: "DatabricksSession.*serverless.*True" min_count: 1 description: "Databricks Connect serverless configuration" + - pattern: "saveAsTable|write.*mode.*append" + min_count: 1 + description: "Delta table output with append capability" + - pattern: "MAX.*event_timestamp|max.*timestamp" + min_count: 1 + description: "Incremental logic reading max timestamp" + - pattern: "append" + min_count: 1 + description: "Append mode for incremental writes" + - pattern: "@F\\.pandas_udf|pandas_udf" + min_count: 1 + description: "Pandas UDF for payload generation" + - pattern: "json\\.dumps|JSON" + min_count: 1 + description: "JSON serialization for variant column" + - pattern: "session_metadata|messages|context|analytics" + min_count: 2 + description: "Deeply nested JSON structure keys" + - pattern: "content.*message|message.*content" + min_count: 1 + description: "Long conversation content in messages" + - pattern: "BATCH_SIZE|batch_size|N_EVENTS" + min_count: 1 + description: "Configurable batch size variable" + - pattern: "TIME_WINDOW|time_window|HOURS" + min_count: 1 + description: "Configurable time window variable" + - pattern: "CREATE SCHEMA IF NOT EXISTS" + min_count: 1 + description: "Infrastructure creation in script" + - pattern: "tableExists|table_exists" + min_count: 1 + description: "Check for existing table before append" guidelines: - "Must use DatabricksSession.builder.serverless(True).getOrCreate()" - - "Must use Spark + Faker + Pandas UDFs for scalable generation" - - "Tickets must reference valid customer_ids from customers table" - - "Priority distribution must vary by customer tier" - - "Resolution hours must use exponential distribution" - - "ARR must use log-normal distribution" - - "Must use high partition count (64+) for large-scale generation" + - "Must use Spark + Faker + Pandas UDFs approach" + - "Must implement incremental logic: read MAX(event_timestamp) from existing table" + - "Must append new data starting from max timestamp + 1 second" + - "Must create new table if it doesn't exist (first run)" + - "Must use append mode for subsequent runs" + - "Must generate deeply nested JSON for variant column (session_metadata, messages, context, analytics)" + - "Messages array must contain multiple turns with role (user/assistant)" + - "Message content must be long strings (50-500 words realistic conversation)" + - "Must have configurable BATCH_SIZE variable (default 20,000)" + - "Must have configurable TIME_WINDOW variable (default 1 hour)" + - "Each batch represents a specific time window of events" + - "Must NOT use .cache() or .persist() (serverless incompatible)" + - "JSON payload should include: session_metadata, messages[], context, analytics" + - "Must create schema infrastructure within the script" metadata: category: "happy_path" - difficulty: "medium" + difficulty: "hard" source: "interactive_execution" execution_date: "2026-02-26" execution_verified: true - tags: ["databricks-connect", "large", "delta", "pandas-udf", "referential-integrity", "executed"] + incremental_test_verified: true + table_created: "dustin_vannoy_catalog.devkit_gen9_test_variant.chatbot_events" + validation_results: + batch_1_events: 20000 + batch_2_events: 20000 + total_events_after_2_batches: 40000 + unique_users_per_batch: 919 + unique_sessions_per_batch: 19995 + payload_min_size_bytes: 2936 + payload_median_size_bytes: 9539 + payload_max_size_bytes: 25561 + payload_avg_size_bytes: 9806 + event_type_distribution: + message: 71.5% + tool_call: 19.9% + session_start: 4.9% + feedback: 3.4% + session_end: 0.2% + json_structure_keys: + session_metadata: ["start_time", "client_info", "user_preferences"] + messages: "array of message objects with role, content, tokens, metadata" + context: ["previous_sessions", "knowledge_base_refs", "tool_calls", "user_profile"] + analytics: ["total_turns", "avg_response_time_ms", "total_tokens", "sentiment_scores", "topics"] + message_content_lengths: + min: 223 + max: 1358 + avg: 725 + tags: ["databricks-connect", "incremental", "delta", "variant", "json", "nested", "chatbot", "event-log", "pandas-udf", "append", "executed"] From c6802695e137ea309b519cb84db314cc60139cc2 Mon Sep 17 00:00:00 2001 From: dustinvannoy-db Date: Fri, 27 Feb 2026 08:35:06 -0800 Subject: [PATCH 15/24] Remove default catalog setting --- .../databricks-data-generation/SKILL.md | 26 +++++++++---------- .../references/2-generation-approaches.md | 8 +++--- .../references/5-output-formats.md | 20 ++++---------- .../scripts/generate_synthetic_data.py | 4 +-- 4 files changed, 24 insertions(+), 34 deletions(-) diff --git a/databricks-skills/databricks-data-generation/SKILL.md b/databricks-skills/databricks-data-generation/SKILL.md index 3702231e..8b58f4ec 100644 --- a/databricks-skills/databricks-data-generation/SKILL.md +++ b/databricks-skills/databricks-data-generation/SKILL.md @@ -3,6 +3,8 @@ name: databricks-data-generation description: "Generate realistic synthetic data using Spark + Faker (strongly recommended). Supports serverless execution, multiple output formats (Parquet/JSON/CSV/Delta), and scales from thousands to millions of rows. For small datasets (<10K rows), can optionally generate locally and upload to volumes. Use when user mentions 'synthetic data', 'test data', 'generate data', 'demo dataset', 'Faker', or 'sample data'." --- +> Catalog and schema are **always user-supplied** — never default to any value. If the user hasn't provided them, ask. For any UC write, **always create the schema if it doesn't exist** before writing data. + # Databricks Synthetic Data Generation Generate realistic, story-driven synthetic data for Databricks using **Spark + Faker + Pandas UDFs** (strongly recommended). @@ -37,7 +39,7 @@ python generate_data.py 1. **Always use Spark + Faker + Pandas UDFs** for data generation (scalable, parallel) 2. **Present a plan for user approval** before generating any code -3. **Ask for catalog/schema** - default to `ai_dev_kit.` +3. **Ask for catalog/schema** - do not default 4. **Use serverless compute** unless user explicitly requests classic cluster 5. **Generate raw data only** - no pre-aggregated fields (unless user requests) 6. **Create master tables first** - then generate related tables with valid FKs @@ -52,7 +54,7 @@ python generate_data.py **You MUST explicitly ask the user which catalog to use.** Do not assume or proceed without confirmation. Example prompt to user: -> "Which Unity Catalog should I use for this data? Default is `ai_dev_kit` but you can specify any catalog you have access to." +> "Which Unity Catalog should I use for this data?" When presenting your plan, always show the selected catalog prominently: ``` @@ -65,21 +67,19 @@ This makes it easy for the user to spot and correct if needed. ### Step 1: Gather Requirements Ask the user about: -- **Catalog/Schema** - Which catalog to use? (default: `ai_dev_kit.`) +- **Catalog/Schema** - Which catalog to use? - What domain/scenario? (e-commerce, support tickets, IoT sensors, etc.) - How many tables? What relationships between them? - Approximate row counts per table? -- Output format preference? (Parquet to Volume is default) +- Output format preference? (Delta table is default) ### Step 2: Present Table Specification Show a clear specification with **YOUR ASSUMPTIONS surfaced**. Always start with the output location: ``` -📍 Output Location: ai_dev_kit.ecommerce_demo - Volume: /Volumes/ai_dev_kit/ecommerce_demo/raw_data/ - - ⬆️ Change this? Just let me know which catalog.schema to use instead. +📍 Output Location: {user_catalog}.ecommerce_demo + Volume: /Volumes/{user_catalog}/ecommerce_demo/raw_data/ ``` | Table | Columns | Rows | Key Assumptions | @@ -158,9 +158,9 @@ customers_df = ( ) # Save to Unity Catalog -spark.sql("CREATE SCHEMA IF NOT EXISTS ai_dev_kit.my_schema") -spark.sql("CREATE VOLUME IF NOT EXISTS ai_dev_kit.my_schema.raw_data") -customers_df.write.mode("overwrite").parquet("/Volumes/ai_dev_kit/my_schema/raw_data/customers") +spark.sql(f"CREATE SCHEMA IF NOT EXISTS {CATALOG}.{SCHEMA}") +spark.sql(f"CREATE VOLUME IF NOT EXISTS {CATALOG}.{SCHEMA}.raw_data") +customers_df.write.mode("overwrite").parquet(f"/Volumes/{CATALOG}/{SCHEMA}/raw_data/customers") ``` ## Common Patterns @@ -223,7 +223,7 @@ See [references/5-output-formats.md](references/5-output-formats.md) for detaile ### Execution - Use serverless (instant start, no cluster wait) -- Ask for catalog/schema (default `ai_dev_kit`) +- Ask for catalog/schema - Present plan before generating ### Data Generation @@ -236,7 +236,7 @@ See [references/5-output-formats.md](references/5-output-formats.md) for detaile ### Output - Create infrastructure in script (`CREATE SCHEMA/VOLUME IF NOT EXISTS`) - Do NOT create catalogs - assume they exist -- Parquet to volumes as default +- Delta tables as default ## Related Skills diff --git a/databricks-skills/databricks-data-generation/references/2-generation-approaches.md b/databricks-skills/databricks-data-generation/references/2-generation-approaches.md index d820cd10..5d6feeca 100644 --- a/databricks-skills/databricks-data-generation/references/2-generation-approaches.md +++ b/databricks-skills/databricks-data-generation/references/2-generation-approaches.md @@ -184,17 +184,17 @@ Otherwise, **always use Spark + Faker + Pandas UDFs**. ### Ask for Catalog and Schema -By default, use the `ai_dev_kit` catalog. Ask the user which schema to use: +Ask the user which catalog and schema to use: -> "I'll save the data to `ai_dev_kit.`. What schema name would you like to use? (You can also specify a different catalog if needed.)" +> "What catalog and schema name would you like to use?" ### Create Infrastructure in Script Always create the schema and volume **inside the Python script** using `spark.sql()`: ```python -CATALOG = "ai_dev_kit" -SCHEMA = "synthetic_data" +CATALOG = "" # MUST ask user - never default +SCHEMA = "" VOLUME_PATH = f"/Volumes/{CATALOG}/{SCHEMA}/raw_data" # Note: Assume catalog exists - do NOT create it diff --git a/databricks-skills/databricks-data-generation/references/5-output-formats.md b/databricks-skills/databricks-data-generation/references/5-output-formats.md index 49214e91..c283a82c 100644 --- a/databricks-skills/databricks-data-generation/references/5-output-formats.md +++ b/databricks-skills/databricks-data-generation/references/5-output-formats.md @@ -2,23 +2,13 @@ Where and how to save generated synthetic data. -## Storage Destination - -### Ask for Catalog and Schema - -By default, use the `ai_dev_kit` catalog. Ask the user which schema to use: - -> "I'll save the data to `ai_dev_kit.`. What schema name would you like to use? (You can also specify a different catalog if needed.)" - -If the user provides just a schema name, use `ai_dev_kit.{schema}`. If they provide `catalog.schema`, use that instead. - -### Create Infrastructure in Script +## Create Infrastructure in Script Always create the schema and volume **inside the Python script** using `spark.sql()`. Do NOT make separate MCP SQL calls - it's much slower. ```python -CATALOG = "ai_dev_kit" -SCHEMA = "synthetic_data" +CATALOG = "" # MUST ask user - never default +SCHEMA = "" VOLUME_PATH = f"/Volumes/{CATALOG}/{SCHEMA}/raw_data" # Note: Assume catalog exists - do NOT create it @@ -34,10 +24,10 @@ spark.sql(f"CREATE VOLUME IF NOT EXISTS {CATALOG}.{SCHEMA}.raw_data") | Format | Use Case | Extension | Best For | |--------|----------|-----------|----------| -| **Parquet** | Default - SDP pipeline input | `.parquet` or none | Best compression, query performance | +| **Parquet** | SDP pipeline input | `.parquet` or none | Best compression, query performance | | **JSON** | Log-style ingestion | `.json` | Simulating external data feeds | | **CSV** | Legacy systems | `.csv` | Human-readable, spreadsheet import | -| **Delta Table** | Direct analytics | N/A | Skip SDP, query immediately | +| **Delta Table** | Default - Direct analytics | N/A | Treat as bronze tables for ETL or skip ETL and query immediately | --- diff --git a/databricks-skills/databricks-data-generation/scripts/generate_synthetic_data.py b/databricks-skills/databricks-data-generation/scripts/generate_synthetic_data.py index d500b15e..778d8098 100644 --- a/databricks-skills/databricks-data-generation/scripts/generate_synthetic_data.py +++ b/databricks-skills/databricks-data-generation/scripts/generate_synthetic_data.py @@ -27,8 +27,8 @@ CLUSTER_ID = None # Only used if USE_SERVERLESS=False # Storage - Update these for your environment -CATALOG = "ai_dev_kit" # Change to your catalog -SCHEMA = "synthetic_data" # Change to your schema +CATALOG = "" # REQUIRED: replace with your catalog +SCHEMA = "" # REQUIRED: replace with your schema VOLUME_PATH = f"/Volumes/{CATALOG}/{SCHEMA}/raw_data" # Data sizes From 09a9cd8fec250710c3cc3531ff2d8e2a7ead6b82 Mon Sep 17 00:00:00 2001 From: dustinvannoy-db Date: Fri, 27 Feb 2026 11:23:17 -0800 Subject: [PATCH 16/24] Add window syntax common issue --- .../databricks-data-generation/SKILL.md | 1 + .../references/6-troubleshooting.md | 29 +++++++++++++++++++ 2 files changed, 30 insertions(+) diff --git a/databricks-skills/databricks-data-generation/SKILL.md b/databricks-skills/databricks-data-generation/SKILL.md index 8b58f4ec..6d78f6af 100644 --- a/databricks-skills/databricks-data-generation/SKILL.md +++ b/databricks-skills/databricks-data-generation/SKILL.md @@ -252,5 +252,6 @@ See [references/5-output-formats.md](references/5-output-formats.md) for detaile | Out of memory | Increase `numPartitions` in `spark.range()` | | Referential integrity errors | Write master table to Delta first, read back for FK joins | | `PERSIST TABLE is not supported on serverless` | **NEVER use `.cache()` or `.persist()` with serverless** - write to Delta table first, then read back | +| `F.window` vs `Window` confusion | Use `from pyspark.sql.window import Window` for `row_number()`, `rank()`, etc. `F.window` is for streaming only. | See [references/6-troubleshooting.md](references/6-troubleshooting.md) for full troubleshooting guide. diff --git a/databricks-skills/databricks-data-generation/references/6-troubleshooting.md b/databricks-skills/databricks-data-generation/references/6-troubleshooting.md index 42961e1d..080b18fd 100644 --- a/databricks-skills/databricks-data-generation/references/6-troubleshooting.md +++ b/databricks-skills/databricks-data-generation/references/6-troubleshooting.md @@ -139,6 +139,35 @@ spark = DatabricksSession.builder.serverless(True).getOrCreate() ## Data Generation Issues +### AttributeError: 'function' object has no attribute 'partitionBy' + +**Problem:** Using `F.window` instead of `Window` for analytical window functions. + +```python +# WRONG - F.window is for time-based tumbling/sliding windows (streaming) +window_spec = F.window.partitionBy("account_id").orderBy("contact_id") +# Error: AttributeError: 'function' object has no attribute 'partitionBy' + +# CORRECT - Window is for analytical window specifications +from pyspark.sql.window import Window +window_spec = Window.partitionBy("account_id").orderBy("contact_id") +``` + +**When to use Window:** For analytical functions like `row_number()`, `rank()`, `lead()`, `lag()`: + +```python +from pyspark.sql.window import Window + +# Mark first contact per account as primary +window_spec = Window.partitionBy("account_id").orderBy("contact_id") +contacts_df = contacts_df.withColumn( + "is_primary", + F.row_number().over(window_spec) == 1 +) +``` + +--- + ### Faker UDF is slow **Problem:** Single-row UDFs don't parallelize well. From c7e335ab6a9b996c1b5e5a0c4ba1a4d060ed42c7 Mon Sep 17 00:00:00 2001 From: dustinvannoy-db Date: Tue, 3 Mar 2026 07:26:50 -0800 Subject: [PATCH 17/24] Rename and overhaul data gen skill and tests timeouts --- .test/README.md | 14 + .../databricks-data-generation/baseline.yaml | 36 - .../baseline.yaml | 21 + .test/scripts/mlflow_eval.py | 51 +- .../candidates.yaml | 7 - .../ground_truth.yaml | 782 ------------ .../candidates.yaml | 7 + .../ground_truth.yaml | 1127 +++++++++++++++++ .../manifest.yaml | 18 +- .test/src/skill_test/config.py | 5 +- .test/src/skill_test/grp/executor.py | 33 + .test/src/skill_test/runners/evaluate.py | 17 +- .../SKILL.md | 17 +- .../references/1-setup-and-execution.md | 0 .../references/2-generation-approaches.md | 0 .../references/3-data-patterns.md | 0 .../references/4-domain-guidance.md | 0 .../references/5-output-formats.md | 0 .../references/6-troubleshooting.md | 0 .../scripts/generate_synthetic_data.py | 0 20 files changed, 1289 insertions(+), 846 deletions(-) delete mode 100644 .test/baselines/databricks-data-generation/baseline.yaml create mode 100644 .test/baselines/databricks-synthetic-data-gen/baseline.yaml delete mode 100644 .test/skills/databricks-data-generation/candidates.yaml delete mode 100644 .test/skills/databricks-data-generation/ground_truth.yaml create mode 100644 .test/skills/databricks-synthetic-data-gen/candidates.yaml create mode 100644 .test/skills/databricks-synthetic-data-gen/ground_truth.yaml rename .test/skills/{databricks-data-generation => databricks-synthetic-data-gen}/manifest.yaml (67%) rename databricks-skills/{databricks-data-generation => databricks-synthetic-data-gen}/SKILL.md (94%) rename databricks-skills/{databricks-data-generation => databricks-synthetic-data-gen}/references/1-setup-and-execution.md (100%) rename databricks-skills/{databricks-data-generation => databricks-synthetic-data-gen}/references/2-generation-approaches.md (100%) rename databricks-skills/{databricks-data-generation => databricks-synthetic-data-gen}/references/3-data-patterns.md (100%) rename databricks-skills/{databricks-data-generation => databricks-synthetic-data-gen}/references/4-domain-guidance.md (100%) rename databricks-skills/{databricks-data-generation => databricks-synthetic-data-gen}/references/5-output-formats.md (100%) rename databricks-skills/{databricks-data-generation => databricks-synthetic-data-gen}/references/6-troubleshooting.md (100%) rename databricks-skills/{databricks-data-generation => databricks-synthetic-data-gen}/scripts/generate_synthetic_data.py (100%) diff --git a/.test/README.md b/.test/README.md index d5c8fe46..d2bbb2db 100644 --- a/.test/README.md +++ b/.test/README.md @@ -233,3 +233,17 @@ uv pip install -e ".test/" uv run pytest .test/tests/ uv run python .test/scripts/regression.py ``` + +--- + +## Troubleshooting + +### MLflow evaluation not returning results + +If `/skill-test mlflow` hangs or doesn't return results, run manually with debug logging: + +```bash +MLFLOW_LOG_LEVEL=DEBUG uv run python .test/scripts/mlflow_eval.py +``` + +This will show detailed MLflow API calls and help identify connection or authentication issues. diff --git a/.test/baselines/databricks-data-generation/baseline.yaml b/.test/baselines/databricks-data-generation/baseline.yaml deleted file mode 100644 index 980a16fb..00000000 --- a/.test/baselines/databricks-data-generation/baseline.yaml +++ /dev/null @@ -1,36 +0,0 @@ -run_id: '20260226_220320' -created_at: '2026-02-26T22:03:20.645758' -skill_name: databricks-data-generation -metrics: - pass_rate: 1.0 - total_tests: 9 - passed_tests: 9 - failed_tests: 0 -test_results: -- id: gen_dbconnect_small_parquet_exec_001 - passed: true - execution_mode: local -- id: gen_dbconnect_large_delta_exec_002 - passed: true - execution_mode: local -- id: gen_serverless_job_catalog_json_003 - passed: true - execution_mode: local -- id: gen_serverless_job_large_delta_financial_004 - passed: true - execution_mode: local -- id: gen_classic_small_delta_hr_005 - passed: true - execution_mode: local -- id: gen_classic_large_parquet_supply_chain_006 - passed: true - execution_mode: local -- id: gen_dbconnect_medium_csv_3tables_retail_007 - passed: true - execution_mode: local -- id: gen_serverless_job_medium_json_3tables_crm_008 - passed: true - execution_mode: local -- id: gen_dbconnect_incremental_delta_variant_009 - passed: true - execution_mode: local diff --git a/.test/baselines/databricks-synthetic-data-gen/baseline.yaml b/.test/baselines/databricks-synthetic-data-gen/baseline.yaml new file mode 100644 index 00000000..b43273c8 --- /dev/null +++ b/.test/baselines/databricks-synthetic-data-gen/baseline.yaml @@ -0,0 +1,21 @@ +run_id: '20260303_071721' +created_at: '2026-03-03T07:17:21.838623' +skill_name: databricks-synthetic-data-gen +metrics: + pass_rate: 1.0 + total_tests: 4 + passed_tests: 4 + failed_tests: 0 +test_results: +- id: grp_20260302_113344 + passed: true + execution_mode: local +- id: gen_serverless_job_catalog_json_002 + passed: true + execution_mode: local +- id: grp_20260302_retail_csv_3tables_003 + passed: true + execution_mode: local +- id: grp_20260303_manufacturing_delta_streaming_004 + passed: true + execution_mode: local diff --git a/.test/scripts/mlflow_eval.py b/.test/scripts/mlflow_eval.py index caa2e45c..54ce7559 100755 --- a/.test/scripts/mlflow_eval.py +++ b/.test/scripts/mlflow_eval.py @@ -2,29 +2,62 @@ """Run MLflow evaluation for a skill. Usage: - python mlflow_eval.py [--filter-category ] [--run-name ] + python mlflow_eval.py [--filter-category ] [--run-name ] [--timeout ] Environment Variables: DATABRICKS_CONFIG_PROFILE - Databricks CLI profile (default: "DEFAULT") MLFLOW_TRACKING_URI - Set to "databricks" for Databricks MLflow MLFLOW_EXPERIMENT_NAME - Experiment path (e.g., "/Users/{user}/skill-test") + MLFLOW_LLM_JUDGE_TIMEOUT - Timeout in seconds for LLM judge evaluation (default: 120) """ +import os import sys +import signal import argparse +# Close stdin and disable tqdm progress bars when run non-interactively +# This fixes hanging issues with tqdm/MLflow progress bars in background tasks +if not sys.stdin.isatty(): + try: + sys.stdin.close() + sys.stdin = open(os.devnull, 'r') + except Exception: + pass + # Disable tqdm progress bars + os.environ.setdefault("TQDM_DISABLE", "1") + # Import common utilities from _common import setup_path, print_result, handle_error +class TimeoutException(Exception): + pass + + +def timeout_handler(signum, frame): + raise TimeoutException("MLflow evaluation timed out") + + def main(): parser = argparse.ArgumentParser(description="Run MLflow evaluation for a skill") parser.add_argument("skill_name", help="Name of skill to evaluate") parser.add_argument("--filter-category", help="Filter by test category") parser.add_argument("--run-name", help="Custom MLflow run name") + parser.add_argument( + "--timeout", + type=int, + default=120, + help="Timeout in seconds for evaluation (default: 120)", + ) args = parser.parse_args() setup_path() + # Set up signal-based timeout (Unix only) + if hasattr(signal, 'SIGALRM'): + signal.signal(signal.SIGALRM, timeout_handler) + signal.alarm(args.timeout) + try: from skill_test.runners import evaluate_skill @@ -34,6 +67,10 @@ def main(): run_name=args.run_name, ) + # Cancel the alarm if we succeeded + if hasattr(signal, 'SIGALRM'): + signal.alarm(0) + # Convert to standard result format if result.get("run_id"): result["success"] = True @@ -42,7 +79,19 @@ def main(): sys.exit(print_result(result)) + except TimeoutException as e: + result = { + "success": False, + "skill_name": args.skill_name, + "error": f"Evaluation timed out after {args.timeout} seconds. This may indicate LLM judge endpoint issues.", + "error_type": "timeout", + } + sys.exit(print_result(result)) + except Exception as e: + # Cancel alarm on any exception + if hasattr(signal, 'SIGALRM'): + signal.alarm(0) sys.exit(handle_error(e, args.skill_name)) diff --git a/.test/skills/databricks-data-generation/candidates.yaml b/.test/skills/databricks-data-generation/candidates.yaml deleted file mode 100644 index fda8a58b..00000000 --- a/.test/skills/databricks-data-generation/candidates.yaml +++ /dev/null @@ -1,7 +0,0 @@ -# Candidates for databricks-data-generation skill -# Test cases pending review before promotion to ground_truth.yaml -# -# Use `/skill-test databricks-data-generation add` to create new candidates -# Use `/skill-test databricks-data-generation review` to promote candidates to ground truth - -candidates: [] diff --git a/.test/skills/databricks-data-generation/ground_truth.yaml b/.test/skills/databricks-data-generation/ground_truth.yaml deleted file mode 100644 index 9094bf4a..00000000 --- a/.test/skills/databricks-data-generation/ground_truth.yaml +++ /dev/null @@ -1,782 +0,0 @@ -test_cases: - # ============================================================================= - # EXECUTED TEST CASES (with verified outputs) - # ============================================================================= - - # Test 1: Databricks Connect + Small + Parquet + 2 tables (EXECUTED) - - id: "gen_dbconnect_small_parquet_exec_001" - inputs: - prompt: | - Generate synthetic e-commerce data locally then save it to Unity Catalog. - Create 2 related tables with referential integrity: - - customers (5,000 rows): customer_id, name, email, tier (Free/Pro/Enterprise weighted 60/30/10), region, created_at - - orders (15,000 rows): order_id, customer_id (FK to customers), amount, order_date, status - - Save as Parquet files to a Unity Catalog volume. Use schema name 'devkit_gen1_test_small_parquet'. - Enterprise customers should generate more orders than Free tier. - expectations: - expected_facts: - - "DatabricksSession" - - "serverless" - - "parquet" - - "customer_id" - - "referential integrity" - - "weighted" - - "log-normal" - - "pandas_udf" - expected_patterns: - - pattern: "DatabricksSession.*serverless.*True" - min_count: 1 - description: "Databricks Connect serverless configuration" - - pattern: "\\.write.*parquet" - min_count: 1 - description: "Parquet output format" - - pattern: "customer_id" - min_count: 3 - description: "Foreign key reference in multiple tables" - - pattern: "lognormal" - min_count: 1 - description: "Log-normal distribution for amounts" - - pattern: "@F\\.pandas_udf|pandas_udf" - min_count: 1 - description: "Pandas UDF for Faker parallelism" - guidelines: - - "Must use DatabricksSession.builder.serverless(True).getOrCreate()" - - "Orders table customer_id must only contain IDs from customers table" - - "Enterprise tier customers must have higher weight for order generation" - - "Amount distribution must use log-normal, not uniform" - - "Must use Spark + Faker + Pandas UDFs approach" - metadata: - category: "happy_path" - difficulty: "easy" - source: "interactive_execution" - execution_date: "2025-02-25" - execution_verified: true - tags: ["databricks-connect", "small", "parquet", "referential-integrity", "pandas-udf", "executed"] - - # Test 2: Databricks Connect + Large + Delta + 2 tables (EXECUTED) - - id: "gen_dbconnect_large_delta_exec_002" - inputs: - prompt: | - Generate large-scale support ticket data. - Create 2 related tables with referential integrity: - - customers (100,000 rows): customer_id, company_name, tier, arr (log-normal), region, signup_date - - tickets (500,000 rows): ticket_id, customer_id (FK), priority (correlates with tier), resolution_hours (exponential), csat_score, created_at - - Save as Delta tables registered in Unity Catalog. Use schema name 'devkit_gen2_test_large_delta'. - Priority should correlate with tier (Enterprise gets more Critical/High priorities). - expectations: - expected_facts: - - "pandas_udf" - - "DatabricksSession" - - "Delta" - - "saveAsTable" - - "customer_id" - - "priority" - - "exponential" - - "lognormal" - - "serverless" - expected_patterns: - - pattern: "@F\\.pandas_udf|pandas_udf" - min_count: 1 - description: "Pandas UDF for Faker parallelism" - - pattern: "saveAsTable" - min_count: 2 - description: "Delta table registration" - - pattern: "numPartitions.*=.*\\d+" - min_count: 1 - description: "Partitioned generation for scale" - - pattern: "exponential" - min_count: 1 - description: "Exponential distribution for resolution times" - - pattern: "lognormal" - min_count: 1 - description: "Log-normal distribution for ARR" - - pattern: "DatabricksSession.*serverless.*True" - min_count: 1 - description: "Databricks Connect serverless configuration" - guidelines: - - "Must use DatabricksSession.builder.serverless(True).getOrCreate()" - - "Must use Spark + Faker + Pandas UDFs for scalable generation" - - "Tickets must reference valid customer_ids from customers table" - - "Priority distribution must vary by customer tier" - - "Resolution hours must use exponential distribution" - - "ARR must use log-normal distribution" - - "Must use high partition count (64+) for large-scale generation" - metadata: - category: "happy_path" - difficulty: "medium" - source: "interactive_execution" - execution_date: "2026-02-26" - execution_verified: true - tags: ["databricks-connect", "large", "delta", "pandas-udf", "referential-integrity", "executed"] - - # Test 3: Serverless Job + Small + JSON + 2 tables (EXECUTED) - - id: "gen_serverless_job_catalog_json_003" - inputs: - prompt: | - Generate synthetic product catalog data that will run as a serverless Databricks job. - Create 2 related tables with referential integrity: - - products (3,000 rows): product_id, name, category (weighted), price (log-normal), inventory_count - - sales (10,000 rows): sale_id, product_id (FK to products), quantity, sale_date, discount_pct - - Save as JSON files to a Unity Catalog volume. Use schema name 'devkit_gen3_test_small_json'. - Create a job definition with environments for dependencies (faker). - Popular product categories should have more sales (weighted sampling). - expectations: - expected_facts: - - "serverless" - - "environments" - - "dependencies" - - "client" - - "json" - - "product_id" - - "weighted" - - "lognormal" - - "pandas_udf" - expected_patterns: - - pattern: "environments.*spec.*dependencies" - min_count: 1 - description: "Serverless job environment configuration" - - pattern: '"client":\\s*"4"' - min_count: 1 - description: "Correct client version for serverless" - - pattern: "\\.write.*json" - min_count: 1 - description: "JSON output format" - - pattern: "product_id" - min_count: 3 - description: "Foreign key reference in multiple places" - - pattern: "@F\\.pandas_udf|pandas_udf" - min_count: 1 - description: "Pandas UDF for Faker parallelism" - - pattern: "lognormal|log-normal|log_normal" - min_count: 1 - description: "Log-normal distribution for prices" - - pattern: "CREATE SCHEMA IF NOT EXISTS|CREATE VOLUME IF NOT EXISTS" - min_count: 1 - description: "Infrastructure creation in script" - guidelines: - - "Must create serverless job with environments parameter for dependencies" - - "Job spec must include client: 4 (not 1)" - - "Sales table product_id must only reference valid products (FK integrity)" - - "Product categories must be weighted (not uniform)" - - "Price distribution must use log-normal, not uniform" - - "Script must create schema and volume infrastructure" - - "Must NOT use .cache() or .persist() (serverless incompatible)" - - "Popular categories should have more sales (weighted sampling)" - metadata: - category: "happy_path" - difficulty: "medium" - source: "interactive_execution" - execution_date: "2026-02-26" - execution_verified: true - job_run_id: "560746964795126" - tags: ["serverless-job", "small", "json", "referential-integrity", "weighted-sampling", "executed"] - - # Test 4: Serverless Job + Large + Delta + 2 tables (Financial Transactions) (EXECUTED) - - id: "gen_serverless_job_large_delta_financial_004" - inputs: - prompt: | - Generate large-scale synthetic financial transaction data as a serverless Databricks job. - Create 2 related tables with referential integrity: - - users (200,000 rows): user_id, username, account_type (Basic/Premium/VIP weighted 70/25/5), country, created_at - - transactions (1,000,000 rows): txn_id, user_id (FK to users), amount (log-normal varies by account_type), txn_type, timestamp - - Save as Delta tables to Unity Catalog. Use schema name 'devkit_gen4_test_large_delta'. - VIP users should have larger transaction amounts. - Create the job with proper environments configuration. - expectations: - expected_facts: - - "serverless" - - "environments" - - "dependencies" - - "client" - - "Delta" - - "saveAsTable" - - "user_id" - - "account_type" - - "lognormal" - - "pandas_udf" - - "VIP" - expected_patterns: - - pattern: "environments.*spec.*dependencies|environments.*dependencies" - min_count: 1 - description: "Serverless job environment configuration" - - pattern: '"client":\\s*"4"' - min_count: 1 - description: "Correct client version for serverless" - - pattern: "saveAsTable" - min_count: 2 - description: "Delta table registration for both tables" - - pattern: "user_id" - min_count: 3 - description: "Foreign key reference across tables" - - pattern: "@F\\.pandas_udf|pandas_udf" - min_count: 1 - description: "Pandas UDF for Faker parallelism at scale" - - pattern: "lognormal" - min_count: 1 - description: "Log-normal distribution for transaction amounts" - - pattern: "numPartitions.*=.*\\d{2,}" - min_count: 1 - description: "High partition count for 1M+ rows" - - pattern: "CREATE SCHEMA IF NOT EXISTS" - min_count: 1 - description: "Infrastructure creation in script" - - pattern: "VIP" - min_count: 2 - description: "VIP account type handling" - guidelines: - - "Must create serverless job with environments parameter for dependencies" - - "Job spec must include client: 4 (not 1)" - - "Transactions table user_id must only reference valid users (FK integrity)" - - "Account types must be weighted: Basic 70%, Premium 25%, VIP 5%" - - "Transaction amounts must vary by account_type (VIP > Premium > Basic)" - - "Must use Spark + Faker + Pandas UDFs for million-row generation" - - "Must NOT use .cache() or .persist() (serverless incompatible)" - - "Script must create schema infrastructure" - - "Must use high partition count (32+) for large-scale generation" - - "Must write users table to Delta first, then read back for FK joins" - metadata: - category: "happy_path" - difficulty: "hard" - source: "interactive_execution" - execution_date: "2026-02-26" - execution_verified: true - job_run_id: "849738704935095" - verified_output: - users_table: "dustin_vannoy_catalog.devkit_gen4_test_large_delta.users" - users_rows: 200000 - users_distribution: - Basic: 140330 - Premium: 56622 - VIP: 3048 - transactions_table: "dustin_vannoy_catalog.devkit_gen4_test_large_delta.transactions" - transactions_rows: 1000000 - amount_by_account_type: - Basic_avg: 39.62 - Premium_avg: 115.22 - VIP_avg: 549.87 - tags: ["serverless-job", "large", "delta", "pandas-udf", "referential-integrity", "financial", "executed"] - - # Test 5: Classic Cluster + Small + Delta + 2 tables (HR Data) - - id: "gen_classic_small_delta_hr_005" - inputs: - prompt: | - Generate synthetic HR data to run on a classic Databricks cluster. - Create 2 related tables with referential integrity: - - employees (2,000 rows): employee_id, name, department (weighted), hire_date, salary (log-normal by dept) - - projects (5,000 rows): project_id, employee_id (FK to employees), project_name, hours_logged, status - - Save as Delta tables in Unity Catalog. Use schema name 'devkit_gen5_test_small_delta'. - Engineering department should have higher salaries than other departments. - expectations: - expected_facts: - - "classic cluster" - - "Delta" - - "saveAsTable" - - "employee_id" - - "department" - - "salary" - - "lognormal" - - "pandas_udf" - - "weighted" - expected_patterns: - - pattern: "DatabricksSession\\.builder\\.remote\\(\\)|DatabricksSession\\.builder\\.clusterId" - min_count: 1 - description: "Classic cluster connection (not serverless)" - - pattern: "saveAsTable" - min_count: 2 - description: "Delta table output for both tables" - - pattern: "@F\\.pandas_udf|pandas_udf" - min_count: 1 - description: "Pandas UDF for Faker parallelism" - - pattern: "employee_id" - min_count: 3 - description: "Foreign key reference across tables" - - pattern: "lognormal" - min_count: 1 - description: "Log-normal distribution for salaries" - - pattern: "Engineering.*higher|Engineering.*\\d{6}" - min_count: 1 - description: "Engineering department has higher salary parameters" - - pattern: "CREATE SCHEMA IF NOT EXISTS" - min_count: 1 - description: "Infrastructure creation in script" - - pattern: "databricks libraries install|pip install" - min_count: 1 - description: "Library installation instructions for classic cluster" - guidelines: - - "Must use DatabricksSession.builder.remote() or clusterId() for classic cluster" - - "Must NOT use serverless(True) - this is classic cluster execution" - - "Must use Spark + Faker + Pandas UDFs approach" - - "Projects must reference valid employee_ids from employees table" - - "Salary must vary by department using log-normal distribution" - - "Engineering department must have highest salary parameters" - - "Must include instructions for installing faker/numpy on classic cluster" - - "Must write master table (employees) before generating child table (projects)" - metadata: - category: "happy_path" - difficulty: "medium" - source: "interactive_execution" - execution_date: "2026-02-26" - execution_verified: true - cluster_id: "0128-180718-qmk3usr4" - tables_created: - - "dustin_vannoy_catalog.devkit_gen5_test_small_delta.employees" - - "dustin_vannoy_catalog.devkit_gen5_test_small_delta.projects" - validation_results: - employees_count: 2000 - projects_count: 5000 - orphan_projects: 0 - engineering_avg_salary: 140349 - hr_avg_salary: 69956 - tags: ["classic-cluster", "small", "delta", "pandas-udf", "referential-integrity", "hr-data", "executed"] - - # Test 6: Classic Cluster + Large + Parquet + 2 tables (Supply Chain Data) - - id: "gen_classic_large_parquet_supply_chain_006" - inputs: - prompt: | - Generate large-scale synthetic supply chain data to run on a classic Databricks cluster. - Create 2 related tables with referential integrity: - - suppliers (50,000 rows): supplier_id, company_name, country, rating (1-5 weighted toward 3-4), active_since - - inventory (300,000 rows): inventory_id, supplier_id (FK to suppliers), product_sku, quantity (Pareto), unit_cost, last_restock_date - - Save as Parquet files to a Unity Catalog volume. Use schema name 'devkit_gen6_test_classic_parquet'. - Higher-rated suppliers should have more inventory items. - expectations: - expected_facts: - - "classic cluster" - - "clusterId" - - "pandas_udf" - - "parquet" - - "supplier_id" - - "inventory" - - "Pareto" - - "lognormal" - - "weighted" - - "rating" - expected_patterns: - - pattern: "DatabricksSession\\.builder\\.clusterId" - min_count: 1 - description: "Classic cluster connection (not serverless)" - - pattern: "@F\\.pandas_udf|pandas_udf" - min_count: 1 - description: "Pandas UDF for Faker parallelism" - - pattern: "\\.write.*parquet|write\\.mode.*parquet" - min_count: 2 - description: "Parquet output for both tables" - - pattern: "supplier_id" - min_count: 3 - description: "Foreign key reference across tables" - - pattern: "pareto|Pareto" - min_count: 1 - description: "Pareto distribution for quantities" - - pattern: "lognormal" - min_count: 1 - description: "Log-normal distribution for unit costs" - - pattern: "rating.*weighted|weights.*rating|\\[0\\.05.*0\\.15.*0\\.35.*0\\.35.*0\\.10\\]" - min_count: 1 - description: "Weighted rating distribution toward 3-4" - - pattern: "CREATE SCHEMA IF NOT EXISTS" - min_count: 1 - description: "Infrastructure creation in script" - - pattern: "CREATE VOLUME IF NOT EXISTS" - min_count: 1 - description: "Volume creation for Parquet output" - - pattern: "numPartitions.*=.*\\d{2}" - min_count: 1 - description: "High partition count for large-scale generation" - guidelines: - - "Must use DatabricksSession.builder.clusterId() for classic cluster (not serverless)" - - "Must NOT use serverless(True) - this is classic cluster execution" - - "Must use Spark + Faker + Pandas UDFs approach" - - "Inventory must reference valid supplier_ids from suppliers table (FK integrity)" - - "Quantity must use Pareto/power-law distribution (right-skewed)" - - "Unit cost must use log-normal distribution" - - "Rating distribution must be weighted toward 3-4 (approximately 5%/15%/35%/35%/10%)" - - "Higher-rated suppliers must have more inventory items on average" - - "Must write master table (suppliers) before generating child table (inventory)" - - "Must create schema and volume infrastructure within the script" - - "Must use high partition count (32+) for large-scale generation" - - "Must include instructions or code for installing faker/numpy on classic cluster" - metadata: - category: "happy_path" - difficulty: "hard" - source: "interactive_execution" - execution_date: "2026-02-26" - execution_verified: true - cluster_id: "0128-180718-qmk3usr4" - tables_created: - - "dustin_vannoy_catalog.devkit_gen6_test_classic_parquet.suppliers (Parquet)" - - "dustin_vannoy_catalog.devkit_gen6_test_classic_parquet.inventory (Parquet)" - validation_results: - suppliers_count: 50000 - inventory_count: 225159 - orphan_inventory: 0 - rating_distribution: "5%/15%/35%/35%/10% for ratings 1-5" - avg_items_rating_1: 2.18 - avg_items_rating_5: 6.07 - quantity_median: 13 - quantity_p95: 33 - quantity_max: 1460 - unit_cost_median: 49.42 - unit_cost_avg: 68.05 - tags: ["classic-cluster", "large", "parquet", "pandas-udf", "referential-integrity", "supply-chain", "pareto", "weighted-rating", "executed"] - - # Test 7: Databricks Connect + Medium + CSV + 3 tables (Retail Orders) - - id: "gen_dbconnect_medium_csv_3tables_retail_007" - inputs: - prompt: | - Generate synthetic retail order data using Databricks Connect with serverless. - Create 3 related tables with full referential integrity: - - customers (10,000 rows): customer_id, name, email, membership_level (Bronze/Silver/Gold/Platinum weighted 50/30/15/5), region - - orders (50,000 rows): order_id, customer_id (FK to customers), order_date, total_amount, status - - line_items (150,000 rows): line_item_id, order_id (FK to orders), product_name, quantity, unit_price - - Save as CSV files with headers to Unity Catalog volume. Use schema name 'devkit_gen7_test_medium_csv'. - Create realistic product names. - Higher membership levels should have more orders. - Order total_amount should equal sum of line_items. - expectations: - expected_facts: - - "DatabricksSession" - - "serverless" - - "CSV" - - "header" - - "customer_id" - - "order_id" - - "line_item" - - "Faker" - - "pandas_udf" - - "membership_level" - - "weighted" - - "total_amount" - - "lognormal" - expected_patterns: - - pattern: "DatabricksSession.*serverless.*True" - min_count: 1 - description: "Databricks Connect serverless configuration" - - pattern: "DatabricksEnv.*withDependencies" - min_count: 1 - description: "Managed dependencies for serverless" - - pattern: "@F\\.pandas_udf|pandas_udf" - min_count: 1 - description: "Pandas UDF for Faker parallelism" - - pattern: "customer_id" - min_count: 5 - description: "FK in customers and orders (multiple references)" - - pattern: "order_id" - min_count: 5 - description: "FK in orders and line_items (multiple references)" - - pattern: "\\.option.*header.*true.*\\.csv|\\.csv.*header" - min_count: 1 - description: "CSV with headers" - - pattern: "Bronze|Silver|Gold|Platinum" - min_count: 4 - description: "All membership levels present" - - pattern: "lognormal" - min_count: 1 - description: "Log-normal distribution for pricing" - - pattern: "CREATE SCHEMA IF NOT EXISTS" - min_count: 1 - description: "Infrastructure creation in script" - - pattern: "CREATE VOLUME IF NOT EXISTS" - min_count: 1 - description: "Volume creation for CSV output" - - pattern: "saveAsTable.*_tmp|write.*saveAsTable" - min_count: 1 - description: "Temp Delta tables for FK integrity (no .cache() on serverless)" - - pattern: "total_amount.*sum|sum.*line_total|computed_total" - min_count: 1 - description: "Order total computed from line items" - guidelines: - - "Must use DatabricksSession.builder.serverless(True).getOrCreate()" - - "Must use DatabricksEnv().withDependencies() for managed dependencies" - - "Must use Spark + Faker + Pandas UDFs approach" - - "Must maintain referential integrity across all 3 tables" - - "orders.customer_id must reference valid customers" - - "line_items.order_id must reference valid orders" - - "Must NOT use .cache() or .persist() (serverless incompatible)" - - "Must write to temp Delta tables for FK integrity, not .cache()" - - "Membership level must be weighted: Bronze 50%, Silver 30%, Gold 15%, Platinum 5%" - - "Higher membership levels must generate more orders per customer" - - "Order total_amount must equal sum of (quantity * unit_price) from line_items" - - "Unit prices should use log-normal distribution for realistic pricing" - - "CSV output must include header row" - - "Must create schema and volume infrastructure within the script" - - "Should use Faker for realistic product names" - metadata: - category: "happy_path" - difficulty: "hard" - source: "interactive_execution" - execution_date: "2026-02-26" - execution_verified: true - verified_output: - customers_table: "dustin_vannoy_catalog.devkit_gen7_test_medium_csv (CSV)" - customers_rows: 10000 - membership_distribution: - Bronze: 4935 - Silver: 4051 - Gold: 965 - Platinum: 49 - orders_rows: 50000 - orders_per_customer_by_tier: - Bronze_avg: 3.5 - Silver_avg: 5.9 - Gold_avg: 8.3 - Platinum_avg: 11.4 - line_items_rows: 119704 - orphan_orders: 0 - orphan_line_items: 0 - total_amount_matches_line_items: true - tags: ["databricks-connect", "medium", "csv", "3-tables", "pandas-udf", "referential-integrity", "retail", "computed-totals", "executed"] - - # Test 8: Serverless Job + Medium + JSON + 3 tables (CRM Data) - - id: "gen_serverless_job_medium_json_3tables_crm_008" - inputs: - prompt: | - Generate synthetic CRM data as a serverless Databricks job. - Create 3 related tables with referential integrity: - - accounts (8,000 rows): account_id, company_name, industry (weighted), annual_revenue (log-normal), tier (SMB/Mid-Market/Enterprise) - - contacts (25,000 rows): contact_id, account_id (FK to accounts), first_name, last_name, email, title, is_primary - - activities (80,000 rows): activity_id, contact_id (FK to contacts), activity_type (Call/Email/Meeting weighted), activity_date, duration_minutes (exponential), notes - - Save as JSON files to Unity Catalog volume. Use schema name 'devkit_gen8_test_medium_json'. - Enterprise accounts should have more contacts. Use realistic time patterns (weekday bias, business hours). - expectations: - expected_facts: - - "serverless" - - "environments" - - "dependencies" - - "client" - - "JSON" - - "account_id" - - "contact_id" - - "activity" - - "weekday" - - "exponential" - - "lognormal" - - "pandas_udf" - - "is_primary" - - "weighted" - expected_patterns: - - pattern: "environments.*spec.*dependencies|environments.*dependencies" - min_count: 1 - description: "Serverless job environment configuration" - - pattern: '"client":\\s*"4"' - min_count: 1 - description: "Correct client version for serverless" - - pattern: "\\.write.*json" - min_count: 1 - description: "JSON output format" - - pattern: "account_id" - min_count: 4 - description: "FK across accounts and contacts tables" - - pattern: "contact_id" - min_count: 4 - description: "FK in contacts and activities tables" - - pattern: "@F\\.pandas_udf|pandas_udf" - min_count: 1 - description: "Pandas UDF for Faker parallelism" - - pattern: "lognormal" - min_count: 1 - description: "Log-normal distribution for annual_revenue" - - pattern: "exponential" - min_count: 1 - description: "Exponential distribution for duration_minutes" - - pattern: "weekday|is_weekend|dayofweek" - min_count: 1 - description: "Weekday bias implementation" - - pattern: "CREATE SCHEMA IF NOT EXISTS" - min_count: 1 - description: "Infrastructure creation in script" - - pattern: "CREATE VOLUME IF NOT EXISTS" - min_count: 1 - description: "Volume creation for JSON output" - - pattern: "is_primary" - min_count: 2 - description: "Primary contact flag" - - pattern: "saveAsTable.*staging|_staging" - min_count: 2 - description: "Staging tables for FK integrity (no .cache() on serverless)" - guidelines: - - "Must create serverless job with environments parameter for dependencies" - - "Job spec must include client: 4 (not 1)" - - "Must use Spark + Faker + Pandas UDFs approach" - - "contacts.account_id must reference valid accounts (FK integrity)" - - "activities.contact_id must reference valid contacts (FK integrity)" - - "Industry must be weighted distribution (Tech, Finance, Healthcare, Retail, Manufacturing, Other)" - - "Tier must be weighted: SMB ~60%, Mid-Market ~30%, Enterprise ~10%" - - "Activity type must be weighted: Email ~50%, Call ~35%, Meeting ~15%" - - "Activity dates must show weekday bias (85%+ Mon-Fri)" - - "Activity dates must show business hours bias (70%+ 9am-5pm)" - - "Duration must use exponential distribution by activity type" - - "Annual revenue must use log-normal distribution by tier" - - "Enterprise accounts must have more contacts on average than SMB" - - "Must mark first contact per account as is_primary=True" - - "Must NOT use .cache() or .persist() (serverless incompatible)" - - "Must NOT use RDDs or broadcast variables (serverless incompatible)" - - "Must write to staging Delta tables for FK integrity, then export to JSON" - - "Must create schema and volume infrastructure within the script" - metadata: - category: "happy_path" - difficulty: "hard" - source: "interactive_execution" - execution_date: "2026-02-26" - execution_verified: true - job_id: 673682369587321 - job_run_id: "129954663948285" - verified_output: - accounts_table: "dustin_vannoy_catalog.devkit_gen8_test_medium_json.accounts_staging" - accounts_rows: 8000 - tier_distribution: - Enterprise: 304 - Mid-Market: 2945 - SMB: 4751 - industry_distribution: - Finance: 2750 - Technology: 2383 - Healthcare: 1803 - Retail: 860 - Manufacturing: 183 - Other: 21 - contacts_table: "dustin_vannoy_catalog.devkit_gen8_test_medium_json.contacts_staging" - contacts_rows: 25000 - contacts_per_tier_avg: - Enterprise: 6.375 - Mid-Market: 3.91 - SMB: 2.71 - activities_table: "dustin_vannoy_catalog.devkit_gen8_test_medium_json.activities_staging" - activities_rows: 80000 - activity_type_distribution: - Email: 39925 - Call: 34131 - Meeting: 5944 - weekday_percentage: 95.76 - weekend_percentage: 4.24 - orphan_contacts: 0 - orphan_activities: 0 - revenue_by_tier: - Enterprise_avg: 78721443.88 - Mid-Market_avg: 7354786.45 - SMB_avg: 684174.54 - duration_by_type_avg: - Email: 4.69 - Call: 14.66 - Meeting: 45.54 - json_output_path: "/Volumes/dustin_vannoy_catalog/devkit_gen8_test_medium_json/raw_data/" - tags: ["serverless-job", "medium", "json", "3-tables", "pandas-udf", "referential-integrity", "crm", "time-patterns", "weekday-bias", "executed"] - - # Test 9: Databricks Connect + Incremental + Delta + Variant Column (Event Log) - - id: "gen_dbconnect_incremental_delta_variant_009" - inputs: - prompt: | - Generate synthetic event log data capturing user activity in a chatbot application. This should include deeply nested json string that will convert to large variant column with very long strings which represent the conversation. - Create this as an incremental script which can append new data based on the max date that already exists. Each batch created should be based on a size and window set by variables. Start with 20,000 rows in the batch representing a 1 hour time window. - Save this to Delta table in UC (representing bronze/raw step in the process). - Save to catalog dustin_vannoy_catalog with schema name 'devkit_gen9_test_variant'. - expectations: - expected_facts: - - "DatabricksSession" - - "serverless" - - "Delta" - - "saveAsTable" - - "variant" - - "JSON" - - "nested" - - "incremental" - - "append" - - "max_timestamp" - - "batch" - - "conversation" - - "messages" - - "pandas_udf" - expected_patterns: - - pattern: "DatabricksSession.*serverless.*True" - min_count: 1 - description: "Databricks Connect serverless configuration" - - pattern: "saveAsTable|write.*mode.*append" - min_count: 1 - description: "Delta table output with append capability" - - pattern: "MAX.*event_timestamp|max.*timestamp" - min_count: 1 - description: "Incremental logic reading max timestamp" - - pattern: "append" - min_count: 1 - description: "Append mode for incremental writes" - - pattern: "@F\\.pandas_udf|pandas_udf" - min_count: 1 - description: "Pandas UDF for payload generation" - - pattern: "json\\.dumps|JSON" - min_count: 1 - description: "JSON serialization for variant column" - - pattern: "session_metadata|messages|context|analytics" - min_count: 2 - description: "Deeply nested JSON structure keys" - - pattern: "content.*message|message.*content" - min_count: 1 - description: "Long conversation content in messages" - - pattern: "BATCH_SIZE|batch_size|N_EVENTS" - min_count: 1 - description: "Configurable batch size variable" - - pattern: "TIME_WINDOW|time_window|HOURS" - min_count: 1 - description: "Configurable time window variable" - - pattern: "CREATE SCHEMA IF NOT EXISTS" - min_count: 1 - description: "Infrastructure creation in script" - - pattern: "tableExists|table_exists" - min_count: 1 - description: "Check for existing table before append" - guidelines: - - "Must use DatabricksSession.builder.serverless(True).getOrCreate()" - - "Must use Spark + Faker + Pandas UDFs approach" - - "Must implement incremental logic: read MAX(event_timestamp) from existing table" - - "Must append new data starting from max timestamp + 1 second" - - "Must create new table if it doesn't exist (first run)" - - "Must use append mode for subsequent runs" - - "Must generate deeply nested JSON for variant column (session_metadata, messages, context, analytics)" - - "Messages array must contain multiple turns with role (user/assistant)" - - "Message content must be long strings (50-500 words realistic conversation)" - - "Must have configurable BATCH_SIZE variable (default 20,000)" - - "Must have configurable TIME_WINDOW variable (default 1 hour)" - - "Each batch represents a specific time window of events" - - "Must NOT use .cache() or .persist() (serverless incompatible)" - - "JSON payload should include: session_metadata, messages[], context, analytics" - - "Must create schema infrastructure within the script" - metadata: - category: "happy_path" - difficulty: "hard" - source: "interactive_execution" - execution_date: "2026-02-26" - execution_verified: true - incremental_test_verified: true - table_created: "dustin_vannoy_catalog.devkit_gen9_test_variant.chatbot_events" - validation_results: - batch_1_events: 20000 - batch_2_events: 20000 - total_events_after_2_batches: 40000 - unique_users_per_batch: 919 - unique_sessions_per_batch: 19995 - payload_min_size_bytes: 2936 - payload_median_size_bytes: 9539 - payload_max_size_bytes: 25561 - payload_avg_size_bytes: 9806 - event_type_distribution: - message: 71.5% - tool_call: 19.9% - session_start: 4.9% - feedback: 3.4% - session_end: 0.2% - json_structure_keys: - session_metadata: ["start_time", "client_info", "user_preferences"] - messages: "array of message objects with role, content, tokens, metadata" - context: ["previous_sessions", "knowledge_base_refs", "tool_calls", "user_profile"] - analytics: ["total_turns", "avg_response_time_ms", "total_tokens", "sentiment_scores", "topics"] - message_content_lengths: - min: 223 - max: 1358 - avg: 725 - tags: ["databricks-connect", "incremental", "delta", "variant", "json", "nested", "chatbot", "event-log", "pandas-udf", "append", "executed"] diff --git a/.test/skills/databricks-synthetic-data-gen/candidates.yaml b/.test/skills/databricks-synthetic-data-gen/candidates.yaml new file mode 100644 index 00000000..c6c2a270 --- /dev/null +++ b/.test/skills/databricks-synthetic-data-gen/candidates.yaml @@ -0,0 +1,7 @@ +# Candidates for databricks-synthetic-data-gen skill +# Test cases pending review before promotion to ground_truth.yaml +# +# Use `/skill-test databricks-synthetic-data-gen add` to create new candidates +# Use `/skill-test databricks-synthetic-data-gen review` to promote candidates to ground truth + +candidates: [] diff --git a/.test/skills/databricks-synthetic-data-gen/ground_truth.yaml b/.test/skills/databricks-synthetic-data-gen/ground_truth.yaml new file mode 100644 index 00000000..bee8cdab --- /dev/null +++ b/.test/skills/databricks-synthetic-data-gen/ground_truth.yaml @@ -0,0 +1,1127 @@ +test_cases: + +- id: grp_20260302_113344 + inputs: + prompt: 'Generate synthetic e-commerce data locally then save it to Unity Catalog. + + Create 2 related tables with referential integrity: + + - customers (5,000 rows): customer_id, name, email, tier (Free/Pro/Enterprise + weighted 60/30/10), region, created_at + + - orders (15,000 rows): order_id, customer_id (FK to customers), amount, order_date, + status + + + Save as Parquet then upload to a Unity Catalog volume. Use schema name ''devkit_gen1_test_small_parquet''. + + Enterprise customers should generate more orders than Free tier.' + outputs: + response: "I'll generate the e-commerce data locally using Polars and Faker, then\ + \ upload to Unity Catalog.\n\n## Generation Plan\n\n**Output Location:** `ai_dev_kit.devkit_gen1_test_small_parquet`\n\ + **Volume:** `/Volumes/devkit/devkit_gen1_test_small_parquet/raw_data/`\n\n|\ + \ Table | Columns | Rows |\n|-------|---------|------|\n| customers | customer_id,\ + \ name, email, tier, region, created_at | 5,000 |\n| orders | order_id, customer_id\ + \ (FK), amount, order_date, status | 15,000 |\n\n**Order Distribution by Tier:**\n\ + - Enterprise (10%): ~8 orders per customer average\n- Pro (30%): ~3 orders per\ + \ customer average\n- Free (60%): ~1.5 orders per customer average\n\nHere's\ + \ the complete script:\n\n```python\nimport polars as pl\nimport numpy as np\n\ + from faker import Faker\nfrom datetime import datetime, timedelta\nimport random\n\ + from databricks.sdk import WorkspaceClient\n\n# Configuration\nCATALOG = \"\ + devkit\"\nSCHEMA = \"devkit_gen1_test_small_parquet\"\nNUM_CUSTOMERS = 5000\n\ + NUM_ORDERS = 15000\n\nfake = Faker()\nFaker.seed(42)\nnp.random.seed(42)\nrandom.seed(42)\n\ + \n# Date ranges\nEND_DATE = datetime.now()\nCUSTOMER_START_DATE = END_DATE -\ + \ timedelta(days=730) # 2 years\nORDER_START_DATE = END_DATE - timedelta(days=180)\ + \ # 6 months\n\n# =============================================================================\n\ + # Generate Customers\n# =============================================================================\n\ + print(\"Generating customers...\")\n\n# Tier distribution: Free 60%, Pro 30%,\ + \ Enterprise 10%\ntier_weights = [0.6, 0.3, 0.1]\ntiers = random.choices([\"\ + Free\", \"Pro\", \"Enterprise\"], weights=tier_weights, k=NUM_CUSTOMERS)\n\n\ + # Region distribution\nregion_weights = [0.4, 0.3, 0.2, 0.1]\nregions = random.choices([\"\ + US-East\", \"US-West\", \"EU\", \"APAC\"], weights=region_weights, k=NUM_CUSTOMERS)\n\ + \ncustomers = pl.DataFrame({\n \"customer_id\": [f\"CUST-{i:05d}\" for i\ + \ in range(NUM_CUSTOMERS)],\n \"name\": [fake.name() for _ in range(NUM_CUSTOMERS)],\n\ + \ \"email\": [fake.email() for _ in range(NUM_CUSTOMERS)],\n \"tier\"\ + : tiers,\n \"region\": regions,\n \"created_at\": [\n fake.date_time_between(start_date=CUSTOMER_START_DATE,\ + \ end_date=END_DATE)\n for _ in range(NUM_CUSTOMERS)\n ],\n})\n\n\ + print(f\"Generated {len(customers)} customers\")\nprint(customers.group_by(\"\ + tier\").len().sort(\"tier\"))\n\n# =============================================================================\n\ + # Generate Orders with Tier-Weighted Distribution\n# =============================================================================\n\ + print(\"\\nGenerating orders with tier-weighted distribution...\")\n\n# Calculate\ + \ orders per tier to achieve ~15,000 total with weighted distribution\n# Enterprise:\ + \ ~8 orders/customer, Pro: ~3, Free: ~1.5\ntier_counts = customers.group_by(\"\ + tier\").len()\nenterprise_count = tier_counts.filter(pl.col(\"tier\") == \"\ + Enterprise\")[\"len\"][0]\npro_count = tier_counts.filter(pl.col(\"tier\") ==\ + \ \"Pro\")[\"len\"][0]\nfree_count = tier_counts.filter(pl.col(\"tier\") ==\ + \ \"Free\")[\"len\"][0]\n\n# Target distribution\nenterprise_orders_per = 8\n\ + pro_orders_per = 3\nfree_orders_per = 1.5\n\ntotal_target = enterprise_count\ + \ * enterprise_orders_per + pro_count * pro_orders_per + free_count * free_orders_per\n\ + scale = NUM_ORDERS / total_target\n\nenterprise_orders = int(enterprise_count\ + \ * enterprise_orders_per * scale)\npro_orders = int(pro_count * pro_orders_per\ + \ * scale)\nfree_orders = NUM_ORDERS - enterprise_orders - pro_orders\n\n# Get\ + \ customer IDs by tier\nenterprise_customers = customers.filter(pl.col(\"tier\"\ + ) == \"Enterprise\")[\"customer_id\"].to_list()\npro_customers = customers.filter(pl.col(\"\ + tier\") == \"Pro\")[\"customer_id\"].to_list()\nfree_customers = customers.filter(pl.col(\"\ + tier\") == \"Free\")[\"customer_id\"].to_list()\n\n# Generate customer_ids for\ + \ orders\norder_customer_ids = (\n random.choices(enterprise_customers, k=enterprise_orders)\ + \ +\n random.choices(pro_customers, k=pro_orders) +\n random.choices(free_customers,\ + \ k=free_orders)\n)\nrandom.shuffle(order_customer_ids)\n\n# Create customer\ + \ tier lookup\ncustomer_tier_map = dict(zip(customers[\"customer_id\"].to_list(),\ + \ customers[\"tier\"].to_list()))\n\n# Generate amounts based on tier (log-normal\ + \ distribution)\ndef generate_amount(tier: str) -> float:\n if tier == \"\ + Enterprise\":\n return round(float(np.random.lognormal(6.2, 0.8)), 2)\ + \ # mean ~$500\n elif tier == \"Pro\":\n return round(float(np.random.lognormal(5.0,\ + \ 0.7)), 2) # mean ~$150\n else:\n return round(float(np.random.lognormal(3.9,\ + \ 0.6)), 2) # mean ~$50\n\namounts = [generate_amount(customer_tier_map[cid])\ + \ for cid in order_customer_ids]\n\n# Status distribution: delivered 65%, shipped\ + \ 15%, processing 10%, pending 5%, cancelled 5%\nstatuses = random.choices(\n\ + \ [\"delivered\", \"shipped\", \"processing\", \"pending\", \"cancelled\"\ + ],\n weights=[0.65, 0.15, 0.10, 0.05, 0.05],\n k=NUM_ORDERS\n)\n\norders\ + \ = pl.DataFrame({\n \"order_id\": [f\"ORD-{i:06d}\" for i in range(NUM_ORDERS)],\n\ + \ \"customer_id\": order_customer_ids,\n \"amount\": amounts,\n \"\ + order_date\": [\n fake.date_between(start_date=ORDER_START_DATE.date(),\ + \ end_date=END_DATE.date())\n for _ in range(NUM_ORDERS)\n ],\n \ + \ \"status\": statuses,\n})\n\nprint(f\"Generated {len(orders)} orders\")\n\ + \n# Verify tier-weighted distribution\norders_with_tier = orders.join(\n \ + \ customers.select([\"customer_id\", \"tier\"]),\n on=\"customer_id\"\n)\n\ + print(\"\\nOrders per tier:\")\nprint(orders_with_tier.group_by(\"tier\").len().sort(\"\ + tier\"))\nprint(\"\\nAverage orders per customer by tier:\")\nprint(\n orders_with_tier.group_by(\"\ + tier\")\n .agg(pl.len().alias(\"total_orders\"))\n .join(tier_counts,\ + \ on=\"tier\")\n .with_columns((pl.col(\"total_orders\") / pl.col(\"len\"\ + )).alias(\"avg_orders_per_customer\"))\n .select([\"tier\", \"avg_orders_per_customer\"\ + ])\n .sort(\"tier\")\n)\n\n# =============================================================================\n\ + # Save Locally as Parquet\n# =============================================================================\n\ + print(\"\\nSaving to local Parquet files...\")\nlocal_path = \"synthetic_data_output\"\ + \nimport os\nos.makedirs(local_path, exist_ok=True)\n\ncustomers.write_parquet(f\"\ + {local_path}/customers.parquet\")\norders.write_parquet(f\"{local_path}/orders.parquet\"\ + )\nprint(f\"Saved to {local_path}/\")\n\n# =============================================================================\n\ + # Upload to Unity Catalog Volume\n# =============================================================================\n\ + print(\"\\nUploading to Unity Catalog...\")\n\nw = WorkspaceClient()\n\n# Create\ + \ schema and volume if needed\nfrom databricks.sdk.service.catalog import VolumeType\n\ + \ntry:\n w.schemas.create(name=SCHEMA, catalog_name=CATALOG)\n print(f\"\ + Created schema {CATALOG}.{SCHEMA}\")\nexcept Exception as e:\n if \"SCHEMA_ALREADY_EXISTS\"\ + \ in str(e):\n print(f\"Schema {CATALOG}.{SCHEMA} already exists\")\n\ + \ else:\n raise\n\ntry:\n w.volumes.create(\n catalog_name=CATALOG,\n\ + \ schema_name=SCHEMA,\n name=\"raw_data\",\n volume_type=VolumeType.MANAGED\n\ + \ )\n print(f\"Created volume {CATALOG}.{SCHEMA}.raw_data\")\nexcept Exception\ + \ as e:\n if \"already exists\" in str(e).lower():\n print(f\"Volume\ + \ {CATALOG}.{SCHEMA}.raw_data already exists\")\n else:\n raise\n\n\ + # Upload files\nvolume_path = f\"/Volumes/{CATALOG}/{SCHEMA}/raw_data\"\n\n\ + with open(f\"{local_path}/customers.parquet\", \"rb\") as f:\n w.files.upload(f\"\ + {volume_path}/customers.parquet\", f, overwrite=True)\nprint(f\"Uploaded customers.parquet\ + \ to {volume_path}/\")\n\nwith open(f\"{local_path}/orders.parquet\", \"rb\"\ + ) as f:\n w.files.upload(f\"{volume_path}/orders.parquet\", f, overwrite=True)\n\ + print(f\"Uploaded orders.parquet to {volume_path}/\")\n\nprint(f\"\\n\u2705\ + \ Complete! Data available at:\")\nprint(f\" {volume_path}/customers.parquet\"\ + )\nprint(f\" {volume_path}/orders.parquet\")\n```\n\nThis script:\n1. Generates\ + \ 5,000 customers with weighted tier distribution (Free 60%, Pro 30%, Enterprise\ + \ 10%)\n2. Generates 15,000 orders with tier-weighted customer assignment (Enterprise\ + \ ~8x, Pro ~3x, Free ~1.5x orders per customer)\n3. Uses log-normal distributions\ + \ for realistic order amounts by tier\n4. Saves locally as Parquet files\n5.\ + \ Creates the schema and volume in Unity Catalog if needed\n6. Uploads the Parquet\ + \ files to the volume\n\nRun with:\n```bash\nuv pip install polars faker numpy\ + \ databricks-sdk\nuv run python generate_ecommerce_data.py\n```" + execution_success: true + expectations: + expected_facts: + - "parquet" + - "customer_id" + - "referential integrity" + - "weighted" + - "log-normal" + expected_patterns: + - pattern: "\\.write.*parquet" + min_count: 1 + description: "Parquet output format" + - pattern: "customer_id" + min_count: 3 + description: "Foreign key reference in multiple tables" + - pattern: "lognormal" + min_count: 1 + description: "Log-normal distribution for amounts" + guidelines: + - "Orders table customer_id must only contain IDs from customers table" + - "Enterprise tier customers must have higher weight for order generation" + - "Amount distribution must use log-normal, not uniform" + metadata: + category: happy_path + source: interactive + created_at: '2026-03-02T11:33:44.621846' + execution_verified: + mode: local + verified_date: '2026-03-02' + +- id: gen_serverless_job_catalog_json_002 + inputs: + prompt: 'Generate synthetic product catalog data that will run as a serverless + Databricks job. + + Create 2 related tables with referential integrity: + + - products (3,000 rows): product_id, name, category (weighted), price (log-normal), + inventory_count + + - sales (10,000 rows): sale_id, product_id (FK to products), quantity, sale_date, + discount_pct + + + Save as JSON files to a Unity Catalog volume. Use schema name ''devkit_gen2_test_small_json''. + + Create a job definition with environments for dependencies (faker). + + Popular product categories should have more sales (weighted sampling). + + ' + outputs: + response: "Here's the Python script to generate product catalog data, followed\ + \ by the serverless job definition:\n\n```python\n\"\"\"\nSynthetic Product\ + \ Catalog Data Generation\nServerless Databricks Job with JSON Output\n\nGenerates:\n\ + - products (3,000 rows): product_id, name, category (weighted), price (log-normal),\ + \ inventory_count\n- sales (10,000 rows): sale_id, product_id (FK), quantity,\ + \ sale_date, discount_pct\n\nPopular product categories have more sales (weighted\ + \ sampling).\n\"\"\"\n\nimport os\nfrom pyspark.sql import SparkSession, functions\ + \ as F\nfrom pyspark.sql.window import Window\nfrom pyspark.sql.types import\ + \ StringType, DoubleType, IntegerType\nimport pandas as pd\nimport numpy as\ + \ np\nfrom datetime import datetime, timedelta\n\n# =============================================================================\n\ + # CONFIGURATION\n# =============================================================================\n\ + CATALOG = \"dustin_vannoy_catalog\"\nSCHEMA = \"devkit_gen2_test_small_json\"\ + \nVOLUME_PATH = f\"/Volumes/{CATALOG}/{SCHEMA}/raw_data\"\n\nNUM_PRODUCTS =\ + \ 3_000\nNUM_SALES = 10_000\nNUM_PARTITIONS = 16\n\nEND_DATE = datetime.now().replace(hour=0,\ + \ minute=0, second=0, microsecond=0)\nSTART_DATE = END_DATE - timedelta(days=180)\n\ + SEED = 42\n\n# =============================================================================\n\ + # SPARK SESSION\n# =============================================================================\n\ + \ndef is_databricks_runtime():\n return \"DATABRICKS_RUNTIME_VERSION\" in\ + \ os.environ\n\nif is_databricks_runtime():\n spark = SparkSession.builder.getOrCreate()\n\ + \ print(\"Running on Databricks Runtime\")\nelse:\n from databricks.connect\ + \ import DatabricksSession\n spark = DatabricksSession.builder.serverless(True).getOrCreate()\n\ + \ print(\"Running with Databricks Connect (serverless)\")\n\nprint(\"=\"\ + \ * 60)\nprint(\"PRODUCT CATALOG DATA GENERATION\")\nprint(\"=\" * 60)\nprint(f\"\ + Catalog: {CATALOG}\")\nprint(f\"Schema: {SCHEMA}\")\nprint(f\"Products: {NUM_PRODUCTS:,}\"\ + )\nprint(f\"Sales: {NUM_SALES:,}\")\nprint(\"=\" * 60)\n\n# =============================================================================\n\ + # CREATE INFRASTRUCTURE\n# =============================================================================\n\ + print(\"\\nCreating infrastructure...\")\nspark.sql(f\"CREATE SCHEMA IF NOT\ + \ EXISTS {CATALOG}.{SCHEMA}\")\nspark.sql(f\"CREATE VOLUME IF NOT EXISTS {CATALOG}.{SCHEMA}.raw_data\"\ + )\n\n# =============================================================================\n\ + # PANDAS UDFs\n# =============================================================================\n\ + \n@F.pandas_udf(StringType())\ndef fake_product_name(categories: pd.Series)\ + \ -> pd.Series:\n from faker import Faker\n fake = Faker()\n\n templates\ + \ = {\n \"Electronics\": [\"Smart\", \"Wireless\", \"Digital\", \"Pro\"\ + , \"Ultra\"],\n \"Home & Garden\": [\"Premium\", \"Deluxe\", \"Classic\"\ + , \"Modern\", \"Natural\"],\n \"Clothing\": [\"Designer\", \"Casual\"\ + , \"Comfort\", \"Luxury\", \"Sport\"],\n \"Sports\": [\"Pro\", \"Elite\"\ + , \"Performance\", \"Outdoor\", \"Active\"],\n \"Books\": [\"Complete\"\ + , \"Essential\", \"Ultimate\", \"Practical\", \"Advanced\"],\n \"Toys\"\ + : [\"Fun\", \"Creative\", \"Educational\", \"Super\", \"Magic\"],\n \"\ + Beauty\": [\"Natural\", \"Premium\", \"Radiance\", \"Pure\", \"Glow\"],\n \ + \ \"Automotive\": [\"Pro\", \"Heavy-Duty\", \"Premium\", \"Performance\"\ + , \"Ultra\"],\n }\n\n products = {\n \"Electronics\": [\"Headphones\"\ + , \"Speaker\", \"Charger\", \"Watch\", \"Camera\"],\n \"Home & Garden\"\ + : [\"Lamp\", \"Planter\", \"Organizer\", \"Rug\", \"Vase\"],\n \"Clothing\"\ + : [\"T-Shirt\", \"Jacket\", \"Pants\", \"Sweater\", \"Dress\"],\n \"\ + Sports\": [\"Ball\", \"Racket\", \"Mat\", \"Gloves\", \"Bag\"],\n \"\ + Books\": [\"Guide\", \"Handbook\", \"Manual\", \"Edition\", \"Collection\"],\n\ + \ \"Toys\": [\"Game\", \"Puzzle\", \"Building Set\", \"Robot\", \"Craft\ + \ Kit\"],\n \"Beauty\": [\"Serum\", \"Cream\", \"Lotion\", \"Mask\",\ + \ \"Oil\"],\n \"Automotive\": [\"Tool Kit\", \"Cover\", \"Mat\", \"Cleaner\"\ + , \"Polish\"],\n }\n\n names = []\n for category in categories:\n \ + \ template_list = templates.get(category, [\"Premium\"])\n product_list\ + \ = products.get(category, [\"Item\"])\n template = np.random.choice(template_list)\n\ + \ product = np.random.choice(product_list)\n color = fake.color_name()\n\ + \ names.append(f\"{template} {color} {product}\")\n\n return pd.Series(names)\n\ + \n\n@F.pandas_udf(DoubleType())\ndef generate_price(categories: pd.Series) ->\ + \ pd.Series:\n price_params = {\n \"Electronics\": (4.5, 0.8),\n \ + \ \"Home & Garden\": (3.8, 0.7),\n \"Clothing\": (3.5, 0.6),\n\ + \ \"Sports\": (4.0, 0.7),\n \"Books\": (2.8, 0.4),\n \"\ + Toys\": (3.2, 0.6),\n \"Beauty\": (3.3, 0.5),\n \"Automotive\"\ + : (4.2, 0.8),\n }\n\n prices = []\n for category in categories:\n \ + \ mu, sigma = price_params.get(category, (3.5, 0.6))\n price =\ + \ float(np.random.lognormal(mu, sigma))\n price = round(price) - 0.01\ + \ if price > 1 else round(price, 2)\n prices.append(max(0.99, price))\n\ + \n return pd.Series(prices)\n\n\n@F.pandas_udf(IntegerType())\ndef generate_inventory(ids:\ + \ pd.Series) -> pd.Series:\n inventory = (np.random.pareto(a=2.0, size=len(ids))\ + \ + 1) * 20\n return pd.Series(inventory.astype(int))\n\n\n# =============================================================================\n\ + # GENERATE PRODUCTS TABLE (Master)\n# =============================================================================\n\ + print(f\"\\nGenerating {NUM_PRODUCTS:,} products...\")\n\nproducts_df = (\n\ + \ spark.range(0, NUM_PRODUCTS, numPartitions=NUM_PARTITIONS)\n .select(\n\ + \ F.concat(F.lit(\"PROD-\"), F.lpad(F.col(\"id\").cast(\"string\"), 5,\ + \ \"0\")).alias(\"product_id\"),\n F.col(\"id\").alias(\"_idx\"),\n \ + \ )\n .withColumn(\n \"category\",\n F.when(F.rand(SEED)\ + \ < 0.25, \"Electronics\")\n .when(F.rand(SEED + 1) < 0.45, \"Home &\ + \ Garden\")\n .when(F.rand(SEED + 2) < 0.63, \"Clothing\")\n \ + \ .when(F.rand(SEED + 3) < 0.75, \"Sports\")\n .when(F.rand(SEED +\ + \ 4) < 0.85, \"Books\")\n .when(F.rand(SEED + 5) < 0.93, \"Toys\")\n\ + \ .when(F.rand(SEED + 6) < 0.98, \"Beauty\")\n .otherwise(\"\ + Automotive\")\n )\n .withColumn(\"name\", fake_product_name(F.col(\"category\"\ + )))\n .withColumn(\"price\", generate_price(F.col(\"category\")))\n .withColumn(\"\ + inventory_count\", generate_inventory(F.col(\"_idx\")))\n .drop(\"_idx\"\ + )\n)\n\nproducts_final = products_df.select(\n \"product_id\", \"name\",\ + \ \"category\", \"price\", \"inventory_count\"\n)\n\nproducts_json_path = f\"\ + {VOLUME_PATH}/products\"\nprint(f\"Writing products to {products_json_path}...\"\ + )\nproducts_final.write.mode(\"overwrite\").json(products_json_path)\n\nproducts_for_fk\ + \ = spark.read.json(products_json_path).select(\"product_id\", \"category\"\ + )\nproduct_count = products_for_fk.count()\nprint(f\"Products written: {product_count:,}\"\ + )\n\n# =============================================================================\n\ + # GENERATE SALES TABLE (with Referential Integrity)\n# =============================================================================\n\ + print(f\"\\nGenerating {NUM_SALES:,} sales with referential integrity...\")\n\ + \nproduct_weights = products_for_fk.select(\n \"product_id\",\n \"category\"\ + ,\n F.when(F.col(\"category\") == \"Electronics\", 3.0)\n .when(F.col(\"\ + category\") == \"Home & Garden\", 2.5)\n .when(F.col(\"category\") == \"\ + Clothing\", 2.0)\n .when(F.col(\"category\") == \"Sports\", 1.5)\n .when(F.col(\"\ + category\") == \"Books\", 1.2)\n .when(F.col(\"category\") == \"Toys\",\ + \ 1.0)\n .when(F.col(\"category\") == \"Beauty\", 0.8)\n .otherwise(0.5).alias(\"\ + weight\")\n)\n\nweighted_products = (\n product_weights\n .select(\n \ + \ F.col(\"product_id\"),\n F.col(\"category\"),\n F.explode(F.array_repeat(F.col(\"\ + product_id\"), F.col(\"weight\").cast(\"int\"))).alias(\"_dup\")\n )\n \ + \ .drop(\"_dup\")\n)\n\nsampled_products = (\n weighted_products\n .orderBy(F.rand(SEED\ + \ + 20))\n .limit(NUM_SALES)\n .withColumn(\"sale_rank\", F.row_number().over(Window.orderBy(F.monotonically_increasing_id())))\n\ + \ .select(\"product_id\", \"category\", \"sale_rank\")\n)\n\nsales_base =\ + \ (\n spark.range(0, NUM_SALES, numPartitions=NUM_PARTITIONS)\n .withColumn(\"\ + sale_rank\", F.row_number().over(Window.orderBy(F.col(\"id\"))))\n .withColumn(\n\ + \ \"sale_id\",\n F.concat(F.lit(\"SALE-\"), F.lpad(F.col(\"id\"\ + ).cast(\"string\"), 6, \"0\"))\n )\n)\n\nsales_with_products = sales_base.join(\n\ + \ sampled_products,\n on=\"sale_rank\",\n how=\"inner\"\n)\n\nsales_df\ + \ = (\n sales_with_products\n .withColumn(\n \"quantity\",\n \ + \ F.when(F.rand(SEED + 21) < 0.60, 1)\n .when(F.rand(SEED + 22)\ + \ < 0.85, 2)\n .when(F.rand(SEED + 23) < 0.95, 3)\n .otherwise(F.floor(F.rand(SEED\ + \ + 24) * 5 + 4).cast(\"int\"))\n )\n .withColumn(\n \"sale_date\"\ + ,\n F.date_add(F.lit(START_DATE.date()), (F.rand(SEED + 25) * 180).cast(\"\ + int\"))\n )\n .withColumn(\n \"discount_pct\",\n F.when(F.rand(SEED\ + \ + 26) < 0.70, 0.0)\n .when(F.rand(SEED + 27) < 0.85, 0.10)\n \ + \ .when(F.rand(SEED + 28) < 0.95, 0.20)\n .otherwise(0.30)\n )\n\ + )\n\nsales_final = sales_df.select(\n \"sale_id\", \"product_id\", \"quantity\"\ + , \"sale_date\", \"discount_pct\"\n)\n\nsales_json_path = f\"{VOLUME_PATH}/sales\"\ + \nprint(f\"Writing sales to {sales_json_path}...\")\nsales_final.write.mode(\"\ + overwrite\").json(sales_json_path)\n\nsales_count = spark.read.json(sales_json_path).count()\n\ + print(f\"Sales written: {sales_count:,}\")\n\n# =============================================================================\n\ + # VALIDATION\n# =============================================================================\n\ + print(\"\\n\" + \"=\" * 60)\nprint(\"VALIDATION\")\nprint(\"=\" * 60)\n\nproducts_check\ + \ = spark.read.json(products_json_path)\nsales_check = spark.read.json(sales_json_path)\n\ + \nprint(f\"\\nRow counts:\")\nprint(f\" Products: {products_check.count():,}\"\ + )\nprint(f\" Sales: {sales_check.count():,}\")\n\nprint(f\"\\nSales per category\ + \ (Electronics should have most):\")\nsales_with_category = sales_check.join(\n\ + \ products_check.select(\"product_id\", \"category\"),\n on=\"product_id\"\ + \n)\nsales_with_category.groupBy(\"category\").agg(\n F.count(\"*\").alias(\"\ + total_sales\"),\n F.sum(\"quantity\").alias(\"total_quantity\")\n).orderBy(F.desc(\"\ + total_sales\")).show()\n\norphan_sales = sales_check.join(\n products_check.select(\"\ + product_id\"),\n on=\"product_id\",\n how=\"left_anti\"\n).count()\nprint(f\"\ + \\nReferential integrity:\")\nprint(f\" Orphan sales (no matching product):\ + \ {orphan_sales}\")\n\nprint(\"\\n\" + \"=\" * 60)\nprint(\"GENERATION COMPLETE\"\ + )\nprint(\"=\" * 60)\nprint(f\"\\nOutput location: {VOLUME_PATH}\")\n```\n\n\ + To run this as a serverless job, create the job with this configuration:\n\n\ + ```json\n{\n \"name\": \"generate_product_catalog_data\",\n \"tasks\": [\n\ + \ {\n \"task_key\": \"generate_data\",\n \"spark_python_task\"\ + : {\n \"python_file\": \"/Workspace/Users/your_email/generate_product_catalog_data.py\"\ + \n },\n \"new_cluster\": {\n \"spark_version\": \"16.4.x-scala2.12\"\ + ,\n \"num_workers\": 0,\n \"spark_conf\": {\n \"spark.databricks.cluster.profile\"\ + : \"serverless\"\n }\n },\n \"environments\": [\n {\n\ + \ \"environment_key\": \"default\",\n \"spec\": {\n \ + \ \"client\": \"4\",\n \"dependencies\": [\n \"\ + faker\",\n \"pandas\",\n \"numpy\"\n ]\n\ + \ }\n }\n ]\n }\n ]\n}\n```\n" + expectations: + expected_facts: + - serverless + - environments + - dependencies + - client + - json + - product_id + - weighted + - lognormal + - pandas_udf + expected_patterns: + - pattern: environment_key.*default + min_count: 1 + description: Serverless job environment configuration + - pattern: client.*4 + min_count: 1 + description: Correct client version for serverless + - pattern: \.write.*json + min_count: 1 + description: JSON output format + - pattern: product_id + min_count: 3 + description: Foreign key reference in multiple places + - pattern: '@F\.pandas_udf|pandas_udf' + min_count: 1 + description: Pandas UDF for Faker parallelism + - pattern: lognormal|log-normal|log_normal + min_count: 1 + description: Log-normal distribution for prices + - pattern: CREATE SCHEMA IF NOT EXISTS|CREATE VOLUME IF NOT EXISTS + min_count: 1 + description: Infrastructure creation in script + guidelines: + - Must create serverless job with environments parameter for dependencies + - 'Job spec must include client: 4 (not 1)' + - Sales table product_id must only reference valid products (FK integrity) + - Product categories must be weighted (not uniform) + - Price distribution must use log-normal, not uniform + - Script must create schema and volume infrastructure + - Must NOT use .cache() or .persist() (serverless incompatible) + - Popular categories should have more sales (weighted sampling) + metadata: + category: happy_path + difficulty: medium + source: interactive_execution + execution_date: '2026-02-26' + execution_verified: true + job_run_id: '560746964795126' + tags: + - serverless-job + - small + - json + - referential-integrity + - weighted-sampling + - executed + +- id: grp_20260302_retail_csv_3tables_003 + inputs: + prompt: | + Generate synthetic retail order data using Databricks Connect with serverless. + Create 3 related tables with full referential integrity: + - customers (10,000 rows): customer_id, name, email, membership_level (Bronze/Silver/Gold/Platinum weighted 50/30/15/5), region + - orders (50,000 rows): order_id, customer_id (FK to customers), order_date, total_amount, status + - line_items (150,000 rows): line_item_id, order_id (FK to orders), product_name, quantity, unit_price + + Save as CSV files with headers to Unity Catalog volume. Use schema name 'devkit_gen3_test_medium_csv'. + Create realistic product names. + Higher membership levels should have more orders. + Order total_amount should equal sum of line_items. + outputs: + response: | + Here's the complete script to generate synthetic retail order data with 3 tables and full referential integrity: + + ```python + """ + Synthetic Retail Order Data Generation + ====================================== + 3 related tables with full referential integrity: + - customers (10,000 rows) + - orders (50,000 rows) + - line_items (150,000 rows) + + Output: CSV files with headers to Unity Catalog volume + Execution: Databricks Connect with serverless compute + """ + + import os + from pyspark.sql import functions as F + from pyspark.sql.types import StringType, DoubleType, IntegerType + from pyspark.sql.window import Window + import pandas as pd + import numpy as np + from datetime import datetime, timedelta + + # ============================================================================= + # CONFIGURATION + # ============================================================================= + CATALOG = "ai_dev_kit" + SCHEMA = "devkit_gen3_test_medium_csv" + VOLUME_PATH = f"/Volumes/{CATALOG}/{SCHEMA}/raw_data" + + NUM_CUSTOMERS = 10_000 + NUM_ORDERS = 50_000 + NUM_LINE_ITEMS = 150_000 + NUM_PARTITIONS = 32 + SEED = 42 + + END_DATE = datetime.now().replace(hour=0, minute=0, second=0, microsecond=0) + START_DATE = END_DATE - timedelta(days=180) + + # ============================================================================= + # SPARK SESSION + # ============================================================================= + def is_databricks_runtime(): + return "DATABRICKS_RUNTIME_VERSION" in os.environ + + if is_databricks_runtime(): + from pyspark.sql import SparkSession + spark = SparkSession.builder.getOrCreate() + print("Running on Databricks Runtime") + else: + from databricks.connect import DatabricksSession + from databricks.connect.session import DatabricksEnv + + # Install dependencies on serverless cluster + env = DatabricksEnv().withDependencies("faker", "pandas", "numpy") + spark = DatabricksSession.builder.withEnvironment(env).serverless(True).getOrCreate() + print("Running with Databricks Connect (serverless with managed dependencies)") + + # ============================================================================= + # CREATE INFRASTRUCTURE + # ============================================================================= + spark.sql(f"CREATE SCHEMA IF NOT EXISTS {CATALOG}.{SCHEMA}") + spark.sql(f"CREATE VOLUME IF NOT EXISTS {CATALOG}.{SCHEMA}.raw_data") + + # ============================================================================= + # PANDAS UDFs + # ============================================================================= + + @F.pandas_udf(StringType()) + def fake_name(ids: pd.Series) -> pd.Series: + from faker import Faker + fake = Faker() + Faker.seed(SEED) + return pd.Series([fake.name() for _ in range(len(ids))]) + + @F.pandas_udf(StringType()) + def fake_email(names: pd.Series) -> pd.Series: + from faker import Faker + fake = Faker() + emails = [] + for name in names: + parts = name.lower().split() + if len(parts) >= 2: + email = f"{parts[0]}.{parts[-1]}@{fake.free_email_domain()}" + else: + email = f"{parts[0]}{np.random.randint(100, 999)}@{fake.free_email_domain()}" + emails.append(email) + return pd.Series(emails) + + @F.pandas_udf(StringType()) + def fake_product_name(ids: pd.Series) -> pd.Series: + from faker import Faker + fake = Faker() + product_types = ["Chair", "Table", "Lamp", "Desk", "Shelf", "Cabinet", "Sofa", "Rug", + "Mirror", "Clock", "Vase", "Frame", "Pillow", "Blanket", "Candle", + "Mug", "Bowl", "Plate", "Glass", "Bottle", "Box", "Bag", "Hat", + "Watch", "Headphones", "Speaker", "Charger", "Cable", "Case"] + products = [] + for _ in range(len(ids)): + color = fake.color_name() + adj = fake.word().capitalize() + product = np.random.choice(product_types) + products.append(f"{color} {adj} {product}") + return pd.Series(products) + + @F.pandas_udf(DoubleType()) + def generate_unit_price(ids: pd.Series) -> pd.Series: + """Log-normal unit prices (median ~$35, range $5-$500)""" + prices = np.random.lognormal(mean=3.5, sigma=0.7, size=len(ids)) + prices = np.clip(prices, 5.0, 500.0) + return pd.Series(np.round(prices, 2)) + + # ============================================================================= + # GENERATE CUSTOMERS TABLE + # ============================================================================= + customers_df = ( + spark.range(0, NUM_CUSTOMERS, numPartitions=NUM_PARTITIONS) + .select( + F.concat(F.lit("CUST-"), F.lpad(F.col("id").cast("string"), 5, "0")).alias("customer_id"), + F.col("id").alias("_idx") + ) + .withColumn("name", fake_name(F.col("_idx"))) + .withColumn("email", fake_email(F.col("name"))) + .withColumn("membership_level", + F.when(F.rand(SEED) < 0.50, "Bronze") + .when(F.rand(SEED + 1) < 0.80, "Silver") + .when(F.rand(SEED + 2) < 0.95, "Gold") + .otherwise("Platinum") + ) + .withColumn("region", + F.when(F.rand(SEED + 3) < 0.30, "US-East") + .when(F.rand(SEED + 4) < 0.55, "US-West") + .when(F.rand(SEED + 5) < 0.80, "EU") + .when(F.rand(SEED + 6) < 0.95, "APAC") + .otherwise("Other") + ) + .drop("_idx") + ) + + # Write to temp Delta table (no .cache() on serverless) + customers_tmp = f"{CATALOG}.{SCHEMA}._tmp_customers" + customers_df.write.mode("overwrite").saveAsTable(customers_tmp) + customers_df = spark.table(customers_tmp) + + # ============================================================================= + # GENERATE ORDERS TABLE (weighted by membership level) + # ============================================================================= + customer_weights = customers_df.select( + "customer_id", + "membership_level", + F.when(F.col("membership_level") == "Platinum", 10.0) + .when(F.col("membership_level") == "Gold", 7.0) + .when(F.col("membership_level") == "Silver", 5.0) + .otherwise(3.0).alias("weight") + ) + + weighted_customers = ( + customer_weights + .withColumn("replicate_count", (F.col("weight") * 2).cast("int")) + .select( + F.col("customer_id"), + F.explode(F.array_repeat(F.col("customer_id"), F.col("replicate_count"))).alias("_dup") + ) + .drop("_dup") + ) + + sampled_customers = ( + weighted_customers + .orderBy(F.rand(SEED + 10)) + .limit(NUM_ORDERS) + .withColumn("_row", F.row_number().over(Window.orderBy(F.monotonically_increasing_id()))) + .select("customer_id", "_row") + ) + + orders_base = ( + spark.range(0, NUM_ORDERS, numPartitions=NUM_PARTITIONS) + .withColumn("order_id", + F.concat(F.lit("ORD-"), F.lpad(F.col("id").cast("string"), 6, "0"))) + .withColumn("_row", F.row_number().over(Window.orderBy(F.col("id")))) + ) + + orders_df = ( + orders_base + .join(sampled_customers, on="_row", how="inner") + .drop("_row", "id") + .withColumn("order_date", + F.date_add(F.lit(START_DATE.date()), (F.rand(SEED + 11) * 180).cast("int"))) + .withColumn("status", + F.when(F.rand(SEED + 12) < 0.60, "Delivered") + .when(F.rand(SEED + 13) < 0.78, "Shipped") + .when(F.rand(SEED + 14) < 0.90, "Processing") + .when(F.rand(SEED + 15) < 0.96, "Pending") + .otherwise("Cancelled") + ) + .withColumn("total_amount", F.lit(0.0)) + ) + + orders_tmp = f"{CATALOG}.{SCHEMA}._tmp_orders" + orders_df.write.mode("overwrite").saveAsTable(orders_tmp) + orders_df = spark.table(orders_tmp) + + # ============================================================================= + # GENERATE LINE_ITEMS TABLE + # ============================================================================= + order_ids = orders_df.select("order_id") + + items_per_order = ( + order_ids + .withColumn("num_items", + F.when(F.rand(SEED + 20) < 0.10, 1) + .when(F.rand(SEED + 21) < 0.25, 2) + .when(F.rand(SEED + 22) < 0.45, 3) + .when(F.rand(SEED + 23) < 0.65, 4) + .when(F.rand(SEED + 24) < 0.80, 5) + .when(F.rand(SEED + 25) < 0.90, 6) + .when(F.rand(SEED + 26) < 0.96, 7) + .otherwise(8) + ) + ) + + line_items_base = ( + items_per_order + .select( + F.col("order_id"), + F.explode(F.sequence(F.lit(1), F.col("num_items"))).alias("item_seq") + ) + .withColumn("_idx", F.monotonically_increasing_id()) + ) + + line_items_df = ( + line_items_base + .limit(NUM_LINE_ITEMS) + .withColumn("line_item_id", + F.concat(F.lit("LI-"), F.lpad(F.col("_idx").cast("string"), 7, "0"))) + .withColumn("product_name", fake_product_name(F.col("_idx"))) + .withColumn("quantity", + F.when(F.rand(SEED + 30) < 0.50, 1) + .when(F.rand(SEED + 31) < 0.80, 2) + .when(F.rand(SEED + 32) < 0.92, 3) + .when(F.rand(SEED + 33) < 0.97, 4) + .otherwise(5) + ) + .withColumn("unit_price", generate_unit_price(F.col("_idx"))) + .drop("_idx", "item_seq") + ) + + line_items_tmp = f"{CATALOG}.{SCHEMA}._tmp_line_items" + line_items_df.write.mode("overwrite").saveAsTable(line_items_tmp) + line_items_df = spark.table(line_items_tmp) + + # ============================================================================= + # COMPUTE ORDER TOTALS FROM LINE ITEMS + # ============================================================================= + order_totals = ( + line_items_df + .withColumn("line_total", F.col("quantity") * F.col("unit_price")) + .groupBy("order_id") + .agg(F.round(F.sum("line_total"), 2).alias("computed_total")) + ) + + orders_final = ( + spark.table(orders_tmp) + .drop("total_amount") + .join(order_totals, on="order_id", how="left") + .withColumn("total_amount", F.coalesce(F.col("computed_total"), F.lit(0.0))) + .drop("computed_total") + ) + + # ============================================================================= + # EXPORT TO CSV + # ============================================================================= + customers_final = spark.table(customers_tmp).select( + "customer_id", "name", "email", "membership_level", "region" + ) + + orders_out = orders_final.select( + "order_id", "customer_id", "order_date", "total_amount", "status" + ) + + line_items_out = spark.table(line_items_tmp).select( + "line_item_id", "order_id", "product_name", "quantity", "unit_price" + ) + + customers_final.coalesce(1).write.mode("overwrite").option("header", "true").csv(f"{VOLUME_PATH}/customers") + orders_out.coalesce(1).write.mode("overwrite").option("header", "true").csv(f"{VOLUME_PATH}/orders") + line_items_out.coalesce(4).write.mode("overwrite").option("header", "true").csv(f"{VOLUME_PATH}/line_items") + + # Cleanup temp tables + spark.sql(f"DROP TABLE IF EXISTS {customers_tmp}") + spark.sql(f"DROP TABLE IF EXISTS {orders_tmp}") + spark.sql(f"DROP TABLE IF EXISTS {line_items_tmp}") + ``` + + Run with: + ```bash + uv pip install "databricks-connect>=16.4,<17.0" faker numpy pandas + uv run python generate_retail_data.py + ``` + execution_success: true + expectations: + expected_facts: + - "DatabricksSession" + - "serverless" + - "CSV" + - "header" + - "customer_id" + - "order_id" + - "line_item" + - "Faker" + - "pandas_udf" + - "membership_level" + - "weighted" + - "total_amount" + - "lognormal" + expected_patterns: + - pattern: "DatabricksSession.*serverless.*True" + min_count: 1 + description: "Databricks Connect serverless configuration" + - pattern: "DatabricksEnv.*withDependencies" + min_count: 1 + description: "Managed dependencies for serverless" + - pattern: "@F\\.pandas_udf|pandas_udf" + min_count: 1 + description: "Pandas UDF for Faker parallelism" + - pattern: "customer_id" + min_count: 5 + description: "FK in customers and orders (multiple references)" + - pattern: "order_id" + min_count: 5 + description: "FK in orders and line_items (multiple references)" + - pattern: "\\.option.*header.*true.*\\.csv|\\.write.*csv" + min_count: 1 + description: "CSV with headers" + - pattern: "Bronze|Silver|Gold|Platinum" + min_count: 4 + description: "All membership levels present" + - pattern: "lognormal" + min_count: 1 + description: "Log-normal distribution for pricing" + - pattern: "CREATE SCHEMA IF NOT EXISTS" + min_count: 1 + description: "Infrastructure creation in script" + - pattern: "CREATE VOLUME IF NOT EXISTS" + min_count: 1 + description: "Volume creation for CSV output" + - pattern: "total_amount.*sum|sum.*line_total|computed_total" + min_count: 1 + description: "Order total computed from line items" + guidelines: + - "Must use DatabricksSession.builder.serverless(True).getOrCreate()" + - "Must use DatabricksEnv().withDependencies() for managed dependencies" + - "Must use Spark + Faker + Pandas UDFs approach" + - "Must maintain referential integrity across all 3 tables" + - "orders.customer_id must reference valid customers" + - "line_items.order_id must reference valid orders" + - "Membership level must be weighted: Bronze 50%, Silver 30%, Gold 15%, Platinum 5%" + - "Higher membership levels must generate more orders per customer" + - "Order total_amount must equal sum of (quantity * unit_price) from line_items" + - "Unit prices should use log-normal distribution for realistic pricing" + - "CSV output must include header row" + - "Must create schema and volume infrastructure within the script" + - "Should use Faker for realistic product names" + metadata: + category: happy_path + difficulty: hard + source: interactive_execution + execution_date: '2026-03-02' + execution_verified: true + verified_output: + customers_rows: 10000 + orders_rows: 50000 + line_items_rows: 150000 + membership_distribution: + Bronze: 5069 + Silver: 3957 + Gold: 919 + Platinum: 55 + orders_per_tier: + Bronze: 18170 + Silver: 23560 + Gold: 7613 + Platinum: 657 + orphan_orders: 0 + orphan_line_items: 0 + tags: + - databricks-connect + - serverless + - medium + - csv + - 3-tables + - pandas-udf + - referential-integrity + - weighted-sampling + - computed-totals + - executed + +- id: grp_20260303_manufacturing_delta_streaming_004 + inputs: + prompt: | + Generate manufacturing data that will run incrementally with Python 3.12 and Databricks Serverless. + Create 2 related tables with referential integrity. + Create a sensor reading table that generates 10,000 rows per batch and configure to run as a streaming job. + Create a lookup table for the sensor device which changes slowly. + Save as Delta tables registered in Unity Catalog. Use catalog 'ai_dev_kit'. Use schema name 'devkit_gen4_test_large_delta'. + outputs: + response: | + Here's the complete script to generate manufacturing IoT sensor data with streaming-ready batch processing: + + ```python + """ + Manufacturing IoT Sensor Data Generation + ======================================== + 2 related tables with referential integrity: + - sensor_devices (500 rows): Slowly changing dimension + - sensor_readings (10,000 rows per batch): Streaming incremental + + Output: Delta tables registered in Unity Catalog + Execution: Databricks Connect with serverless compute (Python 3.12+) + """ + + from databricks.connect import DatabricksSession, DatabricksEnv + from pyspark.sql import functions as F + from pyspark.sql.types import StringType, DoubleType, TimestampType + import pandas as pd + import numpy as np + from datetime import datetime, timedelta + + # Configuration + CATALOG = "ai_dev_kit" + SCHEMA = "devkit_gen4_test_large_delta" + + # Row counts + NUM_DEVICES = 500 + NUM_READINGS_PER_BATCH = 10000 + + # Date range for readings (last 24 hours for streaming simulation) + END_TIME = datetime.now() + START_TIME = END_TIME - timedelta(hours=24) + + # Setup with managed dependencies (databricks-connect 16.4+) + env = DatabricksEnv().withDependencies("faker", "pandas", "numpy") + spark = DatabricksSession.builder.withEnvironment(env).serverless(True).getOrCreate() + + print(f"Connected to Databricks serverless compute") + print(f"Output location: {CATALOG}.{SCHEMA}") + + # Create schema if not exists + spark.sql(f"CREATE SCHEMA IF NOT EXISTS {CATALOG}.{SCHEMA}") + print(f"Schema {CATALOG}.{SCHEMA} ready") + + + # ============================================================================= + # Pandas UDFs for data generation + # ============================================================================= + + @F.pandas_udf(StringType()) + def fake_device_name(ids: pd.Series) -> pd.Series: + """Generate realistic device names like 'SENS-T-00001'.""" + from faker import Faker + fake = Faker() + Faker.seed(42) + + prefixes = {"temperature": "T", "pressure": "P", "vibration": "V", "humidity": "H"} + names = [] + for i, _ in enumerate(ids): + type_key = list(prefixes.keys())[i % 4] + prefix = prefixes[type_key] + names.append(f"SENS-{prefix}-{str(i).zfill(5)}") + return pd.Series(names) + + + @F.pandas_udf(DoubleType()) + def generate_sensor_value(device_types: pd.Series) -> pd.Series: + """Generate realistic sensor values based on device type.""" + values = [] + for dtype in device_types: + if dtype == "temperature": + values.append(float(np.random.normal(70, 15))) # Fahrenheit + elif dtype == "pressure": + values.append(float(np.random.normal(100, 10))) # PSI + elif dtype == "vibration": + values.append(float(np.random.lognormal(1.5, 0.8))) # mm/s with spikes + elif dtype == "humidity": + values.append(float(np.clip(np.random.normal(45, 10), 0, 100))) # Percentage + else: + values.append(float(np.random.normal(50, 10))) + return pd.Series(values) + + + @F.pandas_udf(StringType()) + def generate_unit(device_types: pd.Series) -> pd.Series: + """Generate appropriate unit based on device type.""" + unit_map = { + "temperature": "°F", + "pressure": "PSI", + "vibration": "mm/s", + "humidity": "%" + } + return pd.Series([unit_map.get(dt, "units") for dt in device_types]) + + + # ============================================================================= + # Generate sensor_devices (slowly changing dimension) + # ============================================================================= + + print("\nGenerating sensor_devices table (slowly changing dimension)") + + # Device type weights: temperature 30%, pressure 25%, vibration 25%, humidity 20% + devices_df = ( + spark.range(0, NUM_DEVICES, numPartitions=4) + .select( + F.concat(F.lit("DEV-"), F.lpad(F.col("id").cast("string"), 5, "0")).alias("device_id"), + F.col("id").alias("_id") + ) + .withColumn("device_name", fake_device_name(F.col("_id"))) + .withColumn( + "device_type", + F.when(F.rand() < 0.30, "temperature") + .when(F.rand() < 0.55, "pressure") + .when(F.rand() < 0.80, "vibration") + .otherwise("humidity") + ) + .withColumn( + "location", + F.when(F.rand() < 0.40, "Plant_A") + .when(F.rand() < 0.70, "Plant_B") + .when(F.rand() < 0.90, "Plant_C") + .otherwise("Warehouse") + ) + .withColumn( + "install_date", + F.date_add(F.lit("2020-01-01"), (F.rand() * 1460).cast("int")) + ) + .withColumn( + "status", + F.when(F.rand() < 0.85, "active") + .when(F.rand() < 0.95, "maintenance") + .otherwise("inactive") + ) + .drop("_id") + ) + + # Write devices table + devices_table = f"{CATALOG}.{SCHEMA}.sensor_devices" + devices_df.write.mode("overwrite").saveAsTable(devices_table) + print(f"Created {devices_table}") + + + # ============================================================================= + # Generate sensor_readings (streaming incremental batch) + # ============================================================================= + + print("\nGenerating sensor_readings table (streaming batch)") + + # Read devices back for FK join (no .cache() on serverless!) + devices_for_join = spark.table(devices_table).select("device_id", "device_type") + + # Generate readings with valid device_id FK + readings_df = ( + spark.range(0, NUM_READINGS_PER_BATCH, numPartitions=16) + .select( + F.concat( + F.lit("RDG-"), + F.date_format(F.current_timestamp(), "yyyyMMddHHmmss"), + F.lit("-"), + F.lpad(F.col("id").cast("string"), 6, "0") + ).alias("reading_id"), + (F.abs(F.hash(F.col("id"))) % NUM_DEVICES).alias("device_index"), + F.from_unixtime( + F.unix_timestamp(F.lit(START_TIME)) + + (F.rand() * 86400).cast("long") + ).cast("timestamp").alias("timestamp") + ) + ) + + # Get device IDs with row numbers for joining + devices_indexed = ( + devices_for_join + .withColumn("device_index", F.monotonically_increasing_id() % NUM_DEVICES) + ) + + # Join to get valid device_id and device_type + readings_with_device = ( + readings_df + .join( + devices_indexed.select("device_id", "device_type", "device_index"), + on="device_index", + how="inner" + ) + .drop("device_index") + ) + + # Add sensor values based on device type + readings_final = ( + readings_with_device + .withColumn("value", F.round(generate_sensor_value(F.col("device_type")), 2)) + .withColumn("unit", generate_unit(F.col("device_type"))) + .withColumn( + "quality_flag", + F.when(F.rand() < 0.90, "good") + .when(F.rand() < 0.97, "warning") + .otherwise("error") + ) + .select("reading_id", "device_id", "timestamp", "value", "unit", "quality_flag") + ) + + # Write readings table (append mode for streaming simulation) + readings_table = f"{CATALOG}.{SCHEMA}.sensor_readings" + readings_final.write.mode("overwrite").saveAsTable(readings_table) + print(f"Created {readings_table}") + + + # ============================================================================= + # Validation + # ============================================================================= + + # Check referential integrity + orphan_readings = spark.sql(f""" + SELECT COUNT(*) as orphan_count + FROM {readings_table} r + LEFT JOIN {devices_table} d ON r.device_id = d.device_id + WHERE d.device_id IS NULL + """).collect()[0]["orphan_count"] + + print(f"Orphan readings (should be 0): {orphan_readings}") + print(f"\nSUMMARY") + print(f"Catalog/Schema: {CATALOG}.{SCHEMA}") + print(f"Tables: {devices_table}, {readings_table}") + print(f"Referential integrity: {'PASSED' if orphan_readings == 0 else 'FAILED'}") + ``` + + Run with: + ```bash + uv pip install "databricks-connect>=16.4,<17.0" faker numpy pandas + uv run python generate_manufacturing_data.py + ``` + + For streaming job deployment, this can be scheduled as an incremental job where each run: + 1. Generates a new batch of 10,000 readings with unique timestamps + 2. Appends to the sensor_readings table (change mode to "append") + 3. The sensor_devices table is the slowly-changing dimension that persists + execution_success: true + expectations: + expected_facts: + - "DatabricksSession" + - "serverless" + - "Delta" + - "saveAsTable" + - "device_id" + - "sensor" + - "reading" + - "Faker" + - "pandas_udf" + - "streaming" + - "incremental" + - "batch" + - "slowly changing" + - "lognormal" + expected_patterns: + - pattern: "DatabricksSession.*serverless.*True" + min_count: 1 + description: "Databricks Connect serverless configuration" + - pattern: "DatabricksEnv.*withDependencies" + min_count: 1 + description: "Managed dependencies for serverless" + - pattern: "@F\\.pandas_udf|pandas_udf" + min_count: 1 + description: "Pandas UDF for Faker parallelism" + - pattern: "device_id" + min_count: 3 + description: "FK in devices and readings (multiple references)" + - pattern: "\\.write.*saveAsTable|saveAsTable" + min_count: 2 + description: "Delta table output for both tables" + - pattern: "CREATE SCHEMA IF NOT EXISTS" + min_count: 1 + description: "Infrastructure creation in script" + - pattern: "sensor_devices|sensor_readings" + min_count: 2 + description: "Both sensor tables present" + - pattern: "temperature|pressure|vibration|humidity" + min_count: 4 + description: "All device types present" + - pattern: "lognormal" + min_count: 1 + description: "Log-normal distribution for vibration sensor values" + - pattern: "mode.*overwrite|mode.*append" + min_count: 1 + description: "Write mode for streaming support" + guidelines: + - "Must use DatabricksSession.builder.serverless(True).getOrCreate()" + - "Must use DatabricksEnv().withDependencies() for managed dependencies" + - "Must use Spark + Faker + Pandas UDFs approach" + - "Must maintain referential integrity between devices and readings" + - "readings.device_id must reference valid devices" + - "Must use Delta tables (saveAsTable) not file formats" + - "Must create schema infrastructure within the script" + - "sensor_devices should be designed as slowly changing dimension" + - "sensor_readings should support incremental batch processing" + - "Device types should have weighted distribution" + - "Sensor values should have realistic distributions per type" + - "Vibration should use log-normal for occasional spikes" + - "Must NOT use .cache() or .persist() (serverless incompatible)" + metadata: + category: happy_path + difficulty: medium + source: interactive_execution + execution_date: '2026-03-03' + execution_verified: true + verified_output: + sensor_devices_rows: 500 + sensor_readings_rows: 10013 + device_type_distribution: + temperature: 147 + pressure: 179 + vibration: 140 + humidity: 34 + quality_flag_distribution: + good: 9008 + warning: 979 + error: 26 + orphan_readings: 0 + tags: + - databricks-connect + - serverless + - large + - delta + - 2-tables + - pandas-udf + - referential-integrity + - streaming + - incremental + - iot + - manufacturing + - executed diff --git a/.test/skills/databricks-data-generation/manifest.yaml b/.test/skills/databricks-synthetic-data-gen/manifest.yaml similarity index 67% rename from .test/skills/databricks-data-generation/manifest.yaml rename to .test/skills/databricks-synthetic-data-gen/manifest.yaml index 80b6c21f..f80d77d2 100644 --- a/.test/skills/databricks-data-generation/manifest.yaml +++ b/.test/skills/databricks-synthetic-data-gen/manifest.yaml @@ -1,4 +1,4 @@ -skill_name: databricks-data-generation +skill_name: databricks-synthetic-data-gen description: Tests for Databricks synthetic data generation skill covering Spark + Faker + Pandas UDFs, execution methods, output formats, and referential integrity version: 1.0.0 @@ -18,10 +18,10 @@ scorers: - "Code must use the execution method specified in the prompt" - "Code must save data in the output format specified" - "When generating multiple tables, foreign key columns must use consistent ID formats" - - "Must use non-uniform distributions (log-normal, exponential, weighted) for realistic data" - - "Must include configuration section at top of script with CATALOG, SCHEMA, and size variables" - - "Must create catalog, schema, and volume infrastructure within the Python script" - - "Child tables must reference valid IDs from parent tables for referential integrity" + # - "Must use non-uniform distributions (log-normal, exponential, weighted) for realistic data" + # - "Must include configuration section at top of script with CATALOG, SCHEMA, and size variables" + # - "Must create catalog, schema, and volume infrastructure within the Python script" + # - "Child tables must reference valid IDs from parent tables for referential integrity" quality_gates: syntax_valid: 1.0 # 100% - all Python syntax must be valid @@ -35,10 +35,10 @@ trace_expectations: Read: 20 Write: 15 Edit: 15 - mcp__databricks__run_python_file_on_databricks: 5 - mcp__databricks__execute_databricks_command: 5 - mcp__databricks__create_job: 3 - mcp__databricks__run_job_now: 3 + # mcp__databricks__run_python_file_on_databricks: 5 + # mcp__databricks__execute_databricks_command: 5 + # mcp__databricks__create_job: 3 + # mcp__databricks__run_job_now: 3 token_budget: max_total: 200000 required_tools: [] diff --git a/.test/src/skill_test/config.py b/.test/src/skill_test/config.py index 275e25aa..d7c413aa 100644 --- a/.test/src/skill_test/config.py +++ b/.test/src/skill_test/config.py @@ -83,6 +83,9 @@ class MLflowConfig: tracking_uri: str = field(default_factory=lambda: _get_mlflow_tracking_uri()) experiment_name: str = field(default_factory=lambda: os.getenv("MLFLOW_EXPERIMENT_NAME", "/Shared/skill-tests")) + llm_judge_timeout: int = field( + default_factory=lambda: int(os.getenv("MLFLOW_LLM_JUDGE_TIMEOUT", "120")) + ) # seconds - timeout for LLM judge evaluation def _get_mlflow_tracking_uri() -> str: @@ -118,7 +121,7 @@ class DatabricksExecutionSettings: schema: str = field(default_factory=lambda: os.getenv("SKILL_TEST_SCHEMA", "skill_test")) # Execution settings - timeout: int = 120 # seconds + timeout: int = 480 # seconds preserve_context: bool = True # Reuse context across code blocks diff --git a/.test/src/skill_test/grp/executor.py b/.test/src/skill_test/grp/executor.py index 5cd393bb..6f8dedee 100644 --- a/.test/src/skill_test/grp/executor.py +++ b/.test/src/skill_test/grp/executor.py @@ -1,6 +1,7 @@ """Execute code blocks from skill responses to verify they work.""" import ast +import json import re import time import yaml @@ -192,6 +193,26 @@ def verify_yaml_syntax(code: str) -> ExecutionResult: ) +def verify_json_syntax(code: str) -> ExecutionResult: + """Verify JSON syntax is valid.""" + start_time = time.time() + try: + json.loads(code) + return ExecutionResult( + success=True, + output="JSON syntax valid", + error=None, + execution_time_ms=(time.time() - start_time) * 1000, + ) + except json.JSONDecodeError as e: + return ExecutionResult( + success=False, + output="", + error=f"JSON syntax error: {e.msg} at line {e.lineno}, column {e.colno}", + execution_time_ms=(time.time() - start_time) * 1000, + ) + + def verify_bash_structure(code: str) -> ExecutionResult: """Verify bash code structure (basic validation for examples).""" # For bash examples, just check that it's not empty and looks like shell commands @@ -220,6 +241,8 @@ def execute_code_blocks(response: str) -> Tuple[int, int, List[Dict[str, Any]]]: result = verify_sql_structure(block.code) elif block.language in ("yaml", "yml"): result = verify_yaml_syntax(block.code) + elif block.language == "json": + result = verify_json_syntax(block.code) elif block.language in ("bash", "sh", "shell"): result = verify_bash_structure(block.code) else: @@ -528,6 +551,16 @@ def execute_code_blocks_on_databricks( mcp_execute_sql, mcp_get_best_warehouse, ) + elif block.language == "json": + # JSON blocks are validated locally (e.g., job definitions) + json_result = verify_json_syntax(block.code) + result = DatabricksExecutionResult( + success=json_result.success, + output=json_result.output, + error=json_result.error, + execution_time_ms=json_result.execution_time_ms, + execution_mode="local", + ) else: # Skip unknown languages continue diff --git a/.test/src/skill_test/runners/evaluate.py b/.test/src/skill_test/runners/evaluate.py index 1dff1009..9f42c638 100644 --- a/.test/src/skill_test/runners/evaluate.py +++ b/.test/src/skill_test/runners/evaluate.py @@ -2,6 +2,8 @@ from pathlib import Path from typing import Optional, Dict, Any, List +from concurrent.futures import ProcessPoolExecutor, TimeoutError as FuturesTimeoutError +import multiprocessing import yaml import mlflow from mlflow.genai.scorers import Guidelines, Safety @@ -154,6 +156,7 @@ def evaluate_skill( config: Optional[SkillTestConfig] = None, run_name: Optional[str] = None, filter_category: Optional[str] = None, + timeout: Optional[int] = None, ) -> Dict[str, Any]: """ Evaluate a skill using pre-computed outputs (Pattern 2). @@ -163,6 +166,7 @@ def evaluate_skill( config: Configuration (uses defaults if None) run_name: MLflow run name filter_category: Filter test cases by category + timeout: Timeout in seconds for LLM judge evaluation (overrides config) Returns: Evaluation results dict with metrics and run_id @@ -170,6 +174,9 @@ def evaluate_skill( if config is None: config = SkillTestConfig() + # Use provided timeout or fall back to config + eval_timeout = timeout if timeout is not None else config.mlflow.llm_judge_timeout + setup_mlflow(config) # Load ground truth @@ -192,13 +199,19 @@ def evaluate_skill( else: scorers = get_default_scorers() - # Run evaluation + # Run evaluation with timeout with mlflow.start_run(run_name=run_name or f"{skill_name}_eval"): mlflow.set_tags( - {"skill_name": skill_name, "test_count": len(eval_data), "filter_category": filter_category or "all"} + { + "skill_name": skill_name, + "test_count": len(eval_data), + "filter_category": filter_category or "all", + "timeout_seconds": eval_timeout, + } ) # No predict_fn - using pre-computed outputs + # Run evaluation directly - timeout is handled via signal alarm on Unix results = mlflow.genai.evaluate(data=eval_data, scorers=scorers) return { diff --git a/databricks-skills/databricks-data-generation/SKILL.md b/databricks-skills/databricks-synthetic-data-gen/SKILL.md similarity index 94% rename from databricks-skills/databricks-data-generation/SKILL.md rename to databricks-skills/databricks-synthetic-data-gen/SKILL.md index 6d78f6af..8ec3a469 100644 --- a/databricks-skills/databricks-data-generation/SKILL.md +++ b/databricks-skills/databricks-synthetic-data-gen/SKILL.md @@ -1,5 +1,5 @@ --- -name: databricks-data-generation +name: databricks-synthetic-data-gen description: "Generate realistic synthetic data using Spark + Faker (strongly recommended). Supports serverless execution, multiple output formats (Parquet/JSON/CSV/Delta), and scales from thousands to millions of rows. For small datasets (<10K rows), can optionally generate locally and upload to volumes. Use when user mentions 'synthetic data', 'test data', 'generate data', 'demo dataset', 'Faker', or 'sample data'." --- @@ -37,13 +37,14 @@ python generate_data.py ## Critical Rules -1. **Always use Spark + Faker + Pandas UDFs** for data generation (scalable, parallel) -2. **Present a plan for user approval** before generating any code -3. **Ask for catalog/schema** - do not default -4. **Use serverless compute** unless user explicitly requests classic cluster -5. **Generate raw data only** - no pre-aggregated fields (unless user requests) -6. **Create master tables first** - then generate related tables with valid FKs -7. **NEVER use `.cache()` or `.persist()` with serverless compute** - these operations are NOT supported and will fail with `AnalysisException: PERSIST TABLE is not supported on serverless compute`. Instead, write master tables to Delta first, then read them back for FK joins. +1. **Strongly prefer to use Spark + Faker + Pandas UDFs** for data generation (scalable, parallel) +2. **If user specifies local** then use Polars locally instead of Spark, but suggest Spark if > 30,000 rows. +3. **Present a plan for user approval** before generating any code +4. **Ask for catalog/schema** - do not default +5. **Use serverless compute** unless user explicitly requests classic cluster +6. **Generate raw data only** - no pre-aggregated fields (unless user requests) +7. **Create master tables first** - then generate related tables with valid FKs +8. **NEVER use `.cache()` or `.persist()` with serverless compute** - these operations are NOT supported and will fail with `AnalysisException: PERSIST TABLE is not supported on serverless compute`. Instead, write master tables to Delta first, then read them back for FK joins. ## Generation Planning Workflow diff --git a/databricks-skills/databricks-data-generation/references/1-setup-and-execution.md b/databricks-skills/databricks-synthetic-data-gen/references/1-setup-and-execution.md similarity index 100% rename from databricks-skills/databricks-data-generation/references/1-setup-and-execution.md rename to databricks-skills/databricks-synthetic-data-gen/references/1-setup-and-execution.md diff --git a/databricks-skills/databricks-data-generation/references/2-generation-approaches.md b/databricks-skills/databricks-synthetic-data-gen/references/2-generation-approaches.md similarity index 100% rename from databricks-skills/databricks-data-generation/references/2-generation-approaches.md rename to databricks-skills/databricks-synthetic-data-gen/references/2-generation-approaches.md diff --git a/databricks-skills/databricks-data-generation/references/3-data-patterns.md b/databricks-skills/databricks-synthetic-data-gen/references/3-data-patterns.md similarity index 100% rename from databricks-skills/databricks-data-generation/references/3-data-patterns.md rename to databricks-skills/databricks-synthetic-data-gen/references/3-data-patterns.md diff --git a/databricks-skills/databricks-data-generation/references/4-domain-guidance.md b/databricks-skills/databricks-synthetic-data-gen/references/4-domain-guidance.md similarity index 100% rename from databricks-skills/databricks-data-generation/references/4-domain-guidance.md rename to databricks-skills/databricks-synthetic-data-gen/references/4-domain-guidance.md diff --git a/databricks-skills/databricks-data-generation/references/5-output-formats.md b/databricks-skills/databricks-synthetic-data-gen/references/5-output-formats.md similarity index 100% rename from databricks-skills/databricks-data-generation/references/5-output-formats.md rename to databricks-skills/databricks-synthetic-data-gen/references/5-output-formats.md diff --git a/databricks-skills/databricks-data-generation/references/6-troubleshooting.md b/databricks-skills/databricks-synthetic-data-gen/references/6-troubleshooting.md similarity index 100% rename from databricks-skills/databricks-data-generation/references/6-troubleshooting.md rename to databricks-skills/databricks-synthetic-data-gen/references/6-troubleshooting.md diff --git a/databricks-skills/databricks-data-generation/scripts/generate_synthetic_data.py b/databricks-skills/databricks-synthetic-data-gen/scripts/generate_synthetic_data.py similarity index 100% rename from databricks-skills/databricks-data-generation/scripts/generate_synthetic_data.py rename to databricks-skills/databricks-synthetic-data-gen/scripts/generate_synthetic_data.py From d4a7e3a2a79bdececceb64d4accd8550b50ff61e Mon Sep 17 00:00:00 2001 From: dustinvannoy-db Date: Tue, 3 Mar 2026 11:11:38 -0800 Subject: [PATCH 18/24] Fix skill name mismatch and add missing skills to install scripts - Rename databricks-synthetic-data-generation to databricks-synthetic-data-gen across all install scripts, documentation, and cross-references to match the actual skill directory name - Add missing skills (databricks-iceberg, databricks-parsing) to install.sh and install.ps1 Co-Authored-By: Claude Opus 4.5 --- .test/skills/_routing/ground_truth.yaml | 2 +- .test/src/skill_test/scorers/routing.py | 2 +- .test/tests/test_scorers.py | 6 +++--- databricks-builder-app/.env.example | 4 ++-- databricks-builder-app/README.md | 4 ++-- databricks-builder-app/app.yaml.example | 2 +- databricks-builder-app/client/src/pages/DocPage.tsx | 4 ++-- .../server/services/system_prompt.py | 2 +- databricks-skills/README.md | 2 +- databricks-skills/databricks-agent-bricks/SKILL.md | 6 +++--- databricks-skills/databricks-genie/SKILL.md | 4 ++-- databricks-skills/databricks-genie/spaces.md | 2 +- .../databricks-spark-declarative-pipelines/SKILL.md | 2 +- databricks-skills/databricks-unity-catalog/SKILL.md | 2 +- .../databricks-unstructured-pdf-generation/SKILL.md | 2 +- databricks-skills/databricks-zerobus-ingest/SKILL.md | 2 +- databricks-skills/install_skills.sh | 4 ++-- install.ps1 | 10 +++++----- install.sh | 2 +- 19 files changed, 32 insertions(+), 32 deletions(-) diff --git a/.test/skills/_routing/ground_truth.yaml b/.test/skills/_routing/ground_truth.yaml index a428d5b7..f9948433 100644 --- a/.test/skills/_routing/ground_truth.yaml +++ b/.test/skills/_routing/ground_truth.yaml @@ -99,7 +99,7 @@ test_cases: prompt: "Generate synthetic customer data and evaluate the agent quality with MLflow scorers" expectations: expected_skills: - - "databricks-synthetic-data-generation" + - "databricks-synthetic-data-gen" - "databricks-mlflow-evaluation" is_multi_skill: true metadata: diff --git a/.test/src/skill_test/scorers/routing.py b/.test/src/skill_test/scorers/routing.py index 1a03d698..fad45033 100644 --- a/.test/src/skill_test/scorers/routing.py +++ b/.test/src/skill_test/scorers/routing.py @@ -52,7 +52,7 @@ "rest api", ], "databricks-jobs": ["job", "workflow", "task", "schedule", "trigger"], - "databricks-synthetic-data-generation": [ + "databricks-synthetic-data-gen": [ "synthetic data", "fake data", "generate data", diff --git a/.test/tests/test_scorers.py b/.test/tests/test_scorers.py index 66a39dbf..de5b0c09 100644 --- a/.test/tests/test_scorers.py +++ b/.test/tests/test_scorers.py @@ -52,10 +52,10 @@ def test_detect_mlflow_evaluation(self): assert "databricks-mlflow-evaluation" in skills def test_detect_synthetic_data(self): - """Test detection of databricks-synthetic-data-generation skill.""" + """Test detection of databricks-synthetic-data-gen skill.""" prompt = "Generate synthetic data for testing" skills = detect_skills_from_prompt(prompt) - assert "databricks-synthetic-data-generation" in skills + assert "databricks-synthetic-data-gen" in skills def test_detect_agent_bricks(self): """Test detection of databricks-agent-bricks skill.""" @@ -175,7 +175,7 @@ def test_all_skills_have_triggers(self): "databricks-asset-bundles", "databricks-python-sdk", "databricks-jobs", - "databricks-synthetic-data-generation", + "databricks-synthetic-data-gen", "databricks-mlflow-evaluation", "databricks-agent-bricks", "databricks-lakebase-provisioned", diff --git a/databricks-builder-app/.env.example b/databricks-builder-app/.env.example index c95a818f..f50ed4b6 100644 --- a/databricks-builder-app/.env.example +++ b/databricks-builder-app/.env.example @@ -53,10 +53,10 @@ DATABRICKS_MODEL_MINI=databricks-gemini-3-flash # Skills Configuration # ============================================================================= # Skills to include (comma-separated list of skill folder names) -ENABLED_SKILLS=databricks-agent-bricks,databricks-python-sdk,databricks-spark-declarative-pipelines,databricks-synthetic-data-generation,databricks-unstructured-pdf-generation +ENABLED_SKILLS=databricks-agent-bricks,databricks-python-sdk,databricks-spark-declarative-pipelines,databricks-synthetic-data-gen,databricks-unstructured-pdf-generation # Optional: Add additional skills (example with databricks- prefixed skills) -# ENABLED_SKILLS=databricks-agent-bricks,databricks-python-sdk,databricks-spark-declarative-pipelines,databricks-synthetic-data-generation,databricks-unstructured-pdf-generation +# ENABLED_SKILLS=databricks-agent-bricks,databricks-python-sdk,databricks-spark-declarative-pipelines,databricks-synthetic-data-gen,databricks-unstructured-pdf-generation # Test mode: only enable Skill tool (useful for debugging) SKILLS_ONLY_MODE=false diff --git a/databricks-builder-app/README.md b/databricks-builder-app/README.md index b6a43135..42031cee 100644 --- a/databricks-builder-app/README.md +++ b/databricks-builder-app/README.md @@ -179,7 +179,7 @@ Skills include: - **databricks-python-sdk**: Python SDK patterns - **databricks-mlflow-evaluation**: MLflow evaluation and trace analysis - **databricks-spark-declarative-pipelines**: Spark Declarative Pipelines (SDP) development -- **databricks-synthetic-data-generation**: Creating test datasets +- **databricks-synthetic-data-gen**: Creating test datasets ### 5. Project Persistence @@ -329,7 +329,7 @@ Skills are loaded from `../databricks-skills/` and filtered by the `ENABLED_SKIL - `databricks-python-sdk`: Patterns for using the Databricks Python SDK - `databricks-spark-declarative-pipelines`: SDP/DLT pipeline development -- `databricks-synthetic-data-generation`: Creating test datasets +- `databricks-synthetic-data-gen`: Creating test datasets - `databricks-app-apx`: Full-stack apps with React (APX framework) - `databricks-app-python`: Python apps with Dash, Streamlit, Flask diff --git a/databricks-builder-app/app.yaml.example b/databricks-builder-app/app.yaml.example index 4f77f7a7..8a5c0207 100644 --- a/databricks-builder-app/app.yaml.example +++ b/databricks-builder-app/app.yaml.example @@ -30,7 +30,7 @@ env: # ============================================================================= # Comma-separated list of skills to enable - name: ENABLED_SKILLS - value: "databricks-asset-bundles,databricks-agent-bricks,databricks-aibi-dashboards,databricks-app-apx,databricks-app-python,databricks-config,databricks-docs,databricks-jobs,databricks-python-sdk,databricks-unity-catalog,databricks-mlflow-evaluation,databricks-spark-declarative-pipelines,databricks-synthetic-data-generation,databricks-unstructured-pdf-generation" + value: "databricks-asset-bundles,databricks-agent-bricks,databricks-aibi-dashboards,databricks-app-apx,databricks-app-python,databricks-config,databricks-docs,databricks-jobs,databricks-python-sdk,databricks-unity-catalog,databricks-mlflow-evaluation,databricks-spark-declarative-pipelines,databricks-synthetic-data-gen,databricks-unstructured-pdf-generation" - name: SKILLS_ONLY_MODE value: "false" diff --git a/databricks-builder-app/client/src/pages/DocPage.tsx b/databricks-builder-app/client/src/pages/DocPage.tsx index f8b7b29c..b7ee35ec 100644 --- a/databricks-builder-app/client/src/pages/DocPage.tsx +++ b/databricks-builder-app/client/src/pages/DocPage.tsx @@ -92,7 +92,7 @@ function OverviewSection() { Skills explain how to do things and reference the tools from databricks-tools-core.

- {['databricks-asset-bundles/', 'databricks-app-apx/', 'databricks-app-python/', 'databricks-python-sdk/', 'databricks-mlflow-evaluation/', 'databricks-spark-declarative-pipelines/', 'databricks-synthetic-data-generation/'].map((skill) => ( + {['databricks-asset-bundles/', 'databricks-app-apx/', 'databricks-app-python/', 'databricks-python-sdk/', 'databricks-mlflow-evaluation/', 'databricks-spark-declarative-pipelines/', 'databricks-synthetic-data-gen/'].map((skill) => ( {skill} @@ -204,7 +204,7 @@ function OverviewSection() {

Read Skill

- Claude reads databricks-synthetic-data-generation/ skill to learn best practices + Claude reads databricks-synthetic-data-gen/ skill to learn best practices

{['Non-linear distributions', 'Referential integrity', 'Time patterns', 'Row coherence'].map((item) => ( diff --git a/databricks-builder-app/server/services/system_prompt.py b/databricks-builder-app/server/services/system_prompt.py index 5b7b4fef..fd18f6cf 100644 --- a/databricks-builder-app/server/services/system_prompt.py +++ b/databricks-builder-app/server/services/system_prompt.py @@ -5,7 +5,7 @@ # Mapping of user request patterns to skill names for the selection guide. # Only entries whose skill is enabled will be included in the prompt. _SKILL_GUIDE_ENTRIES = [ - ('Generate data, synthetic data, fake data, test data', 'databricks-synthetic-data-generation'), + ('Generate data, synthetic data, fake data, test data', 'databricks-synthetic-data-gen'), ('Pipeline, ETL, bronze/silver/gold, data transformation', 'databricks-spark-declarative-pipelines'), ('Dashboard, visualization, BI, charts', 'databricks-aibi-dashboards'), ('Job, workflow, schedule, automation', 'databricks-jobs'), diff --git a/databricks-skills/README.md b/databricks-skills/README.md index afaccd9d..29a79ae8 100644 --- a/databricks-skills/README.md +++ b/databricks-skills/README.md @@ -58,7 +58,7 @@ cp -r ai-dev-kit/databricks-skills/databricks-agent-bricks .claude/skills/ - **databricks-iceberg** - Apache Iceberg tables (Managed/Foreign), UniForm, Iceberg REST Catalog, Iceberg Clients Interoperability - **databricks-spark-declarative-pipelines** - SDP (formerly DLT) in SQL/Python - **databricks-jobs** - Multi-task workflows, triggers, schedules -- **databricks-synthetic-data-generation** - Realistic test data with Faker +- **databricks-synthetic-data-gen** - Realistic test data with Faker ### 🚀 Development & Deployment - **databricks-asset-bundles** - DABs for multi-environment deployments diff --git a/databricks-skills/databricks-agent-bricks/SKILL.md b/databricks-skills/databricks-agent-bricks/SKILL.md index 4aff7acb..04be7dad 100644 --- a/databricks-skills/databricks-agent-bricks/SKILL.md +++ b/databricks-skills/databricks-agent-bricks/SKILL.md @@ -28,7 +28,7 @@ Before creating Agent Bricks, ensure you have the required data: ### For Genie Spaces - **See the `databricks-genie` skill** for comprehensive Genie Space guidance - Tables in Unity Catalog with the data to explore -- Generate raw data using the `databricks-synthetic-data-generation` skill +- Generate raw data using the `databricks-synthetic-data-gen` skill - Create tables using the `databricks-spark-declarative-pipelines` skill ### For Supervisor Agents @@ -119,7 +119,7 @@ Before creating Agent Bricks, generate the required source data: **For Genie (SQL exploration)**: ``` -1. Use `databricks-synthetic-data-generation` skill to create raw parquet data +1. Use `databricks-synthetic-data-gen` skill to create raw parquet data 2. Use `databricks-spark-declarative-pipelines` skill to create bronze/silver/gold tables ``` @@ -199,7 +199,7 @@ manage_mas( - **[databricks-genie](../databricks-genie/SKILL.md)** - Comprehensive Genie Space creation, curation, and Conversation API guidance - **[databricks-unstructured-pdf-generation](../databricks-unstructured-pdf-generation/SKILL.md)** - Generate synthetic PDFs to feed into Knowledge Assistants -- **[databricks-synthetic-data-generation](../databricks-synthetic-data-generation/SKILL.md)** - Create raw data for Genie Space tables +- **[databricks-synthetic-data-gen](../databricks-synthetic-data-gen/SKILL.md)** - Create raw data for Genie Space tables - **[databricks-spark-declarative-pipelines](../databricks-spark-declarative-pipelines/SKILL.md)** - Build bronze/silver/gold tables consumed by Genie Spaces - **[databricks-model-serving](../databricks-model-serving/SKILL.md)** - Deploy custom agent endpoints used as MAS agents - **[databricks-vector-search](../databricks-vector-search/SKILL.md)** - Build vector indexes for RAG applications paired with KAs diff --git a/databricks-skills/databricks-genie/SKILL.md b/databricks-skills/databricks-genie/SKILL.md index 576771da..e5b32b6e 100644 --- a/databricks-skills/databricks-genie/SKILL.md +++ b/databricks-skills/databricks-genie/SKILL.md @@ -107,7 +107,7 @@ Before creating a Genie Space: ### Creating Tables Use these skills in sequence: -1. `databricks-synthetic-data-generation` - Generate raw parquet files +1. `databricks-synthetic-data-gen` - Generate raw parquet files 2. `databricks-spark-declarative-pipelines` - Create bronze/silver/gold tables ## Common Issues @@ -121,6 +121,6 @@ Use these skills in sequence: ## Related Skills - **[databricks-agent-bricks](../databricks-agent-bricks/SKILL.md)** - Use Genie Spaces as agents inside Supervisor Agents -- **[databricks-synthetic-data-generation](../databricks-synthetic-data-generation/SKILL.md)** - Generate raw parquet data to populate tables for Genie +- **[databricks-synthetic-data-gen](../databricks-synthetic-data-gen/SKILL.md)** - Generate raw parquet data to populate tables for Genie - **[databricks-spark-declarative-pipelines](../databricks-spark-declarative-pipelines/SKILL.md)** - Build bronze/silver/gold tables consumed by Genie Spaces - **[databricks-unity-catalog](../databricks-unity-catalog/SKILL.md)** - Manage the catalogs, schemas, and tables Genie queries diff --git a/databricks-skills/databricks-genie/spaces.md b/databricks-skills/databricks-genie/spaces.md index 8549d6bd..225efe0e 100644 --- a/databricks-skills/databricks-genie/spaces.md +++ b/databricks-skills/databricks-genie/spaces.md @@ -163,7 +163,7 @@ The tool finds the existing space by name and updates it. ## Example End-to-End Workflow -1. **Generate synthetic data** using `databricks-synthetic-data-generation` skill: +1. **Generate synthetic data** using `databricks-synthetic-data-gen` skill: - Creates parquet files in `/Volumes/catalog/schema/raw_data/` 2. **Create tables** using `databricks-spark-declarative-pipelines` skill: diff --git a/databricks-skills/databricks-spark-declarative-pipelines/SKILL.md b/databricks-skills/databricks-spark-declarative-pipelines/SKILL.md index 48a698f8..60afef0b 100644 --- a/databricks-skills/databricks-spark-declarative-pipelines/SKILL.md +++ b/databricks-skills/databricks-spark-declarative-pipelines/SKILL.md @@ -573,5 +573,5 @@ For advanced configuration options (development mode, continuous pipelines, cust - **[databricks-jobs](../databricks-jobs/SKILL.md)** - for orchestrating and scheduling pipeline runs - **[databricks-asset-bundles](../databricks-asset-bundles/SKILL.md)** - for multi-environment deployment of pipeline projects -- **[databricks-synthetic-data-generation](../databricks-synthetic-data-generation/SKILL.md)** - for generating test data to feed into pipelines +- **[databricks-synthetic-data-gen](../databricks-synthetic-data-gen/SKILL.md)** - for generating test data to feed into pipelines - **[databricks-unity-catalog](../databricks-unity-catalog/SKILL.md)** - for catalog/schema/volume management and governance diff --git a/databricks-skills/databricks-unity-catalog/SKILL.md b/databricks-skills/databricks-unity-catalog/SKILL.md index 553eba97..30f34e3d 100644 --- a/databricks-skills/databricks-unity-catalog/SKILL.md +++ b/databricks-skills/databricks-unity-catalog/SKILL.md @@ -110,7 +110,7 @@ mcp__databricks__execute_sql( - **[databricks-spark-declarative-pipelines](../databricks-spark-declarative-pipelines/SKILL.md)** - for pipelines that write to Unity Catalog tables - **[databricks-jobs](../databricks-jobs/SKILL.md)** - for job execution data visible in system tables -- **[databricks-synthetic-data-generation](../databricks-synthetic-data-generation/SKILL.md)** - for generating data stored in Unity Catalog Volumes +- **[databricks-synthetic-data-gen](../databricks-synthetic-data-gen/SKILL.md)** - for generating data stored in Unity Catalog Volumes - **[databricks-aibi-dashboards](../databricks-aibi-dashboards/SKILL.md)** - for building dashboards on top of Unity Catalog data ## Resources diff --git a/databricks-skills/databricks-unstructured-pdf-generation/SKILL.md b/databricks-skills/databricks-unstructured-pdf-generation/SKILL.md index 7666f21b..ee9abf05 100644 --- a/databricks-skills/databricks-unstructured-pdf-generation/SKILL.md +++ b/databricks-skills/databricks-unstructured-pdf-generation/SKILL.md @@ -190,5 +190,5 @@ AZURE_OPENAI_DEPLOYMENT=gpt-4o - **[databricks-agent-bricks](../databricks-agent-bricks/SKILL.md)** - Create Knowledge Assistants that ingest the generated PDFs - **[databricks-vector-search](../databricks-vector-search/SKILL.md)** - Index generated documents for semantic search and RAG -- **[databricks-synthetic-data-generation](../databricks-synthetic-data-generation/SKILL.md)** - Generate structured tabular data (complement to unstructured PDFs) +- **[databricks-synthetic-data-gen](../databricks-synthetic-data-gen/SKILL.md)** - Generate structured tabular data (complement to unstructured PDFs) - **[databricks-mlflow-evaluation](../databricks-mlflow-evaluation/SKILL.md)** - Evaluate RAG systems using the generated question/guideline pairs diff --git a/databricks-skills/databricks-zerobus-ingest/SKILL.md b/databricks-skills/databricks-zerobus-ingest/SKILL.md index efd52b0d..e3d3f48a 100644 --- a/databricks-skills/databricks-zerobus-ingest/SKILL.md +++ b/databricks-skills/databricks-zerobus-ingest/SKILL.md @@ -218,7 +218,7 @@ The timestamp generation must use microseconds for Databricks. - **[databricks-python-sdk](../databricks-python-sdk/SKILL.md)** - General SDK patterns and WorkspaceClient for table/schema management - **[databricks-spark-declarative-pipelines](../databricks-spark-declarative-pipelines/SKILL.md)** - Downstream pipeline processing of ingested data - **[databricks-unity-catalog](../databricks-unity-catalog/SKILL.md)** - Managing catalogs, schemas, and tables that Zerobus writes to -- **[databricks-synthetic-data-generation](../databricks-synthetic-data-generation/SKILL.md)** - Generate test data to feed into Zerobus producers +- **[databricks-synthetic-data-gen](../databricks-synthetic-data-gen/SKILL.md)** - Generate test data to feed into Zerobus producers - **[databricks-config](../databricks-config/SKILL.md)** - Profile and authentication setup ## Resources diff --git a/databricks-skills/install_skills.sh b/databricks-skills/install_skills.sh index ff8d9b86..763489c8 100755 --- a/databricks-skills/install_skills.sh +++ b/databricks-skills/install_skills.sh @@ -42,7 +42,7 @@ MLFLOW_REPO_RAW_URL="https://raw.githubusercontent.com/mlflow/skills" MLFLOW_REPO_REF="main" # Databricks skills (hosted in this repo) -DATABRICKS_SKILLS="databricks-agent-bricks databricks-aibi-dashboards databricks-asset-bundles databricks-app-apx databricks-app-python databricks-config databricks-dbsql databricks-docs databricks-genie databricks-iceberg databricks-jobs databricks-lakebase-autoscale databricks-lakebase-provisioned databricks-metric-views databricks-mlflow-evaluation databricks-model-serving databricks-parsing databricks-python-sdk databricks-spark-declarative-pipelines databricks-spark-structured-streaming databricks-synthetic-data-generation databricks-unity-catalog databricks-unstructured-pdf-generation databricks-vector-search databricks-zerobus-ingest spark-python-data-source" +DATABRICKS_SKILLS="databricks-agent-bricks databricks-aibi-dashboards databricks-asset-bundles databricks-app-apx databricks-app-python databricks-config databricks-dbsql databricks-docs databricks-genie databricks-iceberg databricks-jobs databricks-lakebase-autoscale databricks-lakebase-provisioned databricks-metric-views databricks-mlflow-evaluation databricks-model-serving databricks-parsing databricks-python-sdk databricks-spark-declarative-pipelines databricks-spark-structured-streaming databricks-synthetic-data-gen databricks-unity-catalog databricks-unstructured-pdf-generation databricks-vector-search databricks-zerobus-ingest spark-python-data-source" # MLflow skills (fetched from mlflow/skills repo) MLFLOW_SKILLS="agent-evaluation analyze-mlflow-chat-session analyze-mlflow-trace instrumenting-with-mlflow-tracing mlflow-onboarding querying-mlflow-metrics retrieving-mlflow-traces searching-mlflow-docs" @@ -76,7 +76,7 @@ get_skill_description() { "databricks-spark-declarative-pipelines") echo "Spark Declarative Pipelines (SDP/LDP/DLT)" ;; "spark-python-data-source") echo "Spark custom Python data sources" ;; "databricks-spark-structured-streaming") echo "Spark Structured Streaming patterns and best practices" ;; - "databricks-synthetic-data-generation") echo "Synthetic test data generation" ;; + "databricks-synthetic-data-gen") echo "Synthetic test data generation" ;; "databricks-unstructured-pdf-generation") echo "Generate synthetic PDFs for RAG" ;; "databricks-vector-search") echo "Vector Search - endpoints, indexes, and queries for RAG" ;; "databricks-zerobus-ingest") echo "Zerobus Ingest - gRPC data ingestion into Delta tables" ;; diff --git a/install.ps1 b/install.ps1 index f144b5ac..38e4a2a0 100644 --- a/install.ps1 +++ b/install.ps1 @@ -78,11 +78,11 @@ $script:ProfileProvided = $false $script:Skills = @( "databricks-agent-bricks", "databricks-aibi-dashboards", "databricks-app-apx", "databricks-app-python", "databricks-asset-bundles", "databricks-config", "databricks-dbsql", "databricks-docs", "databricks-genie", - "databricks-jobs", "databricks-metric-views", "databricks-model-serving", "databricks-python-sdk", - "databricks-unity-catalog", "databricks-vector-search", "databricks-zerobus-ingest", - "databricks-lakebase-autoscale", "databricks-lakebase-provisioned", "databricks-mlflow-evaluation", - "databricks-spark-declarative-pipelines", "spark-python-data-source", "databricks-spark-structured-streaming", - "databricks-synthetic-data-generation", "databricks-unstructured-pdf-generation" + "databricks-iceberg", "databricks-jobs", "databricks-lakebase-autoscale", "databricks-lakebase-provisioned", + "databricks-metric-views", "databricks-mlflow-evaluation", "databricks-model-serving", "databricks-parsing", + "databricks-python-sdk", "databricks-spark-declarative-pipelines", "databricks-spark-structured-streaming", + "databricks-synthetic-data-gen", "databricks-unity-catalog", "databricks-unstructured-pdf-generation", + "databricks-vector-search", "databricks-zerobus-ingest", "spark-python-data-source" ) # MLflow skills (fetched from mlflow/skills repo) diff --git a/install.sh b/install.sh index c347b13e..61b98d42 100755 --- a/install.sh +++ b/install.sh @@ -74,7 +74,7 @@ MIN_SDK_VERSION="0.85.0" G='\033[0;32m' Y='\033[1;33m' R='\033[0;31m' BL='\033[0;34m' B='\033[1m' D='\033[2m' N='\033[0m' # Databricks skills (bundled in repo) -SKILLS="databricks-agent-bricks databricks-aibi-dashboards databricks-app-apx databricks-app-python databricks-asset-bundles databricks-config databricks-dbsql databricks-docs databricks-genie databricks-jobs databricks-lakebase-autoscale databricks-lakebase-provisioned databricks-metric-views databricks-mlflow-evaluation databricks-model-serving databricks-python-sdk databricks-spark-declarative-pipelines databricks-spark-structured-streaming databricks-synthetic-data-generation databricks-unity-catalog databricks-unstructured-pdf-generation databricks-vector-search databricks-zerobus-ingest spark-python-data-source" +SKILLS="databricks-agent-bricks databricks-aibi-dashboards databricks-app-apx databricks-app-python databricks-asset-bundles databricks-config databricks-dbsql databricks-docs databricks-genie databricks-iceberg databricks-jobs databricks-lakebase-autoscale databricks-lakebase-provisioned databricks-metric-views databricks-mlflow-evaluation databricks-model-serving databricks-parsing databricks-python-sdk databricks-spark-declarative-pipelines databricks-spark-structured-streaming databricks-synthetic-data-gen databricks-unity-catalog databricks-unstructured-pdf-generation databricks-vector-search databricks-zerobus-ingest spark-python-data-source" # MLflow skills (fetched from mlflow/skills repo) MLFLOW_SKILLS="agent-evaluation analyze-mlflow-chat-session analyze-mlflow-trace instrumenting-with-mlflow-tracing mlflow-onboarding querying-mlflow-metrics retrieving-mlflow-traces searching-mlflow-docs" From e310b67707f9abd5b52673005ba77b7f6d5ed0f4 Mon Sep 17 00:00:00 2001 From: dustinvannoy-db Date: Tue, 3 Mar 2026 13:43:49 -0800 Subject: [PATCH 19/24] Fix PR review issues for databricks-synthetic-data-gen skill MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bugs: - Remove .cache()/.unpersist() in generate_synthetic_data.py (serverless incompatible) - Fix .gitignore formatting (restore blank line separator) Design: - Refactor ground_truth.yaml to use external response files (1127 → 347 lines) - Change timeout from 480s to 240s with explanatory comment - Add Windows timeout warning in mlflow_eval.py Nits: - Fix hardcoded catalog name (dustin_vannoy_catalog → my_catalog) - Fix DatabricksEnv import path (databricks.connect.session → databricks.connect) - Add EOF newline to 1-setup-and-execution.md - Remove unused imports in evaluate.py Co-Authored-By: Claude Opus 4.5 --- .test/scripts/mlflow_eval.py | 3 + .../gen_serverless_job_catalog_json_002.txt | 337 +++++++ .../grp_20260302_113344.txt | 233 +++++ .../grp_20260302_retail_csv_3tables_003.txt | 297 +++++++ ...0303_manufacturing_delta_streaming_004.txt | 235 +++++ .../ground_truth.yaml | 828 +----------------- .test/src/skill_test/config.py | 2 +- .test/src/skill_test/dataset.py | 21 +- .test/src/skill_test/runners/evaluate.py | 2 - .../references/1-setup-and-execution.md | 2 +- .../scripts/generate_synthetic_data.py | 11 +- 11 files changed, 1157 insertions(+), 814 deletions(-) create mode 100644 .test/skills/databricks-synthetic-data-gen/expected_responses/gen_serverless_job_catalog_json_002.txt create mode 100644 .test/skills/databricks-synthetic-data-gen/expected_responses/grp_20260302_113344.txt create mode 100644 .test/skills/databricks-synthetic-data-gen/expected_responses/grp_20260302_retail_csv_3tables_003.txt create mode 100644 .test/skills/databricks-synthetic-data-gen/expected_responses/grp_20260303_manufacturing_delta_streaming_004.txt diff --git a/.test/scripts/mlflow_eval.py b/.test/scripts/mlflow_eval.py index 54ce7559..93278e4d 100755 --- a/.test/scripts/mlflow_eval.py +++ b/.test/scripts/mlflow_eval.py @@ -57,6 +57,9 @@ def main(): if hasattr(signal, 'SIGALRM'): signal.signal(signal.SIGALRM, timeout_handler) signal.alarm(args.timeout) + else: + # Windows: SIGALRM not available - no timeout enforcement + print("WARNING: Timeout not supported on Windows - test may run indefinitely", file=sys.stderr) try: from skill_test.runners import evaluate_skill diff --git a/.test/skills/databricks-synthetic-data-gen/expected_responses/gen_serverless_job_catalog_json_002.txt b/.test/skills/databricks-synthetic-data-gen/expected_responses/gen_serverless_job_catalog_json_002.txt new file mode 100644 index 00000000..5275d953 --- /dev/null +++ b/.test/skills/databricks-synthetic-data-gen/expected_responses/gen_serverless_job_catalog_json_002.txt @@ -0,0 +1,337 @@ +Here's the Python script to generate product catalog data, followed by the serverless job definition: + +```python +""" +Synthetic Product Catalog Data Generation +Serverless Databricks Job with JSON Output + +Generates: +- products (3,000 rows): product_id, name, category (weighted), price (log-normal), inventory_count +- sales (10,000 rows): sale_id, product_id (FK), quantity, sale_date, discount_pct + +Popular product categories have more sales (weighted sampling). +""" + +import os +from pyspark.sql import SparkSession, functions as F +from pyspark.sql.window import Window +from pyspark.sql.types import StringType, DoubleType, IntegerType +import pandas as pd +import numpy as np +from datetime import datetime, timedelta + +# ============================================================================= +# CONFIGURATION +# ============================================================================= +CATALOG = "my_catalog" +SCHEMA = "devkit_gen2_test_small_json" +VOLUME_PATH = f"/Volumes/{CATALOG}/{SCHEMA}/raw_data" + +NUM_PRODUCTS = 3_000 +NUM_SALES = 10_000 +NUM_PARTITIONS = 16 + +END_DATE = datetime.now().replace(hour=0, minute=0, second=0, microsecond=0) +START_DATE = END_DATE - timedelta(days=180) +SEED = 42 + +# ============================================================================= +# SPARK SESSION +# ============================================================================= + +def is_databricks_runtime(): + return "DATABRICKS_RUNTIME_VERSION" in os.environ + +if is_databricks_runtime(): + spark = SparkSession.builder.getOrCreate() + print("Running on Databricks Runtime") +else: + from databricks.connect import DatabricksSession + spark = DatabricksSession.builder.serverless(True).getOrCreate() + print("Running with Databricks Connect (serverless)") + +print("=" * 60) +print("PRODUCT CATALOG DATA GENERATION") +print("=" * 60) +print(f"Catalog: {CATALOG}") +print(f"Schema: {SCHEMA}") +print(f"Products: {NUM_PRODUCTS:,}") +print(f"Sales: {NUM_SALES:,}") +print("=" * 60) + +# ============================================================================= +# CREATE INFRASTRUCTURE +# ============================================================================= +print("\nCreating infrastructure...") +spark.sql(f"CREATE SCHEMA IF NOT EXISTS {CATALOG}.{SCHEMA}") +spark.sql(f"CREATE VOLUME IF NOT EXISTS {CATALOG}.{SCHEMA}.raw_data") + +# ============================================================================= +# PANDAS UDFs +# ============================================================================= + +@F.pandas_udf(StringType()) +def fake_product_name(categories: pd.Series) -> pd.Series: + from faker import Faker + fake = Faker() + + templates = { + "Electronics": ["Smart", "Wireless", "Digital", "Pro", "Ultra"], + "Home & Garden": ["Premium", "Deluxe", "Classic", "Modern", "Natural"], + "Clothing": ["Designer", "Casual", "Comfort", "Luxury", "Sport"], + "Sports": ["Pro", "Elite", "Performance", "Outdoor", "Active"], + "Books": ["Complete", "Essential", "Ultimate", "Practical", "Advanced"], + "Toys": ["Fun", "Creative", "Educational", "Super", "Magic"], + "Beauty": ["Natural", "Premium", "Radiance", "Pure", "Glow"], + "Automotive": ["Pro", "Heavy-Duty", "Premium", "Performance", "Ultra"], + } + + products = { + "Electronics": ["Headphones", "Speaker", "Charger", "Watch", "Camera"], + "Home & Garden": ["Lamp", "Planter", "Organizer", "Rug", "Vase"], + "Clothing": ["T-Shirt", "Jacket", "Pants", "Sweater", "Dress"], + "Sports": ["Ball", "Racket", "Mat", "Gloves", "Bag"], + "Books": ["Guide", "Handbook", "Manual", "Edition", "Collection"], + "Toys": ["Game", "Puzzle", "Building Set", "Robot", "Craft Kit"], + "Beauty": ["Serum", "Cream", "Lotion", "Mask", "Oil"], + "Automotive": ["Tool Kit", "Cover", "Mat", "Cleaner", "Polish"], + } + + names = [] + for category in categories: + template_list = templates.get(category, ["Premium"]) + product_list = products.get(category, ["Item"]) + template = np.random.choice(template_list) + product = np.random.choice(product_list) + color = fake.color_name() + names.append(f"{template} {color} {product}") + + return pd.Series(names) + + +@F.pandas_udf(DoubleType()) +def generate_price(categories: pd.Series) -> pd.Series: + price_params = { + "Electronics": (4.5, 0.8), + "Home & Garden": (3.8, 0.7), + "Clothing": (3.5, 0.6), + "Sports": (4.0, 0.7), + "Books": (2.8, 0.4), + "Toys": (3.2, 0.6), + "Beauty": (3.3, 0.5), + "Automotive": (4.2, 0.8), + } + + prices = [] + for category in categories: + mu, sigma = price_params.get(category, (3.5, 0.6)) + price = float(np.random.lognormal(mu, sigma)) + price = round(price) - 0.01 if price > 1 else round(price, 2) + prices.append(max(0.99, price)) + + return pd.Series(prices) + + +@F.pandas_udf(IntegerType()) +def generate_inventory(ids: pd.Series) -> pd.Series: + inventory = (np.random.pareto(a=2.0, size=len(ids)) + 1) * 20 + return pd.Series(inventory.astype(int)) + + +# ============================================================================= +# GENERATE PRODUCTS TABLE (Master) +# ============================================================================= +print(f"\nGenerating {NUM_PRODUCTS:,} products...") + +products_df = ( + spark.range(0, NUM_PRODUCTS, numPartitions=NUM_PARTITIONS) + .select( + F.concat(F.lit("PROD-"), F.lpad(F.col("id").cast("string"), 5, "0")).alias("product_id"), + F.col("id").alias("_idx"), + ) + .withColumn( + "category", + F.when(F.rand(SEED) < 0.25, "Electronics") + .when(F.rand(SEED + 1) < 0.45, "Home & Garden") + .when(F.rand(SEED + 2) < 0.63, "Clothing") + .when(F.rand(SEED + 3) < 0.75, "Sports") + .when(F.rand(SEED + 4) < 0.85, "Books") + .when(F.rand(SEED + 5) < 0.93, "Toys") + .when(F.rand(SEED + 6) < 0.98, "Beauty") + .otherwise("Automotive") + ) + .withColumn("name", fake_product_name(F.col("category"))) + .withColumn("price", generate_price(F.col("category"))) + .withColumn("inventory_count", generate_inventory(F.col("_idx"))) + .drop("_idx") +) + +products_final = products_df.select( + "product_id", "name", "category", "price", "inventory_count" +) + +products_json_path = f"{VOLUME_PATH}/products" +print(f"Writing products to {products_json_path}...") +products_final.write.mode("overwrite").json(products_json_path) + +products_for_fk = spark.read.json(products_json_path).select("product_id", "category") +product_count = products_for_fk.count() +print(f"Products written: {product_count:,}") + +# ============================================================================= +# GENERATE SALES TABLE (with Referential Integrity) +# ============================================================================= +print(f"\nGenerating {NUM_SALES:,} sales with referential integrity...") + +product_weights = products_for_fk.select( + "product_id", + "category", + F.when(F.col("category") == "Electronics", 3.0) + .when(F.col("category") == "Home & Garden", 2.5) + .when(F.col("category") == "Clothing", 2.0) + .when(F.col("category") == "Sports", 1.5) + .when(F.col("category") == "Books", 1.2) + .when(F.col("category") == "Toys", 1.0) + .when(F.col("category") == "Beauty", 0.8) + .otherwise(0.5).alias("weight") +) + +weighted_products = ( + product_weights + .select( + F.col("product_id"), + F.col("category"), + F.explode(F.array_repeat(F.col("product_id"), F.col("weight").cast("int"))).alias("_dup") + ) + .drop("_dup") +) + +sampled_products = ( + weighted_products + .orderBy(F.rand(SEED + 20)) + .limit(NUM_SALES) + .withColumn("sale_rank", F.row_number().over(Window.orderBy(F.monotonically_increasing_id()))) + .select("product_id", "category", "sale_rank") +) + +sales_base = ( + spark.range(0, NUM_SALES, numPartitions=NUM_PARTITIONS) + .withColumn("sale_rank", F.row_number().over(Window.orderBy(F.col("id")))) + .withColumn( + "sale_id", + F.concat(F.lit("SALE-"), F.lpad(F.col("id").cast("string"), 6, "0")) + ) +) + +sales_with_products = sales_base.join( + sampled_products, + on="sale_rank", + how="inner" +) + +sales_df = ( + sales_with_products + .withColumn( + "quantity", + F.when(F.rand(SEED + 21) < 0.60, 1) + .when(F.rand(SEED + 22) < 0.85, 2) + .when(F.rand(SEED + 23) < 0.95, 3) + .otherwise(F.floor(F.rand(SEED + 24) * 5 + 4).cast("int")) + ) + .withColumn( + "sale_date", + F.date_add(F.lit(START_DATE.date()), (F.rand(SEED + 25) * 180).cast("int")) + ) + .withColumn( + "discount_pct", + F.when(F.rand(SEED + 26) < 0.70, 0.0) + .when(F.rand(SEED + 27) < 0.85, 0.10) + .when(F.rand(SEED + 28) < 0.95, 0.20) + .otherwise(0.30) + ) +) + +sales_final = sales_df.select( + "sale_id", "product_id", "quantity", "sale_date", "discount_pct" +) + +sales_json_path = f"{VOLUME_PATH}/sales" +print(f"Writing sales to {sales_json_path}...") +sales_final.write.mode("overwrite").json(sales_json_path) + +sales_count = spark.read.json(sales_json_path).count() +print(f"Sales written: {sales_count:,}") + +# ============================================================================= +# VALIDATION +# ============================================================================= +print("\n" + "=" * 60) +print("VALIDATION") +print("=" * 60) + +products_check = spark.read.json(products_json_path) +sales_check = spark.read.json(sales_json_path) + +print(f"\nRow counts:") +print(f" Products: {products_check.count():,}") +print(f" Sales: {sales_check.count():,}") + +print(f"\nSales per category (Electronics should have most):") +sales_with_category = sales_check.join( + products_check.select("product_id", "category"), + on="product_id" +) +sales_with_category.groupBy("category").agg( + F.count("*").alias("total_sales"), + F.sum("quantity").alias("total_quantity") +).orderBy(F.desc("total_sales")).show() + +orphan_sales = sales_check.join( + products_check.select("product_id"), + on="product_id", + how="left_anti" +).count() +print(f"\nReferential integrity:") +print(f" Orphan sales (no matching product): {orphan_sales}") + +print("\n" + "=" * 60) +print("GENERATION COMPLETE") +print("=" * 60) +print(f"\nOutput location: {VOLUME_PATH}") +``` + +To run this as a serverless job, create the job with this configuration: + +```json +{ + "name": "generate_product_catalog_data", + "tasks": [ + { + "task_key": "generate_data", + "spark_python_task": { + "python_file": "/Workspace/Users/your_email/generate_product_catalog_data.py" + }, + "new_cluster": { + "spark_version": "16.4.x-scala2.12", + "num_workers": 0, + "spark_conf": { + "spark.databricks.cluster.profile": "serverless" + } + }, + "environments": [ + { + "environment_key": "default", + "spec": { + "client": "4", + "dependencies": [ + "faker", + "pandas", + "numpy" + ] + } + } + ] + } + ] +} +``` diff --git a/.test/skills/databricks-synthetic-data-gen/expected_responses/grp_20260302_113344.txt b/.test/skills/databricks-synthetic-data-gen/expected_responses/grp_20260302_113344.txt new file mode 100644 index 00000000..39997ba5 --- /dev/null +++ b/.test/skills/databricks-synthetic-data-gen/expected_responses/grp_20260302_113344.txt @@ -0,0 +1,233 @@ +I'll generate the e-commerce data locally using Polars and Faker, then upload to Unity Catalog. + +## Generation Plan + +**Output Location:** `ai_dev_kit.devkit_gen1_test_small_parquet` +**Volume:** `/Volumes/devkit/devkit_gen1_test_small_parquet/raw_data/` + +| Table | Columns | Rows | +|-------|---------|------| +| customers | customer_id, name, email, tier, region, created_at | 5,000 | +| orders | order_id, customer_id (FK), amount, order_date, status | 15,000 | + +**Order Distribution by Tier:** +- Enterprise (10%): ~8 orders per customer average +- Pro (30%): ~3 orders per customer average +- Free (60%): ~1.5 orders per customer average + +Here's the complete script: + +```python +import polars as pl +import numpy as np +from faker import Faker +from datetime import datetime, timedelta +import random +from databricks.sdk import WorkspaceClient + +# Configuration +CATALOG = "devkit" +SCHEMA = "devkit_gen1_test_small_parquet" +NUM_CUSTOMERS = 5000 +NUM_ORDERS = 15000 + +fake = Faker() +Faker.seed(42) +np.random.seed(42) +random.seed(42) + +# Date ranges +END_DATE = datetime.now() +CUSTOMER_START_DATE = END_DATE - timedelta(days=730) # 2 years +ORDER_START_DATE = END_DATE - timedelta(days=180) # 6 months + +# ============================================================================= +# Generate Customers +# ============================================================================= +print("Generating customers...") + +# Tier distribution: Free 60%, Pro 30%, Enterprise 10% +tier_weights = [0.6, 0.3, 0.1] +tiers = random.choices(["Free", "Pro", "Enterprise"], weights=tier_weights, k=NUM_CUSTOMERS) + +# Region distribution +region_weights = [0.4, 0.3, 0.2, 0.1] +regions = random.choices(["US-East", "US-West", "EU", "APAC"], weights=region_weights, k=NUM_CUSTOMERS) + +customers = pl.DataFrame({ + "customer_id": [f"CUST-{i:05d}" for i in range(NUM_CUSTOMERS)], + "name": [fake.name() for _ in range(NUM_CUSTOMERS)], + "email": [fake.email() for _ in range(NUM_CUSTOMERS)], + "tier": tiers, + "region": regions, + "created_at": [ + fake.date_time_between(start_date=CUSTOMER_START_DATE, end_date=END_DATE) + for _ in range(NUM_CUSTOMERS) + ], +}) + +print(f"Generated {len(customers)} customers") +print(customers.group_by("tier").len().sort("tier")) + +# ============================================================================= +# Generate Orders with Tier-Weighted Distribution +# ============================================================================= +print("\nGenerating orders with tier-weighted distribution...") + +# Calculate orders per tier to achieve ~15,000 total with weighted distribution +# Enterprise: ~8 orders/customer, Pro: ~3, Free: ~1.5 +tier_counts = customers.group_by("tier").len() +enterprise_count = tier_counts.filter(pl.col("tier") == "Enterprise")["len"][0] +pro_count = tier_counts.filter(pl.col("tier") == "Pro")["len"][0] +free_count = tier_counts.filter(pl.col("tier") == "Free")["len"][0] + +# Target distribution +enterprise_orders_per = 8 +pro_orders_per = 3 +free_orders_per = 1.5 + +total_target = enterprise_count * enterprise_orders_per + pro_count * pro_orders_per + free_count * free_orders_per +scale = NUM_ORDERS / total_target + +enterprise_orders = int(enterprise_count * enterprise_orders_per * scale) +pro_orders = int(pro_count * pro_orders_per * scale) +free_orders = NUM_ORDERS - enterprise_orders - pro_orders + +# Get customer IDs by tier +enterprise_customers = customers.filter(pl.col("tier") == "Enterprise")["customer_id"].to_list() +pro_customers = customers.filter(pl.col("tier") == "Pro")["customer_id"].to_list() +free_customers = customers.filter(pl.col("tier") == "Free")["customer_id"].to_list() + +# Generate customer_ids for orders +order_customer_ids = ( + random.choices(enterprise_customers, k=enterprise_orders) + + random.choices(pro_customers, k=pro_orders) + + random.choices(free_customers, k=free_orders) +) +random.shuffle(order_customer_ids) + +# Create customer tier lookup +customer_tier_map = dict(zip(customers["customer_id"].to_list(), customers["tier"].to_list())) + +# Generate amounts based on tier (log-normal distribution) +def generate_amount(tier: str) -> float: + if tier == "Enterprise": + return round(float(np.random.lognormal(6.2, 0.8)), 2) # mean ~$500 + elif tier == "Pro": + return round(float(np.random.lognormal(5.0, 0.7)), 2) # mean ~$150 + else: + return round(float(np.random.lognormal(3.9, 0.6)), 2) # mean ~$50 + +amounts = [generate_amount(customer_tier_map[cid]) for cid in order_customer_ids] + +# Status distribution: delivered 65%, shipped 15%, processing 10%, pending 5%, cancelled 5% +statuses = random.choices( + ["delivered", "shipped", "processing", "pending", "cancelled"], + weights=[0.65, 0.15, 0.10, 0.05, 0.05], + k=NUM_ORDERS +) + +orders = pl.DataFrame({ + "order_id": [f"ORD-{i:06d}" for i in range(NUM_ORDERS)], + "customer_id": order_customer_ids, + "amount": amounts, + "order_date": [ + fake.date_between(start_date=ORDER_START_DATE.date(), end_date=END_DATE.date()) + for _ in range(NUM_ORDERS) + ], + "status": statuses, +}) + +print(f"Generated {len(orders)} orders") + +# Verify tier-weighted distribution +orders_with_tier = orders.join( + customers.select(["customer_id", "tier"]), + on="customer_id" +) +print("\nOrders per tier:") +print(orders_with_tier.group_by("tier").len().sort("tier")) +print("\nAverage orders per customer by tier:") +print( + orders_with_tier.group_by("tier") + .agg(pl.len().alias("total_orders")) + .join(tier_counts, on="tier") + .with_columns((pl.col("total_orders") / pl.col("len")).alias("avg_orders_per_customer")) + .select(["tier", "avg_orders_per_customer"]) + .sort("tier") +) + +# ============================================================================= +# Save Locally as Parquet +# ============================================================================= +print("\nSaving to local Parquet files...") +local_path = "synthetic_data_output" +import os +os.makedirs(local_path, exist_ok=True) + +customers.write_parquet(f"{local_path}/customers.parquet") +orders.write_parquet(f"{local_path}/orders.parquet") +print(f"Saved to {local_path}/") + +# ============================================================================= +# Upload to Unity Catalog Volume +# ============================================================================= +print("\nUploading to Unity Catalog...") + +w = WorkspaceClient() + +# Create schema and volume if needed +from databricks.sdk.service.catalog import VolumeType + +try: + w.schemas.create(name=SCHEMA, catalog_name=CATALOG) + print(f"Created schema {CATALOG}.{SCHEMA}") +except Exception as e: + if "SCHEMA_ALREADY_EXISTS" in str(e): + print(f"Schema {CATALOG}.{SCHEMA} already exists") + else: + raise + +try: + w.volumes.create( + catalog_name=CATALOG, + schema_name=SCHEMA, + name="raw_data", + volume_type=VolumeType.MANAGED + ) + print(f"Created volume {CATALOG}.{SCHEMA}.raw_data") +except Exception as e: + if "already exists" in str(e).lower(): + print(f"Volume {CATALOG}.{SCHEMA}.raw_data already exists") + else: + raise + +# Upload files +volume_path = f"/Volumes/{CATALOG}/{SCHEMA}/raw_data" + +with open(f"{local_path}/customers.parquet", "rb") as f: + w.files.upload(f"{volume_path}/customers.parquet", f, overwrite=True) +print(f"Uploaded customers.parquet to {volume_path}/") + +with open(f"{local_path}/orders.parquet", "rb") as f: + w.files.upload(f"{volume_path}/orders.parquet", f, overwrite=True) +print(f"Uploaded orders.parquet to {volume_path}/") + +print(f"\n✅ Complete! Data available at:") +print(f" {volume_path}/customers.parquet") +print(f" {volume_path}/orders.parquet") +``` + +This script: +1. Generates 5,000 customers with weighted tier distribution (Free 60%, Pro 30%, Enterprise 10%) +2. Generates 15,000 orders with tier-weighted customer assignment (Enterprise ~8x, Pro ~3x, Free ~1.5x orders per customer) +3. Uses log-normal distributions for realistic order amounts by tier +4. Saves locally as Parquet files +5. Creates the schema and volume in Unity Catalog if needed +6. Uploads the Parquet files to the volume + +Run with: +```bash +uv pip install polars faker numpy databricks-sdk +uv run python generate_ecommerce_data.py +``` diff --git a/.test/skills/databricks-synthetic-data-gen/expected_responses/grp_20260302_retail_csv_3tables_003.txt b/.test/skills/databricks-synthetic-data-gen/expected_responses/grp_20260302_retail_csv_3tables_003.txt new file mode 100644 index 00000000..ea87365e --- /dev/null +++ b/.test/skills/databricks-synthetic-data-gen/expected_responses/grp_20260302_retail_csv_3tables_003.txt @@ -0,0 +1,297 @@ +Here's the complete script to generate synthetic retail order data with 3 tables and full referential integrity: + +```python +""" +Synthetic Retail Order Data Generation +====================================== +3 related tables with full referential integrity: +- customers (10,000 rows) +- orders (50,000 rows) +- line_items (150,000 rows) + +Output: CSV files with headers to Unity Catalog volume +Execution: Databricks Connect with serverless compute +""" + +import os +from pyspark.sql import functions as F +from pyspark.sql.types import StringType, DoubleType, IntegerType +from pyspark.sql.window import Window +import pandas as pd +import numpy as np +from datetime import datetime, timedelta + +# ============================================================================= +# CONFIGURATION +# ============================================================================= +CATALOG = "ai_dev_kit" +SCHEMA = "devkit_gen3_test_medium_csv" +VOLUME_PATH = f"/Volumes/{CATALOG}/{SCHEMA}/raw_data" + +NUM_CUSTOMERS = 10_000 +NUM_ORDERS = 50_000 +NUM_LINE_ITEMS = 150_000 +NUM_PARTITIONS = 32 +SEED = 42 + +END_DATE = datetime.now().replace(hour=0, minute=0, second=0, microsecond=0) +START_DATE = END_DATE - timedelta(days=180) + +# ============================================================================= +# SPARK SESSION +# ============================================================================= +def is_databricks_runtime(): + return "DATABRICKS_RUNTIME_VERSION" in os.environ + +if is_databricks_runtime(): + from pyspark.sql import SparkSession + spark = SparkSession.builder.getOrCreate() + print("Running on Databricks Runtime") +else: + from databricks.connect import DatabricksSession, DatabricksEnv + + # Install dependencies on serverless cluster + env = DatabricksEnv().withDependencies("faker", "pandas", "numpy") + spark = DatabricksSession.builder.withEnvironment(env).serverless(True).getOrCreate() + print("Running with Databricks Connect (serverless with managed dependencies)") + +# ============================================================================= +# CREATE INFRASTRUCTURE +# ============================================================================= +spark.sql(f"CREATE SCHEMA IF NOT EXISTS {CATALOG}.{SCHEMA}") +spark.sql(f"CREATE VOLUME IF NOT EXISTS {CATALOG}.{SCHEMA}.raw_data") + +# ============================================================================= +# PANDAS UDFs +# ============================================================================= + +@F.pandas_udf(StringType()) +def fake_name(ids: pd.Series) -> pd.Series: + from faker import Faker + fake = Faker() + Faker.seed(SEED) + return pd.Series([fake.name() for _ in range(len(ids))]) + +@F.pandas_udf(StringType()) +def fake_email(names: pd.Series) -> pd.Series: + from faker import Faker + fake = Faker() + emails = [] + for name in names: + parts = name.lower().split() + if len(parts) >= 2: + email = f"{parts[0]}.{parts[-1]}@{fake.free_email_domain()}" + else: + email = f"{parts[0]}{np.random.randint(100, 999)}@{fake.free_email_domain()}" + emails.append(email) + return pd.Series(emails) + +@F.pandas_udf(StringType()) +def fake_product_name(ids: pd.Series) -> pd.Series: + from faker import Faker + fake = Faker() + product_types = ["Chair", "Table", "Lamp", "Desk", "Shelf", "Cabinet", "Sofa", "Rug", + "Mirror", "Clock", "Vase", "Frame", "Pillow", "Blanket", "Candle", + "Mug", "Bowl", "Plate", "Glass", "Bottle", "Box", "Bag", "Hat", + "Watch", "Headphones", "Speaker", "Charger", "Cable", "Case"] + products = [] + for _ in range(len(ids)): + color = fake.color_name() + adj = fake.word().capitalize() + product = np.random.choice(product_types) + products.append(f"{color} {adj} {product}") + return pd.Series(products) + +@F.pandas_udf(DoubleType()) +def generate_unit_price(ids: pd.Series) -> pd.Series: + """Log-normal unit prices (median ~$35, range $5-$500)""" + prices = np.random.lognormal(mean=3.5, sigma=0.7, size=len(ids)) + prices = np.clip(prices, 5.0, 500.0) + return pd.Series(np.round(prices, 2)) + +# ============================================================================= +# GENERATE CUSTOMERS TABLE +# ============================================================================= +customers_df = ( + spark.range(0, NUM_CUSTOMERS, numPartitions=NUM_PARTITIONS) + .select( + F.concat(F.lit("CUST-"), F.lpad(F.col("id").cast("string"), 5, "0")).alias("customer_id"), + F.col("id").alias("_idx") + ) + .withColumn("name", fake_name(F.col("_idx"))) + .withColumn("email", fake_email(F.col("name"))) + .withColumn("membership_level", + F.when(F.rand(SEED) < 0.50, "Bronze") + .when(F.rand(SEED + 1) < 0.80, "Silver") + .when(F.rand(SEED + 2) < 0.95, "Gold") + .otherwise("Platinum") + ) + .withColumn("region", + F.when(F.rand(SEED + 3) < 0.30, "US-East") + .when(F.rand(SEED + 4) < 0.55, "US-West") + .when(F.rand(SEED + 5) < 0.80, "EU") + .when(F.rand(SEED + 6) < 0.95, "APAC") + .otherwise("Other") + ) + .drop("_idx") +) + +# Write to temp Delta table (no .cache() on serverless) +customers_tmp = f"{CATALOG}.{SCHEMA}._tmp_customers" +customers_df.write.mode("overwrite").saveAsTable(customers_tmp) +customers_df = spark.table(customers_tmp) + +# ============================================================================= +# GENERATE ORDERS TABLE (weighted by membership level) +# ============================================================================= +customer_weights = customers_df.select( + "customer_id", + "membership_level", + F.when(F.col("membership_level") == "Platinum", 10.0) + .when(F.col("membership_level") == "Gold", 7.0) + .when(F.col("membership_level") == "Silver", 5.0) + .otherwise(3.0).alias("weight") +) + +weighted_customers = ( + customer_weights + .withColumn("replicate_count", (F.col("weight") * 2).cast("int")) + .select( + F.col("customer_id"), + F.explode(F.array_repeat(F.col("customer_id"), F.col("replicate_count"))).alias("_dup") + ) + .drop("_dup") +) + +sampled_customers = ( + weighted_customers + .orderBy(F.rand(SEED + 10)) + .limit(NUM_ORDERS) + .withColumn("_row", F.row_number().over(Window.orderBy(F.monotonically_increasing_id()))) + .select("customer_id", "_row") +) + +orders_base = ( + spark.range(0, NUM_ORDERS, numPartitions=NUM_PARTITIONS) + .withColumn("order_id", + F.concat(F.lit("ORD-"), F.lpad(F.col("id").cast("string"), 6, "0"))) + .withColumn("_row", F.row_number().over(Window.orderBy(F.col("id")))) +) + +orders_df = ( + orders_base + .join(sampled_customers, on="_row", how="inner") + .drop("_row", "id") + .withColumn("order_date", + F.date_add(F.lit(START_DATE.date()), (F.rand(SEED + 11) * 180).cast("int"))) + .withColumn("status", + F.when(F.rand(SEED + 12) < 0.60, "Delivered") + .when(F.rand(SEED + 13) < 0.78, "Shipped") + .when(F.rand(SEED + 14) < 0.90, "Processing") + .when(F.rand(SEED + 15) < 0.96, "Pending") + .otherwise("Cancelled") + ) + .withColumn("total_amount", F.lit(0.0)) +) + +orders_tmp = f"{CATALOG}.{SCHEMA}._tmp_orders" +orders_df.write.mode("overwrite").saveAsTable(orders_tmp) +orders_df = spark.table(orders_tmp) + +# ============================================================================= +# GENERATE LINE_ITEMS TABLE +# ============================================================================= +order_ids = orders_df.select("order_id") + +items_per_order = ( + order_ids + .withColumn("num_items", + F.when(F.rand(SEED + 20) < 0.10, 1) + .when(F.rand(SEED + 21) < 0.25, 2) + .when(F.rand(SEED + 22) < 0.45, 3) + .when(F.rand(SEED + 23) < 0.65, 4) + .when(F.rand(SEED + 24) < 0.80, 5) + .when(F.rand(SEED + 25) < 0.90, 6) + .when(F.rand(SEED + 26) < 0.96, 7) + .otherwise(8) + ) +) + +line_items_base = ( + items_per_order + .select( + F.col("order_id"), + F.explode(F.sequence(F.lit(1), F.col("num_items"))).alias("item_seq") + ) + .withColumn("_idx", F.monotonically_increasing_id()) +) + +line_items_df = ( + line_items_base + .limit(NUM_LINE_ITEMS) + .withColumn("line_item_id", + F.concat(F.lit("LI-"), F.lpad(F.col("_idx").cast("string"), 7, "0"))) + .withColumn("product_name", fake_product_name(F.col("_idx"))) + .withColumn("quantity", + F.when(F.rand(SEED + 30) < 0.50, 1) + .when(F.rand(SEED + 31) < 0.80, 2) + .when(F.rand(SEED + 32) < 0.92, 3) + .when(F.rand(SEED + 33) < 0.97, 4) + .otherwise(5) + ) + .withColumn("unit_price", generate_unit_price(F.col("_idx"))) + .drop("_idx", "item_seq") +) + +line_items_tmp = f"{CATALOG}.{SCHEMA}._tmp_line_items" +line_items_df.write.mode("overwrite").saveAsTable(line_items_tmp) +line_items_df = spark.table(line_items_tmp) + +# ============================================================================= +# COMPUTE ORDER TOTALS FROM LINE ITEMS +# ============================================================================= +order_totals = ( + line_items_df + .withColumn("line_total", F.col("quantity") * F.col("unit_price")) + .groupBy("order_id") + .agg(F.round(F.sum("line_total"), 2).alias("computed_total")) +) + +orders_final = ( + spark.table(orders_tmp) + .drop("total_amount") + .join(order_totals, on="order_id", how="left") + .withColumn("total_amount", F.coalesce(F.col("computed_total"), F.lit(0.0))) + .drop("computed_total") +) + +# ============================================================================= +# EXPORT TO CSV +# ============================================================================= +customers_final = spark.table(customers_tmp).select( + "customer_id", "name", "email", "membership_level", "region" +) + +orders_out = orders_final.select( + "order_id", "customer_id", "order_date", "total_amount", "status" +) + +line_items_out = spark.table(line_items_tmp).select( + "line_item_id", "order_id", "product_name", "quantity", "unit_price" +) + +customers_final.coalesce(1).write.mode("overwrite").option("header", "true").csv(f"{VOLUME_PATH}/customers") +orders_out.coalesce(1).write.mode("overwrite").option("header", "true").csv(f"{VOLUME_PATH}/orders") +line_items_out.coalesce(4).write.mode("overwrite").option("header", "true").csv(f"{VOLUME_PATH}/line_items") + +# Cleanup temp tables +spark.sql(f"DROP TABLE IF EXISTS {customers_tmp}") +spark.sql(f"DROP TABLE IF EXISTS {orders_tmp}") +spark.sql(f"DROP TABLE IF EXISTS {line_items_tmp}") +``` + +Run with: +```bash +uv pip install "databricks-connect>=16.4,<17.0" faker numpy pandas +uv run python generate_retail_data.py +``` diff --git a/.test/skills/databricks-synthetic-data-gen/expected_responses/grp_20260303_manufacturing_delta_streaming_004.txt b/.test/skills/databricks-synthetic-data-gen/expected_responses/grp_20260303_manufacturing_delta_streaming_004.txt new file mode 100644 index 00000000..06d7f925 --- /dev/null +++ b/.test/skills/databricks-synthetic-data-gen/expected_responses/grp_20260303_manufacturing_delta_streaming_004.txt @@ -0,0 +1,235 @@ +Here's the complete script to generate manufacturing IoT sensor data with streaming-ready batch processing: + +```python +""" +Manufacturing IoT Sensor Data Generation +======================================== +2 related tables with referential integrity: +- sensor_devices (500 rows): Slowly changing dimension +- sensor_readings (10,000 rows per batch): Streaming incremental + +Output: Delta tables registered in Unity Catalog +Execution: Databricks Connect with serverless compute (Python 3.12+) +""" + +from databricks.connect import DatabricksSession, DatabricksEnv +from pyspark.sql import functions as F +from pyspark.sql.types import StringType, DoubleType, TimestampType +import pandas as pd +import numpy as np +from datetime import datetime, timedelta + +# Configuration +CATALOG = "ai_dev_kit" +SCHEMA = "devkit_gen4_test_large_delta" + +# Row counts +NUM_DEVICES = 500 +NUM_READINGS_PER_BATCH = 10000 + +# Date range for readings (last 24 hours for streaming simulation) +END_TIME = datetime.now() +START_TIME = END_TIME - timedelta(hours=24) + +# Setup with managed dependencies (databricks-connect 16.4+) +env = DatabricksEnv().withDependencies("faker", "pandas", "numpy") +spark = DatabricksSession.builder.withEnvironment(env).serverless(True).getOrCreate() + +print(f"Connected to Databricks serverless compute") +print(f"Output location: {CATALOG}.{SCHEMA}") + +# Create schema if not exists +spark.sql(f"CREATE SCHEMA IF NOT EXISTS {CATALOG}.{SCHEMA}") +print(f"Schema {CATALOG}.{SCHEMA} ready") + + +# ============================================================================= +# Pandas UDFs for data generation +# ============================================================================= + +@F.pandas_udf(StringType()) +def fake_device_name(ids: pd.Series) -> pd.Series: + """Generate realistic device names like 'SENS-T-00001'.""" + from faker import Faker + fake = Faker() + Faker.seed(42) + + prefixes = {"temperature": "T", "pressure": "P", "vibration": "V", "humidity": "H"} + names = [] + for i, _ in enumerate(ids): + type_key = list(prefixes.keys())[i % 4] + prefix = prefixes[type_key] + names.append(f"SENS-{prefix}-{str(i).zfill(5)}") + return pd.Series(names) + + +@F.pandas_udf(DoubleType()) +def generate_sensor_value(device_types: pd.Series) -> pd.Series: + """Generate realistic sensor values based on device type.""" + values = [] + for dtype in device_types: + if dtype == "temperature": + values.append(float(np.random.normal(70, 15))) # Fahrenheit + elif dtype == "pressure": + values.append(float(np.random.normal(100, 10))) # PSI + elif dtype == "vibration": + values.append(float(np.random.lognormal(1.5, 0.8))) # mm/s with spikes + elif dtype == "humidity": + values.append(float(np.clip(np.random.normal(45, 10), 0, 100))) # Percentage + else: + values.append(float(np.random.normal(50, 10))) + return pd.Series(values) + + +@F.pandas_udf(StringType()) +def generate_unit(device_types: pd.Series) -> pd.Series: + """Generate appropriate unit based on device type.""" + unit_map = { + "temperature": "°F", + "pressure": "PSI", + "vibration": "mm/s", + "humidity": "%" + } + return pd.Series([unit_map.get(dt, "units") for dt in device_types]) + + +# ============================================================================= +# Generate sensor_devices (slowly changing dimension) +# ============================================================================= + +print("\nGenerating sensor_devices table (slowly changing dimension)") + +# Device type weights: temperature 30%, pressure 25%, vibration 25%, humidity 20% +devices_df = ( + spark.range(0, NUM_DEVICES, numPartitions=4) + .select( + F.concat(F.lit("DEV-"), F.lpad(F.col("id").cast("string"), 5, "0")).alias("device_id"), + F.col("id").alias("_id") + ) + .withColumn("device_name", fake_device_name(F.col("_id"))) + .withColumn( + "device_type", + F.when(F.rand() < 0.30, "temperature") + .when(F.rand() < 0.55, "pressure") + .when(F.rand() < 0.80, "vibration") + .otherwise("humidity") + ) + .withColumn( + "location", + F.when(F.rand() < 0.40, "Plant_A") + .when(F.rand() < 0.70, "Plant_B") + .when(F.rand() < 0.90, "Plant_C") + .otherwise("Warehouse") + ) + .withColumn( + "install_date", + F.date_add(F.lit("2020-01-01"), (F.rand() * 1460).cast("int")) + ) + .withColumn( + "status", + F.when(F.rand() < 0.85, "active") + .when(F.rand() < 0.95, "maintenance") + .otherwise("inactive") + ) + .drop("_id") +) + +# Write devices table +devices_table = f"{CATALOG}.{SCHEMA}.sensor_devices" +devices_df.write.mode("overwrite").saveAsTable(devices_table) +print(f"Created {devices_table}") + + +# ============================================================================= +# Generate sensor_readings (streaming incremental batch) +# ============================================================================= + +print("\nGenerating sensor_readings table (streaming batch)") + +# Read devices back for FK join (no .cache() on serverless!) +devices_for_join = spark.table(devices_table).select("device_id", "device_type") + +# Generate readings with valid device_id FK +readings_df = ( + spark.range(0, NUM_READINGS_PER_BATCH, numPartitions=16) + .select( + F.concat( + F.lit("RDG-"), + F.date_format(F.current_timestamp(), "yyyyMMddHHmmss"), + F.lit("-"), + F.lpad(F.col("id").cast("string"), 6, "0") + ).alias("reading_id"), + (F.abs(F.hash(F.col("id"))) % NUM_DEVICES).alias("device_index"), + F.from_unixtime( + F.unix_timestamp(F.lit(START_TIME)) + + (F.rand() * 86400).cast("long") + ).cast("timestamp").alias("timestamp") + ) +) + +# Get device IDs with row numbers for joining +devices_indexed = ( + devices_for_join + .withColumn("device_index", F.monotonically_increasing_id() % NUM_DEVICES) +) + +# Join to get valid device_id and device_type +readings_with_device = ( + readings_df + .join( + devices_indexed.select("device_id", "device_type", "device_index"), + on="device_index", + how="inner" + ) + .drop("device_index") +) + +# Add sensor values based on device type +readings_final = ( + readings_with_device + .withColumn("value", F.round(generate_sensor_value(F.col("device_type")), 2)) + .withColumn("unit", generate_unit(F.col("device_type"))) + .withColumn( + "quality_flag", + F.when(F.rand() < 0.90, "good") + .when(F.rand() < 0.97, "warning") + .otherwise("error") + ) + .select("reading_id", "device_id", "timestamp", "value", "unit", "quality_flag") +) + +# Write readings table (append mode for streaming simulation) +readings_table = f"{CATALOG}.{SCHEMA}.sensor_readings" +readings_final.write.mode("overwrite").saveAsTable(readings_table) +print(f"Created {readings_table}") + + +# ============================================================================= +# Validation +# ============================================================================= + +# Check referential integrity +orphan_readings = spark.sql(f""" + SELECT COUNT(*) as orphan_count + FROM {readings_table} r + LEFT JOIN {devices_table} d ON r.device_id = d.device_id + WHERE d.device_id IS NULL +""").collect()[0]["orphan_count"] + +print(f"Orphan readings (should be 0): {orphan_readings}") +print(f"\nSUMMARY") +print(f"Catalog/Schema: {CATALOG}.{SCHEMA}") +print(f"Tables: {devices_table}, {readings_table}") +print(f"Referential integrity: {'PASSED' if orphan_readings == 0 else 'FAILED'}") +``` + +Run with: +```bash +uv pip install "databricks-connect>=16.4,<17.0" faker numpy pandas +uv run python generate_manufacturing_data.py +``` + +For streaming job deployment, this can be scheduled as an incremental job where each run: +1. Generates a new batch of 10,000 readings with unique timestamps +2. Appends to the sensor_readings table (change mode to "append") +3. The sensor_devices table is the slowly-changing dimension that persists diff --git a/.test/skills/databricks-synthetic-data-gen/ground_truth.yaml b/.test/skills/databricks-synthetic-data-gen/ground_truth.yaml index bee8cdab..38625b62 100644 --- a/.test/skills/databricks-synthetic-data-gen/ground_truth.yaml +++ b/.test/skills/databricks-synthetic-data-gen/ground_truth.yaml @@ -17,132 +17,29 @@ test_cases: Enterprise customers should generate more orders than Free tier.' outputs: - response: "I'll generate the e-commerce data locally using Polars and Faker, then\ - \ upload to Unity Catalog.\n\n## Generation Plan\n\n**Output Location:** `ai_dev_kit.devkit_gen1_test_small_parquet`\n\ - **Volume:** `/Volumes/devkit/devkit_gen1_test_small_parquet/raw_data/`\n\n|\ - \ Table | Columns | Rows |\n|-------|---------|------|\n| customers | customer_id,\ - \ name, email, tier, region, created_at | 5,000 |\n| orders | order_id, customer_id\ - \ (FK), amount, order_date, status | 15,000 |\n\n**Order Distribution by Tier:**\n\ - - Enterprise (10%): ~8 orders per customer average\n- Pro (30%): ~3 orders per\ - \ customer average\n- Free (60%): ~1.5 orders per customer average\n\nHere's\ - \ the complete script:\n\n```python\nimport polars as pl\nimport numpy as np\n\ - from faker import Faker\nfrom datetime import datetime, timedelta\nimport random\n\ - from databricks.sdk import WorkspaceClient\n\n# Configuration\nCATALOG = \"\ - devkit\"\nSCHEMA = \"devkit_gen1_test_small_parquet\"\nNUM_CUSTOMERS = 5000\n\ - NUM_ORDERS = 15000\n\nfake = Faker()\nFaker.seed(42)\nnp.random.seed(42)\nrandom.seed(42)\n\ - \n# Date ranges\nEND_DATE = datetime.now()\nCUSTOMER_START_DATE = END_DATE -\ - \ timedelta(days=730) # 2 years\nORDER_START_DATE = END_DATE - timedelta(days=180)\ - \ # 6 months\n\n# =============================================================================\n\ - # Generate Customers\n# =============================================================================\n\ - print(\"Generating customers...\")\n\n# Tier distribution: Free 60%, Pro 30%,\ - \ Enterprise 10%\ntier_weights = [0.6, 0.3, 0.1]\ntiers = random.choices([\"\ - Free\", \"Pro\", \"Enterprise\"], weights=tier_weights, k=NUM_CUSTOMERS)\n\n\ - # Region distribution\nregion_weights = [0.4, 0.3, 0.2, 0.1]\nregions = random.choices([\"\ - US-East\", \"US-West\", \"EU\", \"APAC\"], weights=region_weights, k=NUM_CUSTOMERS)\n\ - \ncustomers = pl.DataFrame({\n \"customer_id\": [f\"CUST-{i:05d}\" for i\ - \ in range(NUM_CUSTOMERS)],\n \"name\": [fake.name() for _ in range(NUM_CUSTOMERS)],\n\ - \ \"email\": [fake.email() for _ in range(NUM_CUSTOMERS)],\n \"tier\"\ - : tiers,\n \"region\": regions,\n \"created_at\": [\n fake.date_time_between(start_date=CUSTOMER_START_DATE,\ - \ end_date=END_DATE)\n for _ in range(NUM_CUSTOMERS)\n ],\n})\n\n\ - print(f\"Generated {len(customers)} customers\")\nprint(customers.group_by(\"\ - tier\").len().sort(\"tier\"))\n\n# =============================================================================\n\ - # Generate Orders with Tier-Weighted Distribution\n# =============================================================================\n\ - print(\"\\nGenerating orders with tier-weighted distribution...\")\n\n# Calculate\ - \ orders per tier to achieve ~15,000 total with weighted distribution\n# Enterprise:\ - \ ~8 orders/customer, Pro: ~3, Free: ~1.5\ntier_counts = customers.group_by(\"\ - tier\").len()\nenterprise_count = tier_counts.filter(pl.col(\"tier\") == \"\ - Enterprise\")[\"len\"][0]\npro_count = tier_counts.filter(pl.col(\"tier\") ==\ - \ \"Pro\")[\"len\"][0]\nfree_count = tier_counts.filter(pl.col(\"tier\") ==\ - \ \"Free\")[\"len\"][0]\n\n# Target distribution\nenterprise_orders_per = 8\n\ - pro_orders_per = 3\nfree_orders_per = 1.5\n\ntotal_target = enterprise_count\ - \ * enterprise_orders_per + pro_count * pro_orders_per + free_count * free_orders_per\n\ - scale = NUM_ORDERS / total_target\n\nenterprise_orders = int(enterprise_count\ - \ * enterprise_orders_per * scale)\npro_orders = int(pro_count * pro_orders_per\ - \ * scale)\nfree_orders = NUM_ORDERS - enterprise_orders - pro_orders\n\n# Get\ - \ customer IDs by tier\nenterprise_customers = customers.filter(pl.col(\"tier\"\ - ) == \"Enterprise\")[\"customer_id\"].to_list()\npro_customers = customers.filter(pl.col(\"\ - tier\") == \"Pro\")[\"customer_id\"].to_list()\nfree_customers = customers.filter(pl.col(\"\ - tier\") == \"Free\")[\"customer_id\"].to_list()\n\n# Generate customer_ids for\ - \ orders\norder_customer_ids = (\n random.choices(enterprise_customers, k=enterprise_orders)\ - \ +\n random.choices(pro_customers, k=pro_orders) +\n random.choices(free_customers,\ - \ k=free_orders)\n)\nrandom.shuffle(order_customer_ids)\n\n# Create customer\ - \ tier lookup\ncustomer_tier_map = dict(zip(customers[\"customer_id\"].to_list(),\ - \ customers[\"tier\"].to_list()))\n\n# Generate amounts based on tier (log-normal\ - \ distribution)\ndef generate_amount(tier: str) -> float:\n if tier == \"\ - Enterprise\":\n return round(float(np.random.lognormal(6.2, 0.8)), 2)\ - \ # mean ~$500\n elif tier == \"Pro\":\n return round(float(np.random.lognormal(5.0,\ - \ 0.7)), 2) # mean ~$150\n else:\n return round(float(np.random.lognormal(3.9,\ - \ 0.6)), 2) # mean ~$50\n\namounts = [generate_amount(customer_tier_map[cid])\ - \ for cid in order_customer_ids]\n\n# Status distribution: delivered 65%, shipped\ - \ 15%, processing 10%, pending 5%, cancelled 5%\nstatuses = random.choices(\n\ - \ [\"delivered\", \"shipped\", \"processing\", \"pending\", \"cancelled\"\ - ],\n weights=[0.65, 0.15, 0.10, 0.05, 0.05],\n k=NUM_ORDERS\n)\n\norders\ - \ = pl.DataFrame({\n \"order_id\": [f\"ORD-{i:06d}\" for i in range(NUM_ORDERS)],\n\ - \ \"customer_id\": order_customer_ids,\n \"amount\": amounts,\n \"\ - order_date\": [\n fake.date_between(start_date=ORDER_START_DATE.date(),\ - \ end_date=END_DATE.date())\n for _ in range(NUM_ORDERS)\n ],\n \ - \ \"status\": statuses,\n})\n\nprint(f\"Generated {len(orders)} orders\")\n\ - \n# Verify tier-weighted distribution\norders_with_tier = orders.join(\n \ - \ customers.select([\"customer_id\", \"tier\"]),\n on=\"customer_id\"\n)\n\ - print(\"\\nOrders per tier:\")\nprint(orders_with_tier.group_by(\"tier\").len().sort(\"\ - tier\"))\nprint(\"\\nAverage orders per customer by tier:\")\nprint(\n orders_with_tier.group_by(\"\ - tier\")\n .agg(pl.len().alias(\"total_orders\"))\n .join(tier_counts,\ - \ on=\"tier\")\n .with_columns((pl.col(\"total_orders\") / pl.col(\"len\"\ - )).alias(\"avg_orders_per_customer\"))\n .select([\"tier\", \"avg_orders_per_customer\"\ - ])\n .sort(\"tier\")\n)\n\n# =============================================================================\n\ - # Save Locally as Parquet\n# =============================================================================\n\ - print(\"\\nSaving to local Parquet files...\")\nlocal_path = \"synthetic_data_output\"\ - \nimport os\nos.makedirs(local_path, exist_ok=True)\n\ncustomers.write_parquet(f\"\ - {local_path}/customers.parquet\")\norders.write_parquet(f\"{local_path}/orders.parquet\"\ - )\nprint(f\"Saved to {local_path}/\")\n\n# =============================================================================\n\ - # Upload to Unity Catalog Volume\n# =============================================================================\n\ - print(\"\\nUploading to Unity Catalog...\")\n\nw = WorkspaceClient()\n\n# Create\ - \ schema and volume if needed\nfrom databricks.sdk.service.catalog import VolumeType\n\ - \ntry:\n w.schemas.create(name=SCHEMA, catalog_name=CATALOG)\n print(f\"\ - Created schema {CATALOG}.{SCHEMA}\")\nexcept Exception as e:\n if \"SCHEMA_ALREADY_EXISTS\"\ - \ in str(e):\n print(f\"Schema {CATALOG}.{SCHEMA} already exists\")\n\ - \ else:\n raise\n\ntry:\n w.volumes.create(\n catalog_name=CATALOG,\n\ - \ schema_name=SCHEMA,\n name=\"raw_data\",\n volume_type=VolumeType.MANAGED\n\ - \ )\n print(f\"Created volume {CATALOG}.{SCHEMA}.raw_data\")\nexcept Exception\ - \ as e:\n if \"already exists\" in str(e).lower():\n print(f\"Volume\ - \ {CATALOG}.{SCHEMA}.raw_data already exists\")\n else:\n raise\n\n\ - # Upload files\nvolume_path = f\"/Volumes/{CATALOG}/{SCHEMA}/raw_data\"\n\n\ - with open(f\"{local_path}/customers.parquet\", \"rb\") as f:\n w.files.upload(f\"\ - {volume_path}/customers.parquet\", f, overwrite=True)\nprint(f\"Uploaded customers.parquet\ - \ to {volume_path}/\")\n\nwith open(f\"{local_path}/orders.parquet\", \"rb\"\ - ) as f:\n w.files.upload(f\"{volume_path}/orders.parquet\", f, overwrite=True)\n\ - print(f\"Uploaded orders.parquet to {volume_path}/\")\n\nprint(f\"\\n\u2705\ - \ Complete! Data available at:\")\nprint(f\" {volume_path}/customers.parquet\"\ - )\nprint(f\" {volume_path}/orders.parquet\")\n```\n\nThis script:\n1. Generates\ - \ 5,000 customers with weighted tier distribution (Free 60%, Pro 30%, Enterprise\ - \ 10%)\n2. Generates 15,000 orders with tier-weighted customer assignment (Enterprise\ - \ ~8x, Pro ~3x, Free ~1.5x orders per customer)\n3. Uses log-normal distributions\ - \ for realistic order amounts by tier\n4. Saves locally as Parquet files\n5.\ - \ Creates the schema and volume in Unity Catalog if needed\n6. Uploads the Parquet\ - \ files to the volume\n\nRun with:\n```bash\nuv pip install polars faker numpy\ - \ databricks-sdk\nuv run python generate_ecommerce_data.py\n```" + expected_response_file: expected_responses/grp_20260302_113344.txt execution_success: true expectations: - expected_facts: - - "parquet" - - "customer_id" - - "referential integrity" - - "weighted" - - "log-normal" - expected_patterns: - - pattern: "\\.write.*parquet" - min_count: 1 - description: "Parquet output format" - - pattern: "customer_id" - min_count: 3 - description: "Foreign key reference in multiple tables" - - pattern: "lognormal" - min_count: 1 - description: "Log-normal distribution for amounts" - guidelines: - - "Orders table customer_id must only contain IDs from customers table" - - "Enterprise tier customers must have higher weight for order generation" - - "Amount distribution must use log-normal, not uniform" + expected_facts: + - "parquet" + - "customer_id" + - "referential integrity" + - "weighted" + - "log-normal" + expected_patterns: + - pattern: "\\.write.*parquet" + min_count: 1 + description: "Parquet output format" + - pattern: "customer_id" + min_count: 3 + description: "Foreign key reference in multiple tables" + - pattern: "lognormal" + min_count: 1 + description: "Log-normal distribution for amounts" + guidelines: + - "Orders table customer_id must only contain IDs from customers table" + - "Enterprise tier customers must have higher weight for order generation" + - "Amount distribution must use log-normal, not uniform" metadata: category: happy_path source: interactive @@ -173,151 +70,7 @@ test_cases: ' outputs: - response: "Here's the Python script to generate product catalog data, followed\ - \ by the serverless job definition:\n\n```python\n\"\"\"\nSynthetic Product\ - \ Catalog Data Generation\nServerless Databricks Job with JSON Output\n\nGenerates:\n\ - - products (3,000 rows): product_id, name, category (weighted), price (log-normal),\ - \ inventory_count\n- sales (10,000 rows): sale_id, product_id (FK), quantity,\ - \ sale_date, discount_pct\n\nPopular product categories have more sales (weighted\ - \ sampling).\n\"\"\"\n\nimport os\nfrom pyspark.sql import SparkSession, functions\ - \ as F\nfrom pyspark.sql.window import Window\nfrom pyspark.sql.types import\ - \ StringType, DoubleType, IntegerType\nimport pandas as pd\nimport numpy as\ - \ np\nfrom datetime import datetime, timedelta\n\n# =============================================================================\n\ - # CONFIGURATION\n# =============================================================================\n\ - CATALOG = \"dustin_vannoy_catalog\"\nSCHEMA = \"devkit_gen2_test_small_json\"\ - \nVOLUME_PATH = f\"/Volumes/{CATALOG}/{SCHEMA}/raw_data\"\n\nNUM_PRODUCTS =\ - \ 3_000\nNUM_SALES = 10_000\nNUM_PARTITIONS = 16\n\nEND_DATE = datetime.now().replace(hour=0,\ - \ minute=0, second=0, microsecond=0)\nSTART_DATE = END_DATE - timedelta(days=180)\n\ - SEED = 42\n\n# =============================================================================\n\ - # SPARK SESSION\n# =============================================================================\n\ - \ndef is_databricks_runtime():\n return \"DATABRICKS_RUNTIME_VERSION\" in\ - \ os.environ\n\nif is_databricks_runtime():\n spark = SparkSession.builder.getOrCreate()\n\ - \ print(\"Running on Databricks Runtime\")\nelse:\n from databricks.connect\ - \ import DatabricksSession\n spark = DatabricksSession.builder.serverless(True).getOrCreate()\n\ - \ print(\"Running with Databricks Connect (serverless)\")\n\nprint(\"=\"\ - \ * 60)\nprint(\"PRODUCT CATALOG DATA GENERATION\")\nprint(\"=\" * 60)\nprint(f\"\ - Catalog: {CATALOG}\")\nprint(f\"Schema: {SCHEMA}\")\nprint(f\"Products: {NUM_PRODUCTS:,}\"\ - )\nprint(f\"Sales: {NUM_SALES:,}\")\nprint(\"=\" * 60)\n\n# =============================================================================\n\ - # CREATE INFRASTRUCTURE\n# =============================================================================\n\ - print(\"\\nCreating infrastructure...\")\nspark.sql(f\"CREATE SCHEMA IF NOT\ - \ EXISTS {CATALOG}.{SCHEMA}\")\nspark.sql(f\"CREATE VOLUME IF NOT EXISTS {CATALOG}.{SCHEMA}.raw_data\"\ - )\n\n# =============================================================================\n\ - # PANDAS UDFs\n# =============================================================================\n\ - \n@F.pandas_udf(StringType())\ndef fake_product_name(categories: pd.Series)\ - \ -> pd.Series:\n from faker import Faker\n fake = Faker()\n\n templates\ - \ = {\n \"Electronics\": [\"Smart\", \"Wireless\", \"Digital\", \"Pro\"\ - , \"Ultra\"],\n \"Home & Garden\": [\"Premium\", \"Deluxe\", \"Classic\"\ - , \"Modern\", \"Natural\"],\n \"Clothing\": [\"Designer\", \"Casual\"\ - , \"Comfort\", \"Luxury\", \"Sport\"],\n \"Sports\": [\"Pro\", \"Elite\"\ - , \"Performance\", \"Outdoor\", \"Active\"],\n \"Books\": [\"Complete\"\ - , \"Essential\", \"Ultimate\", \"Practical\", \"Advanced\"],\n \"Toys\"\ - : [\"Fun\", \"Creative\", \"Educational\", \"Super\", \"Magic\"],\n \"\ - Beauty\": [\"Natural\", \"Premium\", \"Radiance\", \"Pure\", \"Glow\"],\n \ - \ \"Automotive\": [\"Pro\", \"Heavy-Duty\", \"Premium\", \"Performance\"\ - , \"Ultra\"],\n }\n\n products = {\n \"Electronics\": [\"Headphones\"\ - , \"Speaker\", \"Charger\", \"Watch\", \"Camera\"],\n \"Home & Garden\"\ - : [\"Lamp\", \"Planter\", \"Organizer\", \"Rug\", \"Vase\"],\n \"Clothing\"\ - : [\"T-Shirt\", \"Jacket\", \"Pants\", \"Sweater\", \"Dress\"],\n \"\ - Sports\": [\"Ball\", \"Racket\", \"Mat\", \"Gloves\", \"Bag\"],\n \"\ - Books\": [\"Guide\", \"Handbook\", \"Manual\", \"Edition\", \"Collection\"],\n\ - \ \"Toys\": [\"Game\", \"Puzzle\", \"Building Set\", \"Robot\", \"Craft\ - \ Kit\"],\n \"Beauty\": [\"Serum\", \"Cream\", \"Lotion\", \"Mask\",\ - \ \"Oil\"],\n \"Automotive\": [\"Tool Kit\", \"Cover\", \"Mat\", \"Cleaner\"\ - , \"Polish\"],\n }\n\n names = []\n for category in categories:\n \ - \ template_list = templates.get(category, [\"Premium\"])\n product_list\ - \ = products.get(category, [\"Item\"])\n template = np.random.choice(template_list)\n\ - \ product = np.random.choice(product_list)\n color = fake.color_name()\n\ - \ names.append(f\"{template} {color} {product}\")\n\n return pd.Series(names)\n\ - \n\n@F.pandas_udf(DoubleType())\ndef generate_price(categories: pd.Series) ->\ - \ pd.Series:\n price_params = {\n \"Electronics\": (4.5, 0.8),\n \ - \ \"Home & Garden\": (3.8, 0.7),\n \"Clothing\": (3.5, 0.6),\n\ - \ \"Sports\": (4.0, 0.7),\n \"Books\": (2.8, 0.4),\n \"\ - Toys\": (3.2, 0.6),\n \"Beauty\": (3.3, 0.5),\n \"Automotive\"\ - : (4.2, 0.8),\n }\n\n prices = []\n for category in categories:\n \ - \ mu, sigma = price_params.get(category, (3.5, 0.6))\n price =\ - \ float(np.random.lognormal(mu, sigma))\n price = round(price) - 0.01\ - \ if price > 1 else round(price, 2)\n prices.append(max(0.99, price))\n\ - \n return pd.Series(prices)\n\n\n@F.pandas_udf(IntegerType())\ndef generate_inventory(ids:\ - \ pd.Series) -> pd.Series:\n inventory = (np.random.pareto(a=2.0, size=len(ids))\ - \ + 1) * 20\n return pd.Series(inventory.astype(int))\n\n\n# =============================================================================\n\ - # GENERATE PRODUCTS TABLE (Master)\n# =============================================================================\n\ - print(f\"\\nGenerating {NUM_PRODUCTS:,} products...\")\n\nproducts_df = (\n\ - \ spark.range(0, NUM_PRODUCTS, numPartitions=NUM_PARTITIONS)\n .select(\n\ - \ F.concat(F.lit(\"PROD-\"), F.lpad(F.col(\"id\").cast(\"string\"), 5,\ - \ \"0\")).alias(\"product_id\"),\n F.col(\"id\").alias(\"_idx\"),\n \ - \ )\n .withColumn(\n \"category\",\n F.when(F.rand(SEED)\ - \ < 0.25, \"Electronics\")\n .when(F.rand(SEED + 1) < 0.45, \"Home &\ - \ Garden\")\n .when(F.rand(SEED + 2) < 0.63, \"Clothing\")\n \ - \ .when(F.rand(SEED + 3) < 0.75, \"Sports\")\n .when(F.rand(SEED +\ - \ 4) < 0.85, \"Books\")\n .when(F.rand(SEED + 5) < 0.93, \"Toys\")\n\ - \ .when(F.rand(SEED + 6) < 0.98, \"Beauty\")\n .otherwise(\"\ - Automotive\")\n )\n .withColumn(\"name\", fake_product_name(F.col(\"category\"\ - )))\n .withColumn(\"price\", generate_price(F.col(\"category\")))\n .withColumn(\"\ - inventory_count\", generate_inventory(F.col(\"_idx\")))\n .drop(\"_idx\"\ - )\n)\n\nproducts_final = products_df.select(\n \"product_id\", \"name\",\ - \ \"category\", \"price\", \"inventory_count\"\n)\n\nproducts_json_path = f\"\ - {VOLUME_PATH}/products\"\nprint(f\"Writing products to {products_json_path}...\"\ - )\nproducts_final.write.mode(\"overwrite\").json(products_json_path)\n\nproducts_for_fk\ - \ = spark.read.json(products_json_path).select(\"product_id\", \"category\"\ - )\nproduct_count = products_for_fk.count()\nprint(f\"Products written: {product_count:,}\"\ - )\n\n# =============================================================================\n\ - # GENERATE SALES TABLE (with Referential Integrity)\n# =============================================================================\n\ - print(f\"\\nGenerating {NUM_SALES:,} sales with referential integrity...\")\n\ - \nproduct_weights = products_for_fk.select(\n \"product_id\",\n \"category\"\ - ,\n F.when(F.col(\"category\") == \"Electronics\", 3.0)\n .when(F.col(\"\ - category\") == \"Home & Garden\", 2.5)\n .when(F.col(\"category\") == \"\ - Clothing\", 2.0)\n .when(F.col(\"category\") == \"Sports\", 1.5)\n .when(F.col(\"\ - category\") == \"Books\", 1.2)\n .when(F.col(\"category\") == \"Toys\",\ - \ 1.0)\n .when(F.col(\"category\") == \"Beauty\", 0.8)\n .otherwise(0.5).alias(\"\ - weight\")\n)\n\nweighted_products = (\n product_weights\n .select(\n \ - \ F.col(\"product_id\"),\n F.col(\"category\"),\n F.explode(F.array_repeat(F.col(\"\ - product_id\"), F.col(\"weight\").cast(\"int\"))).alias(\"_dup\")\n )\n \ - \ .drop(\"_dup\")\n)\n\nsampled_products = (\n weighted_products\n .orderBy(F.rand(SEED\ - \ + 20))\n .limit(NUM_SALES)\n .withColumn(\"sale_rank\", F.row_number().over(Window.orderBy(F.monotonically_increasing_id())))\n\ - \ .select(\"product_id\", \"category\", \"sale_rank\")\n)\n\nsales_base =\ - \ (\n spark.range(0, NUM_SALES, numPartitions=NUM_PARTITIONS)\n .withColumn(\"\ - sale_rank\", F.row_number().over(Window.orderBy(F.col(\"id\"))))\n .withColumn(\n\ - \ \"sale_id\",\n F.concat(F.lit(\"SALE-\"), F.lpad(F.col(\"id\"\ - ).cast(\"string\"), 6, \"0\"))\n )\n)\n\nsales_with_products = sales_base.join(\n\ - \ sampled_products,\n on=\"sale_rank\",\n how=\"inner\"\n)\n\nsales_df\ - \ = (\n sales_with_products\n .withColumn(\n \"quantity\",\n \ - \ F.when(F.rand(SEED + 21) < 0.60, 1)\n .when(F.rand(SEED + 22)\ - \ < 0.85, 2)\n .when(F.rand(SEED + 23) < 0.95, 3)\n .otherwise(F.floor(F.rand(SEED\ - \ + 24) * 5 + 4).cast(\"int\"))\n )\n .withColumn(\n \"sale_date\"\ - ,\n F.date_add(F.lit(START_DATE.date()), (F.rand(SEED + 25) * 180).cast(\"\ - int\"))\n )\n .withColumn(\n \"discount_pct\",\n F.when(F.rand(SEED\ - \ + 26) < 0.70, 0.0)\n .when(F.rand(SEED + 27) < 0.85, 0.10)\n \ - \ .when(F.rand(SEED + 28) < 0.95, 0.20)\n .otherwise(0.30)\n )\n\ - )\n\nsales_final = sales_df.select(\n \"sale_id\", \"product_id\", \"quantity\"\ - , \"sale_date\", \"discount_pct\"\n)\n\nsales_json_path = f\"{VOLUME_PATH}/sales\"\ - \nprint(f\"Writing sales to {sales_json_path}...\")\nsales_final.write.mode(\"\ - overwrite\").json(sales_json_path)\n\nsales_count = spark.read.json(sales_json_path).count()\n\ - print(f\"Sales written: {sales_count:,}\")\n\n# =============================================================================\n\ - # VALIDATION\n# =============================================================================\n\ - print(\"\\n\" + \"=\" * 60)\nprint(\"VALIDATION\")\nprint(\"=\" * 60)\n\nproducts_check\ - \ = spark.read.json(products_json_path)\nsales_check = spark.read.json(sales_json_path)\n\ - \nprint(f\"\\nRow counts:\")\nprint(f\" Products: {products_check.count():,}\"\ - )\nprint(f\" Sales: {sales_check.count():,}\")\n\nprint(f\"\\nSales per category\ - \ (Electronics should have most):\")\nsales_with_category = sales_check.join(\n\ - \ products_check.select(\"product_id\", \"category\"),\n on=\"product_id\"\ - \n)\nsales_with_category.groupBy(\"category\").agg(\n F.count(\"*\").alias(\"\ - total_sales\"),\n F.sum(\"quantity\").alias(\"total_quantity\")\n).orderBy(F.desc(\"\ - total_sales\")).show()\n\norphan_sales = sales_check.join(\n products_check.select(\"\ - product_id\"),\n on=\"product_id\",\n how=\"left_anti\"\n).count()\nprint(f\"\ - \\nReferential integrity:\")\nprint(f\" Orphan sales (no matching product):\ - \ {orphan_sales}\")\n\nprint(\"\\n\" + \"=\" * 60)\nprint(\"GENERATION COMPLETE\"\ - )\nprint(\"=\" * 60)\nprint(f\"\\nOutput location: {VOLUME_PATH}\")\n```\n\n\ - To run this as a serverless job, create the job with this configuration:\n\n\ - ```json\n{\n \"name\": \"generate_product_catalog_data\",\n \"tasks\": [\n\ - \ {\n \"task_key\": \"generate_data\",\n \"spark_python_task\"\ - : {\n \"python_file\": \"/Workspace/Users/your_email/generate_product_catalog_data.py\"\ - \n },\n \"new_cluster\": {\n \"spark_version\": \"16.4.x-scala2.12\"\ - ,\n \"num_workers\": 0,\n \"spark_conf\": {\n \"spark.databricks.cluster.profile\"\ - : \"serverless\"\n }\n },\n \"environments\": [\n {\n\ - \ \"environment_key\": \"default\",\n \"spec\": {\n \ - \ \"client\": \"4\",\n \"dependencies\": [\n \"\ - faker\",\n \"pandas\",\n \"numpy\"\n ]\n\ - \ }\n }\n ]\n }\n ]\n}\n```\n" + expected_response_file: expected_responses/gen_serverless_job_catalog_json_002.txt expectations: expected_facts: - serverless @@ -389,305 +142,7 @@ test_cases: Higher membership levels should have more orders. Order total_amount should equal sum of line_items. outputs: - response: | - Here's the complete script to generate synthetic retail order data with 3 tables and full referential integrity: - - ```python - """ - Synthetic Retail Order Data Generation - ====================================== - 3 related tables with full referential integrity: - - customers (10,000 rows) - - orders (50,000 rows) - - line_items (150,000 rows) - - Output: CSV files with headers to Unity Catalog volume - Execution: Databricks Connect with serverless compute - """ - - import os - from pyspark.sql import functions as F - from pyspark.sql.types import StringType, DoubleType, IntegerType - from pyspark.sql.window import Window - import pandas as pd - import numpy as np - from datetime import datetime, timedelta - - # ============================================================================= - # CONFIGURATION - # ============================================================================= - CATALOG = "ai_dev_kit" - SCHEMA = "devkit_gen3_test_medium_csv" - VOLUME_PATH = f"/Volumes/{CATALOG}/{SCHEMA}/raw_data" - - NUM_CUSTOMERS = 10_000 - NUM_ORDERS = 50_000 - NUM_LINE_ITEMS = 150_000 - NUM_PARTITIONS = 32 - SEED = 42 - - END_DATE = datetime.now().replace(hour=0, minute=0, second=0, microsecond=0) - START_DATE = END_DATE - timedelta(days=180) - - # ============================================================================= - # SPARK SESSION - # ============================================================================= - def is_databricks_runtime(): - return "DATABRICKS_RUNTIME_VERSION" in os.environ - - if is_databricks_runtime(): - from pyspark.sql import SparkSession - spark = SparkSession.builder.getOrCreate() - print("Running on Databricks Runtime") - else: - from databricks.connect import DatabricksSession - from databricks.connect.session import DatabricksEnv - - # Install dependencies on serverless cluster - env = DatabricksEnv().withDependencies("faker", "pandas", "numpy") - spark = DatabricksSession.builder.withEnvironment(env).serverless(True).getOrCreate() - print("Running with Databricks Connect (serverless with managed dependencies)") - - # ============================================================================= - # CREATE INFRASTRUCTURE - # ============================================================================= - spark.sql(f"CREATE SCHEMA IF NOT EXISTS {CATALOG}.{SCHEMA}") - spark.sql(f"CREATE VOLUME IF NOT EXISTS {CATALOG}.{SCHEMA}.raw_data") - - # ============================================================================= - # PANDAS UDFs - # ============================================================================= - - @F.pandas_udf(StringType()) - def fake_name(ids: pd.Series) -> pd.Series: - from faker import Faker - fake = Faker() - Faker.seed(SEED) - return pd.Series([fake.name() for _ in range(len(ids))]) - - @F.pandas_udf(StringType()) - def fake_email(names: pd.Series) -> pd.Series: - from faker import Faker - fake = Faker() - emails = [] - for name in names: - parts = name.lower().split() - if len(parts) >= 2: - email = f"{parts[0]}.{parts[-1]}@{fake.free_email_domain()}" - else: - email = f"{parts[0]}{np.random.randint(100, 999)}@{fake.free_email_domain()}" - emails.append(email) - return pd.Series(emails) - - @F.pandas_udf(StringType()) - def fake_product_name(ids: pd.Series) -> pd.Series: - from faker import Faker - fake = Faker() - product_types = ["Chair", "Table", "Lamp", "Desk", "Shelf", "Cabinet", "Sofa", "Rug", - "Mirror", "Clock", "Vase", "Frame", "Pillow", "Blanket", "Candle", - "Mug", "Bowl", "Plate", "Glass", "Bottle", "Box", "Bag", "Hat", - "Watch", "Headphones", "Speaker", "Charger", "Cable", "Case"] - products = [] - for _ in range(len(ids)): - color = fake.color_name() - adj = fake.word().capitalize() - product = np.random.choice(product_types) - products.append(f"{color} {adj} {product}") - return pd.Series(products) - - @F.pandas_udf(DoubleType()) - def generate_unit_price(ids: pd.Series) -> pd.Series: - """Log-normal unit prices (median ~$35, range $5-$500)""" - prices = np.random.lognormal(mean=3.5, sigma=0.7, size=len(ids)) - prices = np.clip(prices, 5.0, 500.0) - return pd.Series(np.round(prices, 2)) - - # ============================================================================= - # GENERATE CUSTOMERS TABLE - # ============================================================================= - customers_df = ( - spark.range(0, NUM_CUSTOMERS, numPartitions=NUM_PARTITIONS) - .select( - F.concat(F.lit("CUST-"), F.lpad(F.col("id").cast("string"), 5, "0")).alias("customer_id"), - F.col("id").alias("_idx") - ) - .withColumn("name", fake_name(F.col("_idx"))) - .withColumn("email", fake_email(F.col("name"))) - .withColumn("membership_level", - F.when(F.rand(SEED) < 0.50, "Bronze") - .when(F.rand(SEED + 1) < 0.80, "Silver") - .when(F.rand(SEED + 2) < 0.95, "Gold") - .otherwise("Platinum") - ) - .withColumn("region", - F.when(F.rand(SEED + 3) < 0.30, "US-East") - .when(F.rand(SEED + 4) < 0.55, "US-West") - .when(F.rand(SEED + 5) < 0.80, "EU") - .when(F.rand(SEED + 6) < 0.95, "APAC") - .otherwise("Other") - ) - .drop("_idx") - ) - - # Write to temp Delta table (no .cache() on serverless) - customers_tmp = f"{CATALOG}.{SCHEMA}._tmp_customers" - customers_df.write.mode("overwrite").saveAsTable(customers_tmp) - customers_df = spark.table(customers_tmp) - - # ============================================================================= - # GENERATE ORDERS TABLE (weighted by membership level) - # ============================================================================= - customer_weights = customers_df.select( - "customer_id", - "membership_level", - F.when(F.col("membership_level") == "Platinum", 10.0) - .when(F.col("membership_level") == "Gold", 7.0) - .when(F.col("membership_level") == "Silver", 5.0) - .otherwise(3.0).alias("weight") - ) - - weighted_customers = ( - customer_weights - .withColumn("replicate_count", (F.col("weight") * 2).cast("int")) - .select( - F.col("customer_id"), - F.explode(F.array_repeat(F.col("customer_id"), F.col("replicate_count"))).alias("_dup") - ) - .drop("_dup") - ) - - sampled_customers = ( - weighted_customers - .orderBy(F.rand(SEED + 10)) - .limit(NUM_ORDERS) - .withColumn("_row", F.row_number().over(Window.orderBy(F.monotonically_increasing_id()))) - .select("customer_id", "_row") - ) - - orders_base = ( - spark.range(0, NUM_ORDERS, numPartitions=NUM_PARTITIONS) - .withColumn("order_id", - F.concat(F.lit("ORD-"), F.lpad(F.col("id").cast("string"), 6, "0"))) - .withColumn("_row", F.row_number().over(Window.orderBy(F.col("id")))) - ) - - orders_df = ( - orders_base - .join(sampled_customers, on="_row", how="inner") - .drop("_row", "id") - .withColumn("order_date", - F.date_add(F.lit(START_DATE.date()), (F.rand(SEED + 11) * 180).cast("int"))) - .withColumn("status", - F.when(F.rand(SEED + 12) < 0.60, "Delivered") - .when(F.rand(SEED + 13) < 0.78, "Shipped") - .when(F.rand(SEED + 14) < 0.90, "Processing") - .when(F.rand(SEED + 15) < 0.96, "Pending") - .otherwise("Cancelled") - ) - .withColumn("total_amount", F.lit(0.0)) - ) - - orders_tmp = f"{CATALOG}.{SCHEMA}._tmp_orders" - orders_df.write.mode("overwrite").saveAsTable(orders_tmp) - orders_df = spark.table(orders_tmp) - - # ============================================================================= - # GENERATE LINE_ITEMS TABLE - # ============================================================================= - order_ids = orders_df.select("order_id") - - items_per_order = ( - order_ids - .withColumn("num_items", - F.when(F.rand(SEED + 20) < 0.10, 1) - .when(F.rand(SEED + 21) < 0.25, 2) - .when(F.rand(SEED + 22) < 0.45, 3) - .when(F.rand(SEED + 23) < 0.65, 4) - .when(F.rand(SEED + 24) < 0.80, 5) - .when(F.rand(SEED + 25) < 0.90, 6) - .when(F.rand(SEED + 26) < 0.96, 7) - .otherwise(8) - ) - ) - - line_items_base = ( - items_per_order - .select( - F.col("order_id"), - F.explode(F.sequence(F.lit(1), F.col("num_items"))).alias("item_seq") - ) - .withColumn("_idx", F.monotonically_increasing_id()) - ) - - line_items_df = ( - line_items_base - .limit(NUM_LINE_ITEMS) - .withColumn("line_item_id", - F.concat(F.lit("LI-"), F.lpad(F.col("_idx").cast("string"), 7, "0"))) - .withColumn("product_name", fake_product_name(F.col("_idx"))) - .withColumn("quantity", - F.when(F.rand(SEED + 30) < 0.50, 1) - .when(F.rand(SEED + 31) < 0.80, 2) - .when(F.rand(SEED + 32) < 0.92, 3) - .when(F.rand(SEED + 33) < 0.97, 4) - .otherwise(5) - ) - .withColumn("unit_price", generate_unit_price(F.col("_idx"))) - .drop("_idx", "item_seq") - ) - - line_items_tmp = f"{CATALOG}.{SCHEMA}._tmp_line_items" - line_items_df.write.mode("overwrite").saveAsTable(line_items_tmp) - line_items_df = spark.table(line_items_tmp) - - # ============================================================================= - # COMPUTE ORDER TOTALS FROM LINE ITEMS - # ============================================================================= - order_totals = ( - line_items_df - .withColumn("line_total", F.col("quantity") * F.col("unit_price")) - .groupBy("order_id") - .agg(F.round(F.sum("line_total"), 2).alias("computed_total")) - ) - - orders_final = ( - spark.table(orders_tmp) - .drop("total_amount") - .join(order_totals, on="order_id", how="left") - .withColumn("total_amount", F.coalesce(F.col("computed_total"), F.lit(0.0))) - .drop("computed_total") - ) - - # ============================================================================= - # EXPORT TO CSV - # ============================================================================= - customers_final = spark.table(customers_tmp).select( - "customer_id", "name", "email", "membership_level", "region" - ) - - orders_out = orders_final.select( - "order_id", "customer_id", "order_date", "total_amount", "status" - ) - - line_items_out = spark.table(line_items_tmp).select( - "line_item_id", "order_id", "product_name", "quantity", "unit_price" - ) - - customers_final.coalesce(1).write.mode("overwrite").option("header", "true").csv(f"{VOLUME_PATH}/customers") - orders_out.coalesce(1).write.mode("overwrite").option("header", "true").csv(f"{VOLUME_PATH}/orders") - line_items_out.coalesce(4).write.mode("overwrite").option("header", "true").csv(f"{VOLUME_PATH}/line_items") - - # Cleanup temp tables - spark.sql(f"DROP TABLE IF EXISTS {customers_tmp}") - spark.sql(f"DROP TABLE IF EXISTS {orders_tmp}") - spark.sql(f"DROP TABLE IF EXISTS {line_items_tmp}") - ``` - - Run with: - ```bash - uv pip install "databricks-connect>=16.4,<17.0" faker numpy pandas - uv run python generate_retail_data.py - ``` + expected_response_file: expected_responses/grp_20260302_retail_csv_3tables_003.txt execution_success: true expectations: expected_facts: @@ -795,242 +250,7 @@ test_cases: Create a lookup table for the sensor device which changes slowly. Save as Delta tables registered in Unity Catalog. Use catalog 'ai_dev_kit'. Use schema name 'devkit_gen4_test_large_delta'. outputs: - response: | - Here's the complete script to generate manufacturing IoT sensor data with streaming-ready batch processing: - - ```python - """ - Manufacturing IoT Sensor Data Generation - ======================================== - 2 related tables with referential integrity: - - sensor_devices (500 rows): Slowly changing dimension - - sensor_readings (10,000 rows per batch): Streaming incremental - - Output: Delta tables registered in Unity Catalog - Execution: Databricks Connect with serverless compute (Python 3.12+) - """ - - from databricks.connect import DatabricksSession, DatabricksEnv - from pyspark.sql import functions as F - from pyspark.sql.types import StringType, DoubleType, TimestampType - import pandas as pd - import numpy as np - from datetime import datetime, timedelta - - # Configuration - CATALOG = "ai_dev_kit" - SCHEMA = "devkit_gen4_test_large_delta" - - # Row counts - NUM_DEVICES = 500 - NUM_READINGS_PER_BATCH = 10000 - - # Date range for readings (last 24 hours for streaming simulation) - END_TIME = datetime.now() - START_TIME = END_TIME - timedelta(hours=24) - - # Setup with managed dependencies (databricks-connect 16.4+) - env = DatabricksEnv().withDependencies("faker", "pandas", "numpy") - spark = DatabricksSession.builder.withEnvironment(env).serverless(True).getOrCreate() - - print(f"Connected to Databricks serverless compute") - print(f"Output location: {CATALOG}.{SCHEMA}") - - # Create schema if not exists - spark.sql(f"CREATE SCHEMA IF NOT EXISTS {CATALOG}.{SCHEMA}") - print(f"Schema {CATALOG}.{SCHEMA} ready") - - - # ============================================================================= - # Pandas UDFs for data generation - # ============================================================================= - - @F.pandas_udf(StringType()) - def fake_device_name(ids: pd.Series) -> pd.Series: - """Generate realistic device names like 'SENS-T-00001'.""" - from faker import Faker - fake = Faker() - Faker.seed(42) - - prefixes = {"temperature": "T", "pressure": "P", "vibration": "V", "humidity": "H"} - names = [] - for i, _ in enumerate(ids): - type_key = list(prefixes.keys())[i % 4] - prefix = prefixes[type_key] - names.append(f"SENS-{prefix}-{str(i).zfill(5)}") - return pd.Series(names) - - - @F.pandas_udf(DoubleType()) - def generate_sensor_value(device_types: pd.Series) -> pd.Series: - """Generate realistic sensor values based on device type.""" - values = [] - for dtype in device_types: - if dtype == "temperature": - values.append(float(np.random.normal(70, 15))) # Fahrenheit - elif dtype == "pressure": - values.append(float(np.random.normal(100, 10))) # PSI - elif dtype == "vibration": - values.append(float(np.random.lognormal(1.5, 0.8))) # mm/s with spikes - elif dtype == "humidity": - values.append(float(np.clip(np.random.normal(45, 10), 0, 100))) # Percentage - else: - values.append(float(np.random.normal(50, 10))) - return pd.Series(values) - - - @F.pandas_udf(StringType()) - def generate_unit(device_types: pd.Series) -> pd.Series: - """Generate appropriate unit based on device type.""" - unit_map = { - "temperature": "°F", - "pressure": "PSI", - "vibration": "mm/s", - "humidity": "%" - } - return pd.Series([unit_map.get(dt, "units") for dt in device_types]) - - - # ============================================================================= - # Generate sensor_devices (slowly changing dimension) - # ============================================================================= - - print("\nGenerating sensor_devices table (slowly changing dimension)") - - # Device type weights: temperature 30%, pressure 25%, vibration 25%, humidity 20% - devices_df = ( - spark.range(0, NUM_DEVICES, numPartitions=4) - .select( - F.concat(F.lit("DEV-"), F.lpad(F.col("id").cast("string"), 5, "0")).alias("device_id"), - F.col("id").alias("_id") - ) - .withColumn("device_name", fake_device_name(F.col("_id"))) - .withColumn( - "device_type", - F.when(F.rand() < 0.30, "temperature") - .when(F.rand() < 0.55, "pressure") - .when(F.rand() < 0.80, "vibration") - .otherwise("humidity") - ) - .withColumn( - "location", - F.when(F.rand() < 0.40, "Plant_A") - .when(F.rand() < 0.70, "Plant_B") - .when(F.rand() < 0.90, "Plant_C") - .otherwise("Warehouse") - ) - .withColumn( - "install_date", - F.date_add(F.lit("2020-01-01"), (F.rand() * 1460).cast("int")) - ) - .withColumn( - "status", - F.when(F.rand() < 0.85, "active") - .when(F.rand() < 0.95, "maintenance") - .otherwise("inactive") - ) - .drop("_id") - ) - - # Write devices table - devices_table = f"{CATALOG}.{SCHEMA}.sensor_devices" - devices_df.write.mode("overwrite").saveAsTable(devices_table) - print(f"Created {devices_table}") - - - # ============================================================================= - # Generate sensor_readings (streaming incremental batch) - # ============================================================================= - - print("\nGenerating sensor_readings table (streaming batch)") - - # Read devices back for FK join (no .cache() on serverless!) - devices_for_join = spark.table(devices_table).select("device_id", "device_type") - - # Generate readings with valid device_id FK - readings_df = ( - spark.range(0, NUM_READINGS_PER_BATCH, numPartitions=16) - .select( - F.concat( - F.lit("RDG-"), - F.date_format(F.current_timestamp(), "yyyyMMddHHmmss"), - F.lit("-"), - F.lpad(F.col("id").cast("string"), 6, "0") - ).alias("reading_id"), - (F.abs(F.hash(F.col("id"))) % NUM_DEVICES).alias("device_index"), - F.from_unixtime( - F.unix_timestamp(F.lit(START_TIME)) + - (F.rand() * 86400).cast("long") - ).cast("timestamp").alias("timestamp") - ) - ) - - # Get device IDs with row numbers for joining - devices_indexed = ( - devices_for_join - .withColumn("device_index", F.monotonically_increasing_id() % NUM_DEVICES) - ) - - # Join to get valid device_id and device_type - readings_with_device = ( - readings_df - .join( - devices_indexed.select("device_id", "device_type", "device_index"), - on="device_index", - how="inner" - ) - .drop("device_index") - ) - - # Add sensor values based on device type - readings_final = ( - readings_with_device - .withColumn("value", F.round(generate_sensor_value(F.col("device_type")), 2)) - .withColumn("unit", generate_unit(F.col("device_type"))) - .withColumn( - "quality_flag", - F.when(F.rand() < 0.90, "good") - .when(F.rand() < 0.97, "warning") - .otherwise("error") - ) - .select("reading_id", "device_id", "timestamp", "value", "unit", "quality_flag") - ) - - # Write readings table (append mode for streaming simulation) - readings_table = f"{CATALOG}.{SCHEMA}.sensor_readings" - readings_final.write.mode("overwrite").saveAsTable(readings_table) - print(f"Created {readings_table}") - - - # ============================================================================= - # Validation - # ============================================================================= - - # Check referential integrity - orphan_readings = spark.sql(f""" - SELECT COUNT(*) as orphan_count - FROM {readings_table} r - LEFT JOIN {devices_table} d ON r.device_id = d.device_id - WHERE d.device_id IS NULL - """).collect()[0]["orphan_count"] - - print(f"Orphan readings (should be 0): {orphan_readings}") - print(f"\nSUMMARY") - print(f"Catalog/Schema: {CATALOG}.{SCHEMA}") - print(f"Tables: {devices_table}, {readings_table}") - print(f"Referential integrity: {'PASSED' if orphan_readings == 0 else 'FAILED'}") - ``` - - Run with: - ```bash - uv pip install "databricks-connect>=16.4,<17.0" faker numpy pandas - uv run python generate_manufacturing_data.py - ``` - - For streaming job deployment, this can be scheduled as an incremental job where each run: - 1. Generates a new batch of 10,000 readings with unique timestamps - 2. Appends to the sensor_readings table (change mode to "append") - 3. The sensor_devices table is the slowly-changing dimension that persists + expected_response_file: expected_responses/grp_20260303_manufacturing_delta_streaming_004.txt execution_success: true expectations: expected_facts: diff --git a/.test/src/skill_test/config.py b/.test/src/skill_test/config.py index d7c413aa..f4a42cb8 100644 --- a/.test/src/skill_test/config.py +++ b/.test/src/skill_test/config.py @@ -121,7 +121,7 @@ class DatabricksExecutionSettings: schema: str = field(default_factory=lambda: os.getenv("SKILL_TEST_SCHEMA", "skill_test")) # Execution settings - timeout: int = 480 # seconds + timeout: int = 240 # seconds - increased from 120s to handle larger data generation tasks preserve_context: bool = True # Reuse context across code blocks diff --git a/.test/src/skill_test/dataset.py b/.test/src/skill_test/dataset.py index 9941ef12..5c88c330 100644 --- a/.test/src/skill_test/dataset.py +++ b/.test/src/skill_test/dataset.py @@ -41,17 +41,34 @@ class YAMLDatasetSource: yaml_path: Path def load(self) -> List[EvalRecord]: - """Load records from YAML ground_truth.yaml file.""" + """Load records from YAML ground_truth.yaml file. + + Supports external response files via 'expected_response_file' field in outputs. + When present, the response is loaded from the file relative to the YAML directory. + """ with open(self.yaml_path) as f: data = yaml.safe_load(f) + yaml_dir = self.yaml_path.parent + records = [] for case in data.get("test_cases", []): + outputs = case.get("outputs") + + # Load response from external file if specified + if outputs and "expected_response_file" in outputs: + response_file = yaml_dir / outputs["expected_response_file"] + if response_file.exists(): + with open(response_file) as rf: + outputs = dict(outputs) # Copy to avoid modifying original + outputs["response"] = rf.read() + del outputs["expected_response_file"] + records.append( EvalRecord( id=case["id"], inputs=case["inputs"], - outputs=case.get("outputs"), + outputs=outputs, expectations=case.get("expectations"), metadata=case.get("metadata", {}), ) diff --git a/.test/src/skill_test/runners/evaluate.py b/.test/src/skill_test/runners/evaluate.py index 9f42c638..212dd92a 100644 --- a/.test/src/skill_test/runners/evaluate.py +++ b/.test/src/skill_test/runners/evaluate.py @@ -2,8 +2,6 @@ from pathlib import Path from typing import Optional, Dict, Any, List -from concurrent.futures import ProcessPoolExecutor, TimeoutError as FuturesTimeoutError -import multiprocessing import yaml import mlflow from mlflow.genai.scorers import Guidelines, Safety diff --git a/databricks-skills/databricks-synthetic-data-gen/references/1-setup-and-execution.md b/databricks-skills/databricks-synthetic-data-gen/references/1-setup-and-execution.md index f63062f9..e7a6ff18 100644 --- a/databricks-skills/databricks-synthetic-data-gen/references/1-setup-and-execution.md +++ b/databricks-skills/databricks-synthetic-data-gen/references/1-setup-and-execution.md @@ -275,4 +275,4 @@ else: | `ModuleNotFoundError: faker` | Install dependencies per execution mode above | | `DatabricksEnv not found` | Upgrade to databricks-connect >= 16.4 or use job with environments | | `serverless_compute_id` error | Add `serverless_compute_id = auto` to ~/.databrickscfg | -| Classic cluster startup slow | Use serverless instead (instant start) | \ No newline at end of file +| Classic cluster startup slow | Use serverless instead (instant start) | diff --git a/databricks-skills/databricks-synthetic-data-gen/scripts/generate_synthetic_data.py b/databricks-skills/databricks-synthetic-data-gen/scripts/generate_synthetic_data.py index 778d8098..542fa339 100644 --- a/databricks-skills/databricks-synthetic-data-gen/scripts/generate_synthetic_data.py +++ b/databricks-skills/databricks-synthetic-data-gen/scripts/generate_synthetic_data.py @@ -139,7 +139,8 @@ def get_databricks_connect_version(): print("=" * 80) print(f"Missing: {', '.join(missing_deps)}") if on_runtime: - print("\nSolution: Run install using Databricks CLI: Use Databricks CLI to install libraries: databricks libraries install --json '{"cluster_id": "", "libraries": [{"pypi": {"package": "faker"}}, {"pypi": {"package": "holidays"}}]}'") + print('\nSolution: Install libraries via Databricks CLI:') + print(' databricks libraries install --json \'{"cluster_id": "", "libraries": [{"pypi": {"package": "faker"}}, {"pypi": {"package": "holidays"}}]}\'') else: print("\nSolution: Upgrade to databricks-connect >= 16.4 for managed deps") print(" Or create a job with environment settings") @@ -268,8 +269,10 @@ def generate_lognormal_amount(tiers: pd.Series) -> pd.Series: # ============================================================================= print(f"\nGenerating {N_ORDERS:,} orders with referential integrity...") -# Cache customer lookup for FK generation -customer_lookup = customers_df.select("customer_id", "tier").cache() +# Write customer lookup to temp Delta table (no .cache() on serverless!) +customers_tmp_table = f"{CATALOG}.{SCHEMA}._tmp_customers_lookup" +customers_df.select("customer_id", "tier").write.mode("overwrite").saveAsTable(customers_tmp_table) +customer_lookup = spark.table(customers_tmp_table) # Generate orders base orders_df = ( @@ -370,7 +373,7 @@ def generate_lognormal_amount(tiers: pd.Series) -> pd.Series: # ============================================================================= # CLEANUP AND SUMMARY # ============================================================================= -customer_lookup.unpersist() +spark.sql(f"DROP TABLE IF EXISTS {customers_tmp_table}") print("\n" + "=" * 80) print("GENERATION COMPLETE") From d1a8660571fa1452e95e568460cc6f75afecf718 Mon Sep 17 00:00:00 2001 From: dustinvannoy-db Date: Tue, 3 Mar 2026 13:55:39 -0800 Subject: [PATCH 20/24] Simplify serverless job config in test response Remove new_cluster section and use environment_key at task level for cleaner serverless job definition. Co-Authored-By: Claude Opus 4.5 --- .../gen_serverless_job_catalog_json_002.txt | 34 ++++++++----------- 1 file changed, 14 insertions(+), 20 deletions(-) diff --git a/.test/skills/databricks-synthetic-data-gen/expected_responses/gen_serverless_job_catalog_json_002.txt b/.test/skills/databricks-synthetic-data-gen/expected_responses/gen_serverless_job_catalog_json_002.txt index 5275d953..16065794 100644 --- a/.test/skills/databricks-synthetic-data-gen/expected_responses/gen_serverless_job_catalog_json_002.txt +++ b/.test/skills/databricks-synthetic-data-gen/expected_responses/gen_serverless_job_catalog_json_002.txt @@ -311,26 +311,20 @@ To run this as a serverless job, create the job with this configuration: "spark_python_task": { "python_file": "/Workspace/Users/your_email/generate_product_catalog_data.py" }, - "new_cluster": { - "spark_version": "16.4.x-scala2.12", - "num_workers": 0, - "spark_conf": { - "spark.databricks.cluster.profile": "serverless" - } - }, - "environments": [ - { - "environment_key": "default", - "spec": { - "client": "4", - "dependencies": [ - "faker", - "pandas", - "numpy" - ] - } - } - ] + "environment_key": "default" + } + ], + "environments": [ + { + "environment_key": "default", + "spec": { + "client": "4", + "dependencies": [ + "faker", + "pandas", + "numpy" + ] + } } ] } From 9c74e610908f677c68a1a03fcb14b81cca1499ef Mon Sep 17 00:00:00 2001 From: dustinvannoy-db Date: Tue, 3 Mar 2026 14:12:55 -0800 Subject: [PATCH 21/24] Add Python 3.12+ requirement to run instructions DatabricksEnv requires databricks-connect>=16.4 which requires Python 3.12+. Co-Authored-By: Claude Opus 4.5 --- .../expected_responses/grp_20260302_retail_csv_3tables_003.txt | 2 +- .../grp_20260303_manufacturing_delta_streaming_004.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.test/skills/databricks-synthetic-data-gen/expected_responses/grp_20260302_retail_csv_3tables_003.txt b/.test/skills/databricks-synthetic-data-gen/expected_responses/grp_20260302_retail_csv_3tables_003.txt index ea87365e..1eb8738c 100644 --- a/.test/skills/databricks-synthetic-data-gen/expected_responses/grp_20260302_retail_csv_3tables_003.txt +++ b/.test/skills/databricks-synthetic-data-gen/expected_responses/grp_20260302_retail_csv_3tables_003.txt @@ -290,7 +290,7 @@ spark.sql(f"DROP TABLE IF EXISTS {orders_tmp}") spark.sql(f"DROP TABLE IF EXISTS {line_items_tmp}") ``` -Run with: +Run with (requires Python 3.12+): ```bash uv pip install "databricks-connect>=16.4,<17.0" faker numpy pandas uv run python generate_retail_data.py diff --git a/.test/skills/databricks-synthetic-data-gen/expected_responses/grp_20260303_manufacturing_delta_streaming_004.txt b/.test/skills/databricks-synthetic-data-gen/expected_responses/grp_20260303_manufacturing_delta_streaming_004.txt index 06d7f925..644971f9 100644 --- a/.test/skills/databricks-synthetic-data-gen/expected_responses/grp_20260303_manufacturing_delta_streaming_004.txt +++ b/.test/skills/databricks-synthetic-data-gen/expected_responses/grp_20260303_manufacturing_delta_streaming_004.txt @@ -223,7 +223,7 @@ print(f"Tables: {devices_table}, {readings_table}") print(f"Referential integrity: {'PASSED' if orphan_readings == 0 else 'FAILED'}") ``` -Run with: +Run with (requires Python 3.12+): ```bash uv pip install "databricks-connect>=16.4,<17.0" faker numpy pandas uv run python generate_manufacturing_data.py From aa4d8c95c123f0d4eb61e10d56690ff01dc1ca63 Mon Sep 17 00:00:00 2001 From: dustinvannoy-db Date: Tue, 3 Mar 2026 14:27:38 -0800 Subject: [PATCH 22/24] Remove commented out lines from manifest.yaml Co-Authored-By: Claude Opus 4.5 --- .test/skills/databricks-synthetic-data-gen/manifest.yaml | 8 -------- 1 file changed, 8 deletions(-) diff --git a/.test/skills/databricks-synthetic-data-gen/manifest.yaml b/.test/skills/databricks-synthetic-data-gen/manifest.yaml index f80d77d2..f90b1aa8 100644 --- a/.test/skills/databricks-synthetic-data-gen/manifest.yaml +++ b/.test/skills/databricks-synthetic-data-gen/manifest.yaml @@ -18,10 +18,6 @@ scorers: - "Code must use the execution method specified in the prompt" - "Code must save data in the output format specified" - "When generating multiple tables, foreign key columns must use consistent ID formats" - # - "Must use non-uniform distributions (log-normal, exponential, weighted) for realistic data" - # - "Must include configuration section at top of script with CATALOG, SCHEMA, and size variables" - # - "Must create catalog, schema, and volume infrastructure within the Python script" - # - "Child tables must reference valid IDs from parent tables for referential integrity" quality_gates: syntax_valid: 1.0 # 100% - all Python syntax must be valid @@ -35,10 +31,6 @@ trace_expectations: Read: 20 Write: 15 Edit: 15 - # mcp__databricks__run_python_file_on_databricks: 5 - # mcp__databricks__execute_databricks_command: 5 - # mcp__databricks__create_job: 3 - # mcp__databricks__run_job_now: 3 token_budget: max_total: 200000 required_tools: [] From 179856f07e0a9b2d39101d153b573a5410b958b4 Mon Sep 17 00:00:00 2001 From: dustinvannoy-db Date: Tue, 3 Mar 2026 14:59:57 -0800 Subject: [PATCH 23/24] Update databricks-connect version range and fix version detection - Expand version constraint from >=16.4,<17.0 to >=16.4,<17.4 to support databricks-connect 17.x versions - Fix get_databricks_connect_version() to use importlib.metadata.version() instead of non-existent databricks.connect.__version__ attribute Co-Authored-By: Claude Opus 4.5 --- .../grp_20260302_retail_csv_3tables_003.txt | 2 +- .../grp_20260303_manufacturing_delta_streaming_004.txt | 2 +- .../databricks-synthetic-data-gen/SKILL.md | 4 ++-- .../references/1-setup-and-execution.md | 10 +++++----- .../references/6-troubleshooting.md | 4 ++-- .../scripts/generate_synthetic_data.py | 6 +++--- 6 files changed, 14 insertions(+), 14 deletions(-) diff --git a/.test/skills/databricks-synthetic-data-gen/expected_responses/grp_20260302_retail_csv_3tables_003.txt b/.test/skills/databricks-synthetic-data-gen/expected_responses/grp_20260302_retail_csv_3tables_003.txt index 1eb8738c..55be3712 100644 --- a/.test/skills/databricks-synthetic-data-gen/expected_responses/grp_20260302_retail_csv_3tables_003.txt +++ b/.test/skills/databricks-synthetic-data-gen/expected_responses/grp_20260302_retail_csv_3tables_003.txt @@ -292,6 +292,6 @@ spark.sql(f"DROP TABLE IF EXISTS {line_items_tmp}") Run with (requires Python 3.12+): ```bash -uv pip install "databricks-connect>=16.4,<17.0" faker numpy pandas +uv pip install "databricks-connect>=16.4,<17.4" faker numpy pandas uv run python generate_retail_data.py ``` diff --git a/.test/skills/databricks-synthetic-data-gen/expected_responses/grp_20260303_manufacturing_delta_streaming_004.txt b/.test/skills/databricks-synthetic-data-gen/expected_responses/grp_20260303_manufacturing_delta_streaming_004.txt index 644971f9..bd6f8ef4 100644 --- a/.test/skills/databricks-synthetic-data-gen/expected_responses/grp_20260303_manufacturing_delta_streaming_004.txt +++ b/.test/skills/databricks-synthetic-data-gen/expected_responses/grp_20260303_manufacturing_delta_streaming_004.txt @@ -225,7 +225,7 @@ print(f"Referential integrity: {'PASSED' if orphan_readings == 0 else 'FAILED'}" Run with (requires Python 3.12+): ```bash -uv pip install "databricks-connect>=16.4,<17.0" faker numpy pandas +uv pip install "databricks-connect>=16.4,<17.4" faker numpy pandas uv run python generate_manufacturing_data.py ``` diff --git a/databricks-skills/databricks-synthetic-data-gen/SKILL.md b/databricks-skills/databricks-synthetic-data-gen/SKILL.md index 8ec3a469..5bd95e58 100644 --- a/databricks-skills/databricks-synthetic-data-gen/SKILL.md +++ b/databricks-skills/databricks-synthetic-data-gen/SKILL.md @@ -27,11 +27,11 @@ Prefer `uv` for all Python operations. Fall back to `pip` only if `uv` is not av ```bash # Preferred -uv pip install "databricks-connect>=16.4,<17.0" faker numpy pandas holidays +uv pip install "databricks-connect>=16.4,<17.4" faker numpy pandas holidays uv run python generate_data.py # Fallback if uv not available -pip install "databricks-connect>=16.4,<17.0" faker numpy pandas holidays +pip install "databricks-connect>=16.4,<17.4" faker numpy pandas holidays python generate_data.py ``` diff --git a/databricks-skills/databricks-synthetic-data-gen/references/1-setup-and-execution.md b/databricks-skills/databricks-synthetic-data-gen/references/1-setup-and-execution.md index e7a6ff18..3ec36fbc 100644 --- a/databricks-skills/databricks-synthetic-data-gen/references/1-setup-and-execution.md +++ b/databricks-skills/databricks-synthetic-data-gen/references/1-setup-and-execution.md @@ -17,10 +17,10 @@ This guide covers all execution modes for synthetic data generation, organized b **Install locally:** ```bash # Preferred -uv pip install "databricks-connect>=16.4,<17.0" faker numpy pandas holidays +uv pip install "databricks-connect>=16.4,<17.4" faker numpy pandas holidays # Fallback if uv not available -pip install "databricks-connect>=16.4,<17.0" faker numpy pandas holidays +pip install "databricks-connect>=16.4,<17.4" faker numpy pandas holidays ``` **Configure ~/.databrickscfg:** @@ -238,6 +238,7 @@ Use this pattern to auto-detect environment and choose the right session creatio ```python import os +import importlib.metadata def is_databricks_runtime(): """Check if running on Databricks Runtime vs locally.""" @@ -246,11 +247,10 @@ def is_databricks_runtime(): def get_databricks_connect_version(): """Get databricks-connect version as (major, minor) tuple or None.""" try: - import databricks.connect - version_str = databricks.connect.__version__ + version_str = importlib.metadata.version('databricks-connect') parts = version_str.split('.') return (int(parts[0]), int(parts[1])) - except (ImportError, AttributeError, ValueError, IndexError): + except Exception: return None on_runtime = is_databricks_runtime() diff --git a/databricks-skills/databricks-synthetic-data-gen/references/6-troubleshooting.md b/databricks-skills/databricks-synthetic-data-gen/references/6-troubleshooting.md index 080b18fd..420b3500 100644 --- a/databricks-skills/databricks-synthetic-data-gen/references/6-troubleshooting.md +++ b/databricks-skills/databricks-synthetic-data-gen/references/6-troubleshooting.md @@ -33,8 +33,8 @@ spark = DatabricksSession.builder.withEnvironment(env).serverless(True).getOrCre ```bash # Upgrade (prefer uv, fall back to pip) -uv pip install "databricks-connect>=16.4,<17.0" -# or: pip install "databricks-connect>=16.4,<17.0" +uv pip install "databricks-connect>=16.4,<17.4" +# or: pip install "databricks-connect>=16.4,<17.4" # Or use job with environments parameter instead ``` diff --git a/databricks-skills/databricks-synthetic-data-gen/scripts/generate_synthetic_data.py b/databricks-skills/databricks-synthetic-data-gen/scripts/generate_synthetic_data.py index 542fa339..b9f953fa 100644 --- a/databricks-skills/databricks-synthetic-data-gen/scripts/generate_synthetic_data.py +++ b/databricks-skills/databricks-synthetic-data-gen/scripts/generate_synthetic_data.py @@ -71,11 +71,11 @@ def is_databricks_runtime(): def get_databricks_connect_version(): """Get databricks-connect version as (major, minor) tuple or None.""" try: - import databricks.connect - version_str = databricks.connect.__version__ + import importlib.metadata + version_str = importlib.metadata.version('databricks-connect') parts = version_str.split('.') return (int(parts[0]), int(parts[1])) - except (ImportError, AttributeError, ValueError, IndexError): + except Exception: return None # Detect environment From 8265a9b1afe5d4b424d6c3e3325fb6d9914ae8f3 Mon Sep 17 00:00:00 2001 From: dustinvannoy-db Date: Tue, 3 Mar 2026 15:50:38 -0800 Subject: [PATCH 24/24] Reduce guidelines for faster tests with mlflwo --- .../ground_truth.yaml | 20 ------------------- .../manifest.yaml | 1 - 2 files changed, 21 deletions(-) diff --git a/.test/skills/databricks-synthetic-data-gen/ground_truth.yaml b/.test/skills/databricks-synthetic-data-gen/ground_truth.yaml index 38625b62..799c0c19 100644 --- a/.test/skills/databricks-synthetic-data-gen/ground_truth.yaml +++ b/.test/skills/databricks-synthetic-data-gen/ground_truth.yaml @@ -107,12 +107,7 @@ test_cases: guidelines: - Must create serverless job with environments parameter for dependencies - 'Job spec must include client: 4 (not 1)' - - Sales table product_id must only reference valid products (FK integrity) - - Product categories must be weighted (not uniform) - - Price distribution must use log-normal, not uniform - - Script must create schema and volume infrastructure - Must NOT use .cache() or .persist() (serverless incompatible) - - Popular categories should have more sales (weighted sampling) metadata: category: happy_path difficulty: medium @@ -195,18 +190,10 @@ test_cases: description: "Order total computed from line items" guidelines: - "Must use DatabricksSession.builder.serverless(True).getOrCreate()" - - "Must use DatabricksEnv().withDependencies() for managed dependencies" - "Must use Spark + Faker + Pandas UDFs approach" - - "Must maintain referential integrity across all 3 tables" - - "orders.customer_id must reference valid customers" - "line_items.order_id must reference valid orders" - "Membership level must be weighted: Bronze 50%, Silver 30%, Gold 15%, Platinum 5%" - "Higher membership levels must generate more orders per customer" - - "Order total_amount must equal sum of (quantity * unit_price) from line_items" - - "Unit prices should use log-normal distribution for realistic pricing" - - "CSV output must include header row" - - "Must create schema and volume infrastructure within the script" - - "Should use Faker for realistic product names" metadata: category: happy_path difficulty: hard @@ -301,18 +288,11 @@ test_cases: description: "Write mode for streaming support" guidelines: - "Must use DatabricksSession.builder.serverless(True).getOrCreate()" - - "Must use DatabricksEnv().withDependencies() for managed dependencies" - "Must use Spark + Faker + Pandas UDFs approach" - "Must maintain referential integrity between devices and readings" - - "readings.device_id must reference valid devices" - "Must use Delta tables (saveAsTable) not file formats" - - "Must create schema infrastructure within the script" - - "sensor_devices should be designed as slowly changing dimension" - "sensor_readings should support incremental batch processing" - - "Device types should have weighted distribution" - - "Sensor values should have realistic distributions per type" - "Vibration should use log-normal for occasional spikes" - - "Must NOT use .cache() or .persist() (serverless incompatible)" metadata: category: happy_path difficulty: medium diff --git a/.test/skills/databricks-synthetic-data-gen/manifest.yaml b/.test/skills/databricks-synthetic-data-gen/manifest.yaml index f90b1aa8..330f5de7 100644 --- a/.test/skills/databricks-synthetic-data-gen/manifest.yaml +++ b/.test/skills/databricks-synthetic-data-gen/manifest.yaml @@ -17,7 +17,6 @@ scorers: - "Response must generate complete, runnable Python code" - "Code must use the execution method specified in the prompt" - "Code must save data in the output format specified" - - "When generating multiple tables, foreign key columns must use consistent ID formats" quality_gates: syntax_valid: 1.0 # 100% - all Python syntax must be valid