AI-Generated Test Data at Scale

1Template:
2{
3  "user": {
4    "name": "[FIRST_NAME] [LAST_NAME]",
5    "email": "[FIRST_NAME].[LAST_NAME]@example.com",
6    "age": [18-65],
7    "account_created": "[DATE_PAST_YEAR]"
8  }
9}
10
11AI generates:
12{
13  "user": {
14    "name": "Alice Johnson",
15    "email": "alice.johnson@example.com",
16    "age": 34,
17    "account_created": "2025-11-15"
18  }
19}

1Constraints:
2- Order total must be between $10–$500
3- Discount can't exceed 50% of total
4- Shipping must be realistic (US state + valid address)
5- Order date must be before shipping date
6
7AI generates orders that satisfy all constraints + look realistic

1"Generate 50 API requests that might break our payment endpoint"
2
3AI generates:
41. Empty payload: {}
52. Negative amount: { "amount": -100 }
63. Huge amount: { "amount": 999999999 }
74. Special characters: { "name": "<script>alert('xss')</script>" }
85. Unicode: { "name": "José María" }
9... (45 more edge cases)

1import anthropic
2import json
3
4def generate_user_profiles(count: int, demographics: dict):
5    """Generate synthetic user profiles for testing."""
6    
7    client = anthropic.Anthropic()
8    
9    prompt = f"""Generate {count} realistic user profiles for a SaaS app.
10    
11Demographics:
12- Age range: {demographics['age_range']}
13- Location: {demographics['locations']}
14- Account types: {demographics['account_types']}
15- Industry: {demographics.get('industry', 'mixed')}
16
17Requirements:
181. Names should be realistic (first + last)
192. Emails should match naming pattern
203. Account creation dates should span past 2 years
214. Mix of free/premium/enterprise accounts
225. Include mix of active/inactive users
236. Some profiles should have incomplete data (optional fields missing)
24
25Return as JSON array:
26[
27  {{
28    "id": 1,
29    "name": "...",
30    "email": "...",
31    "created_at": "...",
32    "account_type": "...",
33    "region": "...",
34    "last_login": "...",
35    "is_active": true
36  }},
37  ...
38]
39
40Generate exactly {count} profiles. Return ONLY valid JSON."""
41    
42    response = client.messages.create(
43        model="claude-opus-4-7",
44        max_tokens=8000,
45        messages=[
46            {
47                "role": "user",
48                "content": prompt
49            }
50        ]
51    )
52    
53    # Parse and validate
54    data = json.loads(response.content[0].text)
55    return data
56
57# Usage
58profiles = generate_user_profiles(
59    count=100,
60    demographics={
61        "age_range": "18-65",
62        "locations": ["US", "UK", "Canada"],
63        "account_types": ["free", "premium", "enterprise"]
64    }
65)
66
67print(f"Generated {len(profiles)} profiles")
68# Output: Generated 100 profiles

1def generate_payment_api_test_cases():
2    """Generate test payloads for payment API including edge cases."""
3    
4    client = anthropic.Anthropic()
5    
6    prompt = """Generate 30 test cases for a payment processing API.
7
8Include:
91. Happy path (valid payment): 5 cases
102. Input validation failures: 8 cases
11   - Missing fields
12   - Invalid formats (bad email, invalid card)
13   - Type errors (string where number expected)
143. Business rule violations: 6 cases
15   - Amount exceeds limit
16   - Duplicate transaction (same amount/card within 1 min)
174. Edge cases: 8 cases
18   - Unicode characters in name
19   - Maximum/minimum amounts
20   - Expired cards
215. Injection attacks: 3 cases
22   - SQL injection attempts
23   - JSON injection
24
25For each case, provide:
26{
27  "name": "...",
28  "payload": { ... },
29  "expected_result": "...",
30  "http_status": 200 or 400 or 422
31}
32
33Return as JSON array only."""
34    
35    response = client.messages.create(
36        model="claude-opus-4-7",
37        max_tokens=5000,
38        messages=[{"role": "user", "content": prompt}]
39    )
40    
41    return json.loads(response.content[0].text)
42
43# Usage
44test_cases = generate_payment_api_test_cases()
45for case in test_cases:
46    # Run API with case['payload']
47    # Assert response status = case['http_status']
48    pass

1import locust
2import anthropic
3
4class LoadTestWithAIData:
5    def __init__(self):
6        self.ai_client = anthropic.Anthropic()
7        self.test_data = self._generate_load_data()
8    
9    def _generate_load_data(self):
10        """Generate 1000 realistic transactions for load test."""
11        prompt = """Generate 1000 e-commerce transactions suitable for load testing.
12
13Each transaction should include:
14{
15  "user_id": (1-10000),
16  "product_ids": [list of 1-5 product IDs],
17  "quantities": [matching lengths to products],
18  "total_price": (10-500),
19  "payment_method": "card" or "paypal",
20  "country": "US/UK/CA",
21  "timestamp": "2026-05-31T..."
22}
23
24Realistic distribution:
25- 70% small orders ($10-$50)
26- 20% medium orders ($50-$150)
27- 10% large orders ($150-$500)
28- Timestamps spread across past 30 days
29- Repeat customers (same user_id in ~30% of transactions)
30
31Return as JSON array. ONLY JSON, no explanation."""
32        
33        response = self.ai_client.messages.create(
34            model="claude-opus-4-7",
35            max_tokens=10000,
36            messages=[{"role": "user", "content": prompt}]
37        )
38        
39        return json.loads(response.content[0].text)
40    
41    def run_load_test(self):
42        """Run load test with AI-generated data."""
43        for transaction in self.test_data:
44            # Simulate POST /checkout
45            response = self._api_request(
46                method="POST",
47                endpoint="/checkout",
48                payload=transaction
49            )
50            assert response.status_code == 200
51            # Record response time for metrics
52
53# Usage
54load_test = LoadTestWithAIData()
55load_test.run_load_test()

1Old: 10 test users × 1 scenario = 10 test paths
2New: 1000 test users × 1 scenario = 1000 test paths (more coverage)
3New: 100 test users × 10 scenarios = 1000 test paths (more scenarios)

1class TestDataFactory:
2    def __init__(self):
3        self.ai = anthropic.Anthropic()
4    
5    def users(self, count, **kwargs):
6        """Generate user profiles."""
7        return self._generate("user", count, kwargs)
8    
9    def orders(self, count, user_ids, **kwargs):
10        """Generate orders for given users."""
11        return self._generate("order", count, {"user_ids": user_ids, **kwargs})
12    
13    def _generate(self, entity_type, count, params):
14        """Unified generation method."""
15        # Use templated prompts + caching
16        return self.ai.generate(entity_type, count, params)
17
18# Usage in tests
19factory = TestDataFactory()
20users = factory.users(100, age_range="18-35")
21orders = factory.orders(500, user_ids=[u['id'] for u in users])

1users = generate_users(100)
2orders = generate_orders(100, for_users=users)
3run_load_test(users, orders)  # 5 minutes total

1payloads = generate_security_test_cases(50, attack_types=[
2    "sql_injection",
3    "xss",
4    "xxe",
5    "path_traversal"
6])
7# Payload examples:
8# "'; DROP TABLE users; --"
9# "<img src=x onerror=alert('xss')>"
10# "../../../../etc/passwd"

1tenants = generate_tenants(1000, distributions={
2    "company_size": {"small": 0.6, "medium": 0.3, "large": 0.1},
3    "industry": ["tech", "finance", "retail", "healthcare"],
4    "region": ["us", "eu", "asia"]
5})
6
7for tenant in tenants:
8    # Test that tenant A can't see tenant B's data
9    assert isolation_check(tenant)

1   Generate 10 realistic user profiles for an e-commerce platform.
2   
3   Each profile should have:
4   - Name (first + last)
5   - Email
6   - Country
7   - Account creation date (past 2 years)
8   - Account status (active/inactive)
9   - Purchase history (0-50 orders)
10   
11   Return as JSON array. Include a mix of US, UK, and Canada locations.
12   Include both active and inactive accounts.

1   Generate 15 test payloads for a user registration API that accepts:
2   {
3     "email": "...",
4     "password": "...",
5     "first_name": "...",
6     "last_name": "..."
7   }
8   
9   Include:
10   - 3 valid registrations
11   - 5 invalid cases (missing fields, bad email, weak password)
12   - 4 edge cases (special characters, unicode, max length names)
13   - 3 security cases (injection, XSS)
14   
15   For each, include expected HTTP status and error message.

1DON'T:
2- Generate millions of records without validation
3  → Cascades garbage data through your pipeline
4  
5- Use AI data for production-like penetration testing
6  → May miss real attack surfaces
7  
8- Forget to anonymize generated data before sharing
9  → Even synthetic data can reveal patterns
10
11DO:
12- Validate generated data against business rules
13- Start with small datasets, expand after validation
14- Use AI for volume; manual QA for complexity
15- Keep generation prompts versioned (reproduce data)

1def validate_generated_data(records):
2    """Validate generated test data."""
3    
4    for record in records:
5        # Type checks
6        assert isinstance(record['name'], str)
7        assert isinstance(record['created_at'], str)
8        
9        # Format checks
10        assert '@' in record['email']
11        assert len(record['password']) >= 8
12        
13        # Business rule checks
14        assert record['age'] >= 18
15        assert record['account_type'] in ['free', 'premium', 'enterprise']
16        
17        # Realistic checks
18        assert datetime.fromisoformat(record['created_at']) < datetime.now()
19        assert ' ' in record['name']  # Has first + last name
20    
21    print(f"✓ All {len(records)} records passed validation")