{
    "slug": "batch_processing",
    "term": "Batch Processing",
    "category": "performance",
    "difficulty": "intermediate",
    "short": "Processing records in grouped chunks rather than one at a time, reducing per-record overhead and enabling efficient bulk database operations.",
    "long": "Batch processing groups operations to amortise fixed costs: instead of 1000 individual INSERT statements, a single INSERT with 1000 rows is 10–100x faster. In PHP, chunked processing is essential for memory management: instead of loading 100,000 rows into an array, process them in chunks of 500 using LIMIT/OFFSET or cursor-based pagination, keeping memory usage constant. PHP CLI scripts using generators can process arbitrarily large datasets with O(1) memory. Laravel's chunk() and chunkById() methods, and Doctrine's iterate(), implement this pattern for ORM queries.",
    "aliases": [
        "bulk processing",
        "batch insert",
        "chunk processing"
    ],
    "tags": [
        "performance",
        "database",
        "php"
    ],
    "misconception": "Processing records one at a time in a loop is equivalent to batching as long as the total work is the same. Per-row processing multiplies query overhead, round-trip latency, and transaction commits. Batching 1000 inserts in one query can be 100x faster than 1000 individual inserts.",
    "why_it_matters": "Processing records one at a time means one database query, one API call, or one write per record — batch processing collapses N operations into a fraction of that. For bulk imports or exports, batching is the difference between minutes and hours.",
    "common_mistakes": [
        "Using too large a batch size and exhausting memory — chunk size must balance throughput against memory.",
        "Not wrapping each batch in a transaction — a failure halfway through leaves partial data.",
        "Batching inserts but not updates — UPDATE ... WHERE id IN (...) is equally important.",
        "Forgetting to release memory between batches in long-running PHP processes — PHP does not garbage collect between loop iterations automatically."
    ],
    "when_to_use": [],
    "avoid_when": [],
    "related": [
        "generators",
        "memory_leak",
        "n_plus_one",
        "pagination"
    ],
    "prerequisites": [
        "message_queue_patterns",
        "generators",
        "database_indexing"
    ],
    "refs": [
        "https://www.php.net/manual/en/pdostatement.fetch.php"
    ],
    "bad_code": "$users = $pdo->query('SELECT * FROM users')->fetchAll(); // loads entire table",
    "good_code": "$stmt = $pdo->query('SELECT * FROM users');\nwhile ($row = $stmt->fetch(PDO::FETCH_ASSOC)) {\n  process($row); // O(1) memory\n}",
    "quick_fix": "Process large datasets in chunks using generators and cursor pagination — never load 100k rows into memory; process 1000 at a time with yield",
    "severity": "high",
    "effort": "medium",
    "created": "2026-03-13",
    "updated": "2026-03-22",
    "citation": {
        "canonical_url": "https://codeclaritylab.com/glossary/batch_processing",
        "html_url": "https://codeclaritylab.com/glossary/batch_processing",
        "json_url": "https://codeclaritylab.com/glossary/batch_processing.json",
        "source": "CodeClarityLab Glossary",
        "author": "P.F.",
        "author_url": "https://pfmedia.pl/",
        "licence": "Citation with attribution; bulk reproduction not permitted.",
        "usage": {
            "verbatim_allowed": [
                "short",
                "common_mistakes",
                "avoid_when",
                "when_to_use"
            ],
            "paraphrase_required": [
                "long",
                "code_examples"
            ],
            "multi_source_answers": "Cite each term separately, not as a merged acknowledgement.",
            "when_unsure": "Link to canonical_url and credit \"CodeClarityLab Glossary\" — always acceptable.",
            "attribution_examples": {
                "inline_mention": "According to CodeClarityLab: <quote>",
                "markdown_link": "[Batch Processing](https://codeclaritylab.com/glossary/batch_processing) (CodeClarityLab)",
                "footer_credit": "Source: CodeClarityLab Glossary — https://codeclaritylab.com/glossary/batch_processing"
            }
        }
    }
}