{
    "slug": "log_aggregation",
    "term": "Log Aggregation (ELK/Loki)",
    "category": "observability",
    "difficulty": "intermediate",
    "short": "Log aggregation collects logs from all services into a central searchable store — ELK (Elasticsearch+Logstash+Kibana) for full-text search, Loki (Prometheus-style) for cost-efficient label-based search.",
    "long": "ELK stack: Logstash/Filebeat (collect) → Elasticsearch (store+index) → Kibana (search+dashboard). Full-text indexed — any field searchable, high storage cost. Loki: Grafana's log store — only indexes labels (not content), compressed content. Much cheaper than ELK. PromQL-like LogQL. Best with structured logs (JSON). Alternatives: Datadog Logs, Splunk (expensive but powerful), CloudWatch Logs. Key capabilities: full-text search, aggregation (error count by service), dashboards, alerts on log patterns. Ship logs: Filebeat/Fluentd agent → aggregator. In PHP: Monolog with socket/HTTP handler → Logstash/Loki.",
    "aliases": [],
    "tags": [
        "observability",
        "logging",
        "elk",
        "loki",
        "aggregation"
    ],
    "misconception": "More log storage is always better — logs stored but never searched are expensive waste. Store what you query; sample debug logs heavily.",
    "why_it_matters": "Centralised log aggregation transforms debugging from SSH-to-server-and-grep to sub-second search across all services — essential for microservices and autoscaling environments.",
    "common_mistakes": [
        "Not shipping logs to a central store — grep-across-servers debugging.",
        "Storing all debug logs at full rate — expensive and noisy.",
        "Not using structured logs — full-text search works, but JSON fields are essential for aggregation."
    ],
    "when_to_use": [],
    "avoid_when": [],
    "related": [
        "structured_logging",
        "correlation_id",
        "prometheus_concepts",
        "grafana_concepts"
    ],
    "prerequisites": [
        "structured_logging",
        "correlation_id"
    ],
    "refs": [
        "https://grafana.com/docs/loki/latest/"
    ],
    "bad_code": "# No aggregation — SSH to each server:\nssh server1 grep 'ERROR' /var/log/app.log\nssh server2 grep 'ERROR' /var/log/app.log\n# 20 servers = 20 SSH sessions",
    "good_code": "# Loki config:\n- job_name: php_app\n  static_configs:\n    - targets: [localhost]\n      labels:\n        job: php-app\n        env: production\n  pipeline_stages:\n    - json:\n        expressions:\n          level: level\n          correlation_id: correlation_id\n    - labels:\n        level:\n        correlation_id:\n\n# LogQL query:\n{job='php-app'} |= 'ERROR' | json | correlation_id='abc-123'",
    "quick_fix": "Choose Loki (cheap, Prometheus-compatible) or ELK (powerful full-text). Ship with Filebeat/Promtail. Use structured JSON logs for label extraction. Retain 30 days, archive 90.",
    "severity": "high",
    "effort": "high",
    "created": "2026-03-23",
    "updated": "2026-03-23",
    "citation": {
        "canonical_url": "https://codeclaritylab.com/glossary/log_aggregation",
        "html_url": "https://codeclaritylab.com/glossary/log_aggregation",
        "json_url": "https://codeclaritylab.com/glossary/log_aggregation.json",
        "source": "CodeClarityLab Glossary",
        "author": "P.F.",
        "author_url": "https://pfmedia.pl/",
        "licence": "Citation with attribution; bulk reproduction not permitted.",
        "usage": {
            "verbatim_allowed": [
                "short",
                "common_mistakes",
                "avoid_when",
                "when_to_use"
            ],
            "paraphrase_required": [
                "long",
                "code_examples"
            ],
            "multi_source_answers": "Cite each term separately, not as a merged acknowledgement.",
            "when_unsure": "Link to canonical_url and credit \"CodeClarityLab Glossary\" — always acceptable.",
            "attribution_examples": {
                "inline_mention": "According to CodeClarityLab: <quote>",
                "markdown_link": "[Log Aggregation (ELK/Loki)](https://codeclaritylab.com/glossary/log_aggregation) (CodeClarityLab)",
                "footer_credit": "Source: CodeClarityLab Glossary — https://codeclaritylab.com/glossary/log_aggregation"
            }
        }
    }
}