Monitoring, Logging & Observability

# Instrumenting a Flask app with Prometheus metrics
from prometheus_client import (
    Counter, Histogram, Gauge, generate_latest
)
from flask import Flask, request, Response
import time

app = Flask(__name__)

# Define metrics (the Four Golden Signals)

# 1. Traffic - request counter
REQUEST_COUNT = Counter(
    "http_requests_total",
    "Total HTTP requests",
    ["method", "endpoint", "status"]
)

# 2. Latency - response time histogram
REQUEST_LATENCY = Histogram(
    "http_request_duration_seconds",
    "HTTP request duration in seconds",
    ["method", "endpoint"],
    buckets=[.005, .01, .025, .05, .1,
             .25, .5, 1, 2.5, 5, 10]
)

# 3. Errors - tracked via REQUEST_COUNT status label

# 4. Saturation - active requests gauge
IN_PROGRESS = Gauge(
    "http_requests_in_progress",
    "Number of HTTP requests in progress"
)

@app.before_request
def before_request():
    request.start_time = time.time()
    IN_PROGRESS.inc()

@app.after_request
def after_request(response):
    latency = time.time() - request.start_time
    REQUEST_COUNT.labels(
        method=request.method,
        endpoint=request.path,
        status=response.status_code
    ).inc()
    REQUEST_LATENCY.labels(
        method=request.method,
        endpoint=request.path
    ).observe(latency)
    IN_PROGRESS.dec()
    return response

@app.route("/metrics")
def metrics():
    return Response(
        generate_latest(),
        mimetype="text/plain"
    )

# Structured logging with Python's structlog
import structlog
import logging
import uuid

# Configure structured logging
structlog.configure(
    processors=[
        structlog.processors.TimeStamper(fmt="iso"),
        structlog.processors.add_log_level,
        structlog.processors.StackInfoRenderer(),
        structlog.processors.JSONRenderer()
    ],
    wrapper_class=structlog.BoundLogger,
    logger_factory=structlog.PrintLoggerFactory(),
)

def create_request_logger(request):
    """Create a logger bound with request context."""
    return structlog.get_logger().bind(
        request_id=str(uuid.uuid4()),
        trace_id=request.headers.get("X-Trace-ID", "unknown"),
        user_id=request.user_id,
        service="payment-service",
        endpoint=request.path,
    )

# Usage in a request handler
def process_payment(request):
    log = create_request_logger(request)

    log.info("payment_started",
             amount=request.amount,
             currency=request.currency)

    try:
        result = charge_card(request.card_token, request.amount)
        log.info("payment_succeeded",
                 transaction_id=result.id,
                 amount=request.amount,
                 latency_ms=result.latency_ms)
        return result
    except PaymentDeclinedError as e:
        log.warning("payment_declined",
                    reason=e.reason,
                    card_last4=request.card_last4)
        raise
    except Exception as e:
        log.error("payment_error",
                  error_type=type(e).__name__,
                  error_msg=str(e),
                  exc_info=True)
        raise

# Output (one JSON line per log entry):
# {"timestamp":"2024-01-15T10:30:00Z","level":"info",
#  "event":"payment_started","request_id":"abc-123",
#  "trace_id":"trace-456","user_id":42,
#  "service":"payment-service","amount":29.99}