Distributed Consensus & Coordination

# 1. Bank transfer: CP (Consistency)
#    - Showing wrong balance = double-spending risk
#    - Better to reject a transfer than allow overdraft
#    - Use: PostgreSQL with synchronous replication

# 2. Social media likes: AP (Availability)
#    - Showing 999 instead of 1000 is fine
#    - Users expect instant response when clicking "like"
#    - Use: Cassandra with eventual consistency

# 3. Concert tickets: CP (Consistency)
#    - Overselling = angry customers, refund costs
#    - Better to show "temporarily unavailable" than oversell
#    - Use: Redis with distributed lock per seat

# 1. Leader crashes:
# - Followers detect missing heartbeat after election timeout
# - Random timeout: 150-300ms (prevents simultaneous elections)
# - New leader elected in ~300-500ms typically
# - Client retries to new leader after ~1 second total

# 2. Two nodes crash (leader + one follower):
# - 3 remaining nodes = quorum (3/5 = majority)
# - YES, cluster can still elect a leader and accept writes
# - If 3 crash: only 2 left, no quorum, cluster is read-only

# 3. Network partition 2-3:
# - Side with 3 nodes CAN elect a leader (has quorum)
# - Side with 2 nodes CANNOT (no quorum)
# - Old leader on the 2-node side steps down
#   (can't get heartbeat acks from majority)
# - When partition heals, 2-node side rejoins and
#   syncs its log with the new leader

class FencedLockService:
    def __init__(self, etcd_client):
        self.client = etcd_client
        self.token_counter = 0

    def acquire(self, resource, ttl=30):
        """Acquire lock with fencing token."""
        self.token_counter += 1
        token = self.token_counter

        # Atomic compare-and-swap in etcd
        success = self.client.put_if_absent(
            key=f"/locks/{resource}",
            value=f"{token}",
            lease=self.client.lease(ttl=ttl)
        )
        if success:
            return token
        raise LockNotAcquired(resource)

    def release(self, resource, token):
        """Release only if we still hold the lock."""
        current = self.client.get(f"/locks/{resource}")
        if current and int(current) == token:
            self.client.delete(f"/locks/{resource}")

# Payment processor uses fencing token
def process_payment(lock_service, db, order_id):
    token = lock_service.acquire(f"payment:{order_id}")
    try:
        # DB rejects if token < last seen token for this order
        db.execute(
            "UPDATE orders SET status='paid' WHERE id=%s AND fence_token < %s",
            (order_id, token)
        )
    finally:
        lock_service.release(f"payment:{order_id}", token)