Security, IAM & Network Controls

from databricks.sdk import WorkspaceClient
from databricks.sdk.service.iam import ServicePrincipal

# Authenticate using service principal (OAuth M2M)
w = WorkspaceClient(
    host="https://adb-1234567890.12.azuredatabricks.net",
    client_id="your-service-principal-client-id",
    client_secret="your-service-principal-secret"
)

# List clusters using the service principal identity
for cluster in w.clusters.list():
    print(f"{cluster.cluster_name}: {cluster.state}")

# Create a service principal via the Account API
from databricks.sdk import AccountClient

account = AccountClient(
    host="https://accounts.azuredatabricks.net",
    account_id="your-account-id",
    client_id="admin-sp-client-id",
    client_secret="admin-sp-secret"
)

# Create new service principal
sp = account.service_principals.create(
    display_name="etl-pipeline-sp",
    active=True
)
print(f"Created SP: {sp.application_id}")

# Grant workspace access to the service principal
account.workspace_assignment.update(
    workspace_id=1234567890,
    principal_id=sp.id,
    permissions=["USER"]
)

from databricks.sdk import WorkspaceClient
from databricks.sdk.service.settings import CreateIpAccessList

w = WorkspaceClient()

# Create an IP access list (allow list)
ip_list = w.ip_access_lists.create(
    label="Corporate VPN",
    list_type="ALLOW",
    ip_addresses=[
        "10.0.0.0/8",         # Internal network
        "203.0.113.0/24",     # Office IP range
        "198.51.100.50/32",   # VPN gateway
    ]
)

# Enable IP access list enforcement
w.workspace_conf.set_status({
    "enableIpAccessLists": "true"
})

# List all configured IP access lists
for acl in w.ip_access_lists.list():
    print(f"{acl.label}: {acl.ip_addresses}")

from databricks.sdk import WorkspaceClient

w = WorkspaceClient()

# Create a secret scope
w.secrets.create_scope(scope="production-secrets")

# Store a secret
w.secrets.put_secret(
    scope="production-secrets",
    key="database-password",
    string_value="super-secret-password-123"
)

# In a notebook, retrieve the secret
# Secrets are REDACTED in notebook output
password = dbutils.secrets.get(
    scope="production-secrets",
    key="database-password"
)

# Use the secret in a JDBC connection
df = spark.read.format("jdbc").options(
    url="jdbc:postgresql://db-host:5432/mydb",
    dbtable="public.users",
    user="etl_user",
    password=password
).load()

# Grant access to a group
w.secrets.put_acl(
    scope="production-secrets",
    principal="data-engineers",
    permission="READ"
)

# List secrets (keys only, values are never exposed)
for secret in w.secrets.list_secrets(scope="production-secrets"):
    print(f"Key: {secret.key}, Last Updated: {secret.last_updated_timestamp}")

-- Audit logs delivered to your cloud storage (S3/ADLS/GCS)
-- Create an external table over the audit log files

CREATE TABLE IF NOT EXISTS audit.logs
USING JSON
LOCATION 's3://your-audit-bucket/databricks/audit-logs/';

-- Find all failed login attempts in the last 24 hours
SELECT
    timestamp,
    userIdentity.email AS user_email,
    sourceIPAddress,
    requestParams.user AS target_user,
    response.statusCode
FROM audit.logs
WHERE actionName = 'login'
    AND response.statusCode != 200
    AND timestamp >= current_timestamp() - INTERVAL 24 HOURS
ORDER BY timestamp DESC;

-- Track who accessed sensitive tables
SELECT
    timestamp,
    userIdentity.email,
    actionName,
    requestParams.full_name_arg AS table_name
FROM audit.logs
WHERE serviceName = 'unityCatalog'
    AND actionName IN ('getTable', 'readVolume')
    AND requestParams.full_name_arg LIKE '%pii%'
ORDER BY timestamp DESC;