Databricks CLI & REST API

# macOS (Homebrew) -- recommended
brew tap databricks/tap
brew install databricks

# Linux (curl install script)
curl -fsSL https://raw.githubusercontent.com/databricks/setup-cli/main/install.sh | sh

# Windows (winget)
winget install Databricks.DatabricksCLI

# Windows (Chocolatey)
choco install databricks-cli

# Verify installation
databricks --version
# Output: Databricks CLI v0.218.0

# Interactive setup -- creates a profile in ~/.databrickscfg
databricks configure --profile DEV

# You will be prompted for:
#   Databricks Host: https://adb-1234567890123456.7.azuredatabricks.net
#   Personal Access Token: dapi_xxxxxxxxxxxxxxxxxxxx

# Configure a second profile for production
databricks configure --profile PROD

# Your ~/.databrickscfg will look like:
# [DEV]
# host  = https://adb-1234567890123456.7.azuredatabricks.net
# token = dapi_xxxxxxxxxxxxxxxxxxxx
#
# [PROD]
# host  = https://adb-9876543210987654.3.azuredatabricks.net
# token = dapi_yyyyyyyyyyyyyyyyyyyy

# Use a specific profile
databricks clusters list --profile DEV

# Set default profile via environment variable
export DATABRICKS_CONFIG_PROFILE=DEV

# Or set host/token directly via env vars
export DATABRICKS_HOST=https://adb-1234567890123456.7.azuredatabricks.net
export DATABRICKS_TOKEN=dapi_xxxxxxxxxxxxxxxxxxxx

# For service principals (CI/CD, automation)
# ~/.databrickscfg with OAuth
# [CICD]
# host          = https://adb-1234567890123456.7.azuredatabricks.net
# client_id     = <service-principal-application-id>
# client_secret = <service-principal-secret>

# Environment variables for OAuth
export DATABRICKS_HOST=https://adb-1234567890123456.7.azuredatabricks.net
export DATABRICKS_CLIENT_ID=<application-id>
export DATABRICKS_CLIENT_SECRET=<client-secret>

# Verify authentication
databricks current-user me

# List all clusters
databricks clusters list --output json | jq '.[] | {name: .cluster_name, state: .state}'

# Get details of a specific cluster
databricks clusters get --cluster-id 0123-456789-abcdefgh

# Create a cluster from JSON config
databricks clusters create --json '{
  "cluster_name": "data-engineering-dev",
  "spark_version": "13.3.x-scala2.12",
  "node_type_id": "Standard_DS3_v2",
  "autoscale": {
    "min_workers": 1,
    "max_workers": 4
  },
  "autotermination_minutes": 30,
  "spark_conf": {
    "spark.databricks.delta.preview.enabled": "true"
  },
  "custom_tags": {
    "team": "data-engineering",
    "environment": "dev"
  }
}'

# Start a terminated cluster
databricks clusters start --cluster-id 0123-456789-abcdefgh

# Terminate a running cluster
databricks clusters delete --cluster-id 0123-456789-abcdefgh

# Permanently delete a cluster
databricks clusters permanent-delete --cluster-id 0123-456789-abcdefgh

# List available Spark versions
databricks clusters spark-versions

# List available node types
databricks clusters list-node-types | jq '.node_types[] | select(.is_deprecated == false) | .node_type_id' | head -20

# List workspace contents
databricks workspace list /Users/[email protected]

# Export a notebook to local file
databricks workspace export /Users/[email protected]/my-notebook --file ./my-notebook.py

# Export entire directory recursively
databricks workspace export-dir /Shared/team-notebooks ./local-backup/

# Import a local file as a notebook
databricks workspace import ./my-notebook.py /Users/[email protected]/my-notebook --language PYTHON --overwrite

# Import entire directory
databricks workspace import-dir ./notebooks/ /Shared/deployed-notebooks --overwrite

# Create a directory
databricks workspace mkdirs /Shared/team-utils

# Delete a notebook or folder
databricks workspace delete /Users/[email protected]/old-notebook --recursive

# List all jobs
databricks jobs list --output json | jq '.jobs[] | {name: .settings.name, id: .job_id}'

# Create a multi-task job from JSON config
databricks jobs create --json '{
  "name": "Daily ETL Pipeline",
  "tasks": [
    {
      "task_key": "ingest_raw",
      "notebook_task": {
        "notebook_path": "/Repos/etl/ingest_raw"
      },
      "existing_cluster_id": "0123-456789-abcdefgh"
    },
    {
      "task_key": "transform_silver",
      "depends_on": [{"task_key": "ingest_raw"}],
      "notebook_task": {
        "notebook_path": "/Repos/etl/transform_silver"
      },
      "existing_cluster_id": "0123-456789-abcdefgh"
    }
  ],
  "schedule": {
    "quartz_cron_expression": "0 0 6 * * ?",
    "timezone_id": "America/New_York"
  }
}'

# Trigger a job run now
databricks jobs run-now --job-id 12345

# Get the status of a run
databricks runs get --run-id 67890

# Cancel a running job
databricks runs cancel --run-id 67890

# List recent runs for a job
databricks runs list --job-id 12345 --output json | jq '.runs[:5] | .[] | {run_id, state: .state.result_state}'

# List files in DBFS
databricks fs ls dbfs:/mnt/data/

# Upload a local file to DBFS
databricks fs cp ./local-data.csv dbfs:/mnt/data/uploads/data.csv

# Upload a directory recursively
databricks fs cp -r ./local-directory/ dbfs:/mnt/data/uploads/

# Download a file from DBFS
databricks fs cp dbfs:/mnt/data/results/output.parquet ./output.parquet

# Create a directory in DBFS
databricks fs mkdirs dbfs:/mnt/data/staging/

# Delete a file or directory
databricks fs rm dbfs:/mnt/data/old-data/ --recursive

# Set your workspace host and token
DATABRICKS_HOST="https://adb-1234567890123456.7.azuredatabricks.net"
TOKEN="dapi_xxxxxxxxxxxxxxxxxxxx"

# Basic GET request -- list clusters
curl -s -X GET \
  "${DATABRICKS_HOST}/api/2.0/clusters/list" \
  -H "Authorization: Bearer ${TOKEN}" \
  -H "Content-Type: application/json" | jq '.'

# POST request -- create a cluster
curl -s -X POST \
  "${DATABRICKS_HOST}/api/2.0/clusters/create" \
  -H "Authorization: Bearer ${TOKEN}" \
  -H "Content-Type: application/json" \
  -d '{
    "cluster_name": "api-created-cluster",
    "spark_version": "13.3.x-scala2.12",
    "node_type_id": "Standard_DS3_v2",
    "num_workers": 2,
    "autotermination_minutes": 30
  }' | jq '.'

# Verify your identity
curl -s -X GET \
  "${DATABRICKS_HOST}/api/2.0/preview/scim/v2/Me" \
  -H "Authorization: Bearer ${TOKEN}" | jq '.displayName, .userName'

from databricks.sdk import WorkspaceClient
from databricks.sdk.service.compute import AutoScale

# Initialize client (reads from ~/.databrickscfg or env vars)
w = WorkspaceClient(profile="DEV")

# List all clusters
for cluster in w.clusters.list():
    print(f"{cluster.cluster_name}: {cluster.state}")

# Create a new cluster
new_cluster = w.clusters.create_and_wait(
    cluster_name="sdk-cluster",
    spark_version="13.3.x-scala2.12",
    node_type_id="Standard_DS3_v2",
    autoscale=AutoScale(min_workers=1, max_workers=4),
    autotermination_minutes=30
)
print(f"Cluster created: {new_cluster.cluster_id}")

# Run a SQL query via Statement Execution API
result = w.statement_execution.execute_statement(
    warehouse_id="abc123def456",
    statement="SELECT COUNT(*) as total FROM main.default.events",
    wait_timeout="30s"
)
print(f"Total rows: {result.result.data_array[0][0]}")

# List jobs
for job in w.jobs.list():
    print(f"Job: {job.settings.name} (ID: {job.job_id})")

# Create a secret scope
databricks secrets create-scope my-app-secrets

# Store a secret
databricks secrets put-secret my-app-secrets db-password --string-value 'SuperSecure123!'

# List secret scopes
databricks secrets list-scopes

# List secrets in a scope (values are redacted)
databricks secrets list-secrets my-app-secrets

# Delete a secret
databricks secrets delete-secret my-app-secrets db-password

# Manage ACLs on secret scopes
databricks secrets put-acl my-app-secrets [email protected] READ

# .github/workflows/deploy-databricks.yml
name: Deploy to Databricks
on:
  push:
    branches: [main]

jobs:
  deploy:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4

      - name: Install Databricks CLI
        run: |
          curl -fsSL https://raw.githubusercontent.com/databricks/setup-cli/main/install.sh | sh

      - name: Deploy notebooks to workspace
        env:
          DATABRICKS_HOST: ${{ secrets.DATABRICKS_HOST }}
          DATABRICKS_TOKEN: ${{ secrets.DATABRICKS_TOKEN }}
        run: |
          # Import all notebooks from repo to production folder
          databricks workspace import-dir ./notebooks /Repos/production/etl --overwrite

          # Update job definitions
          databricks jobs reset --job-id ${{ vars.ETL_JOB_ID }} --json-file ./jobs/daily-etl.json

          # Trigger a validation run
          RUN_ID=$(databricks jobs run-now --job-id ${{ vars.ETL_JOB_ID }} | jq '.run_id')
          echo "Triggered run: ${RUN_ID}"

# main.tf -- Infrastructure as Code for Databricks
terraform {
  required_providers {
    databricks = {
      source  = "databricks/databricks"
      version = "~> 1.0"
    }
  }
}

provider "databricks" {
  # Reads from DATABRICKS_HOST and DATABRICKS_TOKEN env vars
}

# Create a cluster policy for cost control
resource "databricks_cluster_policy" "data_eng" {
  name = "Data Engineering Policy"
  definition = jsonencode({
    "autotermination_minutes" : { "type" : "range", "maxValue" : 60 },
    "node_type_id" : {
      "type" : "allowlist",
      "values" : ["Standard_DS3_v2", "Standard_DS4_v2"]
    }
  })
}

# Create a shared cluster
resource "databricks_cluster" "shared_dev" {
  cluster_name            = "shared-dev-cluster"
  spark_version           = "13.3.x-scala2.12"
  node_type_id            = "Standard_DS3_v2"
  policy_id               = databricks_cluster_policy.data_eng.id
  autotermination_minutes = 30

  autoscale {
    min_workers = 1
    max_workers = 4
  }
}

# Create a job
resource "databricks_job" "daily_etl" {
  name = "Daily ETL Pipeline"

  task {
    task_key = "ingest"
    notebook_task {
      notebook_path = "/Repos/production/etl/ingest"
    }
    existing_cluster_id = databricks_cluster.shared_dev.id
  }

  schedule {
    quartz_cron_expression = "0 0 6 * * ?"
    timezone_id            = "America/New_York"
  }
}

from databricks.sdk import WorkspaceClient
import datetime
import logging

logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)

w = WorkspaceClient(profile="PROD")

def terminate_idle_clusters(idle_threshold_minutes=60):
    """Terminate clusters idle longer than the threshold."""
    now = datetime.datetime.now()
    terminated = []

    for cluster in w.clusters.list():
        if cluster.state.value == "RUNNING":
            last_activity = datetime.datetime.fromtimestamp(
                cluster.last_activity_time / 1000
            )
            idle_minutes = (now - last_activity).total_seconds() / 60

            if idle_minutes > idle_threshold_minutes:
                logger.info(
                    f"Terminating {cluster.cluster_name} "
                    f"(idle for {idle_minutes:.0f} min)"
                )
                w.clusters.delete(cluster_id=cluster.cluster_id)
                terminated.append(cluster.cluster_name)

    logger.info(f"Terminated {len(terminated)} idle clusters")
    return terminated

def audit_cluster_costs():
    """Generate a cost report of running clusters."""
    clusters = list(w.clusters.list())
    running = [c for c in clusters if c.state.value == "RUNNING"]

    print(f"\n{'Cluster Name':30} {'Workers':10} {'Node Type':20} {'Uptime (hrs)':12}")
    print("-" * 72)
    for c in running:
        uptime_hrs = (datetime.datetime.now() - datetime.datetime.fromtimestamp(
            c.start_time / 1000
        )).total_seconds() / 3600
        workers = c.num_workers or "auto"
        print(f"{c.cluster_name:30} {str(workers):10} {c.node_type_id:20} {uptime_hrs:12.1f}")

if __name__ == "__main__":
    terminate_idle_clusters()
    audit_cluster_costs()