Databricks Workspace & Architecture

# Your data stays in YOUR cloud account
# Databricks control plane only sends instructions

# Example: What happens when you run a query
# 1. You write SQL in the notebook UI (control plane)
# 2. Control plane sends query to your cluster (data plane)
# 3. Cluster reads data from YOUR S3/ADLS/GCS bucket
# 4. Results are computed on YOUR VMs
# 5. Only the result set is sent back to the UI

# This architecture means:
# - Data residency compliance (GDPR, HIPAA)
# - Network isolation via VPC/VNet
# - You control encryption keys
# - Full audit trail in your cloud

import requests
import json

# Configure your workspace connection
DATABRICKS_HOST = "https://adb-1234567890.12.azuredatabricks.net"
TOKEN = "dapi_your_personal_access_token"

headers = {
    "Authorization": f"Bearer {TOKEN}",
    "Content-Type": "application/json"
}

# List workspace contents
def list_workspace(path="/"):
    response = requests.get(
        f"{DATABRICKS_HOST}/api/2.0/workspace/list",
        headers=headers,
        params={"path": path}
    )
    return response.json()

# List all items in /Users directory
result = list_workspace("/Users")
for obj in result.get("objects", []):
    print(f"{obj['object_type']}: {obj['path']}")

# Export a notebook
def export_notebook(path, format="SOURCE"):
    response = requests.get(
        f"{DATABRICKS_HOST}/api/2.0/workspace/export",
        headers=headers,
        params={"path": path, "format": format}
    )
    return response.json()

# Import a notebook
def import_notebook(path, content, language="PYTHON"):
    payload = {
        "path": path,
        "language": language,
        "content": content,  # base64 encoded
        "overwrite": True
    }
    response = requests.post(
        f"{DATABRICKS_HOST}/api/2.0/workspace/import",
        headers=headers,
        json=payload
    )
    return response.json()

# AWS S3 Mount (using instance profile)
dbutils.fs.mount(
    source="s3a://my-data-bucket",
    mount_point="/mnt/data"
)

# Azure ADLS Gen2 Mount (using service principal)
configs = {
    "fs.azure.account.auth.type": "OAuth",
    "fs.azure.account.oauth.provider.type":
        "org.apache.hadoop.fs.azurebfs.oauth2.ClientCredsTokenProvider",
    "fs.azure.account.oauth2.client.id": "<app-id>",
    "fs.azure.account.oauth2.client.secret": dbutils.secrets.get(
        scope="keyvault", key="sp-secret"
    ),
    "fs.azure.account.oauth2.client.endpoint":
        "https://login.microsoftonline.com/<tenant-id>/oauth2/token"
}

dbutils.fs.mount(
    source="abfss://[email protected]/",
    mount_point="/mnt/data",
    extra_configs=configs
)

# List mounted data
display(dbutils.fs.ls("/mnt/data"))

# Read data using the mount
df = spark.read.format("delta").load("/mnt/data/bronze/events")
df.show(5)

# Cell 1: Verify your Spark session
print(f"Spark version: {spark.version}")
print(f"Cluster: {spark.conf.get('spark.databricks.clusterUsageTags.clusterName')}")

# Cell 2: Explore available databases
display(spark.sql("SHOW DATABASES"))

# Cell 3: Create a simple DataFrame
data = [
    (1, "Alice", "Engineering", 95000),
    (2, "Bob", "Data Science", 105000),
    (3, "Carol", "Engineering", 98000),
    (4, "Dave", "Analytics", 87000),
]
columns = ["id", "name", "department", "salary"]

df = spark.createDataFrame(data, columns)
display(df)

# Cell 4: Use dbutils to explore the file system
display(dbutils.fs.ls("/"))

# Cell 5: Check available secrets scopes
print(dbutils.secrets.listScopes())