Multiprocessing

Why Multiprocessing?

Multiprocessing spawns separate Python processes, each with its own interpreter and memory space. This bypasses the GIL and gives true parallelism on multiple cores — ideal for CPU-bound work.

Process and Pool

from multiprocessing import Process

def crunch(n):
    total = sum(i * i for i in range(n))
    print(f"{n}: {total}")

if __name__ == "__main__":
    procs = [Process(target=crunch, args=(10_000_000,)) for _ in range(4)]
    for p in procs: p.start()
    for p in procs: p.join()

ProcessPoolExecutor

from concurrent.futures import ProcessPoolExecutor

def cube(n): return n ** 3

if __name__ == "__main__":
    with ProcessPoolExecutor(max_workers=4) as ex:
        results = list(ex.map(cube, range(1_000_000), chunksize=10_000))

Inter-Process Communication

Queue and Pipe

from multiprocessing import Process, Queue

def worker(q):
    while True:
        item = q.get()
        if item is None: break
        process(item)

if __name__ == "__main__":
    q = Queue()
    workers = [Process(target=worker, args=(q,)) for _ in range(4)]
    for w in workers: w.start()

    for item in jobs: q.put(item)
    for _ in workers: q.put(None)        # shutdown signal
    for w in workers: w.join()

Shared Memory

from multiprocessing import shared_memory
import numpy as np

# Parent — allocate shared block
shm = shared_memory.SharedMemory(create=True, size=1024)
arr = np.ndarray((256,), dtype=np.float64, buffer=shm.buf)
arr[0] = 3.14

# Child — attach by name (shm.name)
# existing = shared_memory.SharedMemory(name=...)

shm.close()
shm.unlink()         # delete the block

Pickling Constraints

Arguments and return values are pickled to cross process boundaries. Lambdas, locally defined functions, and lock instances cannot be pickled.

# BAD — lambda can't be pickled
ex.map(lambda x: x * 2, items)

# GOOD — module-level function
def double(x): return x * 2
ex.map(double, items)