|$ curl https://forge-ai.dev/api/markdown?path=docs/python/stdlib

$cat docs/python-—-standard-library.md

updated Recently·20 min read·published

Python — Standard Library

◆Python◆Intermediate

Introduction

Python's standard library (“batteries included”) provides tools for virtually every common programming task — from filesystem operations and data serialization to functional programming and cryptographic hashing. Mastering these modules eliminates the need for external dependencies in most everyday scripts.

Category	Modules	Common Use Cases
Filesystem & OS	os, sys, pathlib, shutil	File paths, env vars, process info, directory traversal
Serialization	json, csv, pickle	Read/write structured data, object persistence
Date & Time	datetime, time, calendar	Timestamps, formatting, date arithmetic, scheduling
Data Structures	collections, itertools	Counters, deques, grouped iteration, combinatorics
Functional	functools, operator, itertools	Partial application, caching, function composition
Crypto & IDs	hashlib, uuid, secrets	Hashing, unique IDs, secure tokens
Data Science	statistics, math, random	Mean/median, stdev, sampling, probability
System & CLI	argparse, subprocess, tempfile	CLI tools, running shell commands, temp files
Text Processing	re, string, textwrap, difflib	Pattern matching, string ops, diff computation

ℹ

info

Before adding an external dependency, check the standard library first. Modules like json, csv, sqlite3, and pathlib often eliminate the need for third-party libraries entirely.

Filesystem & OS

The os and sys modules provide low-level operating system and interpreter interfaces. For modern path handling, pathlib is the recommended alternative to os.path.

os_module.py

Python

1	import os
2	import sys
3
4	# os — operating system interface
5	os.getcwd() # current working directory → '/Users/alice/project'
6	os.listdir(".") # list files in directory
7	os.environ["HOME"] # access environment variables
8	os.environ.get("DB_URL", "") # safe access with default
9
10	# os.path — path utilities
11	os.path.join("dir", "sub", "file.txt") # → 'dir/sub/file.txt'
12	os.path.exists("/tmp") # → True
13	os.path.isfile("data.csv") # → True
14	os.path.isdir("src") # → True
15	os.path.splitext("photo.jpg") # → ('photo', '.jpg')
16	os.path.basename("/a/b/c.txt") # → 'c.txt'
17	os.path.dirname("/a/b/c.txt") # → '/a/b'
18
19	# Directory operations
20	os.makedirs("a/b/c", exist_ok=True) # recursive mkdir
21	os.remove("temp.txt") # delete file
22	os.rename("old.txt", "new.txt") # rename/move
23	os.system("echo hello") # run shell command (prefer subprocess)

sys_module.py

Python

1	import sys
2
3	# sys — Python interpreter access
4	sys.version # '3.12.3 (main, Apr ...)'
5	sys.version_info # sys.version_info(major=3, minor=12, micro=3)
6	sys.platform # 'darwin', 'linux', 'win32'
7	sys.argv # ['script.py', '--flag', 'value']
8	sys.exit(0) # exit with status code (0 = success, non-zero = error)
9	sys.exit("error msg") # print message and exit with code 1
10
11	# sys.path — module search path
12	sys.path[0] # script directory ('' means cwd)
13	sys.path.append("/my/libs") # add custom import path
14
15	# Memory & garbage collection
16	sys.getsizeof([]) # size of object in bytes
17	sys.getrefcount(obj) # reference count
18
19	# stdin/stdout/stderr
20	data = sys.stdin.read() # read all from stdin
21	sys.stdout.write("output\n") # write to stdout
22	sys.stderr.write("error\n") # write to stderr
23
24	# Recursion limit
25	sys.getrecursionlimit() # default: 1000
26	sys.setrecursionlimit(5000) # increase for deep recursion

pathlib.py

Python

1	from pathlib import Path
2
3	# Path — modern filesystem paths (preferred over os.path)
4	p = Path("/home/user/docs/report.txt")
5
6	# Path components
7	p.parent # → /home/user/docs
8	p.parents[0] # → /home/user/docs (same as p.parent)
9	p.parents[1] # → /home/user
10	p.name # → report.txt
11	p.stem # → report
12	p.suffix # → .txt
13	p.suffixes # → ['.txt']
14	p.anchor # → /
15
16	# Reading & writing
17	Path("hello.txt").write_text("Hello, world!") # write
18	Path("hello.txt").read_text() # → 'Hello, world!'
19	Path("data.bin").write_bytes(b"\\x00\\x01") # binary write
20	Path("data.bin").read_bytes() # binary read
21
22	# File info
23	p.exists() # → bool
24	p.is_file() # → True
25	p.is_dir() # → False
26	p.stat().st_size # file size in bytes
27	p.stat().st_mtime # last modified timestamp
28
29	# Directory operations
30	Path("new_dir").mkdir(exist_ok=True) # create directory
31	Path("a/b/c").mkdir(parents=True, exist_ok=True) # recursive mkdir
32	Path("old.txt").rename("new.txt") # rename
33	Path("tmp.txt").unlink(missing_ok=True) # delete (no error if missing)
34
35	# Globbing
36	list(Path("src").glob("*.py")) # all .py files in src/
37	list(Path("src").rglob("**/__init__.py")) # recursive match
38
39	# Path building with /
40	base = Path("/data")
41	csv_path = base / "2024" / "logs.csv" # → /data/2024/logs.csv
42
43	# Current directory
44	Path.cwd() # current working directory
45	Path.home() # user home directory (~)

✓

best practice

Use pathlib over os.path for all new code. The / operator for path building and the object-oriented API make it significantly more readable. Path objects also work directly with built-in open() and many standard library functions.

Serialization

Serialization converts Python objects to portable formats (JSON, CSV, binary) and back. These modules handle data interchange between systems, file storage, and object persistence.

json_serialization.py

Python

1	import json
2
3	# json.dumps — serialize to string
4	data = {"name": "Alice", "age": 30, "scores": [95, 87, 92]}
5	text = json.dumps(data, indent=2)
6	print(text)
7	# {
8	# "name": "Alice",
9	# "age": 30,
10	# "scores": [95, 87, 92]
11	# }
12
13	# json.loads — deserialize from string
14	restored = json.loads(text)
15	print(restored["name"]) # → Alice
16
17	# json.dump / json.load — file I/O
18	with open("data.json", "w") as f:
19	json.dump(data, f, indent=2)
20
21	with open("data.json") as f:
22	loaded = json.load(f)
23
24	# Handling non-serializable types (datetime, Decimal, etc.)
25	from datetime import datetime
26
27	def json_serializer(obj):
28	if isinstance(obj, datetime):
29	return obj.isoformat()
30	raise TypeError(f"Type {type(obj)} not serializable")
31
32	log = {"event": "login", "time": datetime.now()}
33	json.dumps(log, default=json_serializer)
34	# → '{"event": "login", "time": "2026-07-09T10:30:00"}'
35
36	# Custom JSON encoding with class
37	from json import JSONEncoder
38
39	class CustomEncoder(JSONEncoder):
40	def default(self, obj):
41	if isinstance(obj, datetime):
42	return obj.isoformat()
43	if isinstance(obj, set):
44	return list(obj)
45	return super().default(obj)
46
47	json.dumps({"tags": {"py", "js"}}, cls=CustomEncoder)
48	# → '{"tags": ["py", "js"]}'

csv_io.py

Python

1	import csv
2
3	# csv.reader — read rows as lists
4	with open("employees.csv") as f:
5	reader = csv.reader(f)
6	header = next(reader) # ['name', 'department', 'salary']
7	for row in reader:
8	print(row) # ['Alice', 'Engineering', '120000']
9
10	# csv.DictReader — read rows as dicts
11	with open("employees.csv") as f:
12	reader = csv.DictReader(f)
13	for row in reader:
14	print(row["name"], row["department"])
15
16	# csv.writer — write rows
17	with open("output.csv", "w", newline="") as f:
18	writer = csv.writer(f)
19	writer.writerow(["name", "score"]) # header
20	writer.writerows([ # multiple rows
21	["Alice", 95],
22	["Bob", 87],
23	["Charlie", 92],
24	])
25
26	# csv.DictWriter — write from dicts
27	with open("output.csv", "w", newline="") as f:
28	fieldnames = ["name", "department", "salary"]
29	writer = csv.DictWriter(f, fieldnames=fieldnames)
30	writer.writeheader()
31	writer.writerow({"name": "Alice", "department": "Eng", "salary": "120000"})
32
33	# Custom delimiter (TSV)
34	with open("data.tsv", "w", newline="") as f:
35	writer = csv.writer(f, delimiter="\\t")
36	writer.writerow(["a", "b", "c"])
37
38	# Handling quotes and escaping
39	reader = csv.reader(f, quotechar='"', skipinitialspace=True)

pickle_serialization.py

Python

1	import pickle
2
3	# pickle.dump / pickle.load — serialize/deserialize to file
4	data = {"users": ["Alice", "Bob"], "config": {"theme": "dark", "lang": "en"}}
5
6	with open("state.pkl", "wb") as f:
7	pickle.dump(data, f) # binary write
8
9	with open("state.pkl", "rb") as f:
10	restored = pickle.load(f) # binary read
11
12	# pickle.dumps / pickle.loads — in-memory
13	serialized = pickle.dumps(data)
14	restored = pickle.loads(serialized)
15
16	# Pickle protocol versions (higher = more efficient)
17	pickle.dumps(data, protocol=5) # Python 3.8+ default
18	pickle.DEFAULT_PROTOCOL # → 5 (in 3.12)
19
20	# What can be pickled?
21	# - All native types (int, str, list, dict, set, None, ...)
22	# - Functions and classes (by reference)
23	# - Objects whose class is importable and defines __getstate__/__setstate__
24
25	# Custom serialization
26	class User:
27	def __init__(self, name: str, score: int):
28	self.name = name
29	self.score = score
30
31	def __getstate__(self):
32	# Control what gets serialized
33	return {"name": self.name, "score": self.score}
34
35	def __setstate__(self, state):
36	self.name = state["name"]
37	self.score = state["score"]
38
39	user = User("Alice", 100)
40	data = pickle.dumps(user)
41	restored = pickle.loads(data)
42	print(restored.name) # → Alice

danger

Never unpickle data from untrusted sources. Pickle can execute arbitrary code during deserialization. Use json for data interchange between systems. Pickle is best for local caching and saving program state within your own application.

Date & Time

The datetime module provides classes for manipulating dates, times, and time intervals. It is the go-to module for all date/time operations in Python.

datetime_module.py

Python

1	from datetime import datetime, date, time, timedelta, timezone
2
3	# datetime — combined date + time
4	now = datetime.now() # local time (naive)
5	utc_now = datetime.now(timezone.utc) # timezone-aware UTC
6	specific = datetime(2026, 7, 9, 14, 30, 0) # July 9, 2026, 2:30 PM
7
8	# date — date only (no time)
9	d = date.today() # → 2026-07-09
10	d.year, d.month, d.day # → (2026, 7, 9)
11
12	# time — time only (no date)
13	t = time(14, 30, 0) # → 14:30:00
14	t.hour, t.minute, t.second # → (14, 30, 0)
15
16	# timedelta — duration between two dates/times
17	delta = timedelta(days=7, hours=3)
18	future = now + delta # one week + 3 hours from now
19	past = now - timedelta(days=30) # 30 days ago
20	diff = future - past # → timedelta(days=37, ...)
21
22	# strftime — format datetime to string
23	now.strftime("%Y-%m-%d") # → '2026-07-09'
24	now.strftime("%Y-%m-%d %H:%M:%S") # → '2026-07-09 14:30:00'
25	now.strftime("%A, %B %d, %Y") # → 'Thursday, July 09, 2026'
26	now.strftime("%I:%M %p") # → '02:30 PM'
27	now.strftime("%j") # → day of year (190)
28
29	# strptime — parse string to datetime
30	dt = datetime.strptime("2026-07-09", "%Y-%m-%d")
31	dt2 = datetime.strptime("07/09/26 2:30PM", "%m/%d/%y %I:%M%p")
32
33	# Common format codes
34	# %Y — 4-digit year %m — month (01-12) %d — day (01-31)
35	# %H — hour (00-23) %M — minute (00-59) %S — second (00-59)
36	# %A — weekday full %B — month full %j — day of year
37
38	# Date arithmetic
39	birthday = date(2026, 12, 25)
40	days_until = (birthday - date.today()).days # days until Christmas
41
42	# Timezone handling
43	from zoneinfo import ZoneInfo # Python 3.9+
44	ny_tz = ZoneInfo("America/New_York")
45	ny_now = datetime.now(ny_tz)
46	print(ny_now.strftime("%Y-%m-%d %H:%M:%S %Z")) # → 2026-07-09 ... EDT
47
48	# Replace specific components
49	noon = now.replace(hour=12, minute=0, second=0, microsecond=0)
50
51	# Timestamps
52	now.timestamp() # Unix timestamp (seconds since epoch)
53	datetime.fromtimestamp(0) # → 1970-01-01 00:00:00

📝

note

Use timezone-aware datetimes (datetime.now(timezone.utc) or ZoneInfo) whenever possible. Naive datetimes can cause subtle bugs when handling times across different timezones. Python 3.9+ includes zoneinfo in the standard library.

Collections

The collections module provides specialized container datatypes that extend the built-in dict, list, set, and tuple for common use cases.

collections_part1.py

Python

1	from collections import defaultdict, Counter, namedtuple, deque, OrderedDict
2
3	# defaultdict — dict with default factory
4	dd = defaultdict(int)
5	dd["visits"] += 1 # no KeyError, defaults to 0 first
6	dd["page"] = "/home"
7	print(dd) # → defaultdict(<int>, {'visits': 1, 'page': '/home'})
8
9	# Useful default factories:
10	defaultdict(list) # dd["key"].append(1) — auto create list
11	defaultdict(set) # dd["key"].add(1) — auto create set
12	defaultdict(lambda: 0) # custom default
13
14	# Group items with defaultdict
15	pairs = [("fruit", "apple"), ("fruit", "banana"), ("color", "red")]
16	grouped = defaultdict(list)
17	for key, value in pairs:
18	grouped[key].append(value)
19	print(dict(grouped)) # → {'fruit': ['apple', 'banana'], 'color': ['red']}
20
21	# Counter — count hashable items
22	counter = Counter(["a", "b", "a", "c", "b", "a"])
23	print(counter) # → Counter({'a': 3, 'b': 2, 'c': 1})
24	print(counter["a"]) # → 3
25	print(counter["z"]) # → 0 (no KeyError!)
26
27	counter.update(["a", "b", "d"]) # add more counts
28	counter.most_common(2) # → [('a', 4), ('b', 3)]
29	list(counter.elements()) # → ['a', 'a', 'a', 'a', 'b', 'b', 'b', 'c', 'd']
30
31	# Counter arithmetic
32	c1 = Counter(a=3, b=1)
33	c2 = Counter(a=1, b=2)
34	print(c1 + c2) # → Counter({'a': 4, 'b': 3})
35	print(c1 - c2) # → Counter({'a': 2}) (only positive counts kept)
36
37	# namedtuple — lightweight immutable objects
38	Point = namedtuple("Point", ["x", "y"])
39	p = Point(10, y=20)
40	print(p.x, p.y) # → 10 20
41	print(p[0], p[1]) # → 10 20 (also indexable)
42	x, y = p # unpackable
43
44	# namedtuple methods
45	p._asdict() # → {'x': 10, 'y': 20}
46	p._replace(x=30) # → Point(x=30, y=20) (new instance)
47	Point._make([1, 2]) # → Point(x=1, y=2) (from iterable)
48
49	# Ideal for: database rows, config values, coordinate pairs

collections_part2.py

Python

1	# deque — double-ended queue
2	dq = deque(["a", "b", "c"], maxlen=5)
3	dq.append("d") # add to right end
4	dq.appendleft("z") # add to left end
5	print(dq) # deque(['z', 'a', 'b', 'c', 'd'], maxlen=5)
6
7	dq.pop() # remove from right → 'd'
8	dq.popleft() # remove from left → 'z'
9
10	# Fixed-size buffer (when full, opposite side is discarded)
11	buffer = deque(maxlen=3)
12	for i in range(10):
13	buffer.append(i)
14	print(buffer) # deque([7, 8, 9], maxlen=3)
15
16	# Rotate
17	dq = deque([1, 2, 3, 4, 5])
18	dq.rotate(2) # → deque([4, 5, 1, 2, 3])
19	dq.rotate(-1) # → deque([5, 1, 2, 3, 4])
20
21	# Efficient for: queue, stack, sliding window, undo buffer
22	# O(1) append/pop on both ends vs O(n) for list.pop(0)
23
24	# OrderedDict — dict that preserves insertion order (Python 3.7+ dicts also do)
25	od = OrderedDict()
26	od["z"] = 1
27	od["a"] = 2
28	od["b"] = 3
29	for key in od:
30	print(key) # → z, a, b (insertion order)
31
32	# OrderedDict-specific features:
33	od.move_to_end("z") # move 'z' to the end
34	od.move_to_end("b", last=False) # move 'b' to the front
35	od.popitem(last=True) # pop last (LIFO)
36	od.popitem(last=False) # pop first (FIFO)
37
38	# OrderedDict vs dict: OrderedDict has equality that considers order
39	# Regular dict equality ignores order
40	print({"a": 1, "b": 2} == {"b": 2, "a": 1}) # → True
41	print(OrderedDict({"a": 1, "b": 2}) == OrderedDict({"b": 2, "a": 1})) # → False

Itertools

The itertools module provides fast, memory-efficient iterator building blocks. These functions form an “iterator algebra” that enables constructing specialized iteration patterns.

itertools_part1.py

Python

1	from itertools import chain, product, permutations, combinations, combinations_with_replacement
2
3	# chain — concatenate multiple iterables
4	list(chain([1, 2], [3, 4], [5])) # → [1, 2, 3, 4, 5]
5	list(chain.from_iterable([[1], [2, 3], [4]])) # → [1, 2, 3, 4]
6
7	# product — Cartesian product (nested loop replacement)
8	list(product([1, 2], ["a", "b"]))
9	# → [(1, 'a'), (1, 'b'), (2, 'a'), (2, 'b')]
10
11	# Equivalent nested loop:
12	# for x in [1, 2]:
13	# for y in ["a", "b"]:
14	# ...
15
16	# product with repeat — self product
17	list(product([1, 2, 3], repeat=2))
18	# → [(1,1), (1,2), (1,3), (2,1), (2,2), (2,3), (3,1), (3,2), (3,3)]
19
20	# permutations — all r-length orderings, no repeats
21	list(permutations("ABC", 2))
22	# → [('A', 'B'), ('A', 'C'), ('B', 'A'), ('B', 'C'), ('C', 'A'), ('C', 'B')]
23
24	list(permutations([1, 2, 3]))
25	# → 6 permutations (3!)
26
27	# combinations — r-length subsets, no order, no repeats
28	list(combinations("ABCD", 2))
29	# → [('A','B'), ('A','C'), ('A','D'), ('B','C'), ('B','D'), ('C','D')]
30
31	# combinations_with_replacement — r-length subsets, order ignored, repeats allowed
32	list(combinations_with_replacement([1, 2, 3], 2))
33	# → [(1,1), (1,2), (1,3), (2,2), (2,3), (3,3)]

itertools_part2.py

Python

1	from itertools import groupby, count, cycle, repeat, accumulate, islice
2
3	# groupby — group consecutive elements by key function
4	data = [("fruit", "apple"), ("fruit", "banana"), ("color", "red"), ("color", "blue")]
5	# IMPORTANT: groupby requires sorted data for meaningful results
6	sorted_data = sorted(data, key=lambda x: x[0])
7	for key, group in groupby(sorted_data, key=lambda x: x[0]):
8	print(key, list(group))
9	# fruit [('fruit', 'apple'), ('fruit', 'banana')]
10	# color [('color', 'red'), ('color', 'blue')]
11
12	# count — infinite counter (like range with no end)
13	for i in count(10, 2): # 10, 12, 14, 16, ...
14	if i > 20: break
15	print(i)
16
17	# cycle — infinite repeater
18	c = 0
19	for item in cycle(["A", "B", "C"]): # A, B, C, A, B, C, ...
20	print(item)
21	c += 1
22	if c > 5: break
23
24	# repeat — repeat a value N times (or infinitely)
25	list(repeat("x", 3)) # → ['x', 'x', 'x']
26
27	# accumulate — running sum (or other binary function)
28	list(accumulate([1, 2, 3, 4, 5])) # → [1, 3, 6, 10, 15]
29	list(accumulate([1, 2, 3, 4], lambda a, b: a * b)) # → [1, 2, 6, 24]
30
31	# islice — slice an iterator (like regular slice but for iterators)
32	list(islice(range(100), 10)) # → [0, 1, ..., 9]
33	list(islice(range(100), 10, 20)) # → [10, 11, ..., 19]
34	list(islice(range(100), 10, 20, 2)) # → [10, 12, 14, 16, 18]
35
36	# Practical pattern: pagination with islice
37	def paginate(iterable, page_size):
38	it = iter(iterable)
39	while True:
40	batch = list(islice(it, page_size))
41	if not batch:
42	break
43	yield batch
44
45	for page in paginate(range(100), 10):
46	print(page) # → [0..9], [10..19], ..., [90..99]

🔥

pro tip

itertools functions return lazy iterators — they compute values on demand and use O(1) memory, regardless of the input size. This makes them ideal for working with large datasets that would overflow memory if converted to lists.

Functools

The functools module provides higher-order functions for creating or modifying callable objects. These tools are essential for functional programming patterns in Python.

functools.py

Python

1	from functools import partial, lru_cache, wraps, singledispatch
2
3	# partial — freeze function arguments
4	def power(base, exponent):
5	return base ** exponent
6
7	square = partial(power, exponent=2)
8	cube = partial(power, exponent=3)
9
10	print(square(5)) # → 25
11	print(cube(5)) # → 125
12
13	# Real-world: pre-configure functions
14	import json
15	load_json = partial(json.loads)
16	dumps_pretty = partial(json.dumps, indent=2, sort_keys=True)
17
18	# lru_cache — memoize function results
19	@lru_cache(maxsize=128)
20	def fibonacci(n):
21	if n < 2:
22	return n
23	return fibonacci(n - 1) + fibonacci(n - 2)
24
25	print(fibonacci(50)) # → 12586269025 (instant, not exponential!)
26	print(fibonacci.cache_info())
27	# CacheInfo(hits=48, misses=51, maxsize=128, currsize=51)
28
29	# lru_cache on expensive I/O operations
30	@lru_cache(maxsize=32)
31	def load_config(path):
32	with open(path) as f:
33	return f.read()
34
35	# Clear cache when needed
36	load_config.cache_clear()
37
38	# wraps — preserve metadata in decorators
39	def log_calls(func):
40	@wraps(func) # preserves func.__name__, __doc__, __module__
41	def wrapper(args, *kwargs):
42	print(f"calling {func.__name__}")
43	return func(args, *kwargs)
44	return wrapper
45
46	@log_calls
47	def greet(name):
48	"""Say hello to someone."""
49	return f"Hello, {name}!"
50
51	print(greet.__name__) # → 'greet' (without @wraps: 'wrapper')
52	print(greet.__doc__) # → 'Say hello to someone.' (without @wraps: None)
53
54	# singledispatch — function overloading by type
55	@singledispatch
56	def serialize(obj):
57	"""Serialize an object to string."""
58	raise NotImplementedError(f"unsupported type: {type(obj)}")
59
60	@serialize.register
61	def _(obj: int):
62	return str(obj)
63
64	@serialize.register
65	def _(obj: list):
66	return ",".join(str(x) for x in obj)
67
68	@serialize.register
69	def _(obj: dict):
70	return json.dumps(obj)
71
72	print(serialize(42)) # → '42'
73	print(serialize([1, 2, 3])) # → '1,2,3'
74	print(serialize({"a": 1})) # → '{"a": 1}'

Cryptography & IDs

The hashlib module provides secure hash and message digest algorithms. The uuid module generates universally unique identifiers. For cryptographic-quality randomness, use secrets instead of random.

hashlib_uuid.py

Python

1	import hashlib
2	import uuid
3
4	# hashlib — cryptographic hashing
5
6	# SHA-256 (most common, 32 bytes, 64 hex chars)
7	data = b"hello world"
8	hash_obj = hashlib.sha256(data)
9	print(hash_obj.hexdigest())
10	# → b94d27b9934d3e08a52e52d7da7dabfac484efe37a5380ee9088f7ace2efcde9
11
12	# MD5 (128-bit, faster but not collision-resistant)
13	print(hashlib.md5(data).hexdigest())
14	# → 5eb63bbbe01eeed093cb22bb8f5acdc3
15
16	# SHA-1 (160-bit, deprecated for security)
17	print(hashlib.sha1(data).hexdigest())
18
19	# Update incrementally (for large data)
20	h = hashlib.sha256()
21	h.update(b"hello ")
22	h.update(b"world")
23	print(h.hexdigest()) # same as hashlib.sha256(b"hello world")
24
25	# Available algorithms
26	print(hashlib.algorithms_guaranteed) # always available
27	print(hashlib.algorithms_available) # available on this platform
28
29	# Hashing files
30	def hash_file(path: str) -> str:
31	h = hashlib.sha256()
32	with open(path, "rb") as f:
33	for chunk in iter(lambda: f.read(8192), b""):
34	h.update(chunk)
35	return h.hexdigest()
36
37	# uuid — universally unique identifiers
38
39	# uuid4 — random UUID (most common, 128 bits)
40	uid = uuid.uuid4()
41	print(uid) # → 550e8400-e29b-41d4-a716-446655440000
42	print(str(uid)) # → '550e8400-e29b-41d4-a716-446655440000'
43	print(uid.hex) # → '550e8400e29b41d4a716446655440000'
44
45	# Other UUID versions
46	uuid.uuid1() # based on host ID + clock (privacy concerns)
47	uuid.uuid3(uuid.NAMESPACE_DNS, "example.com") # MD5-based
48	uuid.uuid5(uuid.NAMESPACE_DNS, "example.com") # SHA-1 based
49
50	# Deterministic UUID from string
51	uuid.uuid5(uuid.NAMESPACE_URL, "https://example.com/user/42")
52
53	# secrets — secure random tokens (Python 3.6+)
54	import secrets
55	secrets.token_hex(16) # → 'a1b2...' (32 hex chars, 128 bits)
56	secrets.token_urlsafe(32) # → URL-safe base64 string
57	secrets.choice(["a", "b"]) # cryptographically secure random choice

✓

best practice

Use SHA-256 (or SHA-3) for password hashing and data integrity. Never use MD5 or SHA-1 for security-sensitive applications. For password storage specifically, use bcrypt or hashlib.pbkdf2_hmac with a salt. Use uuid4 for general-purpose unique IDs.

Data Science

The statistics module provides basic statistical functions for small-to-medium datasets. For heavy numerical work, use numpy and scipy, but the standard library covers many common use cases.

statistics.py

Python

1	from statistics import mean, median, mode, stdev, variance, quantiles
2	import math
3
4	# Basic statistics
5	data = [2, 3, 5, 7, 11, 13, 17, 19, 23, 29]
6
7	mean(data) # → 12.9
8	median(data) # → 12.0 (middle value)
9	median([1, 2, 3, 4]) # → 2.5 (average of two middle values)
10
11	mode([1, 1, 2, 3, 3, 3, 4]) # → 3 (most common)
12	try:
13	mode([1, 2, 3]) # raises StatisticsError (no unique mode)
14	except:
15	pass
16
17	# Population vs sample statistics
18	stdev(data) # sample standard deviation (ddof=1)
19	variance(data) # sample variance (ddof=1)
20
21	pstdev(data) # population standard deviation (ddof=0)
22	pvariance(data) # population variance (ddof=0)
23
24	# Quantiles (Python 3.8+)
25	quantiles(data, n=4) # → quartiles [5.0, 12.0, 21.5]
26	quantiles(data, n=100) # → percentiles
27
28	# math module — additional numerical operations
29	math.sqrt(16) # → 4.0
30	math.floor(3.7) # → 3
31	math.ceil(3.2) # → 4
32	math.pi # → 3.14159...
33	math.inf # → infinity (float)
34	math.nan # → NaN (float)
35	math.comb(10, 3) # → 120 (combinations count)
36	math.perm(10, 3) # → 720 (permutations count)
37	math.fsum([0.1] * 10) # → 1.0 (accurate float sum)
38	math.isclose(0.1 + 0.2, 0.3) # → True (safe float comparison)
39
40	# random module
41	import random
42	random.randint(1, 100) # random integer between 1 and 100
43	random.choice(["a", "b", "c"]) # random element
44	random.sample(range(100), 5) # 5 unique random numbers
45	random.shuffle(list(range(10))) # shuffle in-place
46	random.gauss(0, 1) # normally distributed random number

System & CLI

Python excels at system automation. The argparse module builds professional CLI interfaces, subprocess spawns and controls external processes, and tempfile handles temporary files safely.

argparse_demo.py

Python

1	import argparse
2
3	# ArgumentParser — build CLI interfaces
4	parser = argparse.ArgumentParser(
5	description="Process and analyze log files.",
6	epilog="Example: python analyze.py --verbose input.log",
7	)
8
9	# Positional argument (required)
10	parser.add_argument("input", help="input log file path")
11
12	# Optional arguments
13	parser.add_argument("-o", "--output", help="output file (default: stdout)")
14	parser.add_argument("-v", "--verbose", action="store_true", help="increase verbosity")
15	parser.add_argument("-n", "--count", type=int, default=10, help="number of results")
16	parser.add_argument("--format", choices=["json", "csv", "table"], default="table")
17	parser.add_argument("--level", type=int, choices=[1, 2, 3], default=2)
18
19	# Parse arguments (typically from sys.argv)
20	args = parser.parse_args()
21
22	# Access parsed values
23	print(args.input) # → 'input.log'
24	print(args.verbose) # → True/False
25	print(args.count) # → 10 (default)
26	print(args.format) # → 'table' (default)
27
28	# In actual usage:
29	# $ python analyze.py input.log -o results.json -v --count 50 --format json
30
31	# Subcommands (git-like CLI)
32	parser = argparse.ArgumentParser()
33	subparsers = parser.add_subparsers(dest="command", required=True)
34
35	# "run" subcommand
36	run_parser = subparsers.add_parser("run", help="run the process")
37	run_parser.add_argument("--config", required=True)
38
39	# "list" subcommand
40	list_parser = subparsers.add_parser("list", help="list available items")
41	list_parser.add_argument("--all", action="store_true")
42
43	args = parser.parse_args()
44	if args.command == "run":
45	print(f"running with config: {args.config}")

subprocess_demo.py

Python

1	import subprocess
2
3	# subprocess.run — run a command and wait for it (recommended)
4	result = subprocess.run(
5	["echo", "hello"],
6	capture_output=True,
7	text=True,
8	)
9	print(result.stdout) # → 'hello\\n'
10	print(result.returncode) # → 0
11	print(result.stderr) # → '' (empty)
12
13	# Check return code (raises CalledProcessError if non-zero)
14	subprocess.run(["ls", "nonexistent"], check=True)
15	# subprocess.CalledProcessError: ... returned non-zero exit status 1.
16
17	# Shell command (avoid if possible — security risk!)
18	subprocess.run("echo hello \| grep h", shell=True, capture_output=True, text=True)
19
20	# subprocess.Popen — fine-grained control
21	proc = subprocess.Popen(
22	["ping", "-c", "4", "example.com"],
23	stdout=subprocess.PIPE,
24	stderr=subprocess.PIPE,
25	text=True,
26	)
27
28	# Communicate with process (send input, read output)
29	stdout, stderr = proc.communicate(timeout=10)
30	print(stdout)
31
32	# Pipe data through a process
33	proc = subprocess.Popen(
34	["sort"],
35	stdin=subprocess.PIPE,
36	stdout=subprocess.PIPE,
37	text=True,
38	)
39	out, _ = proc.communicate("banana\\napple\\ncherry\\n")
40	print(out) # → 'apple\\nbanana\\ncherry\\n'
41
42	# subprocess.check_output — simple output capture
43	output = subprocess.check_output(["whoami"], text=True).strip()
44	print(output) # → 'alice'
45
46	# Running with environment variables
47	import os
48	env = {**os.environ, "MY_VAR": "value"}
49	subprocess.run(["printenv", "MY_VAR"], env=env, capture_output=True, text=True)

tempfile_demo.py

Python

1	import tempfile
2	import os
3	from pathlib import Path
4
5	# TemporaryFile — file that is deleted when closed
6	with tempfile.TemporaryFile(mode="w+t") as f:
7	f.write("temporary data")
8	f.seek(0)
9	print(f.read()) # → 'temporary data'
10	# File is automatically deleted when exiting the 'with' block
11
12	# NamedTemporaryFile — temp file with a visible name (also auto-deleted)
13	with tempfile.NamedTemporaryFile(suffix=".txt", prefix="prefix_", delete=True) as f:
14	print(f.name) # → '/tmp/prefix_abc123.txt'
15	f.write(b"data")
16
17	# TemporaryDirectory — auto-cleaned temp directory
18	with tempfile.TemporaryDirectory() as tmp_dir:
19	tmp_path = Path(tmp_dir)
20	(tmp_path / "work.txt").write_text("hello")
21	print(list(tmp_path.iterdir())) # → [PosixPath('.../work.txt')]
22	# Directory and all contents are deleted on exit
23
24	# mkdtemp / mkstemp — manual cleanup required
25	temp_dir = tempfile.mkdtemp(prefix="myapp_")
26	print(temp_dir) # → '/tmp/myapp_xyz789'
27
28	# Remember to clean up manually:
29	# import shutil
30	# shutil.rmtree(temp_dir)
31
32	# Temp file descriptor
33	fd, path = tempfile.mkstemp(suffix=".csv")
34	os.write(fd, b"name,value\\nalice,30")
35	os.close(fd)
36	print(path) # → '/tmp/tmp12345.csv'
37	os.remove(path) # manual cleanup required
38
39	# Get system temp directory
40	print(tempfile.gettempdir()) # → '/tmp' (or equivalent)

ℹ

info

Always use subprocess.run with capture_output=True (or check_output) instead of os.system. The subprocess module gives you control over stdin/stdout/stderr, proper argument escaping, and reliable error handling.

Advanced — Regular Expressions

The re module provides regular expression matching operations. Regex patterns are a powerful tool for text processing, validation, extraction, and transformation.

re_module.py

Python

1	import re
2
3	# re.search — find first match anywhere in string
4	match = re.search(r"\\d{3}-\\d{4}", "Call 555-1234 today!")
5	if match:
6	print(match.group()) # → '555-1234'
7	print(match.start()) # → 5 (start index)
8	print(match.end()) # → 13 (end index)
9
10	# re.match — match only at the beginning of string
11	m = re.match(r"\\d{3}", "123-456")
12	print(m.group() if m else None) # → '123'
13
14	m = re.match(r"\\d{3}", "abc-123")
15	print(m) # → None (no match at start)
16
17	# re.findall — find all non-overlapping matches
18	text = "Contact: 555-1234, 555-5678, 555-9012"
19	phones = re.findall(r"\\d{3}-\\d{4}", text)
20	print(phones) # → ['555-1234', '555-5678', '555-9012']
21
22	# re.finditer — iterator of match objects (memory efficient)
23	for m in re.finditer(r"\\d{3}-\\d{4}", text):
24	print(m.group(), m.span())
25
26	# re.sub — substitute matches
27	result = re.sub(r"\\d{3}-\\d{4}", "[REDACTED]", text)
28	print(result) # → 'Contact: [REDACTED], [REDACTED], [REDACTED]'
29
30	# re.sub with function replacement
31	def mask_phone(match):
32	digits = match.group()
33	return f"{digits[:3]}-XXXX"
34
35	result = re.sub(r"\\d{3}-\\d{4}", mask_phone, text)
36	print(result) # → 'Contact: 555-XXXX, 555-XXXX, 555-XXXX'
37
38	# re.split — split string by pattern
39	parts = re.split(r"[,;\\s]+", "a,b;c d")
40	print(parts) # → ['a', 'b', 'c', 'd']
41
42	# re.compile — precompile pattern for reuse (faster)
43	pattern = re.compile(r"\\b[A-Z][a-z]+\\b") # capitalized words
44	matches = pattern.findall("Alice and Bob Went to Paris")
45	print(matches) # → ['Alice', 'Bob', 'Went', 'Paris']
46
47	# Groups — extract specific parts
48	pattern = re.compile(r"(?P<area>\\d{3})-(?P<num>\\d{4})")
49	m = pattern.search("Call 555-1234")
50	print(m.group("area")) # → '555'
51	print(m.group("num")) # → '1234'
52	print(m.groups()) # → ('555', '1234')
53
54	# Common regex patterns
55	EMAIL = r"^[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\\.[a-zA-Z]{2,}$"
56	URL = r"https?://[\\w.-]+(?:\\.[\\w.-]+)+[/\\w\\-.%&=?]*"
57	IP = r"\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}"
58	SLUG = r"^[a-z0-9]+(?:-[a-z0-9]+)*$"
59
60	# Validation example
61	def is_valid_email(email: str) -> bool:
62	return bool(re.match(r"^[\\w._%+-]+@[\\w.-]+\\.[a-zA-Z]{2,}$", email))
63
64	print(is_valid_email("alice@example.com")) # → True
65	print(is_valid_email("invalid@")) # → False

✓

best practice

Always use raw strings (r"pattern") for regex patterns to avoid unintended escape interpretation. Compile patterns with re.compile when using the same pattern repeatedly. Use named groups ((?P<name>...)) for complex patterns to improve readability.

$Blueprint — Engineering Documentation·Section ID: PYTHON-STDLIB·Revision: 1.0

X GitHub YouTube Instagram Facebook Buy me a coffee Sponsor