|$ curl https://forge-ai.dev/api/markdown?path=docs/python/collections-itertools
$cat docs/collections-&-itertools.md
updated Today·20-26 min read·published
collections & itertools
Introduction
The collections and itertools modules are the stdlib power tools for data munging. Master them and you write less code with fewer bugs.
ℹ
info
Compose iterators lazily. Materialize with list() only when you need random access or multiple passes.
Counter
counter.py
Python
| 1 | from collections import Counter |
| 2 | |
| 3 | c = Counter("abracadabra") |
| 4 | print(c.most_common(3)) |
| 5 | print(c.total()) # 3.10+ |
| 6 | |
| 7 | c.update("aaa") |
| 8 | c.subtract("ab") |
| 9 | print(+c) # drop non-positive |
| 10 | |
| 11 | a = Counter(apples=3, oranges=1) |
| 12 | b = Counter(apples=1, bananas=2) |
| 13 | print(a + b, a - b, a & b, a | b) |
defaultdict
defaultdict.py
Python
| 1 | from collections import defaultdict |
| 2 | |
| 3 | by_len: defaultdict[int, list[str]] = defaultdict(list) |
| 4 | for w in ["a", "bb", "ccc", "dd"]: |
| 5 | by_len[len(w)].append(w) |
| 6 | print(dict(by_len)) |
| 7 | |
| 8 | index = defaultdict(set) |
| 9 | for i, w in enumerate(["a", "b", "a"]): |
| 10 | index[w].add(i) |
| 11 | |
| 12 | # nested |
| 13 | tree = defaultdict(lambda: defaultdict(int)) |
| 14 | tree["en"]["hello"] += 1 |
⚠
warning
defaultdict factory is called with zero args. Do not pass list.append — pass list.
deque
deque.py
Python
| 1 | from collections import deque |
| 2 | |
| 3 | q = deque([1, 2, 3], maxlen=5) |
| 4 | q.append(4) |
| 5 | q.appendleft(0) |
| 6 | print(q.pop(), q.popleft()) |
| 7 | |
| 8 | # sliding window average |
| 9 | def moving_avg(xs, k): |
| 10 | window = deque(maxlen=k) |
| 11 | for x in xs: |
| 12 | window.append(x) |
| 13 | if len(window) == k: |
| 14 | yield sum(window) / k |
| 15 | |
| 16 | print(list(moving_avg(range(10), 3))) |
| 17 | |
| 18 | # rotate |
| 19 | d = deque("abcde") |
| 20 | d.rotate(2) |
| 21 | print("".join(d)) |
namedtuple
namedtuple.py
Python
| 1 | from collections import namedtuple |
| 2 | |
| 3 | User = namedtuple("User", ["id", "name", "role"], defaults=["user"]) |
| 4 | u = User(1, "Ada") |
| 5 | print(u.id, u._asdict(), u._replace(role="admin")) |
| 6 | |
| 7 | # Prefer typing.NamedTuple for annotations |
| 8 | from typing import NamedTuple |
| 9 | |
| 10 | class Point(NamedTuple): |
| 11 | x: float |
| 12 | y: float |
| 13 | def dist(self) -> float: |
| 14 | return (self.x ** 2 + self.y ** 2) ** 0.5 |
| 15 | |
| 16 | print(Point(3, 4).dist()) |
📝
note
For mutable records with defaults, prefer dataclasses.
ChainMap
chainmap.py
Python
| 1 | from collections import ChainMap |
| 2 | |
| 3 | defaults = {"color": "red", "user": "guest"} |
| 4 | env = {"user": "ada"} |
| 5 | cli = {"color": "blue"} |
| 6 | cfg = ChainMap(cli, env, defaults) |
| 7 | print(cfg["color"], cfg["user"]) # blue, ada |
| 8 | |
| 9 | # writes hit the first map |
| 10 | cfg["theme"] = "dark" |
| 11 | print(cli) |
itertools — Core Tools
itertools_core.py
Python
| 1 | from itertools import ( |
| 2 | chain, accumulate, compress, dropwhile, takewhile, |
| 3 | filterfalse, islice, pairwise, starmap, zip_longest, |
| 4 | ) |
| 5 | |
| 6 | print(list(chain([1, 2], (3, 4), "ab"))) |
| 7 | print(list(accumulate([1, 2, 3, 4]))) |
| 8 | print(list(compress("abcdef", [1, 0, 1, 0, 1, 1]))) |
| 9 | print(list(dropwhile(lambda x: x < 3, [1, 2, 3, 1]))) |
| 10 | print(list(islice(range(100), 10, 20, 2))) |
| 11 | print(list(pairwise(range(4)))) # 3.10+: (0,1),(1,2),(2,3) |
| 12 | print(list(zip_longest("ab", "xyz", fillvalue="-"))) |
| 13 | print(list(starmap(pow, [(2, 3), (3, 2)]))) |
groupby
Keys must be sorted first — groupby only groups consecutive equal keys.
groupby.py
Python
| 1 | from itertools import groupby |
| 2 | from operator import itemgetter |
| 3 | |
| 4 | rows = [ |
| 5 | {"city": "NYC", "name": "Ada"}, |
| 6 | {"city": "NYC", "name": "Grace"}, |
| 7 | {"city": "SF", "name": "Alan"}, |
| 8 | ] |
| 9 | rows.sort(key=itemgetter("city")) |
| 10 | for city, group in groupby(rows, key=itemgetter("city")): |
| 11 | names = [r["name"] for r in group] |
| 12 | print(city, names) |
product, permutations, combinations
combo.py
Python
| 1 | from itertools import product, permutations, combinations, combinations_with_replacement |
| 2 | |
| 3 | print(list(product("AB", repeat=2))) |
| 4 | print(list(product([0, 1], ["x", "y"]))) |
| 5 | print(list(permutations("ABC", 2))) |
| 6 | print(list(combinations("ABCD", 2))) |
| 7 | print(list(combinations_with_replacement("ABC", 2))) |
| 8 | |
| 9 | # Cartesian grid |
| 10 | for x, y in product(range(2), range(2)): |
| 11 | print(x, y) |
Infinite Iterators
infinite.py
Python
| 1 | from itertools import count, cycle, repeat, islice |
| 2 | |
| 3 | print(list(islice(count(10, 2), 5))) # 10,12,14,16,18 |
| 4 | print(list(islice(cycle("AB"), 5))) # A B A B A |
| 5 | print(list(repeat("x", 3))) |
more-itertools
The third-party more-itertools package adds battle-tested recipes: chunked, windowed, unique_everseen, flatten, spy, and more.
more_itertools_demo.py
Python
| 1 | # pip install more-itertools |
| 2 | from more_itertools import chunked, windowed, unique_everseen, flatten |
| 3 | |
| 4 | print(list(chunked(range(10), 3))) |
| 5 | print(list(windowed(range(5), 3))) |
| 6 | print(list(unique_everseen("AABBCCAB"))) |
| 7 | print(list(flatten([[1, 2], [3], [4, 5]]))) |
Composable Recipes
recipes.py
Python
| 1 | from collections import Counter, deque |
| 2 | from itertools import islice, groupby |
| 3 | from collections.abc import Iterable, Iterator |
| 4 | |
| 5 | def batched(iterable: Iterable, n: int) -> Iterator[tuple]: |
| 6 | """Python 3.12+ has itertools.batched; this works earlier.""" |
| 7 | it = iter(iterable) |
| 8 | while batch := tuple(islice(it, n)): |
| 9 | yield batch |
| 10 | |
| 11 | def top_n(xs: Iterable[str], n: int = 10) -> list[tuple[str, int]]: |
| 12 | return Counter(xs).most_common(n) |
| 13 | |
| 14 | def consume(it: Iterable, n: int | None = None) -> None: |
| 15 | if n is None: |
| 16 | deque(it, maxlen=0) |
| 17 | else: |
| 18 | next(islice(it, n, n), None) |
| 19 | |
| 20 | print(list(batched(range(10), 3))) |
| 21 | print(top_n("abracadabra", 3)) |
OrderedDict (rare)
Since Python 3.7, dict preserves insertion order. OrderedDict still offers move_to_end and equality that respects order.
ordereddict.py
Python
| 1 | from collections import OrderedDict |
| 2 | |
| 3 | od = OrderedDict(a=1, b=2, c=3) |
| 4 | od.move_to_end("a") |
| 5 | print(list(od)) |
| 6 | od.move_to_end("b", last=False) |
| 7 | print(list(od)) |
| 8 | print(OrderedDict(a=1, b=2) == OrderedDict(b=2, a=1)) # False |
| 9 | print(dict(a=1, b=2) == dict(b=2, a=1)) # True |
When to Use What
| Need | Reach for |
|---|---|
| Frequencies | Counter |
| Multi-map / grouping | defaultdict(list/set) |
| Queue / stack / window | deque |
| Immutable record | NamedTuple / dataclass |
| Layered config | ChainMap |
| Lazy pipelines | itertools + generators |
| Cartesian / combos | product / combinations |
| Chunked batches | itertools.batched (3.12) / islice |
Performance Notes
- Counter is a dict subclass — O(1) average updates
- deque append/popleft are O(1); list.pop(0) is O(n)
- groupby without prior sort is a common bug
- product of large ranges can explode — bound inputs
- Prefer generator expressions over list comps when feeding itertools
perf.py
Python
| 1 | from collections import deque |
| 2 | from timeit import timeit |
| 3 | |
| 4 | n = 10_000 |
| 5 | print("list pop(0)", timeit(lambda: (lambda L: [L.pop(0) for _ in range(len(L))])(list(range(n))), number=10)) |
| 6 | print("deque popleft", timeit(lambda: (lambda d: [d.popleft() for _ in range(len(d))])(deque(range(n))), number=10)) |
UserDict / UserList / UserString
Subclass-friendly wrappers when you need to customize dict/list/str behavior safely.
userdict.py
Python
| 1 | from collections import UserDict |
| 2 | |
| 3 | class LowerDict(UserDict): |
| 4 | def __setitem__(self, key, value): |
| 5 | super().__setitem__(str(key).lower(), value) |
| 6 | def __getitem__(self, key): |
| 7 | return super().__getitem__(str(key).lower()) |
| 8 | |
| 9 | d = LowerDict() |
| 10 | d["Host"] = "localhost" |
| 11 | print(d["host"]) |
$Blueprint — Engineering Documentation·Section ID: PYTHON-COLLECTIONS·Revision: 1.0
Community
Get help on Slack, Discord or VIP
Stuck on a guide? Join the community and ask.