|$ curl https://forge-ai.dev/api/markdown?path=docs/python/file-io
$cat docs/python-—-file-i/o.md
updated Recently·14 min read·published
Python — File I/O
◆Python◆Intermediate
Introduction
File I/O is a fundamental part of nearly every Python program. Python provides several layers of abstraction for working with files: the built-in open() function, the pathlib module for modern path handling, and modules like tempfile and shutil for specialized operations.
open() Modes
The open() function takes a mode string that controls how the file is opened. The mode combines a primary access mode with optional modifiers.
modes.py
Python
| 1 | # Primary access modes: |
| 2 | # "r" — read (default). File must exist. |
| 3 | # "w" — write. Creates file or truncates existing. |
| 4 | # "a" — append. Creates file or writes at end. |
| 5 | # "x" — exclusive create. Fails if file exists. |
| 6 | # "r+" — read + write. File must exist. |
| 7 | # "w+" — read + write. Creates or truncates. |
| 8 | # "a+" — read + append. Creates or writes at end. |
| 9 | |
| 10 | # Modifiers (appended to primary mode): |
| 11 | # "b" — binary mode (e.g. "rb", "wb") |
| 12 | # "t" — text mode (default, e.g. "rt", "wt") |
| 13 | |
| 14 | # Examples: |
| 15 | open("file.txt", "r") # text read (default) |
| 16 | open("file.txt", "w") # text write (truncate) |
| 17 | open("file.txt", "a") # text append |
| 18 | open("file.txt", "x") # exclusive create (raises FileExistsError) |
| 19 | open("file.bin", "rb") # binary read |
| 20 | open("file.bin", "wb") # binary write |
| 21 | |
| 22 | # r+ vs w+ vs a+: |
| 23 | # r+: file must exist, write at current position |
| 24 | # w+: always creates new file (or truncates) |
| 25 | # a+: always writes at end, regardless of seek |
| 26 | # (OS-level append — atomic on some systems) |
| 27 | |
| 28 | # Exclusive create (x) is useful for lock files: |
| 29 | try: |
| 30 | with open("lock.pid", "x") as f: |
| 31 | f.write(str(os.getpid())) |
| 32 | except FileExistsError: |
| 33 | print("another instance is running") |
ℹ
info
Text mode ("t") is the default under the hood. You almost never need to write "rt" — just "r" suffices. The "b" modifier is the important one when you need binary access.
Reading Files
reading.py
Python
| 1 | # Read entire file as a single string |
| 2 | with open("data.txt") as f: |
| 3 | content = f.read() |
| 4 | |
| 5 | # Read at most N characters |
| 6 | with open("data.txt") as f: |
| 7 | chunk = f.read(1024) # first 1024 chars |
| 8 | |
| 9 | # Read one line |
| 10 | with open("data.txt") as f: |
| 11 | line = f.readline() # "first line\n" |
| 12 | second = f.readline() # "second line\n" |
| 13 | |
| 14 | # Read all lines into a list |
| 15 | with open("data.txt") as f: |
| 16 | lines = f.readlines() # ["line1\n", "line2\n", ...] |
| 17 | |
| 18 | # Best: iterate over lines (memory-efficient) |
| 19 | with open("data.txt") as f: |
| 20 | for line in f: |
| 21 | print(line.rstrip("\n")) # process one line at a time |
| 22 | |
| 23 | # Read lines with iteration + index |
| 24 | with open("data.txt") as f: |
| 25 | for i, line in enumerate(f, 1): |
| 26 | if "ERROR" in line: |
| 27 | print(f"line {i}: {line}", end="") |
| 28 | |
| 29 | # Read until a delimiter |
| 30 | with open("data.txt") as f: |
| 31 | record = f.read().split("\n---\n") # split on custom delimiter |
✓
best practice
Use for line in f: (iteration) over .readlines() for large files. Iteration reads lazily one line at a time, while readlines() loads the entire file into memory at once.
Writing Files
writing.py
Python
| 1 | # Write a string |
| 2 | with open("output.txt", "w") as f: |
| 3 | f.write("Hello, World!\n") |
| 4 | |
| 5 | # Write multiple strings (no newlines added) |
| 6 | with open("output.txt", "w") as f: |
| 7 | f.writelines([ |
| 8 | "line 1\n", |
| 9 | "line 2\n", |
| 10 | "line 3\n", |
| 11 | ]) |
| 12 | |
| 13 | # writelines with list of strings — no separator added |
| 14 | # This is equivalent to: |
| 15 | # for line in lines: f.write(line) |
| 16 | |
| 17 | # Appending to existing file |
| 18 | with open("log.txt", "a") as f: |
| 19 | f.write(f"[{datetime.now()}] request received\n") |
| 20 | |
| 21 | # Print to file (convenient for formatted output) |
| 22 | with open("output.txt", "w") as f: |
| 23 | print("Hello, World!", file=f) |
| 24 | print(f"Value: {42}", file=f) |
| 25 | |
| 26 | # Flushing — force write buffer to disk |
| 27 | with open("output.txt", "w") as f: |
| 28 | f.write("critical data") |
| 29 | f.flush() # ensure data is written to disk |
| 30 | # File stays open — useful before long operations |
| 31 | |
| 32 | # Buffering control: |
| 33 | # buffering=0 → unbuffered (binary mode only) |
| 34 | # buffering=1 → line-buffered (text mode only) |
| 35 | # buffering=N → buffer N bytes before flushing |
| 36 | # Default: 4096 or 8192 depending on system |
| 37 | with open("data.bin", "wb", buffering=0) as f: |
| 38 | f.write(b"immediate write, no buffer") |
ℹ
info
writelines() does NOT add newline characters. You must include them in each string. Despite the name, it's really "write many strings" — the "lines" naming is historical.
Binary Files & Seek/Tell
Binary mode ("b") reads and writes bytes objects instead of str. No encoding conversion or newline translation is performed.
binary.py
Python
| 1 | # Reading binary files |
| 2 | with open("image.jpg", "rb") as f: |
| 3 | header = f.read(4) # first 4 bytes |
| 4 | if header == b"\xff\xd8\xff\xe0": |
| 5 | print("JPEG detected") |
| 6 | |
| 7 | # Writing binary data |
| 8 | with open("output.bin", "wb") as f: |
| 9 | f.write(b"\x00\x01\x02\x03") |
| 10 | f.write(bytes([0x04, 0x05, 0x06])) |
| 11 | |
| 12 | # seek() — move to a position (offset, whence): |
| 13 | # whence=0: relative to file start (default) |
| 14 | # whence=1: relative to current position |
| 15 | # whence=2: relative to file end |
| 16 | with open("data.bin", "rb") as f: |
| 17 | f.seek(0, 2) # seek to end |
| 18 | size = f.tell() # get current position = file size |
| 19 | print(f"file size: {size} bytes") |
| 20 | |
| 21 | f.seek(0) # seek to beginning |
| 22 | f.seek(100) # skip to byte 100 |
| 23 | data = f.read(50) # read bytes 100-149 |
| 24 | |
| 25 | f.seek(-10, 2) # 10 bytes before end |
| 26 | tail = f.read() # last 10 bytes |
| 27 | |
| 28 | f.seek(10, 1) # skip 10 bytes forward |
| 29 | |
| 30 | # tell() — get current position (bytes from start) |
| 31 | with open("data.bin", "rb") as f: |
| 32 | print(f.tell()) # 0 |
| 33 | f.read(50) |
| 34 | print(f.tell()) # 50 |
| 35 | |
| 36 | # Seek in text mode — only seek(0) or seek(0, 2) allowed |
| 37 | with open("data.txt", "r") as f: |
| 38 | f.seek(0) # OK — go to start |
| 39 | f.seek(0, 2) # OK — go to end |
| 40 | # f.seek(50) # ERROR in text mode (variable-width encoding) |
| 41 | # f.seek(-10, 1) # ERROR in text mode |
✓
best practice
seek() in text mode is fragile because multi-byte encodings (UTF-8) make byte offsets unpredictable. Use binary mode if you need random access positioning, or read line-by-line and track position yourself.
pathlib — Modern Approach
The pathlib module provides an object-oriented interface for filesystem paths, with convenience methods for common I/O operations.
pathlib.py
Python
| 1 | from pathlib import Path |
| 2 | |
| 3 | # Reading and writing text (in one call) |
| 4 | content = Path("data.txt").read_text(encoding="utf-8") |
| 5 | Path("output.txt").write_text("hello world", encoding="utf-8") |
| 6 | |
| 7 | # Reading and writing bytes |
| 8 | data = Path("image.jpg").read_bytes() |
| 9 | Path("copy.jpg").write_bytes(data) |
| 10 | |
| 11 | # Path construction and properties |
| 12 | p = Path("/usr/local/bin/python3") |
| 13 | p.parent # → /usr/local/bin |
| 14 | p.name # → python3 |
| 15 | p.stem # → python (no suffix) |
| 16 | p.suffix # → (empty — no .ext) |
| 17 | p.suffixes # → [] (list of extensions) |
| 18 | |
| 19 | p = Path("archive.tar.gz") |
| 20 | p.suffix # → .gz |
| 21 | p.suffixes # → [".tar", ".gz"] |
| 22 | p.stem # → archive.tar |
| 23 | p.with_suffix(".bz2") # → archive.tar.bz2 |
| 24 | |
| 25 | p = Path("data.txt") |
| 26 | p.exists() # → bool |
| 27 | p.is_file() |
| 28 | p.is_dir() |
| 29 | p.stat().st_size # file size in bytes |
| 30 | p.stat().st_mtime # last modified timestamp |
| 31 | |
| 32 | # Absolute vs relative |
| 33 | Path("data.txt").resolve() # → /Users/me/data.txt |
| 34 | Path("data.txt").absolute() # → /Users/me/data.txt |
| 35 | |
| 36 | # Iterate directory |
| 37 | for child in Path(".").iterdir(): |
| 38 | print(child.name, "dir?" if child.is_dir() else "file") |
| 39 | |
| 40 | # Glob patterns |
| 41 | for py in Path(".").glob("**/*.py"): |
| 42 | print(py) |
| 43 | |
| 44 | # Relative paths |
| 45 | p = Path("/a/b/c/d.txt") |
| 46 | p.relative_to("/a/b") # → PosixPath("c/d.txt") |
| 47 | |
| 48 | # Join paths with / |
| 49 | config = Path.home() / ".config" / "myapp" / "config.json" |
ℹ
info
pathlib is the modern, preferred approach for file I/O in Python 3.6+. The / operator for path joining reads much cleaner than nested os.path.join() calls. Prefer it over os.path for new code.
tempfile Module
The tempfile module creates temporary files and directories that are automatically cleaned up, with secure random names.
tempfile.py
Python
| 1 | import tempfile |
| 2 | from pathlib import Path |
| 3 | |
| 4 | # Temporary file — auto-deleted on close |
| 5 | with tempfile.TemporaryFile() as f: |
| 6 | f.write(b"temporary data") |
| 7 | f.seek(0) |
| 8 | print(f.read()) # b"temporary data" |
| 9 | # File deleted after leaving 'with' block |
| 10 | |
| 11 | # Named temporary file — visible in filesystem |
| 12 | with tempfile.NamedTemporaryFile( |
| 13 | suffix=".txt", |
| 14 | prefix="tmp_", |
| 15 | delete=True, # delete on close (default) |
| 16 | ) as f: |
| 17 | print(f.name) # e.g. /tmp/tmp_abc123.txt |
| 18 | f.write(b"hello") |
| 19 | # File deleted (unless delete=False) |
| 20 | |
| 21 | # Permanent temp file (not auto-deleted) |
| 22 | f = tempfile.NamedTemporaryFile(delete=False) |
| 23 | f.write(b"persistent data") |
| 24 | path = f.name |
| 25 | f.close() |
| 26 | # Must delete manually later: |
| 27 | Path(path).unlink() |
| 28 | |
| 29 | # Temporary directory |
| 30 | with tempfile.TemporaryDirectory() as tmpdir: |
| 31 | path = Path(tmpdir) / "data.txt" |
| 32 | path.write_text("hello") |
| 33 | print(path.read_text()) # "hello" |
| 34 | # Entire directory tree deleted after 'with' |
| 35 | |
| 36 | # Get system temp directory |
| 37 | print(tempfile.gettempdir()) # e.g. /tmp |
| 38 | print(tempfile.gettempprefix()) # e.g. tmp |
| 39 | |
| 40 | # Secure temp file (less predictable names) |
| 41 | # TemporaryFile/NamedTemporaryFile already use |
| 42 | # secure random names by default |
✓
best practice
Always use TemporaryFile() or NamedTemporaryFile() as context managers. If you set delete=False, you are responsible for cleaning up with Path(path).unlink() — otherwise temp files accumulate.
File & Directory Operations
file_ops.py
Python
| 1 | from pathlib import Path |
| 2 | import os |
| 3 | import shutil |
| 4 | |
| 5 | # ── Creating ── |
| 6 | Path("new_dir").mkdir() # single directory |
| 7 | Path("a/b/c").mkdir(parents=True) # recursive (like mkdir -p) |
| 8 | os.makedirs("a/b/c", exist_ok=True) # legacy equivalent |
| 9 | |
| 10 | # ── Removing ── |
| 11 | Path("empty_dir").rmdir() # directory must be empty |
| 12 | Path("file.txt").unlink() # delete file (no-op if missing) |
| 13 | Path("file.txt").unlink(missing_ok=True) # Python 3.8+ — no error |
| 14 | |
| 15 | # ── Rename / Move ── |
| 16 | Path("old.txt").rename("new.txt") |
| 17 | Path("old.txt").replace("new.txt") # overwrites existing |
| 18 | |
| 19 | # ── Copy (shutil) ── |
| 20 | shutil.copy("src.txt", "dst.txt") # file → file |
| 21 | shutil.copy("src.txt", "backup/") # file → directory |
| 22 | shutil.copy2("src.txt", "dst.txt") # preserves metadata |
| 23 | shutil.copytree("src_dir", "dst_dir") # recursive directory copy |
| 24 | shutil.copytree("src", "dst", dirs_exist_ok=True) # Python 3.8+ |
| 25 | |
| 26 | # ── Move (shutil) ── |
| 27 | shutil.move("src.txt", "dest/subdir/") # move file into directory |
| 28 | shutil.move("old_dir", "new_dir") # rename directory |
| 29 | |
| 30 | # ── Delete tree ── |
| 31 | shutil.rmtree("dir_to_delete") # delete directory tree |
| 32 | |
| 33 | # ── File info (stat) ── |
| 34 | stat = Path("file.txt").stat() |
| 35 | stat.st_size # size in bytes |
| 36 | stat.st_mode # permission bits |
| 37 | stat.st_mtime # last modified (timestamp) |
| 38 | stat.st_ctime # creation time (Unix) / metadata change (Windows) |
| 39 | stat.st_atime # last access time |
| 40 | |
| 41 | # ── Glob / Listing ── |
| 42 | for p in Path(".").glob("*.txt"): # current dir only |
| 43 | print(p) |
| 44 | |
| 45 | for p in Path(".").rglob("*.py"): # recursive (**/*.py) |
| 46 | print(p) |
| 47 | |
| 48 | for p in Path(".").iterdir(): # all entries (non-recursive) |
| 49 | pass |
| 50 | |
| 51 | # ── os.path legacy (avoid in new code) ── |
| 52 | os.path.exists("file.txt") |
| 53 | os.path.isfile("file.txt") |
| 54 | os.path.isdir("dir") |
| 55 | os.path.join("a", "b", "c.txt") |
| 56 | os.path.splitext("file.txt") # → ("file", ".txt") |
ℹ
info
Prefer pathlib over os.path for all new code. It provides method chaining, operator overloading (/ for join), and a cleaner API. Fall back to shutil for copy/move/delete-tree operations that pathlib doesn't cover.
Best Practices
best_practices.py
Python
| 1 | # 1. Always use context managers ('with' statement) |
| 2 | # BAD: |
| 3 | f = open("data.txt") |
| 4 | data = f.read() |
| 5 | f.close() # easy to forget → resource leak |
| 6 | |
| 7 | # GOOD: |
| 8 | with open("data.txt") as f: |
| 9 | data = f.read() |
| 10 | # auto-closed, even on exception |
| 11 | |
| 12 | # 2. Specify encoding explicitly |
| 13 | # BAD (platform-dependent default): |
| 14 | with open("data.txt") as f: |
| 15 | ... |
| 16 | |
| 17 | # GOOD (portable): |
| 18 | with open("data.txt", encoding="utf-8") as f: |
| 19 | ... |
| 20 | |
| 21 | # 3. Prefer pathlib for new code |
| 22 | # BAD: |
| 23 | import os |
| 24 | content = open(os.path.join("dir", "file.txt")).read() |
| 25 | |
| 26 | # GOOD: |
| 27 | from pathlib import Path |
| 28 | content = Path("dir/file.txt").read_text(encoding="utf-8") |
| 29 | |
| 30 | # 4. Handle newline differences portably |
| 31 | # Python normalizes to '\n' in text mode by default |
| 32 | # On Windows, '\r\n' → '\n' (reading) and vice versa (writing) |
| 33 | with open("data.txt", newline="") as f: |
| 34 | # newline="" — disable translation, read raw lines |
| 35 | for line in f: |
| 36 | print(repr(line)) # keeps original line endings |
| 37 | |
| 38 | # 5. EAFP for file operations |
| 39 | # LBYL — race condition (file could be deleted between check and open) |
| 40 | if Path("data.txt").exists(): |
| 41 | with open("data.txt") as f: # still might fail! |
| 42 | ... |
| 43 | |
| 44 | # EAFP — Pythonic |
| 45 | try: |
| 46 | with open("data.txt") as f: |
| 47 | data = f.read() |
| 48 | except FileNotFoundError: |
| 49 | data = "" |
| 50 | |
| 51 | # 6. Use file-like objects for testing / mocking |
| 52 | from io import StringIO, BytesIO |
| 53 | |
| 54 | # StringIO acts like a text file in memory |
| 55 | buf = StringIO() |
| 56 | buf.write("hello") |
| 57 | buf.seek(0) |
| 58 | print(buf.read()) # "hello" |
| 59 | |
| 60 | # BytesIO acts like a binary file in memory |
| 61 | bbuf = BytesIO() |
| 62 | bbuf.write(b"\x00\x01\x02") |
| 63 | bbuf.seek(0) |
| 64 | print(bbuf.read()) # b"\x00\x01\x02" |
| 65 | |
| 66 | # Useful for testing — avoid real files in unit tests |
| 67 | def process(f): |
| 68 | return f.read().upper() |
| 69 | |
| 70 | assert process(StringIO("hello")) == "HELLO" |
✓
best practice
File-like objects (StringIO / BytesIO) are invaluable for testing. They implement the same interface as real file objects, so you can pass them to any function that expects a file — no temp files needed in unit tests.
$Blueprint — Engineering Documentation·Section ID: PYTHON-FIO·Revision: 1.0