Learn AI Series):Exercise 1: Pose analysis toolkit.
import numpy as np
# COCO keypoint indices
NOSE, L_EYE, R_EYE = 0, 1, 2
L_EAR, R_EAR = 3, 4
L_SHOULDER, R_SHOULDER = 5, 6
L_ELBOW, R_ELBOW = 7, 8
L_WRIST, R_WRIST = 9, 10
L_HIP, R_HIP = 11, 12
L_KNEE, R_KNEE = 13, 14
L_ANKLE, R_ANKLE = 15, 16
class PoseAnalyzer:
"""Analyze COCO-format pose keypoints."""
def __init__(self, keypoints):
"""keypoints: (17, 3) array -- x, y, vis."""
self.kps = keypoints
def _angle(self, a_idx, b_idx, c_idx):
"""Angle at joint b, segments a-b and b-c."""
a = self.kps[a_idx, :2]
b = self.kps[b_idx, :2]
c = self.kps[c_idx, :2]
ba = a - b
bc = c - b
cos = np.dot(ba, bc) / (
np.linalg.norm(ba) * np.linalg.norm(bc)
+ 1e-8
)
cos = np.clip(cos, -1.0, 1.0)
return np.degrees(np.arccos(cos))
def joint_angles(self):
return {
"left_elbow": self._angle(
L_SHOULDER, L_ELBOW, L_WRIST),
"right_elbow": self._angle(
R_SHOULDER, R_ELBOW, R_WRIST),
"left_knee": self._angle(
L_HIP, L_KNEE, L_ANKLE),
"right_knee": self._angle(
R_HIP, R_KNEE, R_ANKLE),
}
def classify_pose(self):
angles = self.joint_angles()
lk = angles["left_knee"]
rk = angles["right_knee"]
if (self.kps[L_WRIST, 1]
< self.kps[L_SHOULDER, 1]
and self.kps[R_WRIST, 1]
< self.kps[R_SHOULDER, 1]):
return "arms_raised"
if lk > 160 and rk > 160:
return "standing"
if lk < 120 or rk < 120:
return "sitting"
return "unknown"
def symmetry_score(self):
angles = self.joint_angles()
diffs = []
for side in ["elbow", "knee"]:
left = angles[f"left_{side}"]
right = angles[f"right_{side}"]
mx = max(left, right, 1e-8)
diffs.append(abs(left - right) / mx)
return max(0.0, 1.0 - np.mean(diffs))
# Standing pose (legs straight)
standing = np.zeros((17, 3))
standing[:, 2] = 1.0
standing[L_HIP] = [100, 200, 1]
standing[R_HIP] = [140, 200, 1]
standing[L_KNEE] = [100, 350, 1]
standing[R_KNEE] = [140, 350, 1]
standing[L_ANKLE] = [100, 500, 1]
standing[R_ANKLE] = [140, 500, 1]
standing[L_SHOULDER] = [90, 100, 1]
standing[R_SHOULDER] = [150, 100, 1]
standing[L_ELBOW] = [70, 160, 1]
standing[R_ELBOW] = [170, 160, 1]
standing[L_WRIST] = [60, 220, 1]
standing[R_WRIST] = [180, 220, 1]
pa = PoseAnalyzer(standing)
print(f"Standing: {pa.classify_pose()}")
print(f"Symmetry: {pa.symmetry_score():.3f}")
# Sitting (knees bent)
sitting = standing.copy()
sitting[L_KNEE] = [100, 250, 1]
sitting[R_KNEE] = [140, 250, 1]
sitting[L_ANKLE] = [150, 300, 1]
sitting[R_ANKLE] = [190, 300, 1]
pa2 = PoseAnalyzer(sitting)
print(f"Sitting: {pa2.classify_pose()}")
# Arms raised
arms_up = standing.copy()
arms_up[L_WRIST] = [80, 30, 1]
arms_up[R_WRIST] = [160, 30, 1]
arms_up[L_ELBOW] = [85, 60, 1]
arms_up[R_ELBOW] = [155, 60, 1]
pa3 = PoseAnalyzer(arms_up)
print(f"Arms up: {pa3.classify_pose()}")
Pose classification from raw keypoints is pure geometry -- compute angles at joints, apply threshold rules. The symmetry score normalizes by the larger angle to avoid division artifacts when both sides are near zero. Even simple threshold classifiers like this work well for basic pose categories.
Exercise 2: Multi-object tracker with ID management.
import numpy as np
from scipy.optimize import linear_sum_assignment
def compute_iou(box1, box2):
x1 = max(box1[0], box2[0])
y1 = max(box1[1], box2[1])
x2 = min(box1[2], box2[2])
y2 = min(box1[3], box2[3])
inter = max(0, x2 - x1) * max(0, y2 - y1)
a1 = (box1[2] - box1[0]) * (box1[3] - box1[1])
a2 = (box2[2] - box2[0]) * (box2[3] - box2[1])
return inter / max(a1 + a2 - inter, 1e-6)
class MultiObjectTracker:
def __init__(self, min_hits=3, max_age=5,
iou_thresh=0.3):
self.tracks = {}
self.next_id = 0
self.min_hits = min_hits
self.max_age = max_age
self.iou_thresh = iou_thresh
def update(self, detections):
predicted = {
tid: [trk["box"][i] + trk["vel"][i]
for i in range(4)]
for tid, trk in self.tracks.items()
}
if not predicted:
for det in detections:
self._new_track(det)
return self._results()
pids = list(predicted.keys())
pboxes = [predicted[k] for k in pids]
cost = np.zeros((len(pboxes), len(detections)))
for i in range(len(pboxes)):
for j in range(len(detections)):
cost[i, j] = 1.0 - compute_iou(
pboxes[i], detections[j])
row_idx, col_idx = linear_sum_assignment(cost)
m_p, m_d = set(), set()
for r, c in zip(row_idx, col_idx):
if cost[r, c] > (1 - self.iou_thresh):
continue
tid = pids[r]
old = self.tracks[tid]["box"]
new = detections[c]
self.tracks[tid]["vel"] = [
new[i] - old[i] for i in range(4)]
self.tracks[tid]["box"] = new
self.tracks[tid]["age"] = 0
self.tracks[tid]["hits"] += 1
m_p.add(r)
m_d.add(c)
for j in range(len(detections)):
if j not in m_d:
self._new_track(detections[j])
for i in range(len(pids)):
if i not in m_p:
self.tracks[pids[i]]["age"] += 1
self.tracks = {
k: v for k, v in self.tracks.items()
if v["age"] <= self.max_age}
return self._results()
def _new_track(self, box):
self.tracks[self.next_id] = {
"box": box, "vel": [0, 0, 0, 0],
"age": 0, "hits": 1}
self.next_id += 1
def _results(self):
return [
(tid, trk["box"],
"confirmed" if trk["hits"] >= self.min_hits
else "tentative")
for tid, trk in self.tracks.items()]
tracker = MultiObjectTracker()
for frame in range(20):
dets = []
dets.append([50 + frame * 5, 100,
100 + frame * 5, 200])
dets.append([300, 50 + frame * 4,
380, 130 + frame * 4])
if frame < 8 or frame > 10:
dets.append([200, 200 + frame * 3,
260, 260 + frame * 3])
results = tracker.update(dets)
conf = [tid for tid, _, s in results
if s == "confirmed"]
tent = [tid for tid, _, s in results
if s == "tentative"]
print(f"Frame {frame:2d}: confirmed={conf} "
f"tentative={tent}")
Object C disappears at frames 8-10 and reappears at frame 11. With max_age=5, the old track stays alive during the gap. Whether C keeps its old ID or gets a new one depends on whether the predicted position (extrapolated from pre-disappearance velocity) overlaps enough with the reappearance position -- this is exactly the ID switch problem that DeepSORT solves with appearance features.
Exercise 3: OKS-based pose evaluation framework.
import numpy as np
COCO_SIGMAS = np.array([
0.026, 0.025, 0.025, 0.035, 0.035,
0.079, 0.079, 0.072, 0.072, 0.062,
0.062, 0.107, 0.107, 0.087, 0.087,
0.089, 0.089])
class PoseEvaluator:
def compute_oks(self, pred, gt, area):
vis = gt[:, 2] > 0
if vis.sum() == 0:
return 0.0
d2 = (pred[:, 0] - gt[:, 0]) ** 2 + \
(pred[:, 1] - gt[:, 1]) ** 2
s = 2 * (COCO_SIGMAS ** 2) * area
return float(
np.exp(-d2 / (s + 1e-6))[vis].mean())
def compute_ap(self, preds, gts, thresh):
preds = sorted(preds, key=lambda p: p["score"],
reverse=True)
if not gts:
return 0.0
tp = np.zeros(len(preds))
fp = np.zeros(len(preds))
matched = set()
for i, p in enumerate(preds):
best_oks, best_j = 0, -1
for j, g in enumerate(gts):
o = self.compute_oks(
p["kps"], g["kps"], g["area"])
if o > best_oks:
best_oks, best_j = o, j
if best_oks >= thresh and best_j not in matched:
tp[i] = 1
matched.add(best_j)
else:
fp[i] = 1
tc, fc = np.cumsum(tp), np.cumsum(fp)
prec = tc / (tc + fc)
rec = tc / len(gts)
return float(np.trapz(prec, rec))
def evaluate(self, preds, gts):
return {
"AP50": self.compute_ap(preds, gts, 0.50),
"AP75": self.compute_ap(preds, gts, 0.75),
"AP": np.mean([
self.compute_ap(preds, gts, t)
for t in np.arange(0.5, 1.0, 0.05)])}
rng = np.random.RandomState(42)
gts = []
for _ in range(10):
for _ in range(rng.randint(1, 4)):
kps = rng.rand(17, 3) * 400
kps[:, 2] = 1.0
gts.append({"kps": kps, "area": 40000.0})
ev = PoseEvaluator()
perfect = [{"kps": g["kps"].copy(), "score": 1.0}
for g in gts]
good = [{"kps": g["kps"] + np.c_[
rng.randn(17, 2) * 3,
np.zeros((17, 1))],
"score": rng.uniform(0.7, 0.99)} for g in gts]
poor = [{"kps": g["kps"] + np.c_[
rng.randn(17, 2) * 15,
np.zeros((17, 1))],
"score": rng.uniform(0.3, 0.7)}
for g in gts if rng.random() > 0.4]
print(f"{'Detector':<10} {'AP50':>6} {'AP75':>6} "
f"{'AP':>6}")
for name, p in [("perfect", perfect),
("good", good), ("poor", poor)]:
r = ev.evaluate(p, gts)
print(f"{name:<10} {r['AP50']:>6.3f} "
f"{r['AP75']:>6.3f} {r['AP']:>6.3f}")
The perfect detector gets AP=1.0 across all thresholds. The gap between AP50 and AP75 for the "good" detector shows how OKS penalizes imprecise localization -- 3 pixels of noise is tolerable at the 0.50 threshold but starts to hurt at 0.75, especially for low-sigma keypoints like the nose (0.026).
Here we go! Over the last five episodes we've built up quite some visual understanding: image processing fundamentals (#77), object detection in two parts (#78-79), segmentation (#80), and pose estimation with tracking (#81). We can classify images, locate objects with bounding boxes, segment them at the pixel level, and track their skeletal structure through video. But there's one kind of visual content we haven't touched yet -- and it's something humans process so effortlessly that you probably don't even think about it as a "vision task."
Text.
Street signs, handwritten notes, scanned documents, screenshots, receipts, license plates, medicine labels, product packaging -- text is EVERYWHERE in the visual world, and extracting it accurately is one of the oldest and most practically useful problems in all of computer vision. Optical Character Recognition (OCR) converts images of text into machine-readable strings. It sounds simple. A human can read the word "HELLO" from an image in milliseconds. So how hard can it be for a machine?
Turns out: genuinely hard. The text might be rotated, curved, partially occluded, handwritten in terrible penmanship, printed in exotic fonts, embedded in noisy backgrounds, or photographed at odd angles with motion blur. And unlike object detection where "roughly the right bounding box" is good enough, OCR has zero tolerance for error in many applications -- misreading a single digit on a bank cheque or a prescription can have real consequences.
Modern OCR splits the problem into two distinct tasks:
Text detection: find rectangular regions (or polygons) in the image that contain text. This is essentially object detection (episodes #78-79) specialized for text. The output is a set of bounding boxes or rotated rectangles around text regions.
Text recognition: for each detected region, read the characters. This is a sequence prediction problem -- the input is a cropped image of a word or line, the output is a string of characters.
Input image (photo of a receipt)
|
Text Detection -> [box_1, box_2, box_3, ...]
|
Text Recognition -> ["GROCERY STORE", "$4.99",
"TOTAL: $23.47"]
Some newer models are end-to-end: they detect and recognize simultaneously. But the two-stage approach is still dominant because each stage can be optimized and evaluated independently, and you can swap in better components without retraining the whole pipeline.
Text detection is harder than generic object detection because text has unusual geometry. It can be any aspect ratio (a single character vs a paragraph), any orientation (horizontal, vertical, curved along a bottle), and any size (tiny copyright notices vs huge billboards). Standard anchor-based detectors like Faster R-CNN (episode #78) struggle with the extreme aspect ratios -- a text line that's 500 pixels wide and 20 pixels tall doesn't match any reasonable anchor shape ;-)
Two architectures dominate text detection today:
EAST (Efficient and Accurate Scene Text detector, Zhou et al., 2017) handles this by predicting, for each pixel, whether it's inside a text region and the distance to the four edges of the text bounding box. No anchors, no proposals -- just dense per-pixel predictions, similar to how semantic segmentation works (episode #80):
import torch
import torch.nn as nn
class TextDetector(nn.Module):
"""Simplified EAST-style text detector."""
def __init__(self):
super().__init__()
self.backbone = nn.Sequential(
nn.Conv2d(3, 64, 3, padding=1),
nn.BatchNorm2d(64),
nn.ReLU(inplace=True),
nn.Conv2d(64, 128, 3, stride=2, padding=1),
nn.BatchNorm2d(128),
nn.ReLU(inplace=True),
nn.Conv2d(128, 256, 3, stride=2, padding=1),
nn.BatchNorm2d(256),
nn.ReLU(inplace=True),
nn.ConvTranspose2d(256, 128, 4, stride=2,
padding=1),
nn.ReLU(inplace=True),
nn.ConvTranspose2d(128, 64, 4, stride=2,
padding=1),
nn.ReLU(inplace=True),
)
# Score map: is this pixel inside text?
self.score_head = nn.Conv2d(64, 1, 1)
# Geometry: distance to top, right, bottom,
# left edges + rotation angle
self.geo_head = nn.Conv2d(64, 5, 1)
def forward(self, x):
features = self.backbone(x)
score = torch.sigmoid(
self.score_head(features)
)
geometry = self.geo_head(features)
return score, geometry
detector = TextDetector()
img = torch.randn(1, 3, 256, 256)
score, geo = detector(img)
print(f"Score map: {score.shape}")
# (1, 1, 256, 256) -- per-pixel text probability
print(f"Geometry: {geo.shape}")
# (1, 5, 256, 256) -- 4 distances + 1 angle
The score map tells you "is there text at this pixel?" and the geometry tells you "how far is this pixel from each edge of the text bounding box, and at what angle is the text oriented?" At inference time, you threshold the score map, use NMS (episode #78) to merge overlapping predictions, and reconstruct the bounding boxes from the geometry predictions.
DBNet (Differentiable Binarization, Liao et al., 2020) is the current standard for text detection. The clever part is the threshold map -- instead of using a fixed threshold to binarize the probability map, DBNet predicts a per-pixel adaptive threshold. This makes the model aware of text boundaries during training, which produces much sharper text regions:
import torch
import torch.nn as nn
class DBNetHead(nn.Module):
"""Simplified DBNet head with differentiable
binarization."""
def __init__(self, in_channels=256):
super().__init__()
# Probability map (like segmentation)
self.prob_conv = nn.Sequential(
nn.Conv2d(in_channels, 64, 3, padding=1),
nn.BatchNorm2d(64),
nn.ReLU(inplace=True),
nn.ConvTranspose2d(64, 64, 2, stride=2),
nn.BatchNorm2d(64),
nn.ReLU(inplace=True),
nn.ConvTranspose2d(64, 1, 2, stride=2),
nn.Sigmoid(),
)
# Threshold map (adaptive per-pixel threshold)
self.thresh_conv = nn.Sequential(
nn.Conv2d(in_channels, 64, 3, padding=1),
nn.BatchNorm2d(64),
nn.ReLU(inplace=True),
nn.ConvTranspose2d(64, 64, 2, stride=2),
nn.BatchNorm2d(64),
nn.ReLU(inplace=True),
nn.ConvTranspose2d(64, 1, 2, stride=2),
nn.Sigmoid(),
)
def forward(self, features):
prob = self.prob_conv(features)
thresh = self.thresh_conv(features)
# Differentiable binarization
# Approximates step function during training
k = 50 # sharpness factor
binary = torch.reciprocal(
1.0 + torch.exp(-k * (prob - thresh))
)
return prob, thresh, binary
head = DBNetHead(in_channels=256)
feat = torch.randn(1, 256, 64, 64)
prob, thresh, binary = head(feat)
print(f"Probability map: {prob.shape}")
# (1, 1, 256, 256)
print(f"Threshold map: {thresh.shape}")
# (1, 1, 256, 256)
print(f"Binary map: {binary.shape}")
# (1, 1, 256, 256)
The k=50 factor in the differentiable binarization is what makes this work. A standard sigmoid with k=1 gives a smooth transition. With k=50, it approximates a step function (very sharp transition from 0 to 1) while still being differentiable so gradients can flow during training. During inference, you can use the binary map directly or fall back to simple thresholding of the probability map -- both work, but the adaptive threshold version produces sharper text boundaries.
Once you have a cropped image of a text line, you need to decode it into characters. Here's the fundamental challenge: the input is a 2D image but the output is a 1D sequence, and the alignment between image columns and output characters is unknown. The letter "W" takes more horizontal space than "i", ligatures connect multiple characters, and you don't know in advance how many pixels correspond to each character.
There are two main approaches: CTC (Connectionist Temporal Classification) and attention-based decoders.
CRNN (Convolutional Recurrent Neural Network) became the workhorse of OCR. The architecture is elegant: a CNN extracts visual features from the text image, collapsing the height dimension while preserving the width. This gives you a sequence of feature vectors along the horizontal axis. A bidirectional LSTM processes this sequence to capture context between neighboring characters. Finally, a fully connected layer outputs per-position character probabilities:
import torch
import torch.nn as nn
class CRNNRecognizer(nn.Module):
"""CNN + BiLSTM + CTC for text recognition."""
def __init__(self, num_classes=97):
# 96 printable ASCII + blank token
super().__init__()
self.cnn = nn.Sequential(
nn.Conv2d(1, 64, 3, padding=1),
nn.ReLU(inplace=True),
nn.MaxPool2d(2, 2),
nn.Conv2d(64, 128, 3, padding=1),
nn.ReLU(inplace=True),
nn.MaxPool2d(2, 2),
nn.Conv2d(128, 256, 3, padding=1),
nn.BatchNorm2d(256),
nn.ReLU(inplace=True),
nn.Conv2d(256, 256, 3, padding=1),
nn.ReLU(inplace=True),
# Collapse height to 1, keep width
nn.AdaptiveAvgPool2d((1, None)),
)
self.rnn = nn.LSTM(
256, 128, bidirectional=True,
num_layers=2, batch_first=True
)
# 256 because bidirectional: 128 * 2
self.fc = nn.Linear(256, num_classes)
def forward(self, x):
# x: (batch, 1, 32, W) grayscale text image
conv = self.cnn(x) # (B, 256, 1, W')
conv = conv.squeeze(2) # (B, 256, W')
conv = conv.permute(0, 2, 1) # (B, W', 256)
rnn_out, _ = self.rnn(conv) # (B, W', 256)
logits = self.fc(rnn_out) # (B, W', 97)
return logits
model = CRNNRecognizer()
img = torch.randn(1, 1, 32, 128)
logits = model(img)
print(f"Input: {img.shape}")
# (1, 1, 32, 128) -- grayscale, height 32, width 128
print(f"Output: {logits.shape}")
# (1, 32, 97) -- 32 timesteps, 97 character classes
The AdaptiveAvgPool2d((1, None)) is the key trick: it collapses the height dimension to 1 while keeping the width variable. After pooling, a 32x128 image becomes a sequence of 32 feature vectors (128 / pool_size), each representing a vertical column of the input. The bidirectional LSTM reads this sequence left-to-right AND right-to-left, so each position has context from both neighbors -- important because character recognition is heavily context-dependent ("rn" looks like "m" without context).
CTC (Connectionist Temporal Classification, Graves et al., 2006) solves the alignment problem elegantly. The model outputs a probability distribution over characters at each horizontal position (timestep). CTC introduces a special blank token and marginalizes over all possible alignments between the output sequence and the true text:
import torch
import torch.nn as nn
model = CRNNRecognizer()
img = torch.randn(1, 1, 32, 128)
logits = model(img)
# CTC loss for training
ctc_loss = nn.CTCLoss(blank=0, zero_infinity=True)
log_probs = logits.log_softmax(dim=-1)
# CTC expects (T, B, C) not (B, T, C)
log_probs = log_probs.permute(1, 0, 2)
# Target: "HELLO" as character indices
# (skipping blank=0, so chars start at 1)
targets = torch.tensor([8, 5, 12, 12, 15])
input_lengths = torch.tensor([32])
target_lengths = torch.tensor([5])
loss = ctc_loss(
log_probs, targets,
input_lengths, target_lengths
)
print(f"CTC loss: {loss.item():.4f}")
def ctc_decode(logits, blank=0):
"""Greedy CTC decoding: argmax at each timestep,
collapse repeats, remove blanks."""
indices = logits.argmax(dim=-1).tolist()
decoded = []
prev = None
for idx in indices:
if idx != blank and idx != prev:
decoded.append(idx)
prev = idx
return decoded
pred = ctc_decode(logits[0])
print(f"Decoded indices: {pred[:10]}...")
Here's how CTC alignment works for the target "HELLO": the model might output --HH-EE-LL-LLL-OO-- where - is the blank token. CTC collapses this to "HELLO" by removing blanks and merging consecutive identical characters. The blank token is critical for separating repeated characters -- without it, "HELLO" and "HELO" would be indistinguishable because the double-L would get collapsed. CTC marginalizes over all valid alignments during training, so the model doesn't need to be told which exact columns correspond to which characters -- it just needs to know the final text string. That's a really powerful property ;-)
Attention-based decoders (building on episode #51) offer an alternative to CTC. Instead of fixed left-to-right alignment, an attention decoder generates characters one at a time, attending to different parts of the input image at each step:
import torch
import torch.nn as nn
import torch.nn.functional as F
class AttentionDecoder(nn.Module):
"""Attention-based text decoder."""
def __init__(self, feature_dim=256,
hidden_dim=256, num_classes=97,
max_len=50):
super().__init__()
self.embedding = nn.Embedding(
num_classes, hidden_dim)
self.rnn = nn.GRUCell(
hidden_dim + feature_dim, hidden_dim)
self.attention = nn.Linear(
hidden_dim + feature_dim, 1)
self.classifier = nn.Linear(
hidden_dim, num_classes)
self.max_len = max_len
def forward(self, encoder_features):
# encoder_features: (B, T, feature_dim)
B = encoder_features.size(0)
hidden = torch.zeros(
B, 256, device=encoder_features.device)
# Start token (index 1)
input_char = torch.ones(
B, dtype=torch.long,
device=encoder_features.device)
outputs = []
for step in range(self.max_len):
embedded = self.embedding(input_char)
# Attention weights
h_exp = hidden.unsqueeze(1).expand(
-1, encoder_features.size(1), -1)
attn_in = torch.cat(
[h_exp, encoder_features], dim=2)
attn_w = F.softmax(
self.attention(attn_in).squeeze(2),
dim=1)
# Context vector
context = torch.bmm(
attn_w.unsqueeze(1),
encoder_features).squeeze(1)
# Update hidden state
rnn_in = torch.cat(
[embedded, context], dim=1)
hidden = self.rnn(rnn_in, hidden)
output = self.classifier(hidden)
outputs.append(output)
input_char = output.argmax(dim=1)
return torch.stack(outputs, dim=1)
decoder = AttentionDecoder()
enc_features = torch.randn(1, 32, 256)
output = decoder(enc_features)
print(f"Decoder output: {output.shape}")
# (1, 50, 97) -- up to 50 chars, 97 classes
The advantage of attention over CTC: it can handle non-monotonic alignments. CTC assumes left-to-right ordering (column 1 corresponds to the first character, column 2 to the next, etc.). Attention can look anywhere -- useful for perspective-distorted text where the spatial order in the image doesn't perfectly match the reading order, or for bidirectional scripts like Arabic.
Having said that, CTC is simpler, faster (no autoregressive decoding), and works well enough for most horizontal text. Use attention when you need to handle irregular text layouts.
Two tools dominate the OCR landscape in practice:
Tesseract is the oldest and most widely-used OCR engine. Originally developed by HP in the 1980s, now maintained by Google. It's free, works offline, and supports 100+ languages:
import pytesseract
from PIL import Image
# Basic OCR -- give it an image, get text back
image = Image.open("document.png")
text = pytesseract.image_to_string(image)
print(text)
# With bounding box information
data = pytesseract.image_to_data(
image, output_type=pytesseract.Output.DICT
)
for i, word in enumerate(data["text"]):
if word.strip():
x = data["left"][i]
y = data["top"][i]
w = data["width"][i]
h = data["height"][i]
conf = data["conf"][i]
print(f"'{word}' at ({x},{y},{w},{h}), "
f"confidence={conf}%")
# Page segmentation modes control layout
# PSM 6 = assume uniform block of text
# PSM 7 = treat as single text line
# PSM 11 = sparse text, find as much as possible
text = pytesseract.image_to_string(
image, config="--psm 6 --oem 3"
)
Tesseract works well on clean, high-resolution scanned documents with horizontal text and standard fonts. It struggles with complex layouts, curved text, handwriting, low-resolution photos, and noisy backgrounds. If you're processing scanned invoices in a controlled office environment, Tesseract is perfectly fine. If you're reading text from photos taken in the wild -- street signs, restaurant menus, whiteboard notes -- you need something more robust.
PaddleOCR (from Baidu) is the modern go-to for real-world OCR. It packages text detection (DBNet), recognition (CRNN/SVTR), and text direction classification into one pipeline. It's significanly more accurate than Tesseract on natural scene images:
from paddleocr import PaddleOCR
# Initialize with angle classification enabled
ocr = PaddleOCR(use_angle_cls=True, lang="en")
# Run on an image -- returns detection + recognition
result = ocr.ocr("receipt.jpg", cls=True)
for line in result[0]:
# bbox: [[x1,y1], [x2,y2], [x3,y3], [x4,y4]]
bbox = line[0]
text = line[1][0] # recognized text
confidence = line[1][1] # confidence score
# Calculate text region dimensions
xs = [p[0] for p in bbox]
ys = [p[1] for p in bbox]
width = max(xs) - min(xs)
height = max(ys) - min(ys)
print(f"'{text}' (conf: {confidence:.3f}, "
f"size: {width:.0f}x{height:.0f})")
The fact that PaddleOCR returns quadrilateral bounding boxes (4 corner points) instead of axis-aligned rectangles is a big deal for real-world use. A price tag photographed at an angle produces tilted text -- an axis-aligned box would include lots of background pixels, degrading recognition accuracy. The quadrilateral tightly wraps the actual text region regardless of orientation.
To really understand how the pieces fit together, let's build a simplified training data generator. Production OCR systems are trained on millions of synthetic images with varied fonts, backgrounds, distortions, and augmentations -- but the data flow is identical:
import numpy as np
from PIL import Image, ImageDraw, ImageFont
class SyntheticTextDataset:
"""Generate synthetic text images for OCR
training."""
def __init__(self, num_samples=1000,
img_height=32, img_width=128,
seed=42):
self.rng = np.random.RandomState(seed)
self.h = img_height
self.w = img_width
self.charset = (
"ABCDEFGHIJKLMNOPQRSTUVWXYZ"
"abcdefghijklmnopqrstuvwxyz"
"0123456789 "
)
self.char_to_idx = {
c: i + 1
for i, c in enumerate(self.charset)
}
self.idx_to_char = {
i + 1: c
for i, c in enumerate(self.charset)
}
self.data = [
self._generate()
for _ in range(num_samples)
]
def _generate(self):
length = self.rng.randint(3, 12)
text = "".join(
self.rng.choice(list(self.charset))
for _ in range(length)
).strip()
if not text:
text = "hello"
# Create grayscale image with text
img = Image.new("L", (self.w, self.h),
color=255)
draw = ImageDraw.Draw(img)
try:
font = ImageFont.truetype(
"/usr/share/fonts/truetype/"
"dejavu/DejaVuSans.ttf", 20
)
except (IOError, OSError):
font = ImageFont.load_default()
draw.text((5, 4), text, fill=0, font=font)
# Add light noise for realism
arr = np.array(img, dtype=np.float32) / 255.0
noise = self.rng.randn(*arr.shape) * 0.02
arr = np.clip(arr + noise, 0, 1)
# Encode target as character indices
target = [
self.char_to_idx[c] for c in text
if c in self.char_to_idx
]
return arr, target, text
def __len__(self):
return len(self.data)
def __getitem__(self, idx):
return self.data[idx]
dataset = SyntheticTextDataset(num_samples=100)
img, target, text = dataset[0]
print(f"Image shape: {img.shape}")
print(f"Text: '{text}'")
print(f"Target indices: {target}")
print(f"Charset size: {len(dataset.charset)}")
# 63 chars + 1 blank = 64 classes for CTC
In practice you'd add random rotation (up to 15 degrees), perspective warping, color jittering, and multiple font families. The beauty of synthetic data for OCR is that you get perfect labels for free -- no expensive manual annotation required.
Extracting individual words isn't enough for most real applications. A receipt has structure: a header, line items, prices aligned to the right, a total at the bottom. An invoice has tables, fields, addresses, tax numbers. A scientific paper has title, abstract, body columns, references. OCR reads the characters; document understanding interprets the layout.
Layout analysis classifies regions of a document into categories: title, paragraph, table, figure, header, footer. LayoutLM (from Microsoft) and its successors treat this as a multimodal problem: they take both the text content (from OCR) and the spatial positions of each word, and feed both into a transformer that learns relationships between text and layout:
def process_document(image_path, ocr_engine):
"""Conceptual document understanding pipeline."""
# Step 1: OCR extracts words and positions
ocr_results = ocr_engine.ocr(image_path)
words = []
for r in ocr_results[0]:
text = r[1][0]
bbox = r[0]
confidence = r[1][1]
words.append({
"text": text,
"bbox": bbox,
"conf": confidence,
})
# Step 2: layout analysis classifies regions
# LayoutLMv3 takes three inputs:
# - word tokens (text content)
# - bounding box coordinates (spatial layout)
# - image patches (visual appearance)
# and predicts region types per word:
# title, paragraph, table cell, header, etc.
# Step 3: structure extraction groups words
# into semantic units
# For a receipt: find "total", extract nearby price
# For an invoice: detect table grid, assign words
# to rows and columns
# Step 4: key-value extraction
structured = {}
for word_info in words:
text = word_info["text"].lower()
if "total" in text:
structured["total_label"] = word_info
return words, structured
This is where OCR connects back to the LLM world from episodes #57-68. Modern document understanding systems feed OCR output into large language models that can interpret complex layouts, extract structured data, and answer questions about documents. The visual AI and language AI from earlier parts of this series converge here -- LayoutLMv3 is essentially a multimodal transformer (episode #75) applied to document images. Which is actually a pretty elegant full-circle moment if you think about it.
Putting it all together -- a practical receipt parsing pipeline that extracts structured data from OCR output:
import re
def parse_receipt(ocr_results):
"""Parse OCR output from a receipt into
structured data.
ocr_results: list of (bbox, (text, conf)) tuples.
"""
# Sort by vertical position (top to bottom)
lines = sorted(
ocr_results,
key=lambda x: x[0][0][1]
)
store_name = None
items = []
total = None
date = None
for line in lines:
text = line[1][0]
conf = line[1][1]
# First confident text = likely store name
if store_name is None and conf > 0.8:
store_name = text
# Date patterns
date_match = re.search(
r"\d{1,2}[/-]\d{1,2}[/-]\d{2,4}", text
)
if date_match:
date = date_match.group()
# Price patterns
price_match = re.search(
r"\$?(\d+\.\d{2})", text
)
if price_match:
price = float(price_match.group(1))
text_lower = text.lower()
if any(kw in text_lower for kw in
["total", "amount due", "balance"]):
total = price
elif any(kw in text_lower for kw in
["tax", "gst", "vat"]):
continue
else:
item_name = re.sub(
r"\$?\d+\.\d{2}", "", text
).strip()
if item_name and len(item_name) > 1:
items.append({
"name": item_name,
"price": price,
})
return {
"store": store_name,
"date": date,
"items": items,
"total": total,
}
# Simulated OCR output
mock_ocr = [
([[10, 10], [300, 10], [300, 40], [10, 40]],
("FRESH MARKET", 0.98)),
([[10, 50], [200, 50], [200, 70], [10, 70]],
("01/15/2026", 0.95)),
([[10, 100], [250, 100], [250, 120], [10, 120]],
("Organic Milk $4.99", 0.96)),
([[10, 130], [250, 130], [250, 150], [10, 150]],
("Bread Wheat $3.49", 0.94)),
([[10, 160], [250, 160], [250, 180], [10, 180]],
("Coffee Ground $8.99", 0.97)),
([[10, 200], [200, 200], [200, 220], [10, 220]],
("Tax $1.40", 0.93)),
([[10, 240], [250, 240], [250, 270], [10, 270]],
("TOTAL $18.87", 0.99)),
]
result = parse_receipt(mock_ocr)
print(f"Store: {result['store']}")
print(f"Date: {result['date']}")
for item in result["items"]:
print(f" {item['name']}: ${item['price']:.2f}")
print(f"Total: ${result['total']:.2f}")
This is deliberately simple -- production receipt parsers handle multi-line item names, discount lines, tip calculations, different currency formats, and ambiguous layouts. But the pattern is the same everywhere: OCR extracts text and positions, then domain-specific logic interprets the spatial structure. The OCR pipeline doesn't need to understand what a receipt is -- it just needs to accurately read every piece of text in the image. The semantic interpretation happens downstream.
| Use case | Tool | Why |
|---|---|---|
| Clean scanned documents | Tesseract | Free, fast, sufficient for high-quality scans |
| Real-world photos | PaddleOCR | Handles rotation, noise, complex backgrounds |
| Custom text detection | DBNet (fine-tuned) | When you need control over detection quality |
| Document understanding | LayoutLMv3 | When you need to understand page structure |
| Handwriting recognition | Specialized models | Tesseract and PaddleOCR struggle with cursive |
| Multi-language mixed text | PaddleOCR | Supports 80+ languages with auto-detection |
| High-throughput batch processing | Tesseract + parallel | CPU-friendly, easily parallelized |
For most new projects, start with PaddleOCR. It handles the detection-recognition pipeline in three lines of code, works on GPU and CPU, and its accuracy on natural scene images is substantially better than Tesseract. Only reach for Tesseract when you're processing clean scanned documents at scale and want the lightest possible dependency.
We've now covered text in images -- detecting it, recognizing it, and understanding the documents it appears in. This completes a broad visual understanding toolkit: classification tells you what's in an image, detection tells you where things are, segmentation gives you pixel-level boundaries, pose estimation reveals body structure, and now OCR reads the text. There's still a major dimension of visual AI we haven't explored though -- working with video as a temporal medium, understanding not just individual frames but what's happening across sequences of frames over time.
Exercise 1: Build a CTC decoder comparison tool. Create a class CTCDecoder that: (a) implements greedy_decode(logits) that takes per-timestep log probabilities of shape (T, num_classes), selects the argmax at each position, collapses consecutive duplicates, and removes blank tokens (index 0), (b) implements beam_search_decode(logits, beam_width=5) that maintains the top-k most probable partial sequences at each timestep, extending each beam with every possible character and pruning to keep only the best beam_width candidates, (c) generates a synthetic logit matrix for the word "HELLO" where the correct character has high probability at the right positions with blank tokens in between, adds Gaussian noise, and (d) runs both decoders on the same logits and prints the decoded text from each. Verify that both produce "HELLO" on clean logits, and show how beam search handles noisy logits more robustly than greedy decoding by testing with increasing noise levels (sigma=0.1, 0.5, 1.0, 2.0).
Exercise 2: Implement a text detection post-processor. Create a class TextDetectionPostProcessor that: (a) takes a probability map of shape (H, W) with values 0-1 indicating text probability at each pixel, (b) implements threshold_and_contour(prob_map, threshold=0.5) that binarizes the map at the given threshold and finds connected components using flood-fill (no OpenCV -- implement a simple BFS-based connected component labeling), (c) for each connected component, computes the bounding box (min/max row and column), area, and aspect ratio, (d) implements filter_boxes(boxes, min_area=100, min_aspect=0.2, max_aspect=15.0) that removes boxes that are too small or have unreasonable aspect ratios (too square for text, or too extreme), (e) generates a synthetic 200x300 probability map with 5 text-like horizontal strips and 3 noise blobs, runs the full pipeline, and prints which regions were kept vs filtered. Verify that the horizontal strips are kept and the square noise blobs are filtered out.
Exercise 3: Build an OCR accuracy evaluation framework. Create a class OCREvaluator that: (a) implements character_accuracy(predicted, ground_truth) that computes the character-level accuracy using edit distance (Levenshtein distance) -- accuracy = 1 - (edit_distance / max(len(predicted), len(ground_truth))), (b) implements word_accuracy(predicted_words, gt_words) that computes the fraction of words that match exactly (case-insensitive), (c) implements levenshtein_distance(s1, s2) from scratch using dynamic programming (the standard 2D DP table approach), (d) generates 20 test cases with synthetic ground truth strings and three "OCR engine" profiles: "perfect" (exact match), "good" (1-2 random character substitutions per string), and "poor" (30% of characters randomly changed plus random insertions/deletions). Print a comparison table showing per-engine average character accuracy, word accuracy, and average edit distance. Verify that the perfect engine gets 100% on both metrics.
@scipio