Video OCR: get the text out of a video

import collections
import difflib
import subprocess
import sys
import tempfile
from pathlib import Path

FPS = 2  # frames read each second
VF = f"fps={FPS},crop=iw:ih/4:0:ih*3/4,format=gray,lut=y='if(gt(val,245),0,255)'"
MIN_CONF = 30  # drop words Tesseract is unsure of: mostly specks of background


def ocr(png):
    tsv = subprocess.run(["tesseract", str(png), "-", "--psm", "6", "tsv"], capture_output=True, text=True).stdout
    lines = collections.defaultdict(list)
    for row in tsv.splitlines()[1:]:
        f = row.split("\t")
        if len(f) == 12 and f[0] == "5" and float(f[10]) >= MIN_CONF and f[11].strip():
            lines[f[2], f[3], f[4]].append(f[11])
    return "\n".join(" ".join(words) for words in lines.values()).replace("|", "I")


def read_frames(video, vf=VF):
    with tempfile.TemporaryDirectory() as d:
        subprocess.run(["ffmpeg", "-v", "error", "-i", video, "-vf", vf, f"{d}/%06d.png"], check=True)
        return [ocr(p) for p in sorted(Path(d).glob("*.png"))]


def similar(a, b):
    return difflib.SequenceMatcher(None, a, b).ratio()


def to_cues(texts, fps=FPS):
    cues = []  # [first frame, last frame + 1, readings]
    for i, t in enumerate(texts):
        if cues and cues[-1][1] == i and similar(cues[-1][2][-1], t) > 0.6:
            cues[-1][1] = i + 1
            cues[-1][2].append(t)
        elif t:
            cues.append([i, i + 1, [t]])
    # each cue keeps the reading that agrees most with its other readings
    return [(a / fps, b / fps, max(r, key=lambda x: sum(similar(x, y) for y in r))) for a, b, r in cues]


def stamp(s):
    ms = round(s * 1000)
    return f"{ms // 3600000:02}:{ms // 60000 % 60:02}:{ms // 1000 % 60:02},{ms % 1000:03}"


if __name__ == "__main__":
    with open(sys.argv[2], "w") as f:
        for n, (a, b, text) in enumerate(to_cues(read_frames(sys.argv[1])), 1):
            f.write(f"{n}\n{stamp(a)} --> {stamp(b)}\n{text}\n\n")

1
00:00:00,500 --> 00:00:02,000
Morning! Are you open yet?

2
00:00:02,500 --> 00:00:06,000
Almost. The oven isn't hot yet.

3
00:00:06,000 --> 00:00:09,000
What time do you open?

import re
import subprocess
import sys

video = sys.argv[1]
log = subprocess.run(
    ["ffmpeg", "-i", video, "-vf", "freezedetect=d=1", "-an", "-f", "null", "-"], capture_output=True, text=True
).stderr
starts = [float(s) for s in re.findall(r"freeze_start: ([\d.]+)", log)]
ends = [float(s) for s in re.findall(r"freeze_end: ([\d.]+)", log)]
for i, a in enumerate(starts):
    t = (a + ends[i]) / 2 if i < len(ends) else a + 0.5  # the middle of each still stretch
    png = subprocess.run(
        ["ffmpeg", "-v", "error", "-ss", str(t), "-i", video, "-frames:v", "1", "-c:v", "png", "-f", "image2pipe", "-"],
        capture_output=True, check=True,
    ).stdout  # fmt: skip
    print(subprocess.run(["tesseract", "-", "-"], input=png, capture_output=True).stdout.decode())

• Clips: 4 videos I made for this test, 1280x720, 25 frames a second, H.264 at 3 Mbit/s at most. The text is mine. Pillow drew it in Open Sans; this ffmpeg build has no drawtext.
• Subtitles: 24 lines a clip, 1.24 to 3.42 s each, white with a black outline. Fractal: ffmpeg's mandelbrot source, 63 s. Held out: its gradients source, 62 s, made after the settings were fixed.
• Slides: 10 slides of 6 s, 251 words, once with hard cuts and once with half-second crossfades.
• ffmpeg 9.0.2 and Tesseract 5.5.4 from Homebrew, its fast eng model. Python 3.12 on an Apple Silicon Mac, CPU, 2 threads (OMP_THREAD_LIMIT=2), lowest priority. Times are rough.
• Exact: one cue overlaps the true line and has the same text, line breaks as spaces, curly and straight apostrophes the same. Slides: true words found in order.
• The Actor row is its own code from git, build 0.1.65, run on the Mac with its default inputs. Not a run on Apify.

• Real films, yellow or boxed subtitles, subtitles at the top, and languages other than English.
• Text that moves, like credits or tickers.
• GPUs and Windows.

• ffmpeg freezedetect filter
• ffmpeg select filter and scene score
• Tesseract on GitHub (Apache-2.0)
• videocr on GitHub (MIT)
• RapidOCR on GitHub (Apache-2.0)