Extracting HTML tables into pandas DataFrames is one of the most common Python data tasks. pd.read_html(url) makes it look simple. And for some pages, it is — a Wikipedia table, a static financial data page, a simple pricing grid.
But a meaningful fraction of tables you’ll want to extract are on JavaScript-rendered pages, and pd.read_html() only gets the pre-JavaScript HTML. You get an empty result or the wrong table, with no obvious error.
This post covers four approaches in order of complexity, and when to use each.
The Four Approaches
| Approach | Best for | Handles JS? | Handles complex structure? |
|---|---|---|---|
pd.read_html() | Simple static tables | No | Limited |
| BeautifulSoup | Static tables with specific IDs/classes | No | Medium |
| UnWeb Markdown → pandas | Any page, simple table structure | Yes | Medium |
| UnWeb Markdown → Claude → pandas | Complex/irregular tables, mixed content | Yes | High |
Approach 1: pd.read_html() (When It Works)
import pandas as pd
# Simplest case: all tables from a static page
dfs = pd.read_html("https://en.wikipedia.org/wiki/List_of_countries_by_GDP")
# Returns a list of DataFrames, one per table found
print(f"Found {len(dfs)} tables")
df = dfs[0] # First table
print(df.head())
The problem: pd.read_html() uses requests under the hood for URLs, getting only the initial HTML response without JavaScript execution. If the table loads via JS, you get an empty list or the wrong table.
Quick diagnostic: if read_html() returns an empty list or a table that doesn’t look right, the page is probably JS-rendered.
Approach 2: UnWeb Markdown → pandas (JS-Rendered Pages)
UnWeb converts any URL — including JS-rendered pages — to clean Markdown. HTML tables convert to Markdown table syntax (| header | header |), which pandas can parse directly.
import httpx
import pandas as pd
import io
import re
import asyncio
UNWEB_API_KEY = "your_key"
async def fetch_markdown(url: str) -> str:
async with httpx.AsyncClient(timeout=30) as client:
resp = await client.get(
"https://api.unweb.info/v1/convert",
params={"url": url},
headers={"Authorization": f"Bearer {UNWEB_API_KEY}"}
)
resp.raise_for_status()
return resp.json()["markdown"]
def extract_markdown_tables(markdown: str) -> list[str]:
"""Extract individual Markdown table blocks from a larger document."""
# A Markdown table block: consecutive lines with | characters
table_pattern = re.compile(
r'(?:^|\n)((?:\|[^\n]+\|\n?)+)',
re.MULTILINE
)
tables = []
for match in table_pattern.finditer(markdown):
table_text = match.group(1).strip()
# Must have at least a header row, separator, and one data row
lines = [l for l in table_text.split('\n') if l.strip()]
if len(lines) >= 3:
tables.append(table_text)
return tables
def markdown_table_to_dataframe(table_text: str) -> pd.DataFrame:
"""Convert a Markdown table string to a pandas DataFrame."""
lines = [l.strip() for l in table_text.split('\n') if l.strip()]
# Remove separator row (---|---|---)
lines = [l for l in lines if not re.match(r'^[\|\s\-:]+$', l)]
rows = []
for line in lines:
cells = [c.strip() for c in line.strip('|').split('|')]
rows.append(cells)
if not rows:
return pd.DataFrame()
headers = rows[0]
data = rows[1:]
return pd.DataFrame(data, columns=headers)
async def extract_tables_from_url(url: str) -> list[pd.DataFrame]:
markdown = await fetch_markdown(url)
table_texts = extract_markdown_tables(markdown)
return [markdown_table_to_dataframe(t) for t in table_texts]
# Usage
async def main():
tables = await extract_tables_from_url("https://example.com/pricing")
for i, df in enumerate(tables):
print(f"\n--- Table {i+1} ---")
print(df.to_string(index=False))
asyncio.run(main())
Why Markdown tables are more reliable than HTML tables: HTML tables often have complex nested structures, colspan/rowspan attributes, and inline styling that makes parsing fragile. Markdown tables are flat, consistent, and easy to parse. UnWeb’s conversion normalizes table structure during the HTML-to-Markdown step.
Approach 3: LLM-Assisted Extraction for Complex Tables
Some tables don’t convert cleanly to Markdown: tables with merged cells, hierarchical headers, tables embedded in other content, or pages where you need only a specific table by semantic meaning (“the pricing table”, not “the third table on the page”).
For these cases, pass the Markdown to Claude with specific instructions:
import anthropic
import json
import pandas as pd
def extract_table_with_llm(
markdown: str,
table_description: str,
columns: list[str]
) -> pd.DataFrame:
"""
Extract a specific table from Markdown using Claude.
table_description: Natural language description of which table to extract
columns: Expected column names for the output DataFrame
"""
client = anthropic.Anthropic()
schema_hint = json.dumps({col: "string value" for col in columns}, indent=2)
message = client.messages.create(
model="claude-haiku-4-5-20251001",
max_tokens=4096,
messages=[{
"role": "user",
"content": f"""From this Markdown content, extract: {table_description}
Return ONLY a JSON array of objects with these fields:
{schema_hint}
If a field is missing or unclear, use null. Return only the JSON array, no other text.
MARKDOWN:
{markdown[:8000]}"""
}]
)
text = message.content[0].text.strip()
# Handle markdown code blocks if present
if text.startswith("```"):
text = re.sub(r'^```\w*\n?', '', text)
text = re.sub(r'\n?```$', '', text)
data = json.loads(text)
return pd.DataFrame(data)
# Usage: extract the pricing table from a SaaS pricing page
async def extract_pricing_table(url: str) -> pd.DataFrame:
markdown = await fetch_markdown(url)
return extract_table_with_llm(
markdown,
table_description="the main pricing/plans comparison table",
columns=["plan_name", "price_monthly", "price_annual", "included_features", "limits"]
)
Handling Multiple Tables on a Page
Many pages have multiple tables. Approach 2 extracts all of them — but you often want only one. Two filtering strategies:
def find_table_by_column(
tables: list[pd.DataFrame],
required_columns: list[str]
) -> pd.DataFrame | None:
"""Return the first DataFrame that contains all required column names."""
for df in tables:
if all(col.lower() in [c.lower() for c in df.columns] for col in required_columns):
return df
return None
def find_largest_table(tables: list[pd.DataFrame]) -> pd.DataFrame | None:
"""Return the DataFrame with the most rows."""
if not tables:
return None
return max(tables, key=len)
# Usage
tables = await extract_tables_from_url("https://example.com/data")
# Strategy 1: find by expected columns
pricing_df = find_table_by_column(tables, ["Plan", "Price", "Features"])
# Strategy 2: just get the largest table
main_df = find_largest_table(tables)
Cleaning Extracted Table Data
Tables from the web usually need cleaning before analysis:
import re
def clean_table(df: pd.DataFrame) -> pd.DataFrame:
"""Common cleaning operations for web-extracted tables."""
df = df.copy()
for col in df.columns:
if df[col].dtype == object:
# Strip whitespace
df[col] = df[col].str.strip()
# Replace empty strings with NaN
df[col] = df[col].replace('', pd.NA)
# Try to infer numeric columns
for col in df.columns:
# Remove currency symbols, commas, % before trying numeric conversion
cleaned = df[col].astype(str).str.replace(r'[$€£,\s]', '', regex=True)
cleaned = cleaned.str.replace('%', '', regex=False)
try:
df[col + '_numeric'] = pd.to_numeric(cleaned, errors='coerce')
# Only keep the numeric version if most values converted
if df[col + '_numeric'].notna().sum() > len(df) * 0.7:
df[col] = df[col + '_numeric']
except Exception:
pass
finally:
if col + '_numeric' in df.columns and col + '_numeric' != col:
df = df.drop(columns=[col + '_numeric'])
return df
df_clean = clean_table(df)
Batching Multiple URLs
import asyncio
async def extract_tables_batch(
urls: list[str],
target_columns: list[str]
) -> dict[str, pd.DataFrame | None]:
results = {}
for url in urls:
try:
tables = await extract_tables_from_url(url)
results[url] = find_table_by_column(tables, target_columns)
except Exception as e:
print(f"[error] {url}: {e}")
results[url] = None
await asyncio.sleep(0.5) # Rate limiting
return results
# Extract pricing tables from multiple competitor pages
pricing_tables = asyncio.run(extract_tables_batch(
urls=["https://competitor1.com/pricing", "https://competitor2.com/pricing"],
target_columns=["Plan", "Price"]
))
# Combine into one DataFrame
all_pricing = []
for url, df in pricing_tables.items():
if df is not None:
df['source'] = url
all_pricing.append(df)
combined_df = pd.concat(all_pricing, ignore_index=True)
print(combined_df)
UnWeb: Clean Markdown From Any URL
Tables, paragraphs, lists — structured Markdown from any page, including JS-rendered sites. Quality-scored on every response.