MultiNews: Multi-Document News Summarisation EDA

import re
import numpy as np
import pandas as pd
import matplotlib
import matplotlib.pyplot as plt
import matplotlib.ticker as mticker
import warnings
import random

from collections import Counter
from IPython.display import display, HTML
from datasets import load_dataset # pip install datasets
from sklearn.feature_extraction.text import ENGLISH_STOP_WORDS
# pip install scikit-learn 

warnings.filterwarnings(
    'ignore',
    message='.*unauthenticated requests to the HF Hub.*'
)

FL_BLUE   = '#2563eb'
FL_SLATE  = '#64748b'
FL_AMBER  = '#f59e0b'
FL_GREEN  = '#16a34a'
FL_RED    = '#ef4444'
FL_BG     = '#ffffff'
FL_GRID   = '#e2e8f0'
FL_TEXT   = '#0f172a'
FL_TEXT2  = '#334155'
FL_BORDER = '#e2e8f0'

matplotlib.rcParams.update({
    'figure.facecolor':  FL_BG,
    'axes.facecolor':    FL_BG,
    'axes.edgecolor':    FL_BORDER,
    'axes.labelcolor':   FL_TEXT2,
    'axes.spines.top':   False,
    'axes.spines.right': False,
    'axes.grid':         True,
    'grid.color':        FL_GRID,
    'grid.linewidth':    0.7,
    'xtick.color':       FL_TEXT2,
    'ytick.color':       FL_TEXT2,
    'xtick.labelsize':   10,
    'ytick.labelsize':   10,
    'axes.labelsize':    11,
    'axes.titlesize':    12,
    'axes.titlecolor':   FL_TEXT,
    'axes.titlepad':     12,
    'legend.frameon':    False,
    'legend.fontsize':   10,
    'figure.dpi':        300,
    'savefig.bbox':      'tight',
    'font.family':       'sans-serif',
    'font.sans-serif':   ['Inter', 'Helvetica Neue', 'Arial', 'DejaVu Sans'],
})

DATASET_ID   = 'Awesome075/multi_news_parquet'
SPLITS       = ['train', 'validation', 'test']
DOC_SEP      = '|||||'          # separator between source articles
DOC_MIN_CHARS = 80              # minimum chars to count an article as usable

def norm_ws(s):
    return re.sub(r'\s+', ' ', (s or '').strip())

def split_sources(document_field):
    parts = (document_field or '').split(DOC_SEP)
    return [norm_ws(p) for p in parts if norm_ws(p)]

def get_summary(row):
    if isinstance(row, dict):
        return norm_ws(row.get('summary', '') or '')
    return norm_ws(getattr(row, 'summary', '') or '')

def first_sentence(text):
    text = norm_ws(text)
    m = re.search(r'[.!?]\s', text)
    return text[:m.end()].strip() if m else text[:200]

def word_count(text):
    return len(re.findall(r'[a-zA-Z0-9]+', text or ''))

def tok_content(text):
    stops = ENGLISH_STOP_WORDS
    toks = re.findall(r'[a-z0-9]+', (text or '').lower())
    return [t for t in toks if t not in stops and not t.isdigit()]


print('Setup complete - loading dataset from HuggingFace...')

data = {}
for split in SPLITS:
    data[split] = load_dataset(DATASET_ID, split=split)
    print(f'{split:12} {len(data[split]):>7,} examples')

print(f'\nTotal: {sum(len(v) for v in data.values()):,} examples')
print(f'\nColumn names: {data["train"].column_names}')

# Show a single example structure
ex = data['train'][0]
sources = split_sources(ex['document'])
summary = get_summary(ex)
print(f'\nExample 0 - train split:')
print(f'  Source articles:  {len(sources)}')
print(f'  Summary length:   {len(summary)} chars / {word_count(summary)} words')
print(f'  Summary preview:  {summary[:200]}...')

rows = []

train_source_counts = []
train_summary_words = []
train_source_words = []
train_usable_counts = []

for row in data['train']:
    sources = split_sources(row['document'] if isinstance(row, dict) else row.get('document', ''))
    usable = [s for s in sources if len(s) >= DOC_MIN_CHARS]
    summary = get_summary(row)

    train_source_counts.append(len(sources))
    train_usable_counts.append(len(usable))
    train_summary_words.append(word_count(summary))
    train_source_words.append(sum(word_count(s) for s in usable))

sc = np.array(train_source_counts)
uc = np.array(train_usable_counts)
sw = np.array(train_summary_words)
dw = np.array(train_source_words)

rows.append({
    'Split': 'train',
    'Examples': len(sc),
    'Median sources': float(np.median(sc)),
    'Mean sources': float(np.mean(sc)),
    'Median usable': float(np.median(uc)),
    'Median summ words': float(np.median(sw)),
    'Mean summ words': float(np.mean(sw)),
    'Median src words': float(np.median(dw)),
})

report_df = pd.DataFrame(rows)
report_df[['Median sources', 'Mean sources', 'Median usable', 'Median summ words', 'Mean summ words', 'Median src words']] = (
    report_df[['Median sources', 'Mean sources', 'Median usable', 'Median summ words', 'Mean summ words', 'Median src words']]
    .round(1)
)

display(report_df.T)

train_source_counts = []

for row in data['train']:
    sources = split_sources(row['document'] if isinstance(row, dict) else row.get('document', ''))
    train_source_counts.append(len(sources))

cnt = Counter(train_source_counts)
max_show = 15
xs = list(range(1, max_show + 1))
ys = [cnt.get(x, 0) for x in xs]

plt.figure(figsize=(8, 4.5))
plt.bar(xs, ys, color=FL_BLUE, alpha=0.85, width=0.7)
plt.xlabel('Number of source articles')
plt.ylabel('Cluster count')
plt.title('Source article count per cluster - train split')
plt.xticks(xs)
plt.tick_params(length=0)
plt.gca().yaxis.set_major_formatter(
    mticker.FuncFormatter(lambda y, _: f'{y/1000:.0f}k' if y >= 1000 else f'{int(y)}')
)

median_src = np.median(train_source_counts)
plt.axvline(
    median_src,
    color=FL_AMBER,
    linewidth=1.5,
    linestyle='--',
    label=f'Median = {median_src:.0f}'
)
plt.legend(fontsize=9)
plt.tight_layout()
plt.show()

train_summary_words = []
train_source_words = []

for row in data['train']:
    sources = split_sources(row['document'] if isinstance(row, dict) else row.get('document', ''))
    usable = [s for s in sources if len(s) >= DOC_MIN_CHARS]
    summary = get_summary(row)

    train_summary_words.append(word_count(summary))
    train_source_words.append(sum(word_count(s) for s in usable))

sw = np.array(train_summary_words)
dw = np.array(train_source_words)

plt.figure(figsize=(8, 4.5))
plt.hist(
    np.clip(sw, 0, 600),
    bins=40,
    color=FL_BLUE,
    alpha=0.85,
    edgecolor='none'
)
plt.axvline(
    np.median(sw),
    color=FL_AMBER,
    linewidth=1.5,
    linestyle='--',
    label=f'Median = {np.median(sw):.0f}'
)
plt.title('Summary words - train split')
plt.xlabel('Word count (clipped at 600)')
plt.ylabel('Frequency')
plt.legend(fontsize=9)
plt.tick_params(length=0)
plt.tight_layout()
plt.show()

plt.figure(figsize=(8, 4.5))
plt.hist(
    np.clip(dw, 0, 5000),
    bins=40,
    color=FL_SLATE,
    alpha=0.85,
    edgecolor='none'
)
plt.axvline(
    np.median(dw),
    color=FL_AMBER,
    linewidth=1.5,
    linestyle='--',
    label=f'Median = {np.median(dw):.0f}'
)
plt.title('Total source words - train split')
plt.xlabel('Word count (clipped at 5,000)')
plt.ylabel('Frequency')
plt.legend(fontsize=9)
plt.tick_params(length=0)
plt.tight_layout()
plt.show()

sw = np.array(train_summary_words, dtype=float)
dw = np.array(train_source_words, dtype=float)

ratio = np.where(dw > 0, sw / dw, 0)
ratio_clipped = np.clip(ratio, 0, 1)

plt.figure(figsize=(8, 4.5))
plt.hist(
    ratio_clipped,
    bins=40,
    color=FL_GREEN,
    alpha=0.85,
    edgecolor='none'
)
plt.axvline(
    float(np.median(ratio_clipped)),
    color=FL_AMBER,
    linewidth=1.5,
    linestyle='--',
    label=f'Median = {np.median(ratio_clipped):.3f}'
)
plt.title('Compression ratio - train split')
plt.xlabel('Summary / source words')
plt.ylabel('Frequency')
plt.legend(fontsize=9)
plt.tick_params(length=0)
plt.tight_layout()
plt.show()

print(
    f'train        median ratio: {np.median(ratio):.4f}  '
    f'mean: {np.mean(ratio):.4f}  '
    f'p10: {np.percentile(ratio, 10):.4f}  '
    f'p90: {np.percentile(ratio, 90):.4f}'
)

rng = random.Random(42)
sample_indices = rng.sample(range(len(data['train'])), k=min(500, len(data['train'])))

coverages = []
redundancies = []

for idx in sample_indices:
    row = data['train'][idx]
    sources = [
        s for s in split_sources(row['document'] if isinstance(row, dict) else row.get('document', ''))
        if len(s) >= DOC_MIN_CHARS
    ]
    summary = get_summary(row)

    if not sources or not summary:
        continue

    ref_toks = set(tok_content(summary))
    src_union = set()
    src_tok_sets = []

    for s in sources:
        t = set(tok_content(s))
        src_union |= t
        src_tok_sets.append(t)

    cov = len(src_union & ref_toks) / max(1, len(ref_toks)) if ref_toks else 0
    coverages.append(cov)

    pairs = [
        (src_tok_sets[i], src_tok_sets[j])
        for i in range(len(src_tok_sets))
        for j in range(i + 1, len(src_tok_sets))
    ]

    if pairs:
        jac = np.mean([len(a & b) / max(1, len(a | b)) for a, b in pairs])
        redundancies.append(float(jac))

plt.figure(figsize=(8, 4.5))
plt.hist(
    coverages,
    bins=30,
    color=FL_BLUE,
    alpha=0.85,
    edgecolor='none'
)
plt.axvline(
    np.median(coverages),
    color=FL_AMBER,
    linewidth=1.5,
    linestyle='--',
    label=f'Median = {np.median(coverages):.3f}'
)
plt.xlabel('Coverage (summary vocab ∩ source vocab / summary vocab)')
plt.ylabel('Count')
plt.title('Lexical coverage - how much of the summary vocab appears in the source docs')
plt.legend(fontsize=9)
plt.tick_params(length=0)
plt.tight_layout()
plt.show()

plt.figure(figsize=(8, 4.5))
plt.hist(
    redundancies,
    bins=30,
    color=FL_SLATE,
    alpha=0.85,
    edgecolor='none'
)
plt.axvline(
    np.median(redundancies),
    color=FL_AMBER,
    linewidth=1.5,
    linestyle='--',
    label=f'Median = {np.median(redundancies):.3f}'
)
plt.xlabel('Pairwise Jaccard similarity (content tokens)')
plt.ylabel('Count')
plt.title('Source redundancy - average pairwise overlap between source articles')
plt.legend(fontsize=9)
plt.tick_params(length=0)
plt.tight_layout()
plt.show()

print(f'Lexical coverage   - median: {np.median(coverages):.3f}  mean: {np.mean(coverages):.3f}')
print(f'Source redundancy  - median: {np.median(redundancies):.3f}  mean: {np.mean(redundancies):.3f}')

from collections import Counter

src_vocab = Counter()
summ_vocab = Counter()

sample_n = min(1000, len(data['train']))
for idx in range(sample_n):
    row = data['train'][idx]
    sources = [
        s for s in split_sources(row['document'] if isinstance(row, dict) else row.get('document', ''))
        if len(s) >= DOC_MIN_CHARS
    ]
    summary = get_summary(row)

    for s in sources:
        src_vocab.update(tok_content(s))

    summ_vocab.update(tok_content(summary))

top_src = src_vocab.most_common(20)
top_summ = summ_vocab.most_common(20)

words_s, counts_s = zip(*top_src)

plt.figure(figsize=(8, 4.5))
plt.barh(
    list(reversed(words_s)),
    list(reversed(counts_s)),
    color=FL_BLUE,
    alpha=0.85,
    height=0.65
)
plt.xlabel('Term frequency')
plt.title(f'Top 20 content words from source articles (n={sample_n:,})')
plt.tick_params(length=0)
plt.gca().xaxis.set_major_formatter(
    mticker.FuncFormatter(lambda x, _: f'{x/1000:.0f}k' if x >= 1000 else f'{int(x)}')
)
plt.tight_layout()
plt.show()

words_u, counts_u = zip(*top_summ)

plt.figure(figsize=(8, 4.5))
plt.barh(
    list(reversed(words_u)),
    list(reversed(counts_u)),
    color=FL_SLATE,
    alpha=0.85,
    height=0.65
)
plt.xlabel('Term frequency')
plt.title(f'Top 20 content words from summaries (n={sample_n:,})')
plt.tick_params(length=0)
plt.gca().xaxis.set_major_formatter(
    mticker.FuncFormatter(lambda x, _: f'{x/1000:.0f}k' if x >= 1000 else f'{int(x)}')
)
plt.tight_layout()
plt.show()

from IPython.display import display, Markdown

IDX = 10
row = data['train'][IDX]

sources = [
    s for s in split_sources(row['document'] if isinstance(row, dict) else row.get('document', ''))
    if len(s) >= DOC_MIN_CHARS
]
summary = get_summary(row)

display(Markdown('## Summary'))
display(Markdown(summary))

for i, src in enumerate(sources, start=1):
    display(Markdown(f'## Article {i}'))
    display(Markdown(f'Words: {word_count(src)}'))
    display(Markdown(src))