mirror of
https://github.com/theclaymethod/unslop.git
synced 2026-09-19 01:45:09 +08:00
Clayton Kim
14241ed594
# Conflicts: # .github/workflows/evals.yml # scripts/check_suggestions.py # scripts/extract_constraints.py # scripts/readability_metrics.py
296 lines
9.5 KiB
Python
Executable File
296 lines
9.5 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""
|
|
Calculate readability metrics for transformed text.
|
|
|
|
Outputs:
|
|
- Flesch-Kincaid grade level
|
|
- Sentence length variance
|
|
- Word repetition score
|
|
- Paragraph length stats
|
|
|
|
Usage:
|
|
python readability_metrics.py < input.txt
|
|
python readability_metrics.py input.txt
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import sys
|
|
import re
|
|
import json
|
|
from collections import Counter
|
|
from typing import TypedDict
|
|
|
|
|
|
class ReadabilityMetrics(TypedDict):
|
|
flesch_kincaid_grade: float
|
|
flesch_reading_ease: float
|
|
sentence_count: int
|
|
word_count: int
|
|
avg_sentence_length: float
|
|
sentence_length_variance: float
|
|
min_sentence_length: int
|
|
max_sentence_length: int
|
|
consecutive_similar_length: int
|
|
word_repetition_score: float
|
|
top_repeated_words: list[tuple[str, int]]
|
|
paragraph_count: int
|
|
avg_paragraph_length: float
|
|
flags: list[str]
|
|
|
|
|
|
def count_syllables(word: str) -> int:
|
|
"""Count syllables in a word (approximation)."""
|
|
word = word.lower().strip()
|
|
if not word:
|
|
return 0
|
|
|
|
# Handle special cases
|
|
if len(word) <= 3:
|
|
return 1
|
|
|
|
# Count vowel groups
|
|
vowels = "aeiouy"
|
|
count = 0
|
|
prev_is_vowel = False
|
|
|
|
for char in word:
|
|
is_vowel = char in vowels
|
|
if is_vowel and not prev_is_vowel:
|
|
count += 1
|
|
prev_is_vowel = is_vowel
|
|
|
|
# Adjust for silent e
|
|
if word.endswith('e') and count > 1:
|
|
count -= 1
|
|
|
|
# Adjust for -le endings
|
|
if word.endswith('le') and len(word) > 2 and word[-3] not in vowels:
|
|
count += 1
|
|
|
|
return max(1, count)
|
|
|
|
|
|
def split_sentences(text: str) -> list[str]:
|
|
"""Split text into sentences."""
|
|
parts = re.split(r'([.!?]["”]?)\s+(?=["“]?[A-Z])', text)
|
|
sentences = []
|
|
current = ""
|
|
for part in parts:
|
|
if re.match(r'[.!?]["”]?$', part):
|
|
current += part
|
|
sentences.append(current.strip())
|
|
current = ""
|
|
else:
|
|
current += part
|
|
if current.strip():
|
|
sentences.append(current.strip())
|
|
# Filter empty and clean up
|
|
return [s.strip() for s in sentences if s.strip()]
|
|
|
|
|
|
def split_staccato_units(text: str) -> list[str]:
|
|
units = re.split(r'\s*[—;]\s*|(?<=[.!?])\s+', text)
|
|
return [u.strip() for u in units if u.strip()]
|
|
|
|
|
|
def split_words(text: str) -> list[str]:
|
|
"""Split text into words."""
|
|
# Remove punctuation except hyphens in words. Numeric tokens are real words
|
|
# (a data table is not "empty text"), so keep them.
|
|
text = re.sub(r'[^\w\s-]', ' ', text)
|
|
words = text.lower().split()
|
|
return [w for w in words if w]
|
|
|
|
|
|
def calculate_metrics(text: str) -> ReadabilityMetrics:
|
|
"""Calculate all readability metrics."""
|
|
sentences = split_sentences(text)
|
|
words = split_words(text)
|
|
paragraphs = [p.strip() for p in text.split('\n\n') if p.strip()]
|
|
|
|
if not sentences or not words:
|
|
return {
|
|
"flesch_kincaid_grade": 0,
|
|
"flesch_reading_ease": 0,
|
|
"sentence_count": 0,
|
|
"word_count": 0,
|
|
"avg_sentence_length": 0,
|
|
"sentence_length_variance": 0,
|
|
"min_sentence_length": 0,
|
|
"max_sentence_length": 0,
|
|
"consecutive_similar_length": 0,
|
|
"word_repetition_score": 0,
|
|
"top_repeated_words": [],
|
|
"paragraph_count": len(paragraphs),
|
|
"avg_paragraph_length": 0,
|
|
"flags": ["Empty or invalid text"]
|
|
}
|
|
|
|
# Basic counts
|
|
sentence_count = len(sentences)
|
|
word_count = len(words)
|
|
syllable_count = sum(count_syllables(w) for w in words)
|
|
|
|
# Sentence lengths
|
|
sentence_lengths = [len(split_words(s)) for s in sentences]
|
|
avg_sentence_length = word_count / sentence_count if sentence_count else 0
|
|
|
|
# Variance calculation
|
|
if len(sentence_lengths) > 1:
|
|
mean = sum(sentence_lengths) / len(sentence_lengths)
|
|
variance = sum((x - mean) ** 2 for x in sentence_lengths) / len(sentence_lengths)
|
|
else:
|
|
variance = 0
|
|
|
|
# Consecutive similar length sentences
|
|
consecutive_similar = 0
|
|
max_consecutive = 0
|
|
for i in range(1, len(sentence_lengths)):
|
|
# "Similar" = within 3 words of each other
|
|
if abs(sentence_lengths[i] - sentence_lengths[i-1]) <= 3:
|
|
consecutive_similar += 1
|
|
max_consecutive = max(max_consecutive, consecutive_similar)
|
|
else:
|
|
consecutive_similar = 0
|
|
|
|
# Longest run of tiny (<=5-word) sentence-like units — the staccato "anti-slop" cadence.
|
|
# Tracked independently of sentence_count so short de-slopped outputs don't slip.
|
|
staccato_lengths = [len(split_words(s)) for s in split_staccato_units(text)]
|
|
staccato_run = 0
|
|
max_staccato_run = 0
|
|
for length in staccato_lengths:
|
|
if length <= 5:
|
|
staccato_run += 1
|
|
max_staccato_run = max(max_staccato_run, staccato_run)
|
|
else:
|
|
staccato_run = 0
|
|
|
|
# Flesch-Kincaid calculations
|
|
avg_syllables_per_word = syllable_count / word_count if word_count else 0
|
|
|
|
flesch_reading_ease = (
|
|
206.835
|
|
- (1.015 * avg_sentence_length)
|
|
- (84.6 * avg_syllables_per_word)
|
|
)
|
|
|
|
flesch_kincaid_grade = (
|
|
(0.39 * avg_sentence_length)
|
|
+ (11.8 * avg_syllables_per_word)
|
|
- 15.59
|
|
)
|
|
|
|
# Word repetition (excluding common words)
|
|
stop_words = {
|
|
'the', 'a', 'an', 'and', 'or', 'but', 'in', 'on', 'at', 'to', 'for',
|
|
'of', 'with', 'by', 'from', 'is', 'are', 'was', 'were', 'be', 'been',
|
|
'being', 'have', 'has', 'had', 'do', 'does', 'did', 'will', 'would',
|
|
'could', 'should', 'may', 'might', 'must', 'shall', 'can', 'need',
|
|
'it', 'its', 'this', 'that', 'these', 'those', 'i', 'you', 'he', 'she',
|
|
'we', 'they', 'what', 'which', 'who', 'when', 'where', 'why', 'how',
|
|
'not', 'no', 'yes', 'if', 'then', 'else', 'so', 'as', 'than', 'just'
|
|
}
|
|
|
|
content_words = [w for w in words if w not in stop_words and len(w) > 2]
|
|
word_freq = Counter(content_words)
|
|
|
|
# Repetition score: percentage of content words appearing 3+ times
|
|
repeated_words = {w: c for w, c in word_freq.items() if c >= 3}
|
|
repetition_score = (
|
|
sum(repeated_words.values()) / len(content_words) * 100
|
|
if content_words else 0
|
|
)
|
|
|
|
top_repeated = sorted(word_freq.items(), key=lambda x: x[1], reverse=True)[:10]
|
|
|
|
# Paragraph stats
|
|
para_lengths = [len(split_words(p)) for p in paragraphs]
|
|
avg_para_length = sum(para_lengths) / len(para_lengths) if para_lengths else 0
|
|
|
|
# Generate flags
|
|
flags: list[str] = []
|
|
|
|
if flesch_kincaid_grade > 12:
|
|
flags.append(f"High reading level ({flesch_kincaid_grade:.1f} grade)")
|
|
|
|
if variance < 10 and sentence_count > 3:
|
|
flags.append("Low sentence length variance (monotonous rhythm)")
|
|
|
|
if max_consecutive >= 3:
|
|
flags.append(f"{max_consecutive}+ consecutive similar-length sentences")
|
|
|
|
if repetition_score > 15:
|
|
flags.append(f"High word repetition ({repetition_score:.1f}%)")
|
|
|
|
if avg_sentence_length > 25:
|
|
flags.append(f"Long average sentence length ({avg_sentence_length:.1f} words)")
|
|
|
|
if avg_sentence_length < 8 and sentence_count > 3:
|
|
flags.append("Very short sentences (may feel choppy)")
|
|
|
|
# Staccato cadence is the loudest "anti-slop AI" signature; flag it even on
|
|
# short outputs (no sentence_count gate) so de-slopped text can't hide it.
|
|
if max_staccato_run >= 3:
|
|
flags.append(f"Staccato cadence: {max_staccato_run} consecutive tiny sentences (anti-slop AI tell)")
|
|
elif avg_sentence_length < 6 and sentence_count >= 2:
|
|
flags.append(f"Staccato cadence (avg {avg_sentence_length:.1f} words/sentence — anti-slop AI tell)")
|
|
|
|
if max(sentence_lengths) - min(sentence_lengths) < 5 and sentence_count > 5:
|
|
flags.append("Sentences all similar length (AI tell)")
|
|
|
|
return {
|
|
"flesch_kincaid_grade": round(flesch_kincaid_grade, 1),
|
|
"flesch_reading_ease": round(flesch_reading_ease, 1),
|
|
"sentence_count": sentence_count,
|
|
"word_count": word_count,
|
|
"avg_sentence_length": round(avg_sentence_length, 1),
|
|
"sentence_length_variance": round(variance, 1),
|
|
"min_sentence_length": min(sentence_lengths) if sentence_lengths else 0,
|
|
"max_sentence_length": max(sentence_lengths) if sentence_lengths else 0,
|
|
"consecutive_similar_length": max_consecutive,
|
|
"word_repetition_score": round(repetition_score, 1),
|
|
"top_repeated_words": top_repeated,
|
|
"paragraph_count": len(paragraphs),
|
|
"avg_paragraph_length": round(avg_para_length, 1),
|
|
"flags": flags
|
|
}
|
|
|
|
|
|
def parse_args(argv: list[str]) -> argparse.Namespace:
|
|
parser = argparse.ArgumentParser(
|
|
description="Calculate readability metrics for transformed text."
|
|
)
|
|
parser.add_argument("path", nargs="?", help="Path to input text file (default: read stdin)")
|
|
return parser.parse_args(argv)
|
|
|
|
|
|
def main() -> None:
|
|
args = parse_args(sys.argv[1:])
|
|
|
|
# Read input
|
|
if args.path:
|
|
try:
|
|
with open(args.path, 'r', errors="replace") as f:
|
|
text = f.read()
|
|
except OSError as e:
|
|
print(json.dumps({"error": f"Could not read input: {e}"}))
|
|
sys.exit(2)
|
|
else:
|
|
text = sys.stdin.buffer.read().decode("utf-8", errors="replace")
|
|
|
|
if not text.strip():
|
|
print(json.dumps({"error": "No input provided"}))
|
|
sys.exit(1)
|
|
|
|
metrics = calculate_metrics(text)
|
|
print(json.dumps(metrics, indent=2))
|
|
|
|
# Exit with 1 if any flags (warnings)
|
|
sys.exit(1 if metrics["flags"] else 0)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|