jackwener__opencli/clis/substack/utils.ts

import { CommandExecutionError } from '../../errors.js';
import type { IPage } from '../../types.js';

const FEED_POST_LINK_SELECTOR = 'a[href*="/home/post/"], a[href*="/p/"]';
const ARCHIVE_POST_LINK_SELECTOR = 'a[href*="/p/"]';

export function buildSubstackBrowseUrl(category?: string): string {
  if (!category || category === 'all') return 'https://substack.com/';
  const slug = category === 'tech' ? 'technology' : category;
  return `https://substack.com/browse/${slug}`;
}

export async function loadSubstackFeed(page: IPage, url: string, limit: number): Promise<any[]> {
  if (!page) throw new CommandExecutionError('Browser session required for substack feed');
  await page.goto(url);
  await page.wait({ selector: FEED_POST_LINK_SELECTOR, timeout: 5 });
  const data = await page.evaluate(`
    (async () => {
      await new Promise((resolve) => setTimeout(resolve, 3000));
      const limit = ${Math.max(1, Math.min(limit, 50))};
      const normalize = (value) => (value || '').replace(/\\s+/g, ' ').trim();
      const posts = [];
      const seen = new Set();

      const allLinks = Array.from(document.querySelectorAll('a')).filter((link) => {
        const href = link.getAttribute('href') || '';
        return href.includes('/home/post/') || href.includes('/p/');
      });

      for (const linkEl of allLinks) {
        let postUrl = linkEl.getAttribute('href') || '';
        if (!postUrl) continue;
        if (!postUrl.startsWith('http')) postUrl = 'https://substack.com' + postUrl;
        if (seen.has(postUrl)) continue;

        const lines = (linkEl.innerText || '')
          .split('\\n')
          .map((line) => normalize(line))
          .filter(Boolean);

        const readMeta = lines.find((line) => /\\b(read|watch|listen)\\b/i.test(line)) || '';
        if (!readMeta) continue;

        const date = lines.find((line) => /^[A-Z]{3}\\s+\\d{1,2}$/i.test(line)) || '';
        const contentLines = lines.filter((line) =>
          line &&
          line !== date &&
          line !== readMeta &&
          line.toLowerCase() !== 'save' &&
          line.toLowerCase() !== 'more' &&
          !/^(sign in|create account|get app)$/i.test(line),
        );

        const metaParts = readMeta.split('∙').map((part) => normalize(part));
        const author = metaParts[0] || '';
        const readTime = metaParts.slice(1).join(' ∙ ') || readMeta;
        const title = contentLines.length >= 2 ? contentLines[1] : (contentLines[0] || '');
        const description = contentLines.length >= 3 ? contentLines.slice(2).join(' ') : '';
        if (!title) continue;

        seen.add(postUrl);
        posts.push({
          rank: posts.length + 1,
          title,
          author,
          date,
          readTime,
          description: description.slice(0, 150),
          url: postUrl,
        });

        if (posts.length >= limit) break;
      }

      return posts;
    })()
  `);

  return Array.isArray(data) ? data : [];
}

export async function loadSubstackArchive(page: IPage, baseUrl: string, limit: number): Promise<any[]> {
  if (!page) throw new CommandExecutionError('Browser session required for substack archive');
  await page.goto(`${baseUrl}/archive`);
  await page.wait({ selector: ARCHIVE_POST_LINK_SELECTOR, timeout: 5 });
  const data = await page.evaluate(`
    (async () => {
      await new Promise((resolve) => setTimeout(resolve, 3000));
      const normalize = (value) => (value || '').replace(/\\s+/g, ' ').trim();
      const limit = ${Math.max(1, Math.min(limit, 50))};
      const grouped = new Map();

      for (const link of Array.from(document.querySelectorAll('a[href*="/p/"]'))) {
        const rawHref = link.getAttribute('href') || '';
        if (!rawHref || rawHref === '/p/upgrade') continue;

        const url = rawHref.startsWith('http') ? rawHref : ${JSON.stringify(baseUrl)} + rawHref;
        const text = normalize(link.textContent);
        if (!text) continue;
        if (/^(subscribe|paid|home|about|latest|top|discussions)$/i.test(text)) continue;
        if (/^[\\d,]+$/.test(text)) continue;

        const entry = grouped.get(url) || { texts: new Set(), date: '' };
        entry.texts.add(text);

        const container = link.closest('article, section, div') || link.parentElement || link;
        const containerText = normalize(container.textContent);
        if (!entry.date) {
          entry.date = containerText.match(/\\b(?:[A-Z]{3}\\s+\\d{1,2}|[A-Z][a-z]{2}\\s+\\d{1,2})\\b/)?.[0] || '';
        }

        grouped.set(url, entry);
      }

      const posts = [];
      for (const [url, entry] of Array.from(grouped.entries())) {
        const texts = Array.from(entry.texts).map((text) => normalize(text)).filter((text) => text.length > 3).sort((a, b) => a.length - b.length);
        const title = texts[0] || '';
        const description = texts.find((text) => text !== title) || '';
        if (!title) continue;
        posts.push({
          rank: posts.length + 1,
          title,
          date: entry.date,
          description: description.slice(0, 150),
          url,
        });
        if (posts.length >= limit) break;
      }

      return posts;
    })()
  `);

  return Array.isArray(data) ? data : [];
}

export const __test__ = {
  FEED_POST_LINK_SELECTOR,
  ARCHIVE_POST_LINK_SELECTOR,
};