package connector
import (
"context"
"crypto/md5"
"encoding/hex"
"errors"
"io"
"testing"
"time"
)
// TestRSSConnectorOpenSyncFullAndIncremental verifies full and windowed sync.
func TestRSSConnectorOpenSyncFullAndIncremental(t *testing.T) {
connector, err := NewRSSConnector(map[string]any{"feed_url": "https://example.com/feed.xml", "batch_size": 1})
if err != nil {
t.Fatalf("NewRSSConnector failed: %v", err)
}
connector.fetchFeed = staticRSSFeed
fullSession, err := connector.OpenSync(context.Background(), SyncRequest{FromBeginning: true, WindowEnd: mustTime(t, "2026-01-05T00:00:00Z")})
if err != nil {
t.Fatalf("OpenSync full failed: %v", err)
}
first, err := fullSession.NextBatch(context.Background())
if err != nil {
t.Fatalf("NextBatch first failed: %v", err)
}
if len(first.Documents) != 1 || first.Documents[0].SourceID != expectedRSSSourceID("entry-old") {
t.Fatalf("unexpected first full batch: %+v", first.Documents)
}
second, err := fullSession.NextBatch(context.Background())
if err != nil {
t.Fatalf("NextBatch second failed: %v", err)
}
if len(second.Documents) != 1 || second.Documents[0].SourceID != expectedRSSSourceID("entry-new") {
t.Fatalf("unexpected second full batch: %+v", second.Documents)
}
if _, err = fullSession.NextBatch(context.Background()); !errors.Is(err, io.EOF) {
t.Fatalf("NextBatch EOF = %v", err)
}
start := mustTime(t, "2026-01-02T00:00:00Z")
incrementalSession, err := connector.OpenSync(context.Background(), SyncRequest{WindowStart: &start, WindowEnd: mustTime(t, "2026-01-04T00:00:00Z")})
if err != nil {
t.Fatalf("OpenSync incremental failed: %v", err)
}
batch, err := incrementalSession.NextBatch(context.Background())
if err != nil {
t.Fatalf("NextBatch incremental failed: %v", err)
}
if len(batch.Documents) != 1 {
t.Fatalf("incremental batch len = %d, want 1", len(batch.Documents))
}
doc := batch.Documents[0]
if doc.SourceID != expectedRSSSourceID("entry-new") {
t.Fatalf("incremental source id = %s", doc.SourceID)
}
if doc.SemanticIdentifier != "New title" {
t.Fatalf("semantic identifier = %q", doc.SemanticIdentifier)
}
if string(doc.Blob) != "New title\n\nNew body" {
t.Fatalf("blob = %q", string(doc.Blob))
}
if doc.Metadata["link"] != "https://example.com/new" {
t.Fatalf("link metadata = %v", doc.Metadata["link"])
}
}
// TestRSSConnectorOpenPrune verifies complete slim snapshot generation.
func TestRSSConnectorOpenPrune(t *testing.T) {
connector, err := NewRSSConnector(map[string]any{"feed_url": "https://example.com/feed.xml", "batch_size": 10})
if err != nil {
t.Fatalf("NewRSSConnector failed: %v", err)
}
connector.fetchFeed = staticRSSFeed
session, err := connector.OpenPrune(context.Background(), PruneRequest{})
if err != nil {
t.Fatalf("OpenPrune failed: %v", err)
}
batch, err := session.NextBatch(context.Background())
if err != nil {
t.Fatalf("NextBatch failed: %v", err)
}
if len(batch.Documents) != 3 {
t.Fatalf("prune snapshot len = %d, want 3", len(batch.Documents))
}
if batch.Documents[0].SourceID != expectedRSSSourceID("entry-old") ||
batch.Documents[1].SourceID != expectedRSSSourceID("entry-new") ||
batch.Documents[2].SourceID != expectedRSSSourceID("entry-future") {
t.Fatalf("unexpected prune ids: %+v", batch.Documents)
}
}
// staticRSSFeed returns a fixed RSS feed for tests.
func staticRSSFeed(ctx context.Context, feedURL string) ([]byte, error) {
return []byte(`