package connector import ( "context" "crypto/md5" "encoding/hex" "errors" "io" "testing" "time" ) // TestRSSConnectorOpenSyncFullAndIncremental verifies full and windowed sync. func TestRSSConnectorOpenSyncFullAndIncremental(t *testing.T) { connector, err := NewRSSConnector(map[string]any{"feed_url": "https://example.com/feed.xml", "batch_size": 1}) if err != nil { t.Fatalf("NewRSSConnector failed: %v", err) } connector.fetchFeed = staticRSSFeed fullSession, err := connector.OpenSync(context.Background(), SyncRequest{FromBeginning: true, WindowEnd: mustTime(t, "2026-01-05T00:00:00Z")}) if err != nil { t.Fatalf("OpenSync full failed: %v", err) } first, err := fullSession.NextBatch(context.Background()) if err != nil { t.Fatalf("NextBatch first failed: %v", err) } if len(first.Documents) != 1 || first.Documents[0].SourceID != expectedRSSSourceID("entry-old") { t.Fatalf("unexpected first full batch: %+v", first.Documents) } second, err := fullSession.NextBatch(context.Background()) if err != nil { t.Fatalf("NextBatch second failed: %v", err) } if len(second.Documents) != 1 || second.Documents[0].SourceID != expectedRSSSourceID("entry-new") { t.Fatalf("unexpected second full batch: %+v", second.Documents) } if _, err = fullSession.NextBatch(context.Background()); !errors.Is(err, io.EOF) { t.Fatalf("NextBatch EOF = %v", err) } start := mustTime(t, "2026-01-02T00:00:00Z") incrementalSession, err := connector.OpenSync(context.Background(), SyncRequest{WindowStart: &start, WindowEnd: mustTime(t, "2026-01-04T00:00:00Z")}) if err != nil { t.Fatalf("OpenSync incremental failed: %v", err) } batch, err := incrementalSession.NextBatch(context.Background()) if err != nil { t.Fatalf("NextBatch incremental failed: %v", err) } if len(batch.Documents) != 1 { t.Fatalf("incremental batch len = %d, want 1", len(batch.Documents)) } doc := batch.Documents[0] if doc.SourceID != expectedRSSSourceID("entry-new") { t.Fatalf("incremental source id = %s", doc.SourceID) } if doc.SemanticIdentifier != "New title" { t.Fatalf("semantic identifier = %q", doc.SemanticIdentifier) } if string(doc.Blob) != "New title\n\nNew body" { t.Fatalf("blob = %q", string(doc.Blob)) } if doc.Metadata["link"] != "https://example.com/new" { t.Fatalf("link metadata = %v", doc.Metadata["link"]) } } // TestRSSConnectorOpenPrune verifies complete slim snapshot generation. func TestRSSConnectorOpenPrune(t *testing.T) { connector, err := NewRSSConnector(map[string]any{"feed_url": "https://example.com/feed.xml", "batch_size": 10}) if err != nil { t.Fatalf("NewRSSConnector failed: %v", err) } connector.fetchFeed = staticRSSFeed session, err := connector.OpenPrune(context.Background(), PruneRequest{}) if err != nil { t.Fatalf("OpenPrune failed: %v", err) } batch, err := session.NextBatch(context.Background()) if err != nil { t.Fatalf("NextBatch failed: %v", err) } if len(batch.Documents) != 3 { t.Fatalf("prune snapshot len = %d, want 3", len(batch.Documents)) } if batch.Documents[0].SourceID != expectedRSSSourceID("entry-old") || batch.Documents[1].SourceID != expectedRSSSourceID("entry-new") || batch.Documents[2].SourceID != expectedRSSSourceID("entry-future") { t.Fatalf("unexpected prune ids: %+v", batch.Documents) } } // staticRSSFeed returns a fixed RSS feed for tests. func staticRSSFeed(ctx context.Context, feedURL string) ([]byte, error) { return []byte(` Feed entry-old Old title https://example.com/old Thu, 01 Jan 2026 00:00:00 +0000 Old body

]]>
entry-new New title https://example.com/new Sat, 03 Jan 2026 00:00:00 +0000 New body

]]>
news
entry-future Future title https://example.com/future Tue, 06 Jan 2026 00:00:00 +0000 Future body

]]>
`), nil } // expectedRSSSourceID returns Python-compatible RSS source ID. func expectedRSSSourceID(value string) string { sum := md5.Sum([]byte(value)) return "rss:" + hex.EncodeToString(sum[:]) } // mustTime parses a test timestamp. func mustTime(t *testing.T, value string) time.Time { t.Helper() parsed, err := time.Parse(time.RFC3339, value) if err != nil { t.Fatalf("parse time %q: %v", value, err) } return parsed }