feat[syncer]: add data source Confluence (#18501)

### Summary

As title
This commit is contained in:
Haruko386
2026-08-19 18:15:32 +08:00
committed by GitHub
parent 88a72e6a36
commit 91ddbcf60f
3 changed files with 1325 additions and 0 deletions

View File

@@ -27,6 +27,7 @@ import (
// source registers both the task-context factory (used by the syncer runtime)
// and the raw-config factory (used by the test-connection endpoint).
func RegisterBuiltIns(registry *Registry) {
registerBuiltIn(registry, "confluence", NewConfluenceConnector)
registerBuiltIn(registry, "rss", NewRSSConnector)
registerBuiltIn(registry, "bitbucket", NewBitbucketConnector)
registerBuiltIn(registry, "github", NewGitHubConnector)

View File

@@ -0,0 +1,996 @@
//
// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
package connector
import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"html"
"io"
"net/http"
"net/url"
"os"
"path"
"regexp"
"sort"
"strings"
"time"
xhtml "golang.org/x/net/html"
)
const (
defaultConfluenceBatchSize = 32
defaultConfluenceAttachmentThreshold = 10 * 1024 * 1024
confluenceRequestTimeout = 60 * time.Second
maxConfluenceResponseSize = 32 * 1024 * 1024
maxConfluenceSearchPages = 1000
)
var (
confluenceWhitespaceRE = regexp.MustCompile(`[ \t]+`)
confluenceNewlineRE = regexp.MustCompile(`\n{3,}`)
)
// ConfluenceConnector reads pages, comments, and attachments from Confluence.
type ConfluenceConnector struct {
wikiBase string
apiBase string
isCloud bool
indexMode string
space string
pageID string
indexRecursively bool
cqlQuery string
username string
accessToken string
batchSize int
attachmentThreshold int64
client *http.Client
getJSON func(ctx context.Context, path string, out any) error
download func(ctx context.Context, rawURL string) ([]byte, error)
getCurrentUser func(ctx context.Context, userID string) (string, error)
}
// NewConfluenceConnector creates a Confluence connector from Python-compatible config.
func NewConfluenceConnector(config map[string]any) (*ConfluenceConnector, error) {
credentials := configAnyMap(config["credentials"])
wikiBase := strings.TrimRight(strings.TrimSpace(stringConfig(config["wiki_base"])), "/")
isCloud := configBoolDefault(config["is_cloud"], true)
indexMode := strings.ToLower(firstNonEmpty(stringConfig(config["index_mode"]), "everything"))
batchSize := configInt(firstNonEmpty(stringConfig(config["sync_batch_size"]), stringConfig(config["batch_size"])), defaultConfluenceBatchSize)
c := &ConfluenceConnector{
wikiBase: wikiBase,
apiBase: confluenceAPIBase(wikiBase, isCloud),
isCloud: isCloud,
indexMode: indexMode,
space: strings.TrimSpace(stringConfig(config["space"])),
pageID: strings.TrimSpace(stringConfig(config["page_id"])),
indexRecursively: configBoolDefault(config["index_recursively"], false),
cqlQuery: strings.TrimSpace(stringConfig(config["cql_query"])),
username: strings.TrimSpace(stringConfig(credentials["confluence_username"])),
accessToken: stringConfig(credentials["confluence_access_token"]),
batchSize: batchSize,
attachmentThreshold: confluenceAttachmentThreshold(),
client: &http.Client{Timeout: confluenceRequestTimeout},
}
c.getJSON = c.doJSON
c.download = c.downloadURL
c.getCurrentUser = c.currentUserName
return c, nil
}
// Validate validates Confluence settings and credentials.
func (c *ConfluenceConnector) Validate(ctx context.Context) error {
if c == nil {
return fmt.Errorf("confluence connector is nil")
}
if c.wikiBase == "" {
return fmt.Errorf("Confluence wiki_base is required")
}
parsed, err := url.Parse(c.wikiBase)
if err != nil || parsed.Scheme == "" || parsed.Host == "" {
return fmt.Errorf("invalid Confluence wiki_base URL")
}
if parsed.Scheme != "http" && parsed.Scheme != "https" {
return fmt.Errorf("Confluence wiki_base must use HTTP or HTTPS")
}
if c.accessToken == "" {
return fmt.Errorf("Confluence access token is required")
}
if c.isCloud && c.username == "" {
return fmt.Errorf("Confluence Cloud username is required")
}
if c.batchSize <= 0 {
return fmt.Errorf("batch_size must be a positive integer")
}
switch c.indexMode {
case "everything", "space", "page", "":
default:
return fmt.Errorf("invalid Confluence index_mode %q", c.indexMode)
}
if c.indexMode == "space" && c.space == "" {
return fmt.Errorf("Confluence space key is required when index_mode is space")
}
if c.indexMode == "page" && c.pageID == "" {
return fmt.Errorf("Confluence page_id is required when index_mode is page")
}
var spaces confluenceSearchResponse
if err := c.getJSON(ctx, "rest/api/space?limit=1", &spaces); err != nil {
return confluenceValidationError(err)
}
if len(spaces.Results) == 0 {
return fmt.Errorf("no Confluence spaces found")
}
if c.indexMode == "space" {
var space confluenceSpace
if err := c.getJSON(ctx, "rest/api/space/"+url.PathEscape(c.space), &space); err != nil {
return fmt.Errorf("invalid Confluence space key provided: %w", confluenceValidationError(err))
}
}
return nil
}
// ValidateConnectorSetting validates Confluence settings from an unsaved config.
func (c *ConfluenceConnector) ValidateConnectorSetting(ctx context.Context, request map[string]any) error {
ctx, cancel := context.WithTimeout(ctx, connectorSettingValidationTimeout)
defer cancel()
return c.Validate(ctx)
}
// OpenSync opens one Confluence sync session.
func (c *ConfluenceConnector) OpenSync(ctx context.Context, request SyncRequest) (SyncSession, error) {
if err := c.Validate(ctx); err != nil {
return nil, err
}
end := request.WindowEnd
if end.IsZero() {
end = time.Now().UTC()
}
session := &confluenceSyncSession{
connector: c,
batchSize: c.batchSize,
windowStart: request.WindowStart,
windowEnd: end,
fromBeginning: request.FromBeginning,
pageCursor: newConfluenceSearchCursor(c, c.pageCQL(request.WindowStart, end, request.FromBeginning), strings.Join(confluencePageExpansionFields, ",")),
nameCounts: map[string]int{},
}
session.applyResume(request.Resume)
return session, nil
}
// OpenPrune opens one complete Confluence prune snapshot session.
func (c *ConfluenceConnector) OpenPrune(ctx context.Context, request PruneRequest) (PruneSession, error) {
if err := c.Validate(ctx); err != nil {
return nil, err
}
documents, err := c.loadSlimDocuments(ctx)
if err != nil {
return nil, err
}
sort.Slice(documents, func(i, j int) bool { return documents[i].SourceID < documents[j].SourceID })
return &confluencePruneSession{documents: documents, batchSize: c.batchSize}, nil
}
func (c *ConfluenceConnector) loadSlimDocuments(ctx context.Context) ([]SlimDocument, error) {
pages, err := c.searchAll(ctx, c.basePageCQL(), "")
if err != nil {
return nil, err
}
documents := make([]SlimDocument, 0, len(pages))
for _, page := range pages {
documents = append(documents, SlimDocument{SourceID: c.documentURL(page.Links.WebUI)})
attachments, err := c.searchAll(ctx, c.attachmentCQL(page.ID.String(), nil, time.Time{}, true), strings.Join(confluenceAttachmentExpansionFields, ","))
if err != nil {
return nil, err
}
for _, attachment := range attachments {
if !isConfluenceAttachmentAccepted(attachment) {
continue
}
documents = append(documents, SlimDocument{SourceID: c.documentURL(attachment.Links.WebUI)})
}
}
return documents, nil
}
func (c *ConfluenceConnector) pageDocument(ctx context.Context, page confluenceContent) (SourceDocument, error) {
pageURL := c.documentURL(page.Links.WebUI)
content := confluenceHTMLText(page.bodyHTML())
comments, err := c.pageComments(ctx, page.ID.String())
if err != nil {
return SourceDocument{}, err
}
if comments != "" {
content = strings.TrimSpace(content + "\n\nComments:\n" + comments)
}
blob := []byte(content)
updatedAt := parseConfluenceTime(page.Version.When)
if updatedAt.IsZero() {
updatedAt = time.Now().UTC()
}
metadata := map[string]any{}
if page.Space.Name != "" {
metadata["space"] = page.Space.Name
}
if len(page.Metadata.Labels.Results) > 0 {
labels := make([]string, 0, len(page.Metadata.Labels.Results))
for _, label := range page.Metadata.Labels.Results {
if label.Name != "" {
labels = append(labels, label.Name)
}
}
if len(labels) > 0 {
metadata["labels"] = labels
}
}
return SourceDocument{
SourceID: pageURL,
Extension: ".txt",
Blob: blob,
UpdatedAt: updatedAt,
SizeBytes: int64(len(blob)),
Metadata: metadataOrNil(metadata),
Fingerprint: contentFingerprint(blob),
}, nil
}
func (c *ConfluenceConnector) pageComments(ctx context.Context, pageID string) (string, error) {
comments, err := c.searchAll(ctx, fmt.Sprintf("type=comment and container='%s'", confluenceCQLQuote(pageID)), "body.storage.value,body.view.value")
if err != nil {
return "", err
}
parts := make([]string, 0, len(comments))
for _, comment := range comments {
text := confluenceHTMLText(comment.bodyHTML())
if text != "" {
parts = append(parts, text)
}
}
return strings.Join(parts, "\n"), nil
}
func (c *ConfluenceConnector) attachmentDocument(ctx context.Context, page confluenceContent, attachment confluenceContent) (SourceDocument, bool, error) {
if !isConfluenceAttachmentAccepted(attachment) {
return SourceDocument{}, false, nil
}
if attachment.Extensions.FileSize > c.attachmentThreshold {
return SourceDocument{}, false, nil
}
rawURL := attachment.Links.Download
if rawURL == "" {
return SourceDocument{}, false, nil
}
blob, err := c.download(ctx, c.documentURL(rawURL))
if err != nil {
return SourceDocument{}, false, err
}
if len(blob) == 0 {
return SourceDocument{}, false, nil
}
updatedAt := parseConfluenceTime(attachment.Version.When)
if updatedAt.IsZero() {
updatedAt = parseConfluenceTime(page.Version.When)
}
if updatedAt.IsZero() {
updatedAt = time.Now().UTC()
}
metadata := map[string]any{"parent_page_id": c.documentURL(page.Links.WebUI)}
if firstNonEmpty(attachment.Space.Name, page.Space.Name) != "" {
metadata["space"] = firstNonEmpty(attachment.Space.Name, page.Space.Name)
}
if len(attachment.Metadata.Labels.Results) > 0 {
labels := make([]string, 0, len(attachment.Metadata.Labels.Results))
for _, label := range attachment.Metadata.Labels.Results {
if label.Name != "" {
labels = append(labels, label.Name)
}
}
if len(labels) > 0 {
metadata["labels"] = labels
}
}
title := confluenceAttachmentTitle(attachment)
return SourceDocument{
SourceID: c.documentURL(attachment.Links.WebUI),
Extension: confluenceAttachmentExtension(title),
Blob: blob,
UpdatedAt: updatedAt,
SizeBytes: int64(len(blob)),
Metadata: metadata,
Fingerprint: contentFingerprint(blob),
}, true, nil
}
func confluenceFileNameFromURL(rawURL string) string {
parsed, err := url.Parse(rawURL)
if err != nil {
return ""
}
name := path.Base(parsed.Path)
if name == "." || name == "/" {
return ""
}
return name
}
func confluenceAttachmentTitle(attachment confluenceContent) string {
return firstNonEmpty(attachment.Title, confluenceFileNameFromURL(attachment.Links.Download), "attachment")
}
func (c *ConfluenceConnector) searchAll(ctx context.Context, cql, expand string) ([]confluenceContent, error) {
cursor := newConfluenceSearchCursor(c, cql, expand)
var out []confluenceContent
for {
item, ok, err := cursor.next(ctx)
if err != nil {
return nil, err
}
if !ok {
return out, nil
}
out = append(out, item)
}
}
// confluenceSearchCursor iterates one Confluence content search, fetching pages
// on demand while guarding against repeated next links and unbounded pagination.
type confluenceSearchCursor struct {
connector *ConfluenceConnector
nextPath string
seen map[string]struct{}
pages int
results []confluenceContent
index int
}
func newConfluenceSearchCursor(connector *ConfluenceConnector, cql, expand string) *confluenceSearchCursor {
return &confluenceSearchCursor{
connector: connector,
nextPath: confluenceCQLPath(cql, expand, connector.batchSize),
seen: map[string]struct{}{},
}
}
func (cur *confluenceSearchCursor) next(ctx context.Context) (confluenceContent, bool, error) {
for cur.index >= len(cur.results) {
if cur.nextPath == "" {
return confluenceContent{}, false, nil
}
if _, ok := cur.seen[cur.nextPath]; ok {
return confluenceContent{}, false, fmt.Errorf("confluence pagination repeated the same next link")
}
cur.seen[cur.nextPath] = struct{}{}
cur.pages++
if cur.pages > maxConfluenceSearchPages {
return confluenceContent{}, false, fmt.Errorf("confluence search exceeded %d pages", maxConfluenceSearchPages)
}
var page confluenceSearchResponse
if err := cur.connector.getJSON(ctx, cur.nextPath, &page); err != nil {
return confluenceContent{}, false, err
}
cur.results = page.Results
cur.index = 0
cur.nextPath = page.Links.Next
}
item := cur.results[cur.index]
cur.index++
return item, true, nil
}
func (c *ConfluenceConnector) doJSON(ctx context.Context, path string, out any) error {
data, err := c.do(ctx, http.MethodGet, path)
if err != nil {
return err
}
if err := json.Unmarshal(data, out); err != nil {
return fmt.Errorf("decode Confluence response: %w", err)
}
return nil
}
func (c *ConfluenceConnector) downloadURL(ctx context.Context, rawURL string) ([]byte, error) {
return c.do(ctx, http.MethodGet, rawURL)
}
func (c *ConfluenceConnector) do(ctx context.Context, method, rawURL string) ([]byte, error) {
resolved, err := c.resolveURL(rawURL)
if err != nil {
return nil, err
}
req, err := http.NewRequestWithContext(ctx, method, resolved, nil)
if err != nil {
return nil, err
}
req.Header.Set("Accept", "application/json")
if c.isCloud || c.username != "" {
req.SetBasicAuth(c.username, c.accessToken)
} else {
req.Header.Set("Authorization", "Bearer "+c.accessToken)
}
res, err := c.client.Do(req)
if err != nil {
return nil, err
}
defer func() { _ = res.Body.Close() }()
body, err := io.ReadAll(io.LimitReader(res.Body, maxConfluenceResponseSize+1))
if err != nil {
return nil, err
}
if int64(len(body)) > maxConfluenceResponseSize {
return nil, fmt.Errorf("Confluence response exceeds %d bytes", maxConfluenceResponseSize)
}
if res.StatusCode < 200 || res.StatusCode >= 300 {
return nil, &confluenceStatusError{status: res.StatusCode, body: string(body)}
}
return body, nil
}
func (c *ConfluenceConnector) resolveURL(rawURL string) (string, error) {
if c.isCloud && strings.HasPrefix(rawURL, "/rest/") && strings.HasSuffix(c.apiBase, "/wiki") {
rawURL = "/wiki" + rawURL
}
parsed, err := url.Parse(rawURL)
if err != nil {
return "", err
}
base, err := url.Parse(strings.TrimRight(c.apiBase, "/") + "/")
if err != nil {
return "", err
}
if parsed.IsAbs() {
if !strings.EqualFold(parsed.Scheme, base.Scheme) || !strings.EqualFold(parsed.Host, base.Host) {
return "", fmt.Errorf("confluence URL %q targets a different origin than the configured wiki base", rawURL)
}
return parsed.String(), nil
}
return base.ResolveReference(parsed).String(), nil
}
func (c *ConfluenceConnector) documentURL(contentURL string) string {
return buildConfluenceDocumentID(c.wikiBase, contentURL, c.isCloud)
}
func (c *ConfluenceConnector) currentUserName(ctx context.Context, userID string) (string, error) {
var user struct {
DisplayName string `json:"displayName"`
}
field := "key"
if c.isCloud {
field = "accountId"
}
if err := c.getJSON(ctx, "rest/api/user?"+field+"="+url.QueryEscape(userID), &user); err != nil {
return "", err
}
return user.DisplayName, nil
}
func (c *ConfluenceConnector) basePageCQL() string {
if c.cqlQuery != "" {
return c.cqlQuery
}
query := "type=page"
switch c.indexMode {
case "space":
query += fmt.Sprintf(" and space='%s'", confluenceCQLQuote(c.space))
case "page":
if c.indexRecursively {
query += fmt.Sprintf(" and (ancestor='%s' or id='%s')", confluenceCQLQuote(c.pageID), confluenceCQLQuote(c.pageID))
} else {
query += fmt.Sprintf(" and id='%s'", confluenceCQLQuote(c.pageID))
}
}
return query
}
func (c *ConfluenceConnector) pageCQL(start *time.Time, end time.Time, fromBeginning bool) string {
query := c.basePageCQL()
if !fromBeginning && start != nil && !start.IsZero() {
query += " and lastmodified >= '" + confluenceCQLTime(*start) + "'"
}
if !end.IsZero() {
query += " and lastmodified <= '" + confluenceCQLTime(end) + "'"
}
return query + " order by lastmodified asc"
}
func (c *ConfluenceConnector) attachmentCQL(pageID string, start *time.Time, end time.Time, fromBeginning bool) string {
query := fmt.Sprintf("type=attachment and container='%s'", confluenceCQLQuote(pageID))
if !fromBeginning && start != nil && !start.IsZero() {
query += " and lastmodified >= '" + confluenceCQLTime(*start) + "'"
}
if !end.IsZero() {
query += " and lastmodified <= '" + confluenceCQLTime(end) + "'"
}
return query + " order by lastmodified asc"
}
type confluenceSyncSession struct {
connector *ConfluenceConnector
batchSize int
windowStart *time.Time
windowEnd time.Time
fromBeginning bool
pageCursor *confluenceSearchCursor
currentPage confluenceContent
hasCurrentPage bool
pageDocPending bool
attachCursor *confluenceSearchCursor
resumeSourceID string
resumeMatched bool
resumeUpdatedAt *time.Time
nameCounts map[string]int
}
// NextBatch returns the next Confluence document batch, fetching pages,
// comments, and attachments incrementally so session memory stays bounded by
// batchSize.
func (s *confluenceSyncSession) NextBatch(ctx context.Context) (SyncBatch, error) {
documents := make([]SourceDocument, 0, s.batchSize)
var checkpoint *SyncCheckpoint
for len(documents) < s.batchSize {
doc, err := s.nextDocument(ctx)
if errors.Is(err, io.EOF) {
if s.resumeSourceID != "" && !s.resumeMatched {
return SyncBatch{}, fmt.Errorf("confluence sync resume checkpoint %q was not found in the source; refusing to discard unprocessed documents", s.resumeSourceID)
}
if len(documents) == 0 {
return SyncBatch{}, io.EOF
}
break
}
if err != nil {
return SyncBatch{}, err
}
if !s.includeResumed(*doc) {
continue
}
documents = append(documents, *doc)
checkpoint = confluenceSyncCheckpoint(*doc)
}
return SyncBatch{Documents: documents, Checkpoint: checkpoint}, nil
}
// Close closes the Confluence sync session.
func (s *confluenceSyncSession) Close() error {
return nil
}
// nextDocument produces the next document in page order: each page followed by
// its accepted attachments, all filtered by the configured window.
func (s *confluenceSyncSession) nextDocument(ctx context.Context) (*SourceDocument, error) {
for {
if !s.hasCurrentPage {
page, ok, err := s.pageCursor.next(ctx)
if err != nil {
return nil, err
}
if !ok {
return nil, io.EOF
}
s.currentPage = page
s.hasCurrentPage = true
s.pageDocPending = true
s.attachCursor = newConfluenceSearchCursor(s.connector, s.connector.attachmentCQL(page.ID.String(), s.windowStart, s.windowEnd, s.fromBeginning), strings.Join(confluenceAttachmentExpansionFields, ","))
}
if s.pageDocPending {
s.pageDocPending = false
doc, err := s.connector.pageDocument(ctx, s.currentPage)
if err != nil {
return nil, err
}
if s.fromBeginning || inConfluenceWindow(doc.UpdatedAt, s.windowStart, s.windowEnd) {
doc.SemanticIdentifier = confluenceSemanticIdentifier(s.currentPage.Space.Name, s.currentPage.ancestorTitles(), s.currentPage.Title, s.nameCounts)
return &doc, nil
}
continue
}
attachment, ok, err := s.attachCursor.next(ctx)
if err != nil {
return nil, err
}
if !ok {
s.hasCurrentPage = false
continue
}
doc, accepted, err := s.connector.attachmentDocument(ctx, s.currentPage, attachment)
if err != nil {
return nil, err
}
if !accepted {
continue
}
if s.fromBeginning || inConfluenceWindow(doc.UpdatedAt, s.windowStart, s.windowEnd) {
doc.SemanticIdentifier = confluenceSemanticIdentifier(firstNonEmpty(s.currentPage.Space.Name, attachment.Space.Name), nil, s.currentPage.Title+" / "+confluenceAttachmentTitle(attachment), s.nameCounts)
return &doc, nil
}
}
}
func (s *confluenceSyncSession) includeResumed(doc SourceDocument) bool {
if s.resumeMatched || s.resumeSourceID == "" {
return true
}
if doc.SourceID == s.resumeSourceID {
s.resumeMatched = true
return false
}
if s.resumeUpdatedAt != nil && doc.UpdatedAt.After(*s.resumeUpdatedAt) {
s.resumeMatched = true
return true
}
return false
}
func (s *confluenceSyncSession) applyResume(checkpoint *SyncCheckpoint) {
if checkpoint == nil {
return
}
s.resumeSourceID = firstNonEmpty(checkpoint.SourceID, checkpoint.Cursor)
s.resumeUpdatedAt = checkpoint.UpdatedAt
}
func confluenceSyncCheckpoint(doc SourceDocument) *SyncCheckpoint {
updatedAt := doc.UpdatedAt
return &SyncCheckpoint{Cursor: doc.SourceID, SourceID: doc.SourceID, UpdatedAt: &updatedAt}
}
type confluencePruneSession struct {
documents []SlimDocument
batchSize int
index int
}
// NextBatch returns the next Confluence prune snapshot batch.
func (s *confluencePruneSession) NextBatch(ctx context.Context) (PruneBatch, error) {
if s.index >= len(s.documents) {
return PruneBatch{}, io.EOF
}
end := s.index + s.batchSize
if end > len(s.documents) {
end = len(s.documents)
}
batch := PruneBatch{Documents: s.documents[s.index:end]}
s.index = end
return batch, nil
}
// Close closes the Confluence prune session.
func (s *confluencePruneSession) Close() error {
return nil
}
type confluenceSearchResponse struct {
Results []confluenceContent `json:"results"`
Links struct {
Next string `json:"next"`
} `json:"_links"`
}
type confluenceSpace struct {
Key string `json:"key"`
Name string `json:"name"`
}
type confluenceContent struct {
ID confluenceString `json:"id"`
Title string `json:"title"`
Type string `json:"type"`
Body struct {
Storage confluenceBody `json:"storage"`
View confluenceBody `json:"view"`
} `json:"body"`
Space confluenceSpace `json:"space"`
Ancestors []struct {
Title string `json:"title"`
} `json:"ancestors"`
Version struct {
When string `json:"when"`
By struct {
DisplayName string `json:"displayName"`
Email string `json:"email"`
} `json:"by"`
} `json:"version"`
Metadata struct {
MediaType string `json:"mediaType"`
Labels struct {
Results []struct {
Name string `json:"name"`
} `json:"results"`
} `json:"labels"`
} `json:"metadata"`
Extensions struct {
FileSize int64 `json:"fileSize"`
} `json:"extensions"`
Links struct {
WebUI string `json:"webui"`
Download string `json:"download"`
} `json:"_links"`
}
type confluenceBody struct {
Value string `json:"value"`
}
type confluenceString string
func (s *confluenceString) UnmarshalJSON(data []byte) error {
var text string
if err := json.Unmarshal(data, &text); err == nil {
*s = confluenceString(text)
return nil
}
var number json.Number
decoder := json.NewDecoder(bytes.NewReader(data))
decoder.UseNumber()
if err := decoder.Decode(&number); err == nil {
*s = confluenceString(number.String())
return nil
}
return fmt.Errorf("Confluence string field must be string or number")
}
func (s confluenceString) String() string {
return string(s)
}
func (c confluenceContent) bodyHTML() string {
return firstNonEmpty(c.Body.Storage.Value, c.Body.View.Value)
}
func (c confluenceContent) ancestorTitles() []string {
out := make([]string, 0, len(c.Ancestors))
for _, ancestor := range c.Ancestors {
if ancestor.Title != "" {
out = append(out, ancestor.Title)
}
}
return out
}
func confluenceAPIBase(wikiBase string, isCloud bool) string {
base := strings.TrimRight(wikiBase, "/")
if isCloud && base != "" && !strings.HasSuffix(base, "/wiki") {
base += "/wiki"
}
return base
}
func buildConfluenceDocumentID(baseURL, contentURL string, isCloud bool) string {
finalBase := strings.TrimRight(baseURL, "/") + "/"
if isCloud && !strings.HasSuffix(finalBase, "/wiki/") {
finalBase += "wiki/"
}
base, err := url.Parse(finalBase)
if err != nil {
return strings.TrimRight(finalBase, "/") + "/" + strings.TrimLeft(contentURL, "/")
}
ref, err := url.Parse(strings.TrimLeft(contentURL, "/"))
if err != nil {
return strings.TrimRight(finalBase, "/") + "/" + strings.TrimLeft(contentURL, "/")
}
return base.ResolveReference(ref).String()
}
func confluenceCQLPath(cql, expand string, limit int) string {
values := url.Values{}
values.Set("cql", cql)
if expand != "" {
values.Set("expand", expand)
}
values.Set("limit", fmt.Sprint(limit))
return "rest/api/content/search?" + values.Encode()
}
func confluenceCQLQuote(value string) string {
value = strings.ReplaceAll(value, `\`, `\\`)
return strings.ReplaceAll(value, "'", "\\'")
}
func confluenceCQLTime(value time.Time) string {
return value.UTC().Format("2006-01-02 15:04")
}
func inConfluenceWindow(updatedAt time.Time, start *time.Time, end time.Time) bool {
if !end.IsZero() && updatedAt.After(end) {
return false
}
if start != nil && !updatedAt.After(*start) {
return false
}
return true
}
func parseConfluenceTime(value string) time.Time {
value = strings.TrimSpace(value)
if value == "" {
return time.Time{}
}
layouts := []string{
time.RFC3339Nano,
time.RFC3339,
"2006-01-02T15:04:05.000-0700",
"2006-01-02T15:04:05-0700",
"2006-01-02 15:04:05",
}
for _, layout := range layouts {
if parsed, err := time.Parse(layout, value); err == nil {
return parsed.UTC()
}
}
return time.Time{}
}
func confluenceSemanticIdentifier(space string, ancestors []string, title string, counts map[string]int) string {
title = firstNonEmpty(title, "Untitled")
parts := make([]string, 0, len(ancestors)+2)
if space != "" {
parts = append(parts, space)
}
parts = append(parts, ancestors...)
parts = append(parts, title)
fullPath := strings.Join(parts, " / ")
counts[title]++
if counts[title] > 1 && fullPath != "" {
return fullPath
}
return title
}
func confluenceHTMLText(raw string) string {
raw = strings.TrimSpace(raw)
if raw == "" {
return ""
}
root, err := xhtml.Parse(strings.NewReader(raw))
if err != nil {
return html.UnescapeString(raw)
}
var buffer bytes.Buffer
var walk func(*xhtml.Node)
walk = func(node *xhtml.Node) {
if node.Type == xhtml.ElementNode {
switch strings.ToLower(node.Data) {
case "script", "style":
return
case "br", "p", "div", "tr", "li", "table", "h1", "h2", "h3", "h4", "h5", "h6":
buffer.WriteByte('\n')
}
}
if node.Type == xhtml.TextNode {
text := strings.TrimSpace(html.UnescapeString(node.Data))
if text != "" {
if buffer.Len() > 0 {
buffer.WriteByte(' ')
}
buffer.WriteString(text)
}
}
for child := node.FirstChild; child != nil; child = child.NextSibling {
walk(child)
}
if node.Type == xhtml.ElementNode {
switch strings.ToLower(node.Data) {
case "p", "div", "tr", "li", "table", "h1", "h2", "h3", "h4", "h5", "h6":
buffer.WriteByte('\n')
}
}
}
walk(root)
text := strings.ReplaceAll(buffer.String(), "\r\n", "\n")
text = confluenceWhitespaceRE.ReplaceAllString(text, " ")
text = strings.ReplaceAll(text, " \n", "\n")
text = strings.ReplaceAll(text, "\n ", "\n")
text = confluenceNewlineRE.ReplaceAllString(text, "\n\n")
return strings.TrimSpace(text)
}
func metadataOrNil(metadata map[string]any) map[string]any {
if len(metadata) == 0 {
return nil
}
return metadata
}
func isConfluenceAttachmentAccepted(attachment confluenceContent) bool {
mediaType := strings.ToLower(attachment.Metadata.MediaType)
if strings.HasPrefix(mediaType, "image/") {
switch mediaType {
case "image/jpeg", "image/jpg", "image/png", "image/gif", "image/bmp", "image/tiff", "image/webp":
return true
default:
return false
}
}
ext := confluenceAttachmentExtension(attachment.Title)
if _, ok := webdavTextExtensions[ext]; ok {
return true
}
if _, ok := webdavDocumentExtensions[ext]; ok {
return true
}
return false
}
func confluenceAttachmentExtension(title string) string {
ext := strings.ToLower(path.Ext(title))
if ext == "" {
return ".unknown"
}
return ext
}
func confluenceAttachmentThreshold() int64 {
if raw := strings.TrimSpace(os.Getenv("CONFLUENCE_CONNECTOR_ATTACHMENT_SIZE_THRESHOLD")); raw != "" {
if parsed := configInt(raw, defaultConfluenceAttachmentThreshold); parsed > 0 {
return int64(parsed)
}
}
return defaultConfluenceAttachmentThreshold
}
func confluenceValidationError(err error) error {
var statusErr *confluenceStatusError
if errors.As(err, &statusErr) {
switch statusErr.status {
case http.StatusUnauthorized:
return fmt.Errorf("invalid or expired Confluence credentials")
case http.StatusForbidden:
return fmt.Errorf("insufficient permissions to access Confluence resources")
}
}
return err
}
type confluenceStatusError struct {
status int
body string
}
func (e *confluenceStatusError) Error() string {
return fmt.Sprintf("Confluence request failed with status %d: %s", e.status, strings.TrimSpace(e.body))
}
var confluencePageExpansionFields = []string{
"body.storage.value",
"body.view.value",
"space",
"ancestors",
"version",
"metadata.labels",
}
var confluenceAttachmentExpansionFields = []string{
"metadata",
"metadata.labels",
"extensions",
"version",
"space",
}

View File

@@ -0,0 +1,328 @@
package connector
import (
"context"
"encoding/base64"
"encoding/json"
"errors"
"io"
"net/http"
"net/http/httptest"
"net/url"
"strings"
"testing"
"time"
)
func TestConfluenceConnectorSyncPagesCommentsAndAttachments(t *testing.T) {
server := newConfluenceFixtureServer(t, http.StatusOK)
defer server.Close()
connector := newTestConfluenceConnector(t, server.URL, map[string]any{
"index_mode": "page",
"page_id": "123",
"index_recursively": true,
"batch_size": 10,
})
session, err := connector.OpenSync(context.Background(), SyncRequest{
FromBeginning: true,
WindowEnd: confluenceTestTime(t, "2026-01-03T00:00:00Z"),
})
if err != nil {
t.Fatalf("OpenSync() error = %v", err)
}
defer func() { _ = session.Close() }()
batch, err := session.NextBatch(context.Background())
if err != nil {
t.Fatalf("NextBatch() error = %v", err)
}
if len(batch.Documents) != 2 {
t.Fatalf("len(documents) = %d, want 2", len(batch.Documents))
}
page := batch.Documents[0]
if page.SourceID != server.URL+"/wiki/display/SPACE/Page" {
t.Fatalf("page SourceID = %q", page.SourceID)
}
if page.SemanticIdentifier != "Page" {
t.Fatalf("page SemanticIdentifier = %q", page.SemanticIdentifier)
}
if !strings.Contains(string(page.Blob), "Hello Confluence") || !strings.Contains(string(page.Blob), "Comment text") {
t.Fatalf("page blob did not include page and comment text: %q", string(page.Blob))
}
if page.Extension != ".txt" {
t.Fatalf("page Extension = %q", page.Extension)
}
if page.Metadata["space"] != "Engineering" {
t.Fatalf("page metadata = %#v", page.Metadata)
}
if page.Fingerprint != contentFingerprint(page.Blob) {
t.Fatalf("page fingerprint mismatch")
}
attachment := batch.Documents[1]
if attachment.SourceID != server.URL+"/wiki/download/attachments/file.txt?version=1" {
t.Fatalf("attachment SourceID = %q", attachment.SourceID)
}
if attachment.Extension != ".txt" {
t.Fatalf("attachment Extension = %q", attachment.Extension)
}
if string(attachment.Blob) != "attachment bytes" {
t.Fatalf("attachment Blob = %q", string(attachment.Blob))
}
if attachment.Metadata["parent_page_id"] != page.SourceID {
t.Fatalf("attachment metadata = %#v", attachment.Metadata)
}
if batch.Checkpoint == nil || batch.Checkpoint.SourceID != attachment.SourceID {
t.Fatalf("checkpoint = %#v", batch.Checkpoint)
}
_, err = session.NextBatch(context.Background())
if !errors.Is(err, io.EOF) {
t.Fatalf("second NextBatch() error = %v, want EOF", err)
}
}
func TestConfluenceConnectorResume(t *testing.T) {
server := newConfluenceFixtureServer(t, http.StatusOK)
defer server.Close()
connector := newTestConfluenceConnector(t, server.URL, map[string]any{"batch_size": 1})
session, err := connector.OpenSync(context.Background(), SyncRequest{
FromBeginning: true,
WindowEnd: confluenceTestTime(t, "2026-01-03T00:00:00Z"),
})
if err != nil {
t.Fatalf("OpenSync() error = %v", err)
}
first, err := session.NextBatch(context.Background())
if err != nil {
t.Fatalf("first NextBatch() error = %v", err)
}
resumed, err := connector.OpenSync(context.Background(), SyncRequest{
FromBeginning: true,
WindowEnd: confluenceTestTime(t, "2026-01-03T00:00:00Z"),
Resume: first.Checkpoint,
})
if err != nil {
t.Fatalf("resumed OpenSync() error = %v", err)
}
batch, err := resumed.NextBatch(context.Background())
if err != nil {
t.Fatalf("resumed NextBatch() error = %v", err)
}
if len(batch.Documents) != 1 || batch.Documents[0].Extension != ".txt" || batch.Documents[0].SourceID == first.Documents[0].SourceID {
t.Fatalf("resumed batch = %#v first = %#v", batch.Documents, first.Documents)
}
}
func TestConfluenceConnectorPrune(t *testing.T) {
server := newConfluenceFixtureServer(t, http.StatusOK)
defer server.Close()
connector := newTestConfluenceConnector(t, server.URL, nil)
session, err := connector.OpenPrune(context.Background(), PruneRequest{})
if err != nil {
t.Fatalf("OpenPrune() error = %v", err)
}
defer func() { _ = session.Close() }()
batch, err := session.NextBatch(context.Background())
if err != nil {
t.Fatalf("NextBatch() error = %v", err)
}
got := map[string]bool{}
for _, doc := range batch.Documents {
got[doc.SourceID] = true
}
for _, want := range []string{
server.URL + "/wiki/display/SPACE/Page",
server.URL + "/wiki/download/attachments/file.txt?version=1",
} {
if !got[want] {
t.Fatalf("missing prune source id %q in %#v", want, batch.Documents)
}
}
}
func TestConfluenceConnectorValidateMapsUnauthorized(t *testing.T) {
server := newConfluenceFixtureServer(t, http.StatusUnauthorized)
defer server.Close()
connector := newTestConfluenceConnector(t, server.URL, nil)
err := connector.ValidateConnectorSetting(context.Background(), nil)
if err == nil || !strings.Contains(err.Error(), "invalid or expired") {
t.Fatalf("ValidateConnectorSetting() error = %v", err)
}
}
func newTestConfluenceConnector(t *testing.T, wikiBase string, overrides map[string]any) *ConfluenceConnector {
t.Helper()
config := map[string]any{
"wiki_base": wikiBase,
"is_cloud": true,
"batch_size": 10,
"credentials": map[string]any{
"confluence_username": "user@example.com",
"confluence_access_token": "token",
},
}
for key, value := range overrides {
config[key] = value
}
connector, err := NewConfluenceConnector(config)
if err != nil {
t.Fatalf("NewConfluenceConnector() error = %v", err)
}
return connector
}
func newConfluenceFixtureServer(t *testing.T, validateStatus int) *httptest.Server {
t.Helper()
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if validateStatus != http.StatusOK && strings.HasPrefix(r.URL.Path, "/wiki/rest/api/space") {
http.Error(w, "unauthorized", validateStatus)
return
}
if !confluenceHasBasicAuth(r, "user@example.com", "token") {
http.Error(w, "bad auth", http.StatusUnauthorized)
return
}
switch {
case r.URL.Path == "/wiki/rest/api/space":
writeConfluenceJSON(t, w, map[string]any{"results": []map[string]any{{"key": "SPACE", "name": "Engineering"}}})
case r.URL.Path == "/wiki/rest/api/space/SPACE":
writeConfluenceJSON(t, w, map[string]any{"key": "SPACE", "name": "Engineering"})
case r.URL.Path == "/wiki/rest/api/content/search":
cql := r.URL.Query().Get("cql")
switch {
case strings.Contains(cql, "type=page"):
writeConfluenceJSON(t, w, map[string]any{"results": []map[string]any{confluenceFixturePage()}})
case strings.Contains(cql, "type=comment"):
writeConfluenceJSON(t, w, map[string]any{"results": []map[string]any{confluenceFixtureComment()}})
case strings.Contains(cql, "type=attachment"):
writeConfluenceJSON(t, w, map[string]any{"results": []map[string]any{confluenceFixtureAttachment()}})
default:
t.Errorf("Unexpected CQL: %q", cql)
http.Error(w, "unexpected cql", http.StatusInternalServerError)
}
case r.URL.Path == "/wiki/download/attachments/file.txt":
w.WriteHeader(http.StatusOK)
_, _ = w.Write([]byte("attachment bytes"))
default:
t.Errorf("unexpected request %s", r.URL.String())
http.Error(w, "unexpected request", http.StatusInternalServerError)
}
}))
return server
}
func confluenceHasBasicAuth(r *http.Request, username, password string) bool {
const prefix = "Basic "
raw := r.Header.Get("Authorization")
if !strings.HasPrefix(raw, prefix) {
return false
}
decoded, err := base64.StdEncoding.DecodeString(strings.TrimPrefix(raw, prefix))
if err != nil {
return false
}
return string(decoded) == username+":"+password
}
func writeConfluenceJSON(t *testing.T, w http.ResponseWriter, payload any) {
t.Helper()
w.Header().Set("Content-Type", "application/json")
if err := json.NewEncoder(w).Encode(payload); err != nil {
t.Errorf("encode fixture: %v", err)
}
}
func confluenceFixturePage() map[string]any {
return map[string]any{
"id": 123,
"type": "page",
"title": "Page",
"body": map[string]any{"storage": map[string]any{"value": "<p>Hello <strong>Confluence</strong></p>"}},
"space": map[string]any{"key": "SPACE", "name": "Engineering"},
"ancestors": []map[string]any{
{"title": "Root"},
},
"version": map[string]any{"when": "2026-01-01T10:00:00Z"},
"metadata": map[string]any{"labels": map[string]any{"results": []map[string]any{{"name": "docs"}}}},
"_links": map[string]any{"webui": "/display/SPACE/Page"},
}
}
func confluenceFixtureComment() map[string]any {
return map[string]any{
"id": 456,
"type": "comment",
"title": "Comment",
"body": map[string]any{"storage": map[string]any{"value": "<p>Comment text</p>"}},
"version": map[string]any{"when": "2026-01-01T11:00:00Z"},
"_links": map[string]any{"webui": "/display/SPACE/Page?focusedCommentId=c1"},
}
}
func confluenceFixtureAttachment() map[string]any {
return map[string]any{
"id": 789,
"type": "attachment",
"title": "file.txt",
"space": map[string]any{"key": "SPACE", "name": "Engineering"},
"metadata": map[string]any{
"mediaType": "text/plain",
"labels": map[string]any{"results": []map[string]any{{"name": "file"}}},
},
"extensions": map[string]any{"fileSize": 16},
"version": map[string]any{"when": "2026-01-01T12:00:00Z"},
"_links": map[string]any{
"webui": "/download/attachments/file.txt?version=1",
"download": "/download/attachments/file.txt?version=1",
},
}
}
func confluenceTestTime(t *testing.T, value string) time.Time {
t.Helper()
parsed, err := time.Parse(time.RFC3339, value)
if err != nil {
t.Fatalf("parse test time %q: %v", value, err)
}
return parsed
}
func TestBuildConfluenceDocumentIDAddsWikiForCloud(t *testing.T) {
got := buildConfluenceDocumentID("https://example.atlassian.net", "/display/SPACE/Page", true)
want := "https://example.atlassian.net/wiki/display/SPACE/Page"
if got != want {
t.Fatalf("buildConfluenceDocumentID() = %q, want %q", got, want)
}
got = buildConfluenceDocumentID("https://example.atlassian.net/wiki", "/display/SPACE/Page", true)
if got != want {
t.Fatalf("buildConfluenceDocumentID() with wiki base = %q, want %q", got, want)
}
}
func TestConfluenceCQLPathEscapesQuery(t *testing.T) {
raw := confluenceCQLPath("type=page and space='A B'", "version", 10)
parsed, err := url.Parse(raw)
if err != nil {
t.Fatalf("parse path: %v", err)
}
if parsed.Query().Get("cql") != "type=page and space='A B'" {
t.Fatalf("cql = %q", parsed.Query().Get("cql"))
}
}
func TestConfluenceCQLQuoteEscapesBackslashes(t *testing.T) {
got := confluenceCQLQuote(`path\name's`)
want := `path\\name\'s`
if got != want {
t.Fatalf("confluenceCQLQuote() = %q, want %q", got, want)
}
}