mirror of
https://github.com/infiniflow/ragflow.git
synced 2026-08-22 16:23:12 +08:00
@@ -27,6 +27,7 @@ import (
|
||||
// source registers both the task-context factory (used by the syncer runtime)
|
||||
// and the raw-config factory (used by the test-connection endpoint).
|
||||
func RegisterBuiltIns(registry *Registry) {
|
||||
registerBuiltIn(registry, "confluence", NewConfluenceConnector)
|
||||
registerBuiltIn(registry, "rss", NewRSSConnector)
|
||||
registerBuiltIn(registry, "bitbucket", NewBitbucketConnector)
|
||||
registerBuiltIn(registry, "github", NewGitHubConnector)
|
||||
|
||||
996
internal/syncer/connector/confluence.go
Normal file
996
internal/syncer/connector/confluence.go
Normal file
@@ -0,0 +1,996 @@
|
||||
//
|
||||
// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
//
|
||||
|
||||
package connector
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"html"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"path"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
xhtml "golang.org/x/net/html"
|
||||
)
|
||||
|
||||
const (
|
||||
defaultConfluenceBatchSize = 32
|
||||
defaultConfluenceAttachmentThreshold = 10 * 1024 * 1024
|
||||
confluenceRequestTimeout = 60 * time.Second
|
||||
maxConfluenceResponseSize = 32 * 1024 * 1024
|
||||
maxConfluenceSearchPages = 1000
|
||||
)
|
||||
|
||||
var (
|
||||
confluenceWhitespaceRE = regexp.MustCompile(`[ \t]+`)
|
||||
confluenceNewlineRE = regexp.MustCompile(`\n{3,}`)
|
||||
)
|
||||
|
||||
// ConfluenceConnector reads pages, comments, and attachments from Confluence.
|
||||
type ConfluenceConnector struct {
|
||||
wikiBase string
|
||||
apiBase string
|
||||
isCloud bool
|
||||
indexMode string
|
||||
space string
|
||||
pageID string
|
||||
indexRecursively bool
|
||||
cqlQuery string
|
||||
username string
|
||||
accessToken string
|
||||
batchSize int
|
||||
attachmentThreshold int64
|
||||
client *http.Client
|
||||
|
||||
getJSON func(ctx context.Context, path string, out any) error
|
||||
download func(ctx context.Context, rawURL string) ([]byte, error)
|
||||
getCurrentUser func(ctx context.Context, userID string) (string, error)
|
||||
}
|
||||
|
||||
// NewConfluenceConnector creates a Confluence connector from Python-compatible config.
|
||||
func NewConfluenceConnector(config map[string]any) (*ConfluenceConnector, error) {
|
||||
credentials := configAnyMap(config["credentials"])
|
||||
wikiBase := strings.TrimRight(strings.TrimSpace(stringConfig(config["wiki_base"])), "/")
|
||||
isCloud := configBoolDefault(config["is_cloud"], true)
|
||||
indexMode := strings.ToLower(firstNonEmpty(stringConfig(config["index_mode"]), "everything"))
|
||||
batchSize := configInt(firstNonEmpty(stringConfig(config["sync_batch_size"]), stringConfig(config["batch_size"])), defaultConfluenceBatchSize)
|
||||
c := &ConfluenceConnector{
|
||||
wikiBase: wikiBase,
|
||||
apiBase: confluenceAPIBase(wikiBase, isCloud),
|
||||
isCloud: isCloud,
|
||||
indexMode: indexMode,
|
||||
space: strings.TrimSpace(stringConfig(config["space"])),
|
||||
pageID: strings.TrimSpace(stringConfig(config["page_id"])),
|
||||
indexRecursively: configBoolDefault(config["index_recursively"], false),
|
||||
cqlQuery: strings.TrimSpace(stringConfig(config["cql_query"])),
|
||||
username: strings.TrimSpace(stringConfig(credentials["confluence_username"])),
|
||||
accessToken: stringConfig(credentials["confluence_access_token"]),
|
||||
batchSize: batchSize,
|
||||
attachmentThreshold: confluenceAttachmentThreshold(),
|
||||
client: &http.Client{Timeout: confluenceRequestTimeout},
|
||||
}
|
||||
c.getJSON = c.doJSON
|
||||
c.download = c.downloadURL
|
||||
c.getCurrentUser = c.currentUserName
|
||||
return c, nil
|
||||
}
|
||||
|
||||
// Validate validates Confluence settings and credentials.
|
||||
func (c *ConfluenceConnector) Validate(ctx context.Context) error {
|
||||
if c == nil {
|
||||
return fmt.Errorf("confluence connector is nil")
|
||||
}
|
||||
if c.wikiBase == "" {
|
||||
return fmt.Errorf("Confluence wiki_base is required")
|
||||
}
|
||||
parsed, err := url.Parse(c.wikiBase)
|
||||
if err != nil || parsed.Scheme == "" || parsed.Host == "" {
|
||||
return fmt.Errorf("invalid Confluence wiki_base URL")
|
||||
}
|
||||
if parsed.Scheme != "http" && parsed.Scheme != "https" {
|
||||
return fmt.Errorf("Confluence wiki_base must use HTTP or HTTPS")
|
||||
}
|
||||
if c.accessToken == "" {
|
||||
return fmt.Errorf("Confluence access token is required")
|
||||
}
|
||||
if c.isCloud && c.username == "" {
|
||||
return fmt.Errorf("Confluence Cloud username is required")
|
||||
}
|
||||
if c.batchSize <= 0 {
|
||||
return fmt.Errorf("batch_size must be a positive integer")
|
||||
}
|
||||
switch c.indexMode {
|
||||
case "everything", "space", "page", "":
|
||||
default:
|
||||
return fmt.Errorf("invalid Confluence index_mode %q", c.indexMode)
|
||||
}
|
||||
if c.indexMode == "space" && c.space == "" {
|
||||
return fmt.Errorf("Confluence space key is required when index_mode is space")
|
||||
}
|
||||
if c.indexMode == "page" && c.pageID == "" {
|
||||
return fmt.Errorf("Confluence page_id is required when index_mode is page")
|
||||
}
|
||||
|
||||
var spaces confluenceSearchResponse
|
||||
if err := c.getJSON(ctx, "rest/api/space?limit=1", &spaces); err != nil {
|
||||
return confluenceValidationError(err)
|
||||
}
|
||||
if len(spaces.Results) == 0 {
|
||||
return fmt.Errorf("no Confluence spaces found")
|
||||
}
|
||||
if c.indexMode == "space" {
|
||||
var space confluenceSpace
|
||||
if err := c.getJSON(ctx, "rest/api/space/"+url.PathEscape(c.space), &space); err != nil {
|
||||
return fmt.Errorf("invalid Confluence space key provided: %w", confluenceValidationError(err))
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ValidateConnectorSetting validates Confluence settings from an unsaved config.
|
||||
func (c *ConfluenceConnector) ValidateConnectorSetting(ctx context.Context, request map[string]any) error {
|
||||
ctx, cancel := context.WithTimeout(ctx, connectorSettingValidationTimeout)
|
||||
defer cancel()
|
||||
return c.Validate(ctx)
|
||||
}
|
||||
|
||||
// OpenSync opens one Confluence sync session.
|
||||
func (c *ConfluenceConnector) OpenSync(ctx context.Context, request SyncRequest) (SyncSession, error) {
|
||||
if err := c.Validate(ctx); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
end := request.WindowEnd
|
||||
if end.IsZero() {
|
||||
end = time.Now().UTC()
|
||||
}
|
||||
session := &confluenceSyncSession{
|
||||
connector: c,
|
||||
batchSize: c.batchSize,
|
||||
windowStart: request.WindowStart,
|
||||
windowEnd: end,
|
||||
fromBeginning: request.FromBeginning,
|
||||
pageCursor: newConfluenceSearchCursor(c, c.pageCQL(request.WindowStart, end, request.FromBeginning), strings.Join(confluencePageExpansionFields, ",")),
|
||||
nameCounts: map[string]int{},
|
||||
}
|
||||
session.applyResume(request.Resume)
|
||||
return session, nil
|
||||
}
|
||||
|
||||
// OpenPrune opens one complete Confluence prune snapshot session.
|
||||
func (c *ConfluenceConnector) OpenPrune(ctx context.Context, request PruneRequest) (PruneSession, error) {
|
||||
if err := c.Validate(ctx); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
documents, err := c.loadSlimDocuments(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
sort.Slice(documents, func(i, j int) bool { return documents[i].SourceID < documents[j].SourceID })
|
||||
return &confluencePruneSession{documents: documents, batchSize: c.batchSize}, nil
|
||||
}
|
||||
|
||||
func (c *ConfluenceConnector) loadSlimDocuments(ctx context.Context) ([]SlimDocument, error) {
|
||||
pages, err := c.searchAll(ctx, c.basePageCQL(), "")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
documents := make([]SlimDocument, 0, len(pages))
|
||||
for _, page := range pages {
|
||||
documents = append(documents, SlimDocument{SourceID: c.documentURL(page.Links.WebUI)})
|
||||
attachments, err := c.searchAll(ctx, c.attachmentCQL(page.ID.String(), nil, time.Time{}, true), strings.Join(confluenceAttachmentExpansionFields, ","))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, attachment := range attachments {
|
||||
if !isConfluenceAttachmentAccepted(attachment) {
|
||||
continue
|
||||
}
|
||||
documents = append(documents, SlimDocument{SourceID: c.documentURL(attachment.Links.WebUI)})
|
||||
}
|
||||
}
|
||||
return documents, nil
|
||||
}
|
||||
|
||||
func (c *ConfluenceConnector) pageDocument(ctx context.Context, page confluenceContent) (SourceDocument, error) {
|
||||
pageURL := c.documentURL(page.Links.WebUI)
|
||||
content := confluenceHTMLText(page.bodyHTML())
|
||||
comments, err := c.pageComments(ctx, page.ID.String())
|
||||
if err != nil {
|
||||
return SourceDocument{}, err
|
||||
}
|
||||
if comments != "" {
|
||||
content = strings.TrimSpace(content + "\n\nComments:\n" + comments)
|
||||
}
|
||||
blob := []byte(content)
|
||||
updatedAt := parseConfluenceTime(page.Version.When)
|
||||
if updatedAt.IsZero() {
|
||||
updatedAt = time.Now().UTC()
|
||||
}
|
||||
metadata := map[string]any{}
|
||||
if page.Space.Name != "" {
|
||||
metadata["space"] = page.Space.Name
|
||||
}
|
||||
if len(page.Metadata.Labels.Results) > 0 {
|
||||
labels := make([]string, 0, len(page.Metadata.Labels.Results))
|
||||
for _, label := range page.Metadata.Labels.Results {
|
||||
if label.Name != "" {
|
||||
labels = append(labels, label.Name)
|
||||
}
|
||||
}
|
||||
if len(labels) > 0 {
|
||||
metadata["labels"] = labels
|
||||
}
|
||||
}
|
||||
return SourceDocument{
|
||||
SourceID: pageURL,
|
||||
Extension: ".txt",
|
||||
Blob: blob,
|
||||
UpdatedAt: updatedAt,
|
||||
SizeBytes: int64(len(blob)),
|
||||
Metadata: metadataOrNil(metadata),
|
||||
Fingerprint: contentFingerprint(blob),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (c *ConfluenceConnector) pageComments(ctx context.Context, pageID string) (string, error) {
|
||||
comments, err := c.searchAll(ctx, fmt.Sprintf("type=comment and container='%s'", confluenceCQLQuote(pageID)), "body.storage.value,body.view.value")
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
parts := make([]string, 0, len(comments))
|
||||
for _, comment := range comments {
|
||||
text := confluenceHTMLText(comment.bodyHTML())
|
||||
if text != "" {
|
||||
parts = append(parts, text)
|
||||
}
|
||||
}
|
||||
return strings.Join(parts, "\n"), nil
|
||||
}
|
||||
|
||||
func (c *ConfluenceConnector) attachmentDocument(ctx context.Context, page confluenceContent, attachment confluenceContent) (SourceDocument, bool, error) {
|
||||
if !isConfluenceAttachmentAccepted(attachment) {
|
||||
return SourceDocument{}, false, nil
|
||||
}
|
||||
if attachment.Extensions.FileSize > c.attachmentThreshold {
|
||||
return SourceDocument{}, false, nil
|
||||
}
|
||||
rawURL := attachment.Links.Download
|
||||
if rawURL == "" {
|
||||
return SourceDocument{}, false, nil
|
||||
}
|
||||
blob, err := c.download(ctx, c.documentURL(rawURL))
|
||||
if err != nil {
|
||||
return SourceDocument{}, false, err
|
||||
}
|
||||
if len(blob) == 0 {
|
||||
return SourceDocument{}, false, nil
|
||||
}
|
||||
updatedAt := parseConfluenceTime(attachment.Version.When)
|
||||
if updatedAt.IsZero() {
|
||||
updatedAt = parseConfluenceTime(page.Version.When)
|
||||
}
|
||||
if updatedAt.IsZero() {
|
||||
updatedAt = time.Now().UTC()
|
||||
}
|
||||
metadata := map[string]any{"parent_page_id": c.documentURL(page.Links.WebUI)}
|
||||
if firstNonEmpty(attachment.Space.Name, page.Space.Name) != "" {
|
||||
metadata["space"] = firstNonEmpty(attachment.Space.Name, page.Space.Name)
|
||||
}
|
||||
if len(attachment.Metadata.Labels.Results) > 0 {
|
||||
labels := make([]string, 0, len(attachment.Metadata.Labels.Results))
|
||||
for _, label := range attachment.Metadata.Labels.Results {
|
||||
if label.Name != "" {
|
||||
labels = append(labels, label.Name)
|
||||
}
|
||||
}
|
||||
if len(labels) > 0 {
|
||||
metadata["labels"] = labels
|
||||
}
|
||||
}
|
||||
title := confluenceAttachmentTitle(attachment)
|
||||
return SourceDocument{
|
||||
SourceID: c.documentURL(attachment.Links.WebUI),
|
||||
Extension: confluenceAttachmentExtension(title),
|
||||
Blob: blob,
|
||||
UpdatedAt: updatedAt,
|
||||
SizeBytes: int64(len(blob)),
|
||||
Metadata: metadata,
|
||||
Fingerprint: contentFingerprint(blob),
|
||||
}, true, nil
|
||||
}
|
||||
|
||||
func confluenceFileNameFromURL(rawURL string) string {
|
||||
parsed, err := url.Parse(rawURL)
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
name := path.Base(parsed.Path)
|
||||
if name == "." || name == "/" {
|
||||
return ""
|
||||
}
|
||||
return name
|
||||
}
|
||||
|
||||
func confluenceAttachmentTitle(attachment confluenceContent) string {
|
||||
return firstNonEmpty(attachment.Title, confluenceFileNameFromURL(attachment.Links.Download), "attachment")
|
||||
}
|
||||
|
||||
func (c *ConfluenceConnector) searchAll(ctx context.Context, cql, expand string) ([]confluenceContent, error) {
|
||||
cursor := newConfluenceSearchCursor(c, cql, expand)
|
||||
var out []confluenceContent
|
||||
for {
|
||||
item, ok, err := cursor.next(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !ok {
|
||||
return out, nil
|
||||
}
|
||||
out = append(out, item)
|
||||
}
|
||||
}
|
||||
|
||||
// confluenceSearchCursor iterates one Confluence content search, fetching pages
|
||||
// on demand while guarding against repeated next links and unbounded pagination.
|
||||
type confluenceSearchCursor struct {
|
||||
connector *ConfluenceConnector
|
||||
nextPath string
|
||||
seen map[string]struct{}
|
||||
pages int
|
||||
results []confluenceContent
|
||||
index int
|
||||
}
|
||||
|
||||
func newConfluenceSearchCursor(connector *ConfluenceConnector, cql, expand string) *confluenceSearchCursor {
|
||||
return &confluenceSearchCursor{
|
||||
connector: connector,
|
||||
nextPath: confluenceCQLPath(cql, expand, connector.batchSize),
|
||||
seen: map[string]struct{}{},
|
||||
}
|
||||
}
|
||||
|
||||
func (cur *confluenceSearchCursor) next(ctx context.Context) (confluenceContent, bool, error) {
|
||||
for cur.index >= len(cur.results) {
|
||||
if cur.nextPath == "" {
|
||||
return confluenceContent{}, false, nil
|
||||
}
|
||||
if _, ok := cur.seen[cur.nextPath]; ok {
|
||||
return confluenceContent{}, false, fmt.Errorf("confluence pagination repeated the same next link")
|
||||
}
|
||||
cur.seen[cur.nextPath] = struct{}{}
|
||||
cur.pages++
|
||||
if cur.pages > maxConfluenceSearchPages {
|
||||
return confluenceContent{}, false, fmt.Errorf("confluence search exceeded %d pages", maxConfluenceSearchPages)
|
||||
}
|
||||
var page confluenceSearchResponse
|
||||
if err := cur.connector.getJSON(ctx, cur.nextPath, &page); err != nil {
|
||||
return confluenceContent{}, false, err
|
||||
}
|
||||
cur.results = page.Results
|
||||
cur.index = 0
|
||||
cur.nextPath = page.Links.Next
|
||||
}
|
||||
item := cur.results[cur.index]
|
||||
cur.index++
|
||||
return item, true, nil
|
||||
}
|
||||
|
||||
func (c *ConfluenceConnector) doJSON(ctx context.Context, path string, out any) error {
|
||||
data, err := c.do(ctx, http.MethodGet, path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := json.Unmarshal(data, out); err != nil {
|
||||
return fmt.Errorf("decode Confluence response: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *ConfluenceConnector) downloadURL(ctx context.Context, rawURL string) ([]byte, error) {
|
||||
return c.do(ctx, http.MethodGet, rawURL)
|
||||
}
|
||||
|
||||
func (c *ConfluenceConnector) do(ctx context.Context, method, rawURL string) ([]byte, error) {
|
||||
resolved, err := c.resolveURL(rawURL)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
req, err := http.NewRequestWithContext(ctx, method, resolved, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
req.Header.Set("Accept", "application/json")
|
||||
if c.isCloud || c.username != "" {
|
||||
req.SetBasicAuth(c.username, c.accessToken)
|
||||
} else {
|
||||
req.Header.Set("Authorization", "Bearer "+c.accessToken)
|
||||
}
|
||||
res, err := c.client.Do(req)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer func() { _ = res.Body.Close() }()
|
||||
body, err := io.ReadAll(io.LimitReader(res.Body, maxConfluenceResponseSize+1))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if int64(len(body)) > maxConfluenceResponseSize {
|
||||
return nil, fmt.Errorf("Confluence response exceeds %d bytes", maxConfluenceResponseSize)
|
||||
}
|
||||
if res.StatusCode < 200 || res.StatusCode >= 300 {
|
||||
return nil, &confluenceStatusError{status: res.StatusCode, body: string(body)}
|
||||
}
|
||||
return body, nil
|
||||
}
|
||||
|
||||
func (c *ConfluenceConnector) resolveURL(rawURL string) (string, error) {
|
||||
if c.isCloud && strings.HasPrefix(rawURL, "/rest/") && strings.HasSuffix(c.apiBase, "/wiki") {
|
||||
rawURL = "/wiki" + rawURL
|
||||
}
|
||||
parsed, err := url.Parse(rawURL)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
base, err := url.Parse(strings.TrimRight(c.apiBase, "/") + "/")
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if parsed.IsAbs() {
|
||||
if !strings.EqualFold(parsed.Scheme, base.Scheme) || !strings.EqualFold(parsed.Host, base.Host) {
|
||||
return "", fmt.Errorf("confluence URL %q targets a different origin than the configured wiki base", rawURL)
|
||||
}
|
||||
return parsed.String(), nil
|
||||
}
|
||||
return base.ResolveReference(parsed).String(), nil
|
||||
}
|
||||
|
||||
func (c *ConfluenceConnector) documentURL(contentURL string) string {
|
||||
return buildConfluenceDocumentID(c.wikiBase, contentURL, c.isCloud)
|
||||
}
|
||||
|
||||
func (c *ConfluenceConnector) currentUserName(ctx context.Context, userID string) (string, error) {
|
||||
var user struct {
|
||||
DisplayName string `json:"displayName"`
|
||||
}
|
||||
field := "key"
|
||||
if c.isCloud {
|
||||
field = "accountId"
|
||||
}
|
||||
if err := c.getJSON(ctx, "rest/api/user?"+field+"="+url.QueryEscape(userID), &user); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return user.DisplayName, nil
|
||||
}
|
||||
|
||||
func (c *ConfluenceConnector) basePageCQL() string {
|
||||
if c.cqlQuery != "" {
|
||||
return c.cqlQuery
|
||||
}
|
||||
query := "type=page"
|
||||
switch c.indexMode {
|
||||
case "space":
|
||||
query += fmt.Sprintf(" and space='%s'", confluenceCQLQuote(c.space))
|
||||
case "page":
|
||||
if c.indexRecursively {
|
||||
query += fmt.Sprintf(" and (ancestor='%s' or id='%s')", confluenceCQLQuote(c.pageID), confluenceCQLQuote(c.pageID))
|
||||
} else {
|
||||
query += fmt.Sprintf(" and id='%s'", confluenceCQLQuote(c.pageID))
|
||||
}
|
||||
}
|
||||
return query
|
||||
}
|
||||
|
||||
func (c *ConfluenceConnector) pageCQL(start *time.Time, end time.Time, fromBeginning bool) string {
|
||||
query := c.basePageCQL()
|
||||
if !fromBeginning && start != nil && !start.IsZero() {
|
||||
query += " and lastmodified >= '" + confluenceCQLTime(*start) + "'"
|
||||
}
|
||||
if !end.IsZero() {
|
||||
query += " and lastmodified <= '" + confluenceCQLTime(end) + "'"
|
||||
}
|
||||
return query + " order by lastmodified asc"
|
||||
}
|
||||
|
||||
func (c *ConfluenceConnector) attachmentCQL(pageID string, start *time.Time, end time.Time, fromBeginning bool) string {
|
||||
query := fmt.Sprintf("type=attachment and container='%s'", confluenceCQLQuote(pageID))
|
||||
if !fromBeginning && start != nil && !start.IsZero() {
|
||||
query += " and lastmodified >= '" + confluenceCQLTime(*start) + "'"
|
||||
}
|
||||
if !end.IsZero() {
|
||||
query += " and lastmodified <= '" + confluenceCQLTime(end) + "'"
|
||||
}
|
||||
return query + " order by lastmodified asc"
|
||||
}
|
||||
|
||||
type confluenceSyncSession struct {
|
||||
connector *ConfluenceConnector
|
||||
batchSize int
|
||||
windowStart *time.Time
|
||||
windowEnd time.Time
|
||||
fromBeginning bool
|
||||
|
||||
pageCursor *confluenceSearchCursor
|
||||
currentPage confluenceContent
|
||||
hasCurrentPage bool
|
||||
pageDocPending bool
|
||||
attachCursor *confluenceSearchCursor
|
||||
|
||||
resumeSourceID string
|
||||
resumeMatched bool
|
||||
resumeUpdatedAt *time.Time
|
||||
|
||||
nameCounts map[string]int
|
||||
}
|
||||
|
||||
// NextBatch returns the next Confluence document batch, fetching pages,
|
||||
// comments, and attachments incrementally so session memory stays bounded by
|
||||
// batchSize.
|
||||
func (s *confluenceSyncSession) NextBatch(ctx context.Context) (SyncBatch, error) {
|
||||
documents := make([]SourceDocument, 0, s.batchSize)
|
||||
var checkpoint *SyncCheckpoint
|
||||
for len(documents) < s.batchSize {
|
||||
doc, err := s.nextDocument(ctx)
|
||||
if errors.Is(err, io.EOF) {
|
||||
if s.resumeSourceID != "" && !s.resumeMatched {
|
||||
return SyncBatch{}, fmt.Errorf("confluence sync resume checkpoint %q was not found in the source; refusing to discard unprocessed documents", s.resumeSourceID)
|
||||
}
|
||||
if len(documents) == 0 {
|
||||
return SyncBatch{}, io.EOF
|
||||
}
|
||||
break
|
||||
}
|
||||
if err != nil {
|
||||
return SyncBatch{}, err
|
||||
}
|
||||
if !s.includeResumed(*doc) {
|
||||
continue
|
||||
}
|
||||
documents = append(documents, *doc)
|
||||
checkpoint = confluenceSyncCheckpoint(*doc)
|
||||
}
|
||||
return SyncBatch{Documents: documents, Checkpoint: checkpoint}, nil
|
||||
}
|
||||
|
||||
// Close closes the Confluence sync session.
|
||||
func (s *confluenceSyncSession) Close() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// nextDocument produces the next document in page order: each page followed by
|
||||
// its accepted attachments, all filtered by the configured window.
|
||||
func (s *confluenceSyncSession) nextDocument(ctx context.Context) (*SourceDocument, error) {
|
||||
for {
|
||||
if !s.hasCurrentPage {
|
||||
page, ok, err := s.pageCursor.next(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !ok {
|
||||
return nil, io.EOF
|
||||
}
|
||||
s.currentPage = page
|
||||
s.hasCurrentPage = true
|
||||
s.pageDocPending = true
|
||||
s.attachCursor = newConfluenceSearchCursor(s.connector, s.connector.attachmentCQL(page.ID.String(), s.windowStart, s.windowEnd, s.fromBeginning), strings.Join(confluenceAttachmentExpansionFields, ","))
|
||||
}
|
||||
|
||||
if s.pageDocPending {
|
||||
s.pageDocPending = false
|
||||
doc, err := s.connector.pageDocument(ctx, s.currentPage)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if s.fromBeginning || inConfluenceWindow(doc.UpdatedAt, s.windowStart, s.windowEnd) {
|
||||
doc.SemanticIdentifier = confluenceSemanticIdentifier(s.currentPage.Space.Name, s.currentPage.ancestorTitles(), s.currentPage.Title, s.nameCounts)
|
||||
return &doc, nil
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
attachment, ok, err := s.attachCursor.next(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !ok {
|
||||
s.hasCurrentPage = false
|
||||
continue
|
||||
}
|
||||
doc, accepted, err := s.connector.attachmentDocument(ctx, s.currentPage, attachment)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !accepted {
|
||||
continue
|
||||
}
|
||||
if s.fromBeginning || inConfluenceWindow(doc.UpdatedAt, s.windowStart, s.windowEnd) {
|
||||
doc.SemanticIdentifier = confluenceSemanticIdentifier(firstNonEmpty(s.currentPage.Space.Name, attachment.Space.Name), nil, s.currentPage.Title+" / "+confluenceAttachmentTitle(attachment), s.nameCounts)
|
||||
return &doc, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (s *confluenceSyncSession) includeResumed(doc SourceDocument) bool {
|
||||
if s.resumeMatched || s.resumeSourceID == "" {
|
||||
return true
|
||||
}
|
||||
if doc.SourceID == s.resumeSourceID {
|
||||
s.resumeMatched = true
|
||||
return false
|
||||
}
|
||||
if s.resumeUpdatedAt != nil && doc.UpdatedAt.After(*s.resumeUpdatedAt) {
|
||||
s.resumeMatched = true
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func (s *confluenceSyncSession) applyResume(checkpoint *SyncCheckpoint) {
|
||||
if checkpoint == nil {
|
||||
return
|
||||
}
|
||||
s.resumeSourceID = firstNonEmpty(checkpoint.SourceID, checkpoint.Cursor)
|
||||
s.resumeUpdatedAt = checkpoint.UpdatedAt
|
||||
}
|
||||
|
||||
func confluenceSyncCheckpoint(doc SourceDocument) *SyncCheckpoint {
|
||||
updatedAt := doc.UpdatedAt
|
||||
return &SyncCheckpoint{Cursor: doc.SourceID, SourceID: doc.SourceID, UpdatedAt: &updatedAt}
|
||||
}
|
||||
|
||||
type confluencePruneSession struct {
|
||||
documents []SlimDocument
|
||||
batchSize int
|
||||
index int
|
||||
}
|
||||
|
||||
// NextBatch returns the next Confluence prune snapshot batch.
|
||||
func (s *confluencePruneSession) NextBatch(ctx context.Context) (PruneBatch, error) {
|
||||
if s.index >= len(s.documents) {
|
||||
return PruneBatch{}, io.EOF
|
||||
}
|
||||
end := s.index + s.batchSize
|
||||
if end > len(s.documents) {
|
||||
end = len(s.documents)
|
||||
}
|
||||
batch := PruneBatch{Documents: s.documents[s.index:end]}
|
||||
s.index = end
|
||||
return batch, nil
|
||||
}
|
||||
|
||||
// Close closes the Confluence prune session.
|
||||
func (s *confluencePruneSession) Close() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
type confluenceSearchResponse struct {
|
||||
Results []confluenceContent `json:"results"`
|
||||
Links struct {
|
||||
Next string `json:"next"`
|
||||
} `json:"_links"`
|
||||
}
|
||||
|
||||
type confluenceSpace struct {
|
||||
Key string `json:"key"`
|
||||
Name string `json:"name"`
|
||||
}
|
||||
|
||||
type confluenceContent struct {
|
||||
ID confluenceString `json:"id"`
|
||||
Title string `json:"title"`
|
||||
Type string `json:"type"`
|
||||
Body struct {
|
||||
Storage confluenceBody `json:"storage"`
|
||||
View confluenceBody `json:"view"`
|
||||
} `json:"body"`
|
||||
Space confluenceSpace `json:"space"`
|
||||
Ancestors []struct {
|
||||
Title string `json:"title"`
|
||||
} `json:"ancestors"`
|
||||
Version struct {
|
||||
When string `json:"when"`
|
||||
By struct {
|
||||
DisplayName string `json:"displayName"`
|
||||
Email string `json:"email"`
|
||||
} `json:"by"`
|
||||
} `json:"version"`
|
||||
Metadata struct {
|
||||
MediaType string `json:"mediaType"`
|
||||
Labels struct {
|
||||
Results []struct {
|
||||
Name string `json:"name"`
|
||||
} `json:"results"`
|
||||
} `json:"labels"`
|
||||
} `json:"metadata"`
|
||||
Extensions struct {
|
||||
FileSize int64 `json:"fileSize"`
|
||||
} `json:"extensions"`
|
||||
Links struct {
|
||||
WebUI string `json:"webui"`
|
||||
Download string `json:"download"`
|
||||
} `json:"_links"`
|
||||
}
|
||||
|
||||
type confluenceBody struct {
|
||||
Value string `json:"value"`
|
||||
}
|
||||
|
||||
type confluenceString string
|
||||
|
||||
func (s *confluenceString) UnmarshalJSON(data []byte) error {
|
||||
var text string
|
||||
if err := json.Unmarshal(data, &text); err == nil {
|
||||
*s = confluenceString(text)
|
||||
return nil
|
||||
}
|
||||
|
||||
var number json.Number
|
||||
decoder := json.NewDecoder(bytes.NewReader(data))
|
||||
decoder.UseNumber()
|
||||
if err := decoder.Decode(&number); err == nil {
|
||||
*s = confluenceString(number.String())
|
||||
return nil
|
||||
}
|
||||
|
||||
return fmt.Errorf("Confluence string field must be string or number")
|
||||
}
|
||||
|
||||
func (s confluenceString) String() string {
|
||||
return string(s)
|
||||
}
|
||||
|
||||
func (c confluenceContent) bodyHTML() string {
|
||||
return firstNonEmpty(c.Body.Storage.Value, c.Body.View.Value)
|
||||
}
|
||||
|
||||
func (c confluenceContent) ancestorTitles() []string {
|
||||
out := make([]string, 0, len(c.Ancestors))
|
||||
for _, ancestor := range c.Ancestors {
|
||||
if ancestor.Title != "" {
|
||||
out = append(out, ancestor.Title)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func confluenceAPIBase(wikiBase string, isCloud bool) string {
|
||||
base := strings.TrimRight(wikiBase, "/")
|
||||
if isCloud && base != "" && !strings.HasSuffix(base, "/wiki") {
|
||||
base += "/wiki"
|
||||
}
|
||||
return base
|
||||
}
|
||||
|
||||
func buildConfluenceDocumentID(baseURL, contentURL string, isCloud bool) string {
|
||||
finalBase := strings.TrimRight(baseURL, "/") + "/"
|
||||
if isCloud && !strings.HasSuffix(finalBase, "/wiki/") {
|
||||
finalBase += "wiki/"
|
||||
}
|
||||
base, err := url.Parse(finalBase)
|
||||
if err != nil {
|
||||
return strings.TrimRight(finalBase, "/") + "/" + strings.TrimLeft(contentURL, "/")
|
||||
}
|
||||
ref, err := url.Parse(strings.TrimLeft(contentURL, "/"))
|
||||
if err != nil {
|
||||
return strings.TrimRight(finalBase, "/") + "/" + strings.TrimLeft(contentURL, "/")
|
||||
}
|
||||
return base.ResolveReference(ref).String()
|
||||
}
|
||||
|
||||
func confluenceCQLPath(cql, expand string, limit int) string {
|
||||
values := url.Values{}
|
||||
values.Set("cql", cql)
|
||||
if expand != "" {
|
||||
values.Set("expand", expand)
|
||||
}
|
||||
values.Set("limit", fmt.Sprint(limit))
|
||||
return "rest/api/content/search?" + values.Encode()
|
||||
}
|
||||
|
||||
func confluenceCQLQuote(value string) string {
|
||||
value = strings.ReplaceAll(value, `\`, `\\`)
|
||||
return strings.ReplaceAll(value, "'", "\\'")
|
||||
}
|
||||
|
||||
func confluenceCQLTime(value time.Time) string {
|
||||
return value.UTC().Format("2006-01-02 15:04")
|
||||
}
|
||||
|
||||
func inConfluenceWindow(updatedAt time.Time, start *time.Time, end time.Time) bool {
|
||||
if !end.IsZero() && updatedAt.After(end) {
|
||||
return false
|
||||
}
|
||||
if start != nil && !updatedAt.After(*start) {
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func parseConfluenceTime(value string) time.Time {
|
||||
value = strings.TrimSpace(value)
|
||||
if value == "" {
|
||||
return time.Time{}
|
||||
}
|
||||
layouts := []string{
|
||||
time.RFC3339Nano,
|
||||
time.RFC3339,
|
||||
"2006-01-02T15:04:05.000-0700",
|
||||
"2006-01-02T15:04:05-0700",
|
||||
"2006-01-02 15:04:05",
|
||||
}
|
||||
for _, layout := range layouts {
|
||||
if parsed, err := time.Parse(layout, value); err == nil {
|
||||
return parsed.UTC()
|
||||
}
|
||||
}
|
||||
return time.Time{}
|
||||
}
|
||||
|
||||
func confluenceSemanticIdentifier(space string, ancestors []string, title string, counts map[string]int) string {
|
||||
title = firstNonEmpty(title, "Untitled")
|
||||
parts := make([]string, 0, len(ancestors)+2)
|
||||
if space != "" {
|
||||
parts = append(parts, space)
|
||||
}
|
||||
parts = append(parts, ancestors...)
|
||||
parts = append(parts, title)
|
||||
fullPath := strings.Join(parts, " / ")
|
||||
counts[title]++
|
||||
if counts[title] > 1 && fullPath != "" {
|
||||
return fullPath
|
||||
}
|
||||
return title
|
||||
}
|
||||
|
||||
func confluenceHTMLText(raw string) string {
|
||||
raw = strings.TrimSpace(raw)
|
||||
if raw == "" {
|
||||
return ""
|
||||
}
|
||||
root, err := xhtml.Parse(strings.NewReader(raw))
|
||||
if err != nil {
|
||||
return html.UnescapeString(raw)
|
||||
}
|
||||
var buffer bytes.Buffer
|
||||
var walk func(*xhtml.Node)
|
||||
walk = func(node *xhtml.Node) {
|
||||
if node.Type == xhtml.ElementNode {
|
||||
switch strings.ToLower(node.Data) {
|
||||
case "script", "style":
|
||||
return
|
||||
case "br", "p", "div", "tr", "li", "table", "h1", "h2", "h3", "h4", "h5", "h6":
|
||||
buffer.WriteByte('\n')
|
||||
}
|
||||
}
|
||||
if node.Type == xhtml.TextNode {
|
||||
text := strings.TrimSpace(html.UnescapeString(node.Data))
|
||||
if text != "" {
|
||||
if buffer.Len() > 0 {
|
||||
buffer.WriteByte(' ')
|
||||
}
|
||||
buffer.WriteString(text)
|
||||
}
|
||||
}
|
||||
for child := node.FirstChild; child != nil; child = child.NextSibling {
|
||||
walk(child)
|
||||
}
|
||||
if node.Type == xhtml.ElementNode {
|
||||
switch strings.ToLower(node.Data) {
|
||||
case "p", "div", "tr", "li", "table", "h1", "h2", "h3", "h4", "h5", "h6":
|
||||
buffer.WriteByte('\n')
|
||||
}
|
||||
}
|
||||
}
|
||||
walk(root)
|
||||
text := strings.ReplaceAll(buffer.String(), "\r\n", "\n")
|
||||
text = confluenceWhitespaceRE.ReplaceAllString(text, " ")
|
||||
text = strings.ReplaceAll(text, " \n", "\n")
|
||||
text = strings.ReplaceAll(text, "\n ", "\n")
|
||||
text = confluenceNewlineRE.ReplaceAllString(text, "\n\n")
|
||||
return strings.TrimSpace(text)
|
||||
}
|
||||
|
||||
func metadataOrNil(metadata map[string]any) map[string]any {
|
||||
if len(metadata) == 0 {
|
||||
return nil
|
||||
}
|
||||
return metadata
|
||||
}
|
||||
|
||||
func isConfluenceAttachmentAccepted(attachment confluenceContent) bool {
|
||||
mediaType := strings.ToLower(attachment.Metadata.MediaType)
|
||||
if strings.HasPrefix(mediaType, "image/") {
|
||||
switch mediaType {
|
||||
case "image/jpeg", "image/jpg", "image/png", "image/gif", "image/bmp", "image/tiff", "image/webp":
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
ext := confluenceAttachmentExtension(attachment.Title)
|
||||
if _, ok := webdavTextExtensions[ext]; ok {
|
||||
return true
|
||||
}
|
||||
if _, ok := webdavDocumentExtensions[ext]; ok {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func confluenceAttachmentExtension(title string) string {
|
||||
ext := strings.ToLower(path.Ext(title))
|
||||
if ext == "" {
|
||||
return ".unknown"
|
||||
}
|
||||
return ext
|
||||
}
|
||||
|
||||
func confluenceAttachmentThreshold() int64 {
|
||||
if raw := strings.TrimSpace(os.Getenv("CONFLUENCE_CONNECTOR_ATTACHMENT_SIZE_THRESHOLD")); raw != "" {
|
||||
if parsed := configInt(raw, defaultConfluenceAttachmentThreshold); parsed > 0 {
|
||||
return int64(parsed)
|
||||
}
|
||||
}
|
||||
return defaultConfluenceAttachmentThreshold
|
||||
}
|
||||
|
||||
func confluenceValidationError(err error) error {
|
||||
var statusErr *confluenceStatusError
|
||||
if errors.As(err, &statusErr) {
|
||||
switch statusErr.status {
|
||||
case http.StatusUnauthorized:
|
||||
return fmt.Errorf("invalid or expired Confluence credentials")
|
||||
case http.StatusForbidden:
|
||||
return fmt.Errorf("insufficient permissions to access Confluence resources")
|
||||
}
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
type confluenceStatusError struct {
|
||||
status int
|
||||
body string
|
||||
}
|
||||
|
||||
func (e *confluenceStatusError) Error() string {
|
||||
return fmt.Sprintf("Confluence request failed with status %d: %s", e.status, strings.TrimSpace(e.body))
|
||||
}
|
||||
|
||||
var confluencePageExpansionFields = []string{
|
||||
"body.storage.value",
|
||||
"body.view.value",
|
||||
"space",
|
||||
"ancestors",
|
||||
"version",
|
||||
"metadata.labels",
|
||||
}
|
||||
|
||||
var confluenceAttachmentExpansionFields = []string{
|
||||
"metadata",
|
||||
"metadata.labels",
|
||||
"extensions",
|
||||
"version",
|
||||
"space",
|
||||
}
|
||||
328
internal/syncer/connector/confluence_test.go
Normal file
328
internal/syncer/connector/confluence_test.go
Normal file
@@ -0,0 +1,328 @@
|
||||
package connector
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/url"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestConfluenceConnectorSyncPagesCommentsAndAttachments(t *testing.T) {
|
||||
server := newConfluenceFixtureServer(t, http.StatusOK)
|
||||
defer server.Close()
|
||||
|
||||
connector := newTestConfluenceConnector(t, server.URL, map[string]any{
|
||||
"index_mode": "page",
|
||||
"page_id": "123",
|
||||
"index_recursively": true,
|
||||
"batch_size": 10,
|
||||
})
|
||||
session, err := connector.OpenSync(context.Background(), SyncRequest{
|
||||
FromBeginning: true,
|
||||
WindowEnd: confluenceTestTime(t, "2026-01-03T00:00:00Z"),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("OpenSync() error = %v", err)
|
||||
}
|
||||
defer func() { _ = session.Close() }()
|
||||
|
||||
batch, err := session.NextBatch(context.Background())
|
||||
if err != nil {
|
||||
t.Fatalf("NextBatch() error = %v", err)
|
||||
}
|
||||
if len(batch.Documents) != 2 {
|
||||
t.Fatalf("len(documents) = %d, want 2", len(batch.Documents))
|
||||
}
|
||||
|
||||
page := batch.Documents[0]
|
||||
if page.SourceID != server.URL+"/wiki/display/SPACE/Page" {
|
||||
t.Fatalf("page SourceID = %q", page.SourceID)
|
||||
}
|
||||
if page.SemanticIdentifier != "Page" {
|
||||
t.Fatalf("page SemanticIdentifier = %q", page.SemanticIdentifier)
|
||||
}
|
||||
if !strings.Contains(string(page.Blob), "Hello Confluence") || !strings.Contains(string(page.Blob), "Comment text") {
|
||||
t.Fatalf("page blob did not include page and comment text: %q", string(page.Blob))
|
||||
}
|
||||
if page.Extension != ".txt" {
|
||||
t.Fatalf("page Extension = %q", page.Extension)
|
||||
}
|
||||
if page.Metadata["space"] != "Engineering" {
|
||||
t.Fatalf("page metadata = %#v", page.Metadata)
|
||||
}
|
||||
if page.Fingerprint != contentFingerprint(page.Blob) {
|
||||
t.Fatalf("page fingerprint mismatch")
|
||||
}
|
||||
|
||||
attachment := batch.Documents[1]
|
||||
if attachment.SourceID != server.URL+"/wiki/download/attachments/file.txt?version=1" {
|
||||
t.Fatalf("attachment SourceID = %q", attachment.SourceID)
|
||||
}
|
||||
if attachment.Extension != ".txt" {
|
||||
t.Fatalf("attachment Extension = %q", attachment.Extension)
|
||||
}
|
||||
if string(attachment.Blob) != "attachment bytes" {
|
||||
t.Fatalf("attachment Blob = %q", string(attachment.Blob))
|
||||
}
|
||||
if attachment.Metadata["parent_page_id"] != page.SourceID {
|
||||
t.Fatalf("attachment metadata = %#v", attachment.Metadata)
|
||||
}
|
||||
if batch.Checkpoint == nil || batch.Checkpoint.SourceID != attachment.SourceID {
|
||||
t.Fatalf("checkpoint = %#v", batch.Checkpoint)
|
||||
}
|
||||
|
||||
_, err = session.NextBatch(context.Background())
|
||||
if !errors.Is(err, io.EOF) {
|
||||
t.Fatalf("second NextBatch() error = %v, want EOF", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestConfluenceConnectorResume(t *testing.T) {
|
||||
server := newConfluenceFixtureServer(t, http.StatusOK)
|
||||
defer server.Close()
|
||||
|
||||
connector := newTestConfluenceConnector(t, server.URL, map[string]any{"batch_size": 1})
|
||||
session, err := connector.OpenSync(context.Background(), SyncRequest{
|
||||
FromBeginning: true,
|
||||
WindowEnd: confluenceTestTime(t, "2026-01-03T00:00:00Z"),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("OpenSync() error = %v", err)
|
||||
}
|
||||
first, err := session.NextBatch(context.Background())
|
||||
if err != nil {
|
||||
t.Fatalf("first NextBatch() error = %v", err)
|
||||
}
|
||||
|
||||
resumed, err := connector.OpenSync(context.Background(), SyncRequest{
|
||||
FromBeginning: true,
|
||||
WindowEnd: confluenceTestTime(t, "2026-01-03T00:00:00Z"),
|
||||
Resume: first.Checkpoint,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("resumed OpenSync() error = %v", err)
|
||||
}
|
||||
batch, err := resumed.NextBatch(context.Background())
|
||||
if err != nil {
|
||||
t.Fatalf("resumed NextBatch() error = %v", err)
|
||||
}
|
||||
if len(batch.Documents) != 1 || batch.Documents[0].Extension != ".txt" || batch.Documents[0].SourceID == first.Documents[0].SourceID {
|
||||
t.Fatalf("resumed batch = %#v first = %#v", batch.Documents, first.Documents)
|
||||
}
|
||||
}
|
||||
|
||||
func TestConfluenceConnectorPrune(t *testing.T) {
|
||||
server := newConfluenceFixtureServer(t, http.StatusOK)
|
||||
defer server.Close()
|
||||
|
||||
connector := newTestConfluenceConnector(t, server.URL, nil)
|
||||
session, err := connector.OpenPrune(context.Background(), PruneRequest{})
|
||||
if err != nil {
|
||||
t.Fatalf("OpenPrune() error = %v", err)
|
||||
}
|
||||
defer func() { _ = session.Close() }()
|
||||
|
||||
batch, err := session.NextBatch(context.Background())
|
||||
if err != nil {
|
||||
t.Fatalf("NextBatch() error = %v", err)
|
||||
}
|
||||
got := map[string]bool{}
|
||||
for _, doc := range batch.Documents {
|
||||
got[doc.SourceID] = true
|
||||
}
|
||||
for _, want := range []string{
|
||||
server.URL + "/wiki/display/SPACE/Page",
|
||||
server.URL + "/wiki/download/attachments/file.txt?version=1",
|
||||
} {
|
||||
if !got[want] {
|
||||
t.Fatalf("missing prune source id %q in %#v", want, batch.Documents)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestConfluenceConnectorValidateMapsUnauthorized(t *testing.T) {
|
||||
server := newConfluenceFixtureServer(t, http.StatusUnauthorized)
|
||||
defer server.Close()
|
||||
|
||||
connector := newTestConfluenceConnector(t, server.URL, nil)
|
||||
err := connector.ValidateConnectorSetting(context.Background(), nil)
|
||||
if err == nil || !strings.Contains(err.Error(), "invalid or expired") {
|
||||
t.Fatalf("ValidateConnectorSetting() error = %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func newTestConfluenceConnector(t *testing.T, wikiBase string, overrides map[string]any) *ConfluenceConnector {
|
||||
t.Helper()
|
||||
config := map[string]any{
|
||||
"wiki_base": wikiBase,
|
||||
"is_cloud": true,
|
||||
"batch_size": 10,
|
||||
"credentials": map[string]any{
|
||||
"confluence_username": "user@example.com",
|
||||
"confluence_access_token": "token",
|
||||
},
|
||||
}
|
||||
for key, value := range overrides {
|
||||
config[key] = value
|
||||
}
|
||||
connector, err := NewConfluenceConnector(config)
|
||||
if err != nil {
|
||||
t.Fatalf("NewConfluenceConnector() error = %v", err)
|
||||
}
|
||||
return connector
|
||||
}
|
||||
|
||||
func newConfluenceFixtureServer(t *testing.T, validateStatus int) *httptest.Server {
|
||||
t.Helper()
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if validateStatus != http.StatusOK && strings.HasPrefix(r.URL.Path, "/wiki/rest/api/space") {
|
||||
http.Error(w, "unauthorized", validateStatus)
|
||||
return
|
||||
}
|
||||
if !confluenceHasBasicAuth(r, "user@example.com", "token") {
|
||||
http.Error(w, "bad auth", http.StatusUnauthorized)
|
||||
return
|
||||
}
|
||||
switch {
|
||||
case r.URL.Path == "/wiki/rest/api/space":
|
||||
writeConfluenceJSON(t, w, map[string]any{"results": []map[string]any{{"key": "SPACE", "name": "Engineering"}}})
|
||||
case r.URL.Path == "/wiki/rest/api/space/SPACE":
|
||||
writeConfluenceJSON(t, w, map[string]any{"key": "SPACE", "name": "Engineering"})
|
||||
case r.URL.Path == "/wiki/rest/api/content/search":
|
||||
cql := r.URL.Query().Get("cql")
|
||||
switch {
|
||||
case strings.Contains(cql, "type=page"):
|
||||
writeConfluenceJSON(t, w, map[string]any{"results": []map[string]any{confluenceFixturePage()}})
|
||||
case strings.Contains(cql, "type=comment"):
|
||||
writeConfluenceJSON(t, w, map[string]any{"results": []map[string]any{confluenceFixtureComment()}})
|
||||
case strings.Contains(cql, "type=attachment"):
|
||||
writeConfluenceJSON(t, w, map[string]any{"results": []map[string]any{confluenceFixtureAttachment()}})
|
||||
default:
|
||||
t.Errorf("Unexpected CQL: %q", cql)
|
||||
http.Error(w, "unexpected cql", http.StatusInternalServerError)
|
||||
}
|
||||
case r.URL.Path == "/wiki/download/attachments/file.txt":
|
||||
w.WriteHeader(http.StatusOK)
|
||||
_, _ = w.Write([]byte("attachment bytes"))
|
||||
default:
|
||||
t.Errorf("unexpected request %s", r.URL.String())
|
||||
http.Error(w, "unexpected request", http.StatusInternalServerError)
|
||||
}
|
||||
}))
|
||||
return server
|
||||
}
|
||||
|
||||
func confluenceHasBasicAuth(r *http.Request, username, password string) bool {
|
||||
const prefix = "Basic "
|
||||
raw := r.Header.Get("Authorization")
|
||||
if !strings.HasPrefix(raw, prefix) {
|
||||
return false
|
||||
}
|
||||
decoded, err := base64.StdEncoding.DecodeString(strings.TrimPrefix(raw, prefix))
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
return string(decoded) == username+":"+password
|
||||
}
|
||||
|
||||
func writeConfluenceJSON(t *testing.T, w http.ResponseWriter, payload any) {
|
||||
t.Helper()
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
if err := json.NewEncoder(w).Encode(payload); err != nil {
|
||||
t.Errorf("encode fixture: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func confluenceFixturePage() map[string]any {
|
||||
return map[string]any{
|
||||
"id": 123,
|
||||
"type": "page",
|
||||
"title": "Page",
|
||||
"body": map[string]any{"storage": map[string]any{"value": "<p>Hello <strong>Confluence</strong></p>"}},
|
||||
"space": map[string]any{"key": "SPACE", "name": "Engineering"},
|
||||
"ancestors": []map[string]any{
|
||||
{"title": "Root"},
|
||||
},
|
||||
"version": map[string]any{"when": "2026-01-01T10:00:00Z"},
|
||||
"metadata": map[string]any{"labels": map[string]any{"results": []map[string]any{{"name": "docs"}}}},
|
||||
"_links": map[string]any{"webui": "/display/SPACE/Page"},
|
||||
}
|
||||
}
|
||||
|
||||
func confluenceFixtureComment() map[string]any {
|
||||
return map[string]any{
|
||||
"id": 456,
|
||||
"type": "comment",
|
||||
"title": "Comment",
|
||||
"body": map[string]any{"storage": map[string]any{"value": "<p>Comment text</p>"}},
|
||||
"version": map[string]any{"when": "2026-01-01T11:00:00Z"},
|
||||
"_links": map[string]any{"webui": "/display/SPACE/Page?focusedCommentId=c1"},
|
||||
}
|
||||
}
|
||||
|
||||
func confluenceFixtureAttachment() map[string]any {
|
||||
return map[string]any{
|
||||
"id": 789,
|
||||
"type": "attachment",
|
||||
"title": "file.txt",
|
||||
"space": map[string]any{"key": "SPACE", "name": "Engineering"},
|
||||
"metadata": map[string]any{
|
||||
"mediaType": "text/plain",
|
||||
"labels": map[string]any{"results": []map[string]any{{"name": "file"}}},
|
||||
},
|
||||
"extensions": map[string]any{"fileSize": 16},
|
||||
"version": map[string]any{"when": "2026-01-01T12:00:00Z"},
|
||||
"_links": map[string]any{
|
||||
"webui": "/download/attachments/file.txt?version=1",
|
||||
"download": "/download/attachments/file.txt?version=1",
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func confluenceTestTime(t *testing.T, value string) time.Time {
|
||||
t.Helper()
|
||||
parsed, err := time.Parse(time.RFC3339, value)
|
||||
if err != nil {
|
||||
t.Fatalf("parse test time %q: %v", value, err)
|
||||
}
|
||||
return parsed
|
||||
}
|
||||
|
||||
func TestBuildConfluenceDocumentIDAddsWikiForCloud(t *testing.T) {
|
||||
got := buildConfluenceDocumentID("https://example.atlassian.net", "/display/SPACE/Page", true)
|
||||
want := "https://example.atlassian.net/wiki/display/SPACE/Page"
|
||||
if got != want {
|
||||
t.Fatalf("buildConfluenceDocumentID() = %q, want %q", got, want)
|
||||
}
|
||||
|
||||
got = buildConfluenceDocumentID("https://example.atlassian.net/wiki", "/display/SPACE/Page", true)
|
||||
if got != want {
|
||||
t.Fatalf("buildConfluenceDocumentID() with wiki base = %q, want %q", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestConfluenceCQLPathEscapesQuery(t *testing.T) {
|
||||
raw := confluenceCQLPath("type=page and space='A B'", "version", 10)
|
||||
parsed, err := url.Parse(raw)
|
||||
if err != nil {
|
||||
t.Fatalf("parse path: %v", err)
|
||||
}
|
||||
if parsed.Query().Get("cql") != "type=page and space='A B'" {
|
||||
t.Fatalf("cql = %q", parsed.Query().Get("cql"))
|
||||
}
|
||||
}
|
||||
|
||||
func TestConfluenceCQLQuoteEscapesBackslashes(t *testing.T) {
|
||||
got := confluenceCQLQuote(`path\name's`)
|
||||
want := `path\\name\'s`
|
||||
if got != want {
|
||||
t.Fatalf("confluenceCQLQuote() = %q, want %q", got, want)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user