mirror of
https://github.com/larksuite/cli.git
synced 2026-09-14 18:42:53 +08:00
a6f3e635d4
* feat(docs): add local authoring and resource workflows Add docs +script workflows for isolated draft initialization and tolerant XML/Markdown profiling. Support local and remote document resources across create and update flows with safe, bounded-concurrency uploads, binding verification, and cleanup. Synchronize shared credential-source selection during concurrent uploads, refresh lark-doc guidance, and expand unit, dry-run, and live E2E coverage. * docs(lark-doc): clarify genre reference paths * fix(docs): address PR validation feedback * docs(lark-doc): clarify remote image handling * feat: streamline docs draft workflow * fix(docs): clarify script input and resource cleanup * fix(docs): align script dry-run test with auth flow * fix(docs): authenticate local script e2e test * fix(docs): refine script diagnostics and image preflight * fix(docs): align draft workspace cleanup with VFS * fix(docs): route workspace cleanup through FileIO * docs(lark-doc): simplify profile check guidance
2842 lines
89 KiB
Go
2842 lines
89 KiB
Go
// Copyright (c) 2026 Lark Technologies Pte. Ltd.
|
|
// SPDX-License-Identifier: MIT
|
|
|
|
package doc
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"crypto/rand"
|
|
"encoding/hex"
|
|
"encoding/json"
|
|
"encoding/xml"
|
|
"errors"
|
|
"fmt"
|
|
"image"
|
|
"io"
|
|
"math"
|
|
"math/big"
|
|
"mime"
|
|
"net/http"
|
|
"net/url"
|
|
"path/filepath"
|
|
"regexp"
|
|
"strconv"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/google/uuid"
|
|
|
|
"github.com/larksuite/cli/errs"
|
|
"github.com/larksuite/cli/internal/validate"
|
|
"github.com/larksuite/cli/shortcuts/common"
|
|
)
|
|
|
|
const (
|
|
localDocResourceBindBatchSize = 20
|
|
localDocResourceCleanupBatchSize = 200
|
|
localDocResourceBindAttempts = 3
|
|
localDocResourceUploadAttempts = 3
|
|
remoteDocImageDownloadAttempts = 3
|
|
remoteDocImageUploadConcurrency = 10
|
|
localDocResourceUploadInterval = 220 * time.Millisecond
|
|
localDocResourceBindInterval = 350 * time.Millisecond
|
|
localDocResourceVerifyInterval = 220 * time.Millisecond
|
|
localDocResourceUploadConflictCode = 1061045
|
|
localDocResourceUploadRateLimitCode = 99991400
|
|
localDocImageMaxDisplayWidthPx = 1020
|
|
localDocImageScalePrecision = 1000000
|
|
remoteDocImageMaxBytes = int64(20 * 1024 * 1024)
|
|
)
|
|
|
|
var waitLocalDocResourceRequest = func(ctx context.Context, delay time.Duration) error {
|
|
timer := time.NewTimer(delay)
|
|
defer timer.Stop()
|
|
select {
|
|
case <-timer.C:
|
|
return nil
|
|
case <-ctx.Done():
|
|
return ctx.Err()
|
|
}
|
|
}
|
|
|
|
func localDocResourceRetryDelay(base time.Duration, attempt int) time.Duration {
|
|
delay := base * time.Duration(1<<attempt)
|
|
jitterLimit := delay / 4
|
|
if jitterLimit <= 0 {
|
|
return delay
|
|
}
|
|
jitter, err := rand.Int(rand.Reader, big.NewInt(int64(jitterLimit)+1))
|
|
if err != nil {
|
|
return delay
|
|
}
|
|
return delay + time.Duration(jitter.Int64())
|
|
}
|
|
|
|
type localDocResourceKind string
|
|
|
|
const (
|
|
localDocResourceImage localDocResourceKind = "image"
|
|
localDocResourceFile localDocResourceKind = "file"
|
|
)
|
|
|
|
var localDocResourceMarkerPattern = regexp.MustCompile(`^@lcli_(?:img|file)_[0-9a-f]{32}$`)
|
|
|
|
func isReservedLocalDocResourceMarker(value string) bool {
|
|
value = strings.TrimSpace(value)
|
|
if localDocResourceMarkerPattern.MatchString(value) {
|
|
return true
|
|
}
|
|
return strings.HasPrefix(value, "@lcli_img_") || strings.HasPrefix(value, "@lcli_file_")
|
|
}
|
|
|
|
type localDocResource struct {
|
|
Occurrence int
|
|
Kind localDocResourceKind
|
|
Marker string
|
|
Path string
|
|
FileName string
|
|
Size int64
|
|
ImageWidth int
|
|
ImageHeight int
|
|
ImageAlign string
|
|
ImageScale float64
|
|
HasScale bool
|
|
RemoteURL string
|
|
RequestedImagePresentation localDocImagePresentation
|
|
Content []byte
|
|
}
|
|
|
|
type localDocImagePresentation struct {
|
|
Width string
|
|
Height string
|
|
Align string
|
|
Scale string
|
|
}
|
|
|
|
type localDocResourceTag struct {
|
|
Name string
|
|
Attrs []html5BlockAttr
|
|
SelfClosing bool
|
|
}
|
|
|
|
type localDocResourceOutcome struct {
|
|
Resource localDocResource
|
|
BlockID string
|
|
Block map[string]interface{}
|
|
CleanupBlockIDs []string
|
|
FileToken string
|
|
Status string
|
|
CleanupStatus string
|
|
Err error
|
|
ServerWarnings []interface{}
|
|
SafeToCleanup bool
|
|
}
|
|
|
|
func prepareLocalDocResources(runtime *common.RuntimeContext, format, content string) (string, []localDocResource, error) {
|
|
if !strings.Contains(content, "<img") && !strings.Contains(content, "<source") &&
|
|
!(strings.TrimSpace(format) == "markdown" && strings.Contains(content, "![")) {
|
|
return content, nil, nil
|
|
}
|
|
|
|
resources := make([]localDocResource, 0)
|
|
markdownMode := strings.TrimSpace(format) == "markdown"
|
|
rewriteSegment := func(segment string, localRefs map[string]struct{}) (string, error) {
|
|
return rewriteLocalDocResourceSegment(runtime, segment, markdownMode, localRefs, &resources)
|
|
}
|
|
|
|
if !markdownMode {
|
|
out, err := rewriteSegment(content, nil)
|
|
return out, resources, err
|
|
}
|
|
|
|
content, inertSpans := protectMarkdownLocalDocResourceMarkup(content)
|
|
localRefs := collectLocalMarkdownImageReferences(content)
|
|
var (
|
|
out strings.Builder
|
|
segment strings.Builder
|
|
rewriteErr error
|
|
fenceChar byte
|
|
fenceLen int
|
|
indentCtx markdownIndentContext
|
|
)
|
|
flush := func() {
|
|
if segment.Len() == 0 {
|
|
return
|
|
}
|
|
if rewriteErr != nil {
|
|
out.WriteString(segment.String())
|
|
segment.Reset()
|
|
return
|
|
}
|
|
rewritten, err := rewriteSegment(segment.String(), localRefs)
|
|
if err != nil {
|
|
rewriteErr = err
|
|
out.WriteString(segment.String())
|
|
} else {
|
|
out.WriteString(rewritten)
|
|
}
|
|
segment.Reset()
|
|
}
|
|
|
|
for _, line := range strings.SplitAfter(content, "\n") {
|
|
indentedCode := false
|
|
if fenceChar == 0 {
|
|
indentedCode = indentCtx.isIndentedCodeLine(line)
|
|
}
|
|
char, run, isFence := markdownFence(line)
|
|
if fenceChar == 0 && isFence {
|
|
flush()
|
|
fenceChar, fenceLen = char, run
|
|
out.WriteString(line)
|
|
continue
|
|
}
|
|
if fenceChar != 0 {
|
|
out.WriteString(line)
|
|
if isFence && char == fenceChar && run >= fenceLen && markdownFenceCloses(line, char, run) {
|
|
fenceChar, fenceLen = 0, 0
|
|
}
|
|
continue
|
|
}
|
|
if indentedCode {
|
|
flush()
|
|
out.WriteString(line)
|
|
continue
|
|
}
|
|
segment.WriteString(line)
|
|
}
|
|
flush()
|
|
if rewriteErr != nil {
|
|
return "", nil, rewriteErr
|
|
}
|
|
result := out.String()
|
|
for _, inert := range inertSpans {
|
|
result = strings.ReplaceAll(result, inert.Protected, inert.Original)
|
|
}
|
|
return result, resources, nil
|
|
}
|
|
|
|
type protectedLocalDocResourceSpan struct {
|
|
Protected string
|
|
Original string
|
|
}
|
|
|
|
func protectMarkdownLocalDocResourceMarkup(content string) (string, []protectedLocalDocResourceSpan) {
|
|
var out strings.Builder
|
|
spans := make([]protectedLocalDocResourceSpan, 0)
|
|
nextToken := 0
|
|
for i := 0; i < len(content); {
|
|
end := -1
|
|
switch {
|
|
case strings.HasPrefix(content[i:], "<!--"):
|
|
if offset := strings.Index(content[i+4:], "-->"); offset >= 0 {
|
|
end = i + 4 + offset + 3
|
|
} else {
|
|
end = len(content)
|
|
}
|
|
case strings.HasPrefix(content[i:], "<![CDATA["):
|
|
if offset := strings.Index(content[i+9:], "]]>"); offset >= 0 {
|
|
end = i + 9 + offset + 3
|
|
} else {
|
|
end = len(content)
|
|
}
|
|
default:
|
|
if rawEnd, ok := findMarkdownRawHTMLInertEnd(content, i); ok {
|
|
end = rawEnd
|
|
}
|
|
}
|
|
if end < 0 {
|
|
out.WriteByte(content[i])
|
|
i++
|
|
continue
|
|
}
|
|
|
|
original := content[i:end]
|
|
protected := ""
|
|
for {
|
|
protected = fmt.Sprintf("\ue000lark_cli_inert_%d\ue001", nextToken)
|
|
nextToken++
|
|
if !strings.Contains(content, protected) {
|
|
break
|
|
}
|
|
}
|
|
protected += strings.Repeat("\n", strings.Count(original, "\n"))
|
|
out.WriteString(protected)
|
|
spans = append(spans, protectedLocalDocResourceSpan{Protected: protected, Original: original})
|
|
i = end
|
|
}
|
|
return out.String(), spans
|
|
}
|
|
|
|
func rewriteLocalDocResourceSegment(runtime *common.RuntimeContext, segment string, markdownMode bool, localRefs map[string]struct{}, resources *[]localDocResource) (string, error) {
|
|
var out strings.Builder
|
|
for i := 0; i < len(segment); {
|
|
if markdownMode {
|
|
if end, ok := findMarkdownRawHTMLInertEnd(segment, i); ok {
|
|
out.WriteString(segment[i:end])
|
|
i = end
|
|
continue
|
|
}
|
|
}
|
|
if strings.HasPrefix(segment[i:], "<!--") {
|
|
end := strings.Index(segment[i+4:], "-->")
|
|
if end < 0 {
|
|
out.WriteString(segment[i:])
|
|
break
|
|
}
|
|
end += i + 7
|
|
out.WriteString(segment[i:end])
|
|
i = end
|
|
continue
|
|
}
|
|
if strings.HasPrefix(segment[i:], "<![CDATA[") {
|
|
end := strings.Index(segment[i+9:], "]]>")
|
|
if end < 0 {
|
|
out.WriteString(segment[i:])
|
|
break
|
|
}
|
|
end += i + 12
|
|
out.WriteString(segment[i:end])
|
|
i = end
|
|
continue
|
|
}
|
|
if markdownMode && segment[i] == '`' {
|
|
run := countByteRun(segment, i, '`')
|
|
end := findMatchingBacktickRun(segment, i+run, run)
|
|
if end < 0 {
|
|
out.WriteString(segment[i : i+run])
|
|
i += run
|
|
continue
|
|
}
|
|
out.WriteString(segment[i:end])
|
|
i = end
|
|
continue
|
|
}
|
|
|
|
if segment[i] == '<' && !(markdownMode && isEscapedMarkdownByte(segment, i)) {
|
|
name := localResourceTagNameAt(segment, i)
|
|
if name != "" {
|
|
end := findXMLStartTagEnd(segment, i)
|
|
if end < 0 {
|
|
return "", common.ValidationErrorf("invalid <%s> local resource tag", name).WithParam(name)
|
|
}
|
|
raw := segment[i:end]
|
|
rewritten, resource, changed, err := rewriteRawLocalResourceTag(runtime, raw, name, len(*resources)+1)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
out.WriteString(rewritten)
|
|
if changed {
|
|
*resources = append(*resources, resource)
|
|
}
|
|
i = end
|
|
continue
|
|
}
|
|
}
|
|
|
|
if segment[i] == '!' && i+1 < len(segment) && segment[i+1] == '[' && !isEscapedMarkdownByte(segment, i) {
|
|
image, ok := parseMarkdownImageAt(segment, i)
|
|
if ok {
|
|
if image.ReferenceLabel != "" {
|
|
if _, local := localRefs[normalizeMarkdownReferenceLabel(image.ReferenceLabel)]; local {
|
|
return "", common.ValidationErrorf("local Markdown reference-style images are not supported; use  instead").WithParam("content")
|
|
}
|
|
out.WriteString(segment[i:image.End])
|
|
i = image.End
|
|
continue
|
|
}
|
|
if strings.HasPrefix(image.Destination, "@") {
|
|
resource, err := newLocalDocResource(runtime, localDocResourceImage, image.Destination, len(*resources)+1)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
out.WriteString(`<img path="`)
|
|
out.WriteString(escapeXMLAttr(resource.Marker))
|
|
out.WriteByte('"')
|
|
if strings.TrimSpace(image.Alt) != "" {
|
|
out.WriteString(` caption="`)
|
|
out.WriteString(escapeXMLAttr(unescapeMarkdownText(image.Alt)))
|
|
out.WriteByte('"')
|
|
}
|
|
if strings.TrimSpace(image.Title) != "" {
|
|
out.WriteString(` title="`)
|
|
out.WriteString(escapeXMLAttr(image.Title))
|
|
out.WriteByte('"')
|
|
}
|
|
out.WriteString("/>")
|
|
*resources = append(*resources, resource)
|
|
i = image.End
|
|
continue
|
|
}
|
|
out.WriteString(segment[i:image.End])
|
|
i = image.End
|
|
continue
|
|
}
|
|
}
|
|
|
|
out.WriteByte(segment[i])
|
|
i++
|
|
}
|
|
return out.String(), nil
|
|
}
|
|
|
|
func rewriteRawLocalResourceTag(runtime *common.RuntimeContext, raw, name string, occurrence int) (string, localDocResource, bool, error) {
|
|
tag, err := parseLocalDocResourceTag(raw, name)
|
|
if err != nil {
|
|
return "", localDocResource{}, false, common.ValidationErrorf("invalid <%s> local resource tag: %v", name, err).WithParam(name).WithCause(err)
|
|
}
|
|
pathValue, hasPath := tag.attr("path")
|
|
if !hasPath {
|
|
if name != "img" {
|
|
return raw, localDocResource{}, false, nil
|
|
}
|
|
href, hasHref := tag.attr("href")
|
|
if !hasHref {
|
|
return raw, localDocResource{}, false, nil
|
|
}
|
|
conflicts := []string{"src", "token", "img_key", "img-key", "url"}
|
|
for _, conflict := range conflicts {
|
|
if tag.hasAttr(conflict) {
|
|
return "", localDocResource{}, false, common.ValidationErrorf("<img> href cannot be combined with %s", conflict).WithParam("img")
|
|
}
|
|
}
|
|
resource, err := newRemoteDocImageResource(href, occurrence)
|
|
if err != nil {
|
|
return "", localDocResource{}, false, err
|
|
}
|
|
tag.renameAttr("href", "path")
|
|
tag.setAttr("path", resource.Marker)
|
|
if _, hasCaption := tag.attr("caption"); !hasCaption {
|
|
tag.renameAttr("alt", "caption")
|
|
}
|
|
resource.RequestedImagePresentation = captureLocalDocImagePresentation(tag)
|
|
resource.captureImagePresentation(tag)
|
|
for _, name := range []string{"width", "height", "align", "scale"} {
|
|
tag.deleteAttr(name)
|
|
}
|
|
return tag.render(), resource, true, nil
|
|
}
|
|
|
|
conflicts := []string{"src", "href", "token", "img_key", "img-key", "url"}
|
|
for _, conflict := range conflicts {
|
|
if tag.hasAttr(conflict) {
|
|
return "", localDocResource{}, false, common.ValidationErrorf("<%s> local path cannot be combined with %s", name, conflict).WithParam(name)
|
|
}
|
|
}
|
|
|
|
kind := localDocResourceImage
|
|
if name == "source" {
|
|
kind = localDocResourceFile
|
|
}
|
|
resource, err := newLocalDocResource(runtime, kind, pathValue, occurrence)
|
|
if err != nil {
|
|
return "", localDocResource{}, false, err
|
|
}
|
|
tag.setAttr("path", resource.Marker)
|
|
if kind == localDocResourceImage {
|
|
if _, hasCaption := tag.attr("caption"); !hasCaption {
|
|
tag.renameAttr("alt", "caption")
|
|
}
|
|
if err := normalizeLocalDocImagePresentation(runtime, &tag, &resource); err != nil {
|
|
return "", localDocResource{}, false, localResourceValidationErrorWithCause(kind, occurrence, "file is not a supported BMP, GIF, JPEG, PNG, TIFF, or WebP image", err)
|
|
}
|
|
resource.captureImagePresentation(tag)
|
|
} else if rawName, hasName := tag.attr("name"); hasName {
|
|
fileName := strings.TrimSpace(rawName)
|
|
if fileName == "" || fileName == "." || fileName == ".." || strings.ContainsAny(fileName, `/\\`) {
|
|
return "", localDocResource{}, false, common.ValidationErrorf("<source> name must be a non-empty file name without path separators").WithParam("source")
|
|
}
|
|
resource.FileName = fileName
|
|
tag.setAttr("name", fileName)
|
|
}
|
|
return tag.render(), resource, true, nil
|
|
}
|
|
|
|
func newRemoteDocImageResource(rawURL string, occurrence int) (localDocResource, error) {
|
|
rawURL = strings.TrimSpace(rawURL)
|
|
u, err := url.Parse(rawURL)
|
|
if err != nil || u == nil || (u.Scheme != "http" && u.Scheme != "https") || strings.TrimSpace(u.Hostname()) == "" || u.User != nil {
|
|
return localDocResource{}, errs.NewValidationError(
|
|
errs.SubtypeInvalidArgument,
|
|
"remote image #%d href must be an absolute HTTP(S) URL without userinfo",
|
|
occurrence,
|
|
).WithParam("href")
|
|
}
|
|
marker, err := newLocalDocResourceMarker(localDocResourceImage)
|
|
if err != nil {
|
|
return localDocResource{}, errs.NewInternalError(errs.SubtypeUnknown, "failed to generate remote image marker").WithCause(err)
|
|
}
|
|
return localDocResource{
|
|
Occurrence: occurrence,
|
|
Kind: localDocResourceImage,
|
|
Marker: marker,
|
|
RemoteURL: u.String(),
|
|
}, nil
|
|
}
|
|
|
|
type remoteDocImageDownload struct {
|
|
Content []byte
|
|
FileName string
|
|
Width int
|
|
Height int
|
|
}
|
|
|
|
var (
|
|
downloadRemoteDocImage = downloadRemoteDocImageContent
|
|
doRemoteDocImageRequest = func(client remoteDocImageHTTPDoer, req *http.Request) (*http.Response, error) {
|
|
return client.Do(req)
|
|
}
|
|
)
|
|
|
|
type remoteDocImageHTTPDoer interface {
|
|
Do(*http.Request) (*http.Response, error)
|
|
}
|
|
|
|
func downloadRemoteDocImageContent(runtime *common.RuntimeContext, rawURL string, occurrence int) (remoteDocImageDownload, error) {
|
|
if err := validateRemoteDocImageSource(runtime.Ctx(), rawURL, occurrence); err != nil {
|
|
return remoteDocImageDownload{}, err
|
|
}
|
|
baseClient, err := runtime.Factory.ExternalHTTPClient()
|
|
if err != nil {
|
|
return remoteDocImageDownload{}, errs.NewInternalError(errs.SubtypeSDKError, "initialize remote image HTTP client: %v", err).WithCause(err)
|
|
}
|
|
client := validate.NewDownloadHTTPClient(baseClient, validate.DownloadHTTPClientOptions{AllowHTTP: true, MaxRedirects: 5})
|
|
req, err := http.NewRequestWithContext(runtime.Ctx(), http.MethodGet, rawURL, nil) //nolint:forbidigo // guarded download of user-provided image URL
|
|
if err != nil {
|
|
return remoteDocImageDownload{}, errs.NewValidationError(errs.SubtypeInvalidArgument, "invalid remote image #%d href: %v", occurrence, err).WithParam("href").WithCause(err)
|
|
}
|
|
resp, err := doRemoteDocImageRequest(client, req)
|
|
if err != nil {
|
|
return remoteDocImageDownload{}, remoteDocImageNetworkError(err, occurrence, "request failed")
|
|
}
|
|
defer resp.Body.Close()
|
|
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
|
|
subtype := errs.SubtypeNetworkTransport
|
|
retryable := false
|
|
switch {
|
|
case resp.StatusCode == http.StatusTooManyRequests:
|
|
subtype = errs.SubtypeRateLimit
|
|
retryable = true
|
|
case resp.StatusCode >= 500:
|
|
subtype = errs.SubtypeNetworkServer
|
|
retryable = true
|
|
}
|
|
httpErr := errs.NewNetworkError(subtype, "download remote image #%d failed: HTTP %d", occurrence, resp.StatusCode).WithCode(resp.StatusCode)
|
|
if retryable {
|
|
httpErr.WithRetryable()
|
|
}
|
|
return remoteDocImageDownload{}, httpErr
|
|
}
|
|
mediaType, _, err := mime.ParseMediaType(resp.Header.Get("Content-Type"))
|
|
if err != nil {
|
|
return remoteDocImageDownload{}, errs.NewValidationError(errs.SubtypeInvalidArgument, "remote image #%d response has an invalid Content-Type", occurrence).WithParam("href").WithCause(err)
|
|
}
|
|
ext, ok := docCoverAllowedContentTypes[strings.ToLower(mediaType)]
|
|
if !ok {
|
|
return remoteDocImageDownload{}, errs.NewValidationError(errs.SubtypeInvalidArgument, "remote image #%d response Content-Type %q is not supported", occurrence, mediaType).WithParam("href")
|
|
}
|
|
if resp.ContentLength > remoteDocImageMaxBytes {
|
|
return remoteDocImageDownload{}, errs.NewValidationError(errs.SubtypeInvalidArgument, "remote image #%d exceeds 20MiB limit", occurrence).WithParam("href")
|
|
}
|
|
|
|
body := &remoteDocImageResponseReader{reader: resp.Body}
|
|
content, err := io.ReadAll(io.LimitReader(body, remoteDocImageMaxBytes+1))
|
|
if err != nil {
|
|
if body.err != nil {
|
|
return remoteDocImageDownload{}, remoteDocImageNetworkError(body.err, occurrence, "response body failed")
|
|
}
|
|
if contextErr := runtime.Ctx().Err(); contextErr != nil {
|
|
return remoteDocImageDownload{}, remoteDocImageNetworkError(contextErr, occurrence, "was interrupted")
|
|
}
|
|
return remoteDocImageDownload{}, remoteDocImageNetworkError(err, occurrence, "response body failed")
|
|
}
|
|
if len(content) == 0 {
|
|
return remoteDocImageDownload{}, errs.NewValidationError(errs.SubtypeInvalidArgument, "remote image #%d response is empty", occurrence).WithParam("href")
|
|
}
|
|
if int64(len(content)) > remoteDocImageMaxBytes {
|
|
return remoteDocImageDownload{}, errs.NewValidationError(errs.SubtypeInvalidArgument, "remote image #%d exceeds 20MiB limit", occurrence).WithParam("href")
|
|
}
|
|
config, detectedFormat, err := image.DecodeConfig(bytes.NewReader(content))
|
|
if err != nil {
|
|
return remoteDocImageDownload{}, errs.NewValidationError(
|
|
errs.SubtypeInvalidArgument,
|
|
"remote image #%d response body is not a valid %s image",
|
|
occurrence,
|
|
mediaType,
|
|
).WithParam("href").WithCause(err)
|
|
}
|
|
expectedFormat := strings.TrimPrefix(strings.ToLower(mediaType), "image/")
|
|
if expectedFormat == "jpeg" && detectedFormat == "jpg" {
|
|
detectedFormat = "jpeg"
|
|
}
|
|
if detectedFormat != expectedFormat {
|
|
return remoteDocImageDownload{}, errs.NewValidationError(
|
|
errs.SubtypeInvalidArgument,
|
|
"remote image #%d response declared %s but contains %s image data",
|
|
occurrence,
|
|
mediaType,
|
|
detectedFormat,
|
|
).WithParam("href")
|
|
}
|
|
return remoteDocImageDownload{
|
|
Content: content,
|
|
FileName: "image" + ext,
|
|
Width: config.Width,
|
|
Height: config.Height,
|
|
}, nil
|
|
}
|
|
|
|
type remoteDocImageResponseReader struct {
|
|
reader io.Reader
|
|
err error
|
|
}
|
|
|
|
func (r *remoteDocImageResponseReader) Read(p []byte) (int, error) {
|
|
n, err := r.reader.Read(p)
|
|
if err != nil && !errors.Is(err, io.EOF) {
|
|
r.err = err
|
|
}
|
|
return n, err
|
|
}
|
|
|
|
func remoteDocImageNetworkError(err error, occurrence int, action string) error {
|
|
if _, ok := errs.ProblemOf(err); ok {
|
|
return err
|
|
}
|
|
subtype := errs.SubtypeNetworkTransport
|
|
if errors.Is(err, context.DeadlineExceeded) {
|
|
subtype = errs.SubtypeNetworkTimeout
|
|
}
|
|
networkErr := errs.NewNetworkError(subtype, "download remote image #%d %s", occurrence, action).WithCause(err)
|
|
if !errors.Is(err, context.Canceled) {
|
|
networkErr.WithRetryable()
|
|
}
|
|
return networkErr
|
|
}
|
|
|
|
func validateRemoteDocImageSource(ctx context.Context, rawURL string, occurrence int) error {
|
|
if err := validate.ValidateDownloadSourceURL(ctx, rawURL); err != nil {
|
|
return errs.NewValidationError(errs.SubtypeInvalidArgument,
|
|
"remote image #%d href is not allowed: %v", occurrence, err).
|
|
WithParam("href").
|
|
WithHint("use a public HTTP(S) image URL, or save the image in the draft workspace and reference it with <img path=\"@relative/path\"/>").
|
|
WithCause(err)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func validateRemoteDocImageSources(ctx context.Context, resources []localDocResource) error {
|
|
for _, resource := range resources {
|
|
if resource.RemoteURL == "" {
|
|
continue
|
|
}
|
|
if err := validateRemoteDocImageSource(ctx, resource.RemoteURL, resource.Occurrence); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func probeRemoteDocImageDownload(runtime *common.RuntimeContext, rawURL string, occurrence int) error {
|
|
if err := validateRemoteDocImageSource(runtime.Ctx(), rawURL, occurrence); err != nil {
|
|
return err
|
|
}
|
|
baseClient, err := runtime.Factory.ExternalHTTPClient()
|
|
if err != nil {
|
|
return errs.NewInternalError(errs.SubtypeSDKError, "initialize remote image HTTP client: %v", err).WithCause(err)
|
|
}
|
|
client := validate.NewDownloadHTTPClient(baseClient, validate.DownloadHTTPClientOptions{AllowHTTP: true, MaxRedirects: 5})
|
|
return probeRemoteDocImageRequest(runtime, client, rawURL, occurrence)
|
|
}
|
|
|
|
func probeRemoteDocImageRequest(runtime *common.RuntimeContext, client remoteDocImageHTTPDoer, rawURL string, occurrence int) error {
|
|
req, err := http.NewRequestWithContext(runtime.Ctx(), http.MethodGet, rawURL, nil) //nolint:forbidigo // guarded probe of user-provided image URL
|
|
if err != nil {
|
|
return errs.NewValidationError(errs.SubtypeInvalidArgument, "invalid remote image #%d href: %v", occurrence, err).WithParam("href").WithCause(err)
|
|
}
|
|
req.Header.Set("Range", "bytes=0-0")
|
|
resp, err := doRemoteDocImageRequest(client, req)
|
|
if err != nil {
|
|
return remoteDocImageNetworkError(err, occurrence, "availability probe failed")
|
|
}
|
|
defer resp.Body.Close()
|
|
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
|
|
return remoteDocImageHTTPStatusError(occurrence, resp.StatusCode, "probe")
|
|
}
|
|
if resp.ContentLength > remoteDocImageMaxBytes {
|
|
return errs.NewValidationError(errs.SubtypeInvalidArgument, "remote image #%d exceeds 20MiB limit", occurrence).WithParam("href")
|
|
}
|
|
contentType := strings.TrimSpace(resp.Header.Get("Content-Type"))
|
|
if contentType == "" {
|
|
return nil
|
|
}
|
|
mediaType, _, err := mime.ParseMediaType(contentType)
|
|
if err != nil {
|
|
return errs.NewValidationError(errs.SubtypeInvalidArgument, "remote image #%d response has an invalid Content-Type", occurrence).WithParam("href").WithCause(err)
|
|
}
|
|
if _, ok := docCoverAllowedContentTypes[strings.ToLower(mediaType)]; !ok {
|
|
return errs.NewValidationError(errs.SubtypeInvalidArgument, "remote image #%d response Content-Type %q is not supported", occurrence, mediaType).WithParam("href")
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func remoteDocImageHTTPStatusError(occurrence, statusCode int, action string) error {
|
|
subtype := errs.SubtypeNetworkTransport
|
|
retryable := false
|
|
switch {
|
|
case statusCode == http.StatusTooManyRequests:
|
|
subtype = errs.SubtypeRateLimit
|
|
retryable = true
|
|
case statusCode >= 500:
|
|
subtype = errs.SubtypeNetworkServer
|
|
retryable = true
|
|
}
|
|
httpErr := errs.NewNetworkError(subtype, "%s remote image #%d failed: HTTP %d", action, occurrence, statusCode).WithCode(statusCode)
|
|
if retryable {
|
|
httpErr.WithRetryable()
|
|
}
|
|
return httpErr
|
|
}
|
|
|
|
func applyRemoteDocImageDownload(resource *localDocResource, download remoteDocImageDownload) error {
|
|
if len(download.Content) == 0 || download.Width <= 0 || download.Height <= 0 {
|
|
return errs.NewInternalError(errs.SubtypeInvalidResponse, "remote image download is missing content or dimensions")
|
|
}
|
|
resource.FileName = download.FileName
|
|
resource.Size = int64(len(download.Content))
|
|
tag := resource.RequestedImagePresentation.tag()
|
|
normalizeDocImagePresentation(&tag, resource, download.Width, download.Height)
|
|
resource.ImageWidth = 0
|
|
resource.ImageHeight = 0
|
|
resource.ImageAlign = ""
|
|
resource.ImageScale = 0
|
|
resource.HasScale = false
|
|
resource.captureImagePresentation(tag)
|
|
return nil
|
|
}
|
|
|
|
func captureLocalDocImagePresentation(tag localDocResourceTag) localDocImagePresentation {
|
|
presentation := localDocImagePresentation{}
|
|
presentation.Width, _ = tag.attr("width")
|
|
presentation.Height, _ = tag.attr("height")
|
|
presentation.Align, _ = tag.attr("align")
|
|
presentation.Scale, _ = tag.attr("scale")
|
|
return presentation
|
|
}
|
|
|
|
func (p localDocImagePresentation) tag() localDocResourceTag {
|
|
tag := localDocResourceTag{Name: "img", SelfClosing: true}
|
|
for _, attr := range []struct {
|
|
name string
|
|
value string
|
|
}{
|
|
{name: "width", value: p.Width},
|
|
{name: "height", value: p.Height},
|
|
{name: "align", value: p.Align},
|
|
{name: "scale", value: p.Scale},
|
|
} {
|
|
if attr.value != "" {
|
|
tag.setAttr(attr.name, attr.value)
|
|
}
|
|
}
|
|
return tag
|
|
}
|
|
|
|
func (r *localDocResource) captureImagePresentation(tag localDocResourceTag) {
|
|
if raw, ok := tag.attr("width"); ok {
|
|
if value, err := strconv.Atoi(strings.TrimSpace(raw)); err == nil && value > 0 {
|
|
r.ImageWidth = value
|
|
}
|
|
}
|
|
if raw, ok := tag.attr("height"); ok {
|
|
if value, err := strconv.Atoi(strings.TrimSpace(raw)); err == nil && value > 0 {
|
|
r.ImageHeight = value
|
|
}
|
|
}
|
|
if raw, ok := tag.attr("align"); ok {
|
|
align := strings.ToLower(strings.TrimSpace(raw))
|
|
if _, supported := alignMap[align]; supported {
|
|
r.ImageAlign = align
|
|
}
|
|
}
|
|
if raw, ok := tag.attr("scale"); ok {
|
|
if value, err := strconv.ParseFloat(strings.TrimSpace(raw), 64); err == nil && value > 0 {
|
|
r.ImageScale = value
|
|
r.HasScale = true
|
|
}
|
|
}
|
|
}
|
|
|
|
// normalizeLocalDocImagePresentation mirrors the SDK's network-image
|
|
// normalization before the local path is replaced by an opaque marker. The
|
|
// stored width/height are always the source image's intrinsic pixel
|
|
// dimensions; model-provided display dimensions are converted to scale.
|
|
func normalizeLocalDocImagePresentation(runtime *common.RuntimeContext, tag *localDocResourceTag, resource *localDocResource) error {
|
|
nativeWidth, nativeHeight, err := detectImageDimensionsFromPath(runtime.FileIO(), resource.Path)
|
|
if err != nil || nativeWidth <= 0 || nativeHeight <= 0 {
|
|
if err != nil {
|
|
return err
|
|
}
|
|
return invalidLocalDocImageDimensionsError()
|
|
}
|
|
normalizeDocImagePresentation(tag, resource, nativeWidth, nativeHeight)
|
|
return nil
|
|
}
|
|
|
|
func normalizeDocImagePresentation(tag *localDocResourceTag, resource *localDocResource, nativeWidth, nativeHeight int) {
|
|
modelScale, hasModelScale := positiveLocalDocImageFloatAttr(*tag, "scale")
|
|
modelWidth, hasModelWidth := positiveLocalDocImageDisplaySizeAttr(*tag, "width", nativeWidth)
|
|
modelHeight, hasModelHeight := positiveLocalDocImageDisplaySizeAttr(*tag, "height", nativeHeight)
|
|
|
|
tag.setAttr("width", strconv.Itoa(nativeWidth))
|
|
tag.setAttr("height", strconv.Itoa(nativeHeight))
|
|
|
|
scale, hasScale := resolveLocalDocImageScale(
|
|
nativeWidth,
|
|
nativeHeight,
|
|
modelScale,
|
|
hasModelScale,
|
|
modelWidth,
|
|
hasModelWidth,
|
|
modelHeight,
|
|
hasModelHeight,
|
|
)
|
|
if !hasScale {
|
|
tag.deleteAttr("scale")
|
|
return
|
|
}
|
|
tag.setAttr("scale", strconv.FormatFloat(scale, 'f', 6, 64))
|
|
}
|
|
|
|
func resolveLocalDocImageScale(nativeWidth, nativeHeight int, modelScale float64, hasModelScale bool, modelWidth float64, hasModelWidth bool, modelHeight float64, hasModelHeight bool) (float64, bool) {
|
|
var scale float64
|
|
switch {
|
|
case hasModelScale:
|
|
scale = modelScale
|
|
case hasModelWidth:
|
|
scale = modelWidth / float64(nativeWidth)
|
|
case hasModelHeight:
|
|
scale = modelHeight / float64(nativeHeight)
|
|
case nativeWidth >= localDocImageMaxDisplayWidthPx:
|
|
scale = 1
|
|
default:
|
|
return 0, false
|
|
}
|
|
scale = floorLocalDocImageScalePrecision(scale)
|
|
return capLocalDocImageScaleBelowPageWidth(nativeWidth, scale), true
|
|
}
|
|
|
|
func positiveLocalDocImageFloatAttr(tag localDocResourceTag, name string) (float64, bool) {
|
|
value, ok := tag.attr(name)
|
|
if !ok {
|
|
return 0, false
|
|
}
|
|
return positiveLocalDocImageFloat(strings.TrimSpace(value))
|
|
}
|
|
|
|
func positiveLocalDocImageDisplaySizeAttr(tag localDocResourceTag, name string, nativeSize int) (float64, bool) {
|
|
value, ok := tag.attr(name)
|
|
if !ok {
|
|
return 0, false
|
|
}
|
|
value = strings.TrimSpace(value)
|
|
if strings.HasSuffix(value, "%") {
|
|
percentage, valid := positiveLocalDocImageFloat(strings.TrimSpace(strings.TrimSuffix(value, "%")))
|
|
if !valid {
|
|
return 0, false
|
|
}
|
|
return float64(nativeSize) * percentage / 100, true
|
|
}
|
|
return positiveLocalDocImageFloat(value)
|
|
}
|
|
|
|
func positiveLocalDocImageFloat(value string) (float64, bool) {
|
|
if value == "" {
|
|
return 0, false
|
|
}
|
|
parsed, err := strconv.ParseFloat(value, 64)
|
|
if err != nil || parsed <= 0 || math.IsNaN(parsed) || math.IsInf(parsed, 0) {
|
|
return 0, false
|
|
}
|
|
return parsed, true
|
|
}
|
|
|
|
func floorLocalDocImageScalePrecision(scale float64) float64 {
|
|
floored := math.Floor(scale*localDocImageScalePrecision) / localDocImageScalePrecision
|
|
if floored <= 0 {
|
|
return scale
|
|
}
|
|
return floored
|
|
}
|
|
|
|
func capLocalDocImageScaleBelowPageWidth(nativeWidth int, scale float64) float64 {
|
|
maxScale := float64(localDocImageMaxDisplayWidthPx) / float64(nativeWidth)
|
|
if scale < maxScale {
|
|
return scale
|
|
}
|
|
capped := math.Floor(maxScale*localDocImageScalePrecision) / localDocImageScalePrecision
|
|
if capped >= maxScale {
|
|
capped -= 1.0 / localDocImageScalePrecision
|
|
}
|
|
if capped <= 0 {
|
|
return math.Nextafter(maxScale, 0)
|
|
}
|
|
return capped
|
|
}
|
|
|
|
func newLocalDocResource(runtime *common.RuntimeContext, kind localDocResourceKind, pathValue string, occurrence int) (localDocResource, error) {
|
|
pathValue = strings.TrimSpace(pathValue)
|
|
if !strings.HasPrefix(pathValue, "@") {
|
|
return localDocResource{}, localResourceValidationError(kind, occurrence, "path must start with @")
|
|
}
|
|
if isReservedLocalDocResourceMarker(pathValue) {
|
|
return localDocResource{}, localResourceValidationError(kind, occurrence, "path uses a reserved lark-cli marker")
|
|
}
|
|
relPath := strings.TrimSpace(strings.TrimPrefix(pathValue, "@"))
|
|
clean := filepath.Clean(relPath)
|
|
if relPath == "" || filepath.IsAbs(clean) || clean == "." || clean == ".." || strings.HasPrefix(clean, ".."+string(filepath.Separator)) {
|
|
return localDocResource{}, localResourceValidationError(kind, occurrence, "path must be a relative file inside the current working directory")
|
|
}
|
|
|
|
info, err := runtime.FileIO().Stat(clean)
|
|
if err != nil {
|
|
return localDocResource{}, localResourceValidationErrorWithCause(kind, occurrence, "file does not exist or its path is unsafe", err)
|
|
}
|
|
if !info.Mode().IsRegular() {
|
|
return localDocResource{}, localResourceValidationError(kind, occurrence, "path must point to a regular file")
|
|
}
|
|
if info.Size() <= 0 {
|
|
return localDocResource{}, localResourceValidationError(kind, occurrence, "file must not be empty")
|
|
}
|
|
file, err := runtime.FileIO().Open(clean)
|
|
if err != nil {
|
|
return localDocResource{}, localResourceValidationErrorWithCause(kind, occurrence, "file is not readable", err)
|
|
}
|
|
if err := file.Close(); err != nil {
|
|
return localDocResource{}, localResourceValidationErrorWithCause(kind, occurrence, "file could not be closed after validation", err)
|
|
}
|
|
var imageWidth, imageHeight int
|
|
if kind == localDocResourceImage {
|
|
imageWidth, imageHeight, _, err = detectImageConfigFromPath(runtime.FileIO(), clean)
|
|
if err != nil || imageWidth <= 0 || imageHeight <= 0 {
|
|
if err == nil {
|
|
err = invalidLocalDocImageDimensionsError()
|
|
}
|
|
return localDocResource{}, localResourceValidationErrorWithCause(kind, occurrence, "file is not a supported BMP, GIF, JPEG, PNG, TIFF, or WebP image", err)
|
|
}
|
|
}
|
|
|
|
marker, err := newLocalDocResourceMarker(kind)
|
|
if err != nil {
|
|
return localDocResource{}, errs.NewInternalError(errs.SubtypeUnknown, "failed to generate local resource marker").WithCause(err)
|
|
}
|
|
return localDocResource{
|
|
Occurrence: occurrence,
|
|
Kind: kind,
|
|
Marker: marker,
|
|
Path: clean,
|
|
FileName: filepath.Base(clean),
|
|
Size: info.Size(),
|
|
ImageWidth: imageWidth,
|
|
ImageHeight: imageHeight,
|
|
}, nil
|
|
}
|
|
|
|
func invalidLocalDocImageDimensionsError() error {
|
|
return errs.NewValidationError(errs.SubtypeInvalidArgument, "decoded image has invalid dimensions")
|
|
}
|
|
|
|
func newLocalDocResourceMarker(kind localDocResourceKind) (string, error) {
|
|
raw := make([]byte, 16)
|
|
if _, err := io.ReadFull(rand.Reader, raw); err != nil {
|
|
return "", err
|
|
}
|
|
prefix := "@lcli_img_"
|
|
if kind == localDocResourceFile {
|
|
prefix = "@lcli_file_"
|
|
}
|
|
return prefix + hex.EncodeToString(raw), nil
|
|
}
|
|
|
|
func localResourceValidationError(kind localDocResourceKind, occurrence int, reason string) error {
|
|
return errs.NewValidationError(errs.SubtypeInvalidArgument, "local %s #%d: %s", kind, occurrence, reason).WithParam("path")
|
|
}
|
|
|
|
func localResourceValidationErrorWithCause(kind localDocResourceKind, occurrence int, reason string, cause error) error {
|
|
return errs.NewValidationError(errs.SubtypeInvalidArgument, "local %s #%d: %s", kind, occurrence, reason).WithParam("path").WithCause(cause)
|
|
}
|
|
|
|
func validateLocalDocResourceUpdateCommand(command string, resources []localDocResource) error {
|
|
if len(resources) == 0 || command == "append" || command == "block_insert_after" || command == "block_replace" || command == "overwrite" {
|
|
return nil
|
|
}
|
|
return errs.NewValidationError(
|
|
errs.SubtypeInvalidArgument,
|
|
"local images and files are only supported with --command append, block_insert_after, block_replace, or overwrite",
|
|
).WithParams(
|
|
errs.InvalidParam{Name: "--command", Reason: "use append, block_insert_after, block_replace, or overwrite for local resources"},
|
|
errs.InvalidParam{Name: "--content", Reason: "contains local image or file input"},
|
|
)
|
|
}
|
|
|
|
func finalizeLocalDocResources(runtime *common.RuntimeContext, documentKey string, data map[string]interface{}, resources []localDocResource) error {
|
|
if len(resources) == 0 {
|
|
return nil
|
|
}
|
|
if data == nil {
|
|
data = map[string]interface{}{}
|
|
}
|
|
outcomes := correlateLocalDocResources(data, resources)
|
|
if strings.TrimSpace(documentKey) == "" {
|
|
for _, outcome := range outcomes {
|
|
outcome.Status = "correlation_failed"
|
|
outcome.CleanupStatus = "skipped"
|
|
outcome.Err = errs.NewInternalError(errs.SubtypeInvalidResponse, "document response is missing document_id")
|
|
}
|
|
appendLocalDocResourceFailures(data, outcomes)
|
|
scrubLocalDocResourceResponseMarkers(data)
|
|
return runtime.OutPartialFailure(data, nil)
|
|
}
|
|
|
|
uploadLocalDocResources(runtime, documentKey, outcomes)
|
|
lastRevision := localDocResourceRevisionFromDocsAI(data)
|
|
revisionKnown := lastRevision != nil
|
|
bindRevision, bindRevisionKnown := bindLocalDocResources(runtime, documentKey, outcomes)
|
|
if bindRevision != nil {
|
|
lastRevision = bindRevision
|
|
revisionKnown = true
|
|
} else if !bindRevisionKnown {
|
|
lastRevision = nil
|
|
revisionKnown = false
|
|
}
|
|
cleanupRevision, cleanupRevisionKnown := cleanupLocalDocResourcePlaceholders(runtime, documentKey, outcomes, lastRevision)
|
|
if cleanupRevision != nil {
|
|
lastRevision = cleanupRevision
|
|
revisionKnown = true
|
|
} else if !cleanupRevisionKnown {
|
|
lastRevision = nil
|
|
revisionKnown = false
|
|
}
|
|
if revisionKnown && lastRevision != nil {
|
|
setLocalDocResourceRevision(data, lastRevision)
|
|
} else if !revisionKnown {
|
|
clearLocalDocResourceRevision(data)
|
|
}
|
|
|
|
failed := false
|
|
for _, outcome := range outcomes {
|
|
if outcome.Status == "bound" {
|
|
if outcome.Block != nil {
|
|
outcome.Block["block_token"] = outcome.FileToken
|
|
}
|
|
continue
|
|
}
|
|
failed = true
|
|
if outcome.Block != nil {
|
|
delete(outcome.Block, "block_token")
|
|
}
|
|
}
|
|
scrubLocalDocResourceResponseMarkers(data)
|
|
if !failed {
|
|
return nil
|
|
}
|
|
appendLocalDocResourceFailures(data, outcomes)
|
|
return runtime.OutPartialFailure(data, nil)
|
|
}
|
|
|
|
func appendLocalDocResourceFailures(data map[string]interface{}, outcomes []*localDocResourceOutcome) {
|
|
failures := make([]interface{}, 0)
|
|
for _, outcome := range outcomes {
|
|
if outcome == nil || outcome.Status == "bound" {
|
|
continue
|
|
}
|
|
failure := map[string]interface{}{
|
|
"occurrence": outcome.Resource.Occurrence,
|
|
"kind": outcome.Resource.Kind,
|
|
"status": outcome.Status,
|
|
"cleanup_status": outcome.CleanupStatus,
|
|
}
|
|
if problem, ok := errs.ProblemOf(outcome.Err); ok {
|
|
detail := map[string]interface{}{
|
|
"type": problem.Category,
|
|
"subtype": problem.Subtype,
|
|
}
|
|
if problem.Code != 0 {
|
|
detail["code"] = problem.Code
|
|
}
|
|
if problem.Retryable {
|
|
detail["retryable"] = true
|
|
}
|
|
failure["error"] = detail
|
|
}
|
|
if len(outcome.ServerWarnings) > 0 {
|
|
failure["server_warnings"] = outcome.ServerWarnings
|
|
}
|
|
failures = append(failures, failure)
|
|
}
|
|
if len(failures) > 0 {
|
|
data["local_resource_failures"] = failures
|
|
}
|
|
}
|
|
|
|
func scrubLocalDocResourceResponseMarkers(data map[string]interface{}) {
|
|
doc, _ := data["document"].(map[string]interface{})
|
|
for _, raw := range common.GetSlice(doc, "new_blocks") {
|
|
block, _ := raw.(map[string]interface{})
|
|
if block == nil {
|
|
continue
|
|
}
|
|
marker := strings.TrimSpace(common.GetString(block, "block_token"))
|
|
if isReservedLocalDocResourceMarker(marker) {
|
|
delete(block, "block_token")
|
|
}
|
|
}
|
|
}
|
|
|
|
func correlateLocalDocResources(data map[string]interface{}, resources []localDocResource) []*localDocResourceOutcome {
|
|
outcomes := make([]*localDocResourceOutcome, len(resources))
|
|
byMarker := make(map[string][]map[string]interface{}, len(resources))
|
|
expectedMarkers := make(map[string]struct{}, len(resources))
|
|
for _, resource := range resources {
|
|
expectedMarkers[resource.Marker] = struct{}{}
|
|
}
|
|
unknownBlocks := make([]map[string]interface{}, 0)
|
|
doc, _ := data["document"].(map[string]interface{})
|
|
for _, raw := range common.GetSlice(doc, "new_blocks") {
|
|
block, _ := raw.(map[string]interface{})
|
|
if block == nil {
|
|
continue
|
|
}
|
|
marker := strings.TrimSpace(common.GetString(block, "block_token"))
|
|
if marker == "" {
|
|
continue
|
|
}
|
|
if _, expected := expectedMarkers[marker]; expected {
|
|
byMarker[marker] = append(byMarker[marker], block)
|
|
} else if isReservedLocalDocResourceMarker(marker) {
|
|
unknownBlocks = append(unknownBlocks, block)
|
|
}
|
|
}
|
|
|
|
for i, resource := range resources {
|
|
outcome := &localDocResourceOutcome{
|
|
Resource: resource,
|
|
Status: "pending",
|
|
CleanupStatus: "not_needed",
|
|
}
|
|
matches := byMarker[resource.Marker]
|
|
if len(matches) != 1 {
|
|
outcome.Status = "correlation_failed"
|
|
outcome.Err = errs.NewInternalError(errs.SubtypeInvalidResponse, "SDK returned %d blocks for local %s #%d; expected exactly one", len(matches), resource.Kind, resource.Occurrence)
|
|
allBlocksMatchKind := true
|
|
for _, block := range matches {
|
|
if !localDocResourceBlockMatchesKind(block, resource.Kind) {
|
|
allBlocksMatchKind = false
|
|
}
|
|
if id := strings.TrimSpace(common.GetString(block, "block_id")); id != "" {
|
|
outcome.CleanupBlockIDs = append(outcome.CleanupBlockIDs, id)
|
|
}
|
|
}
|
|
outcome.SafeToCleanup = len(outcome.CleanupBlockIDs) > 0 && allBlocksMatchKind
|
|
if len(matches) == 0 || !allBlocksMatchKind {
|
|
outcome.CleanupStatus = "skipped_ambiguous"
|
|
}
|
|
outcomes[i] = outcome
|
|
continue
|
|
}
|
|
|
|
block := matches[0]
|
|
blockID := strings.TrimSpace(common.GetString(block, "block_id"))
|
|
blockType := strings.TrimSpace(common.GetString(block, "block_type"))
|
|
outcome.Block = block
|
|
outcome.BlockID = blockID
|
|
blockMatchesKind := localDocResourceBlockMatchesKind(block, resource.Kind)
|
|
if blockID != "" && blockMatchesKind {
|
|
outcome.CleanupBlockIDs = []string{blockID}
|
|
outcome.SafeToCleanup = true
|
|
}
|
|
if blockID == "" || !blockMatchesKind {
|
|
outcome.Status = "correlation_failed"
|
|
outcome.Err = errs.NewInternalError(errs.SubtypeInvalidResponse, "SDK returned an invalid block for local %s #%d", resource.Kind, resource.Occurrence)
|
|
if blockID != "" && blockType != string(resource.Kind) {
|
|
outcome.CleanupStatus = "skipped_ambiguous"
|
|
}
|
|
}
|
|
outcomes[i] = outcome
|
|
}
|
|
if len(unknownBlocks) > 0 {
|
|
for _, outcome := range outcomes {
|
|
outcome.Status = "correlation_failed"
|
|
outcome.CleanupStatus = "skipped_ambiguous"
|
|
outcome.SafeToCleanup = false
|
|
outcome.Err = errs.NewInternalError(errs.SubtypeInvalidResponse, "SDK returned unrecognized local resource markers; no local uploads were started")
|
|
}
|
|
return outcomes
|
|
}
|
|
byBlockID := make(map[string][]*localDocResourceOutcome, len(outcomes))
|
|
for _, outcome := range outcomes {
|
|
if outcome.BlockID != "" {
|
|
byBlockID[outcome.BlockID] = append(byBlockID[outcome.BlockID], outcome)
|
|
}
|
|
}
|
|
for blockID, conflicts := range byBlockID {
|
|
if len(conflicts) < 2 {
|
|
continue
|
|
}
|
|
blockIDSafeToCleanup := true
|
|
for _, outcome := range conflicts {
|
|
if !outcome.SafeToCleanup {
|
|
blockIDSafeToCleanup = false
|
|
break
|
|
}
|
|
}
|
|
for i, outcome := range conflicts {
|
|
outcome.Status = "correlation_failed"
|
|
outcome.Err = errs.NewInternalError(errs.SubtypeInvalidResponse, "SDK correlated %d local resources to the same block_id", len(conflicts))
|
|
if i == 0 && blockIDSafeToCleanup {
|
|
found := false
|
|
for _, cleanupBlockID := range outcome.CleanupBlockIDs {
|
|
if cleanupBlockID == blockID {
|
|
found = true
|
|
break
|
|
}
|
|
}
|
|
if !found {
|
|
outcome.CleanupBlockIDs = append(outcome.CleanupBlockIDs, blockID)
|
|
}
|
|
continue
|
|
}
|
|
outcome.CleanupBlockIDs = nil
|
|
outcome.SafeToCleanup = false
|
|
if blockIDSafeToCleanup {
|
|
outcome.CleanupStatus = "skipped_duplicate"
|
|
} else {
|
|
outcome.CleanupStatus = "skipped_ambiguous"
|
|
}
|
|
}
|
|
}
|
|
return outcomes
|
|
}
|
|
|
|
func uploadLocalDocResources(runtime *common.RuntimeContext, documentKey string, outcomes []*localDocResourceOutcome) {
|
|
remoteOutcomes := make([]*localDocResourceOutcome, 0)
|
|
localOutcomes := make([]*localDocResourceOutcome, 0, len(outcomes))
|
|
for _, outcome := range outcomes {
|
|
if outcome.Status != "pending" {
|
|
continue
|
|
}
|
|
if outcome.Resource.RemoteURL != "" {
|
|
remoteOutcomes = append(remoteOutcomes, outcome)
|
|
continue
|
|
}
|
|
localOutcomes = append(localOutcomes, outcome)
|
|
}
|
|
uploadRemoteDocImages(runtime, documentKey, remoteOutcomes)
|
|
uploadLocalDocResourcesSerially(runtime, documentKey, localOutcomes)
|
|
}
|
|
|
|
func uploadRemoteDocImages(runtime *common.RuntimeContext, documentKey string, outcomes []*localDocResourceOutcome) {
|
|
if len(outcomes) == 0 {
|
|
return
|
|
}
|
|
// Resolve credentials before fan-out so lazy credential-source selection
|
|
// completes on the caller goroutine. Upload requests still resolve the
|
|
// current token through the standard API path.
|
|
if _, err := runtime.AccessToken(); err != nil {
|
|
for _, outcome := range outcomes {
|
|
markLocalDocResourceUploadFailed(outcome, err)
|
|
}
|
|
return
|
|
}
|
|
workerCount := min(remoteDocImageUploadConcurrency, len(outcomes))
|
|
jobs := make(chan *localDocResourceOutcome)
|
|
done := make(chan struct{}, workerCount)
|
|
for range workerCount {
|
|
go func() {
|
|
defer func() { done <- struct{}{} }()
|
|
for outcome := range jobs {
|
|
uploadLocalDocResource(runtime, documentKey, outcome)
|
|
}
|
|
}()
|
|
}
|
|
for _, outcome := range outcomes {
|
|
jobs <- outcome
|
|
}
|
|
close(jobs)
|
|
for range workerCount {
|
|
<-done
|
|
}
|
|
}
|
|
|
|
func uploadLocalDocResourcesSerially(runtime *common.RuntimeContext, documentKey string, outcomes []*localDocResourceOutcome) {
|
|
started := false
|
|
for _, outcome := range outcomes {
|
|
if started {
|
|
if err := waitLocalDocResourceRequest(runtime.Ctx(), localDocResourceUploadInterval); err != nil {
|
|
markLocalDocResourceUploadFailed(outcome, err)
|
|
continue
|
|
}
|
|
}
|
|
started = true
|
|
uploadLocalDocResource(runtime, documentKey, outcome)
|
|
}
|
|
}
|
|
|
|
func uploadLocalDocResource(runtime *common.RuntimeContext, documentKey string, outcome *localDocResourceOutcome) {
|
|
var content []byte
|
|
if outcome.Resource.RemoteURL != "" {
|
|
outcome.Resource.Content = nil
|
|
download, err := downloadRemoteDocImageWithRetry(runtime, outcome.Resource.RemoteURL, outcome.Resource.Occurrence)
|
|
if err != nil {
|
|
markLocalDocResourceUploadFailed(outcome, err)
|
|
return
|
|
}
|
|
if err := applyRemoteDocImageDownload(&outcome.Resource, download); err != nil {
|
|
markLocalDocResourceUploadFailed(outcome, err)
|
|
return
|
|
}
|
|
content = download.Content
|
|
outcome.Resource.Content = nil
|
|
defer func() {
|
|
content = nil
|
|
outcome.Resource.Content = nil
|
|
}()
|
|
} else if len(outcome.Resource.Content) > 0 {
|
|
content = outcome.Resource.Content
|
|
}
|
|
|
|
var uploadErr error
|
|
for attempt := 0; attempt < localDocResourceUploadAttempts; attempt++ {
|
|
upload := UploadDocMediaFileConfig{
|
|
FilePath: outcome.Resource.Path,
|
|
FileName: outcome.Resource.FileName,
|
|
FileSize: outcome.Resource.Size,
|
|
ParentType: parentTypeForMediaType(string(outcome.Resource.Kind)),
|
|
ParentNode: outcome.BlockID,
|
|
DocID: documentKey,
|
|
}
|
|
if len(content) > 0 {
|
|
upload.Reader = bytes.NewReader(content)
|
|
}
|
|
token, err := uploadDocMediaFile(runtime, upload)
|
|
if err == nil {
|
|
outcome.FileToken = token
|
|
outcome.Status = "uploaded"
|
|
return
|
|
}
|
|
uploadErr = err
|
|
if !isRetryableLocalDocResourceUpload(err) || attempt+1 >= localDocResourceUploadAttempts {
|
|
break
|
|
}
|
|
delay := localDocResourceRetryDelay(localDocResourceUploadInterval, attempt)
|
|
if waitErr := waitLocalDocResourceRequest(runtime.Ctx(), delay); waitErr != nil {
|
|
uploadErr = errors.Join(uploadErr, waitErr)
|
|
break
|
|
}
|
|
}
|
|
markLocalDocResourceUploadFailed(outcome, uploadErr)
|
|
}
|
|
|
|
func downloadRemoteDocImageWithRetry(runtime *common.RuntimeContext, rawURL string, occurrence int) (remoteDocImageDownload, error) {
|
|
var downloadErr error
|
|
for attempt := 0; attempt < remoteDocImageDownloadAttempts; attempt++ {
|
|
download, err := downloadRemoteDocImage(runtime, rawURL, occurrence)
|
|
if err == nil {
|
|
return download, nil
|
|
}
|
|
downloadErr = err
|
|
if !isRetryableLocalDocResourceNetwork(err) || attempt+1 >= remoteDocImageDownloadAttempts {
|
|
break
|
|
}
|
|
delay := localDocResourceRetryDelay(localDocResourceUploadInterval, attempt)
|
|
if waitErr := waitLocalDocResourceRequest(runtime.Ctx(), delay); waitErr != nil {
|
|
downloadErr = errors.Join(downloadErr, waitErr)
|
|
break
|
|
}
|
|
}
|
|
return remoteDocImageDownload{}, downloadErr
|
|
}
|
|
|
|
func markLocalDocResourceUploadFailed(outcome *localDocResourceOutcome, err error) {
|
|
outcome.Status = "upload_failed"
|
|
outcome.CleanupStatus = "pending"
|
|
outcome.Err = err
|
|
}
|
|
|
|
func isRetryableLocalDocResourceUpload(err error) bool {
|
|
if isRetryableLocalDocResourceNetwork(err) {
|
|
return true
|
|
}
|
|
problem, ok := errs.ProblemOf(err)
|
|
return ok && (problem.Code == localDocResourceUploadConflictCode || problem.Code == localDocResourceUploadRateLimitCode)
|
|
}
|
|
|
|
func isRetryableLocalDocResourceNetwork(err error) bool {
|
|
if errors.Is(err, context.Canceled) {
|
|
return false
|
|
}
|
|
if errs.IsRetryable(err) {
|
|
return true
|
|
}
|
|
problem, ok := errs.ProblemOf(err)
|
|
if !ok {
|
|
return false
|
|
}
|
|
if problem.Category != errs.CategoryNetwork {
|
|
return false
|
|
}
|
|
return problem.Subtype == errs.SubtypeNetworkTransport ||
|
|
problem.Subtype == errs.SubtypeNetworkTimeout ||
|
|
problem.Subtype == errs.SubtypeNetworkServer ||
|
|
problem.Subtype == errs.SubtypeRateLimit ||
|
|
problem.Code == http.StatusTooManyRequests ||
|
|
problem.Code >= http.StatusInternalServerError
|
|
}
|
|
|
|
func bindLocalDocResources(runtime *common.RuntimeContext, documentKey string, outcomes []*localDocResourceOutcome) (interface{}, bool) {
|
|
ready := make([]*localDocResourceOutcome, 0, len(outcomes))
|
|
for _, outcome := range outcomes {
|
|
if outcome.Status == "uploaded" {
|
|
ready = append(ready, outcome)
|
|
}
|
|
}
|
|
var lastRevision interface{}
|
|
revisionKnown := true
|
|
for start := 0; start < len(ready); start += localDocResourceBindBatchSize {
|
|
end := min(start+localDocResourceBindBatchSize, len(ready))
|
|
chunk := ready[start:end]
|
|
if start > 0 {
|
|
if err := waitLocalDocResourceRequest(runtime.Ctx(), localDocResourceBindInterval); err != nil {
|
|
markLocalDocResourceBindFailed(ready[start:], err)
|
|
break
|
|
}
|
|
}
|
|
revision, chunkRevisionKnown := bindLocalDocResourceChunk(runtime, documentKey, chunk)
|
|
if revision != nil {
|
|
lastRevision = revision
|
|
revisionKnown = true
|
|
} else if !chunkRevisionKnown {
|
|
lastRevision = nil
|
|
revisionKnown = false
|
|
}
|
|
}
|
|
return lastRevision, revisionKnown
|
|
}
|
|
|
|
func bindLocalDocResourceChunk(runtime *common.RuntimeContext, documentKey string, chunk []*localDocResourceOutcome) (interface{}, bool) {
|
|
clientToken := uuid.NewString()
|
|
body := buildLocalDocResourceBatchUpdate(chunk)
|
|
url := fmt.Sprintf("/open-apis/docx/v1/documents/%s/blocks/batch_update", validate.EncodePathSegment(documentKey))
|
|
var lastErr error
|
|
for attempt := 0; attempt < localDocResourceBindAttempts; attempt++ {
|
|
data, err := runtime.CallAPITyped("PATCH", url, map[string]interface{}{"client_token": clientToken}, body)
|
|
if err == nil {
|
|
for _, outcome := range chunk {
|
|
outcome.Status = "bound"
|
|
outcome.CleanupStatus = "not_needed"
|
|
outcome.SafeToCleanup = false
|
|
}
|
|
revision := localDocResourceRevisionFromBatch(data)
|
|
return revision, revision != nil
|
|
}
|
|
lastErr = err
|
|
allBound, hasConflict, hasUnknown := verifyLocalDocResourceChunk(runtime, documentKey, chunk, err)
|
|
if allBound {
|
|
return nil, false
|
|
}
|
|
if hasConflict || hasUnknown || !errs.IsRetryable(err) {
|
|
break
|
|
}
|
|
if attempt+1 < localDocResourceBindAttempts {
|
|
delay := localDocResourceBindInterval * time.Duration(1<<attempt)
|
|
if waitErr := waitLocalDocResourceRequest(runtime.Ctx(), delay); waitErr != nil {
|
|
lastErr = errors.Join(lastErr, waitErr)
|
|
break
|
|
}
|
|
}
|
|
}
|
|
|
|
markLocalDocResourceBindFailed(chunk, lastErr)
|
|
for _, outcome := range chunk {
|
|
if outcome.Status == "bound" || outcome.Status == "bind_conflict" || outcome.Status == "bind_ambiguous" {
|
|
return nil, false
|
|
}
|
|
}
|
|
return nil, true
|
|
}
|
|
|
|
func markLocalDocResourceBindFailed(chunk []*localDocResourceOutcome, bindErr error) {
|
|
for _, outcome := range chunk {
|
|
if outcome.Status == "bound" || outcome.Status == "bind_conflict" || outcome.Status == "bind_ambiguous" {
|
|
continue
|
|
}
|
|
outcome.Status = "bind_failed"
|
|
outcome.CleanupStatus = "pending"
|
|
outcome.SafeToCleanup = true
|
|
outcome.Err = bindErr
|
|
}
|
|
}
|
|
|
|
func verifyLocalDocResourceChunk(runtime *common.RuntimeContext, documentKey string, chunk []*localDocResourceOutcome, bindErr error) (allBound, hasConflict, hasUnknown bool) {
|
|
allBound = true
|
|
for i, outcome := range chunk {
|
|
if i > 0 {
|
|
if err := waitLocalDocResourceRequest(runtime.Ctx(), localDocResourceVerifyInterval); err != nil {
|
|
outcome.Status = "bind_ambiguous"
|
|
outcome.CleanupStatus = "skipped_ambiguous"
|
|
outcome.SafeToCleanup = false
|
|
outcome.Err = errors.Join(bindErr, err)
|
|
allBound = false
|
|
hasUnknown = true
|
|
continue
|
|
}
|
|
}
|
|
data, err := runtime.CallAPITyped("GET", fmt.Sprintf("/open-apis/docx/v1/documents/%s/blocks/%s", validate.EncodePathSegment(documentKey), validate.EncodePathSegment(outcome.BlockID)), nil, nil)
|
|
if err != nil {
|
|
outcome.Status = "bind_ambiguous"
|
|
outcome.CleanupStatus = "skipped_ambiguous"
|
|
outcome.SafeToCleanup = false
|
|
outcome.Err = errors.Join(bindErr, err)
|
|
allBound = false
|
|
hasUnknown = true
|
|
continue
|
|
}
|
|
actualToken := localDocResourceBlockToken(common.GetMap(data, "block"), outcome.Resource.Kind)
|
|
switch actualToken {
|
|
case outcome.FileToken:
|
|
outcome.Status = "bound"
|
|
outcome.CleanupStatus = "not_needed"
|
|
outcome.SafeToCleanup = false
|
|
case "":
|
|
outcome.Status = "uploaded"
|
|
outcome.SafeToCleanup = true
|
|
allBound = false
|
|
default:
|
|
outcome.Status = "bind_conflict"
|
|
outcome.CleanupStatus = "skipped_conflict"
|
|
outcome.SafeToCleanup = false
|
|
outcome.Err = errs.NewInternalError(errs.SubtypeInvalidResponse, "local %s #%d block token changed unexpectedly; placeholder was preserved", outcome.Resource.Kind, outcome.Resource.Occurrence)
|
|
allBound = false
|
|
hasConflict = true
|
|
}
|
|
}
|
|
return allBound, hasConflict, hasUnknown
|
|
}
|
|
|
|
func buildLocalDocResourceBatchUpdate(chunk []*localDocResourceOutcome) map[string]interface{} {
|
|
requests := make([]interface{}, 0, len(chunk))
|
|
for _, outcome := range chunk {
|
|
request := map[string]interface{}{"block_id": outcome.BlockID}
|
|
if outcome.Resource.Kind == localDocResourceFile {
|
|
request["replace_file"] = map[string]interface{}{"token": outcome.FileToken}
|
|
} else {
|
|
replaceImage := map[string]interface{}{"token": outcome.FileToken}
|
|
if outcome.Resource.ImageWidth > 0 {
|
|
replaceImage["width"] = outcome.Resource.ImageWidth
|
|
}
|
|
if outcome.Resource.ImageHeight > 0 {
|
|
replaceImage["height"] = outcome.Resource.ImageHeight
|
|
}
|
|
if align, ok := alignMap[outcome.Resource.ImageAlign]; ok {
|
|
replaceImage["align"] = align
|
|
}
|
|
if outcome.Resource.HasScale {
|
|
replaceImage["scale"] = outcome.Resource.ImageScale
|
|
}
|
|
request["replace_image"] = replaceImage
|
|
}
|
|
requests = append(requests, request)
|
|
}
|
|
return map[string]interface{}{"requests": requests}
|
|
}
|
|
|
|
func cleanupLocalDocResourcePlaceholders(runtime *common.RuntimeContext, documentKey string, outcomes []*localDocResourceOutcome, baseRevision interface{}) (interface{}, bool) {
|
|
type cleanupTarget struct {
|
|
BlockID string
|
|
Owner *localDocResourceOutcome
|
|
}
|
|
targets := make([]cleanupTarget, 0)
|
|
seenBlockIDs := make(map[string]struct{})
|
|
for _, outcome := range outcomes {
|
|
if outcome.Status == "bound" || !outcome.SafeToCleanup {
|
|
continue
|
|
}
|
|
for _, blockID := range outcome.CleanupBlockIDs {
|
|
blockID = strings.TrimSpace(blockID)
|
|
if blockID == "" {
|
|
continue
|
|
}
|
|
if _, duplicate := seenBlockIDs[blockID]; duplicate {
|
|
continue
|
|
}
|
|
seenBlockIDs[blockID] = struct{}{}
|
|
targets = append(targets, cleanupTarget{BlockID: blockID, Owner: outcome})
|
|
}
|
|
if len(outcome.CleanupBlockIDs) == 0 && outcome.CleanupStatus == "not_needed" {
|
|
outcome.CleanupStatus = "skipped"
|
|
}
|
|
}
|
|
if len(targets) == 0 {
|
|
return nil, true
|
|
}
|
|
baseRevision = normalizeLocalDocResourceRevision(baseRevision)
|
|
if baseRevision == nil {
|
|
err := errs.NewInternalError(errs.SubtypeInvalidResponse, "document response is missing a revision; local resource placeholders were preserved")
|
|
for _, target := range targets {
|
|
target.Owner.CleanupStatus = "skipped_ambiguous"
|
|
target.Owner.SafeToCleanup = false
|
|
appendLocalDocResourceOutcomeError(target.Owner, err)
|
|
}
|
|
return nil, false
|
|
}
|
|
|
|
verifiedTargets := make([]cleanupTarget, 0, len(targets))
|
|
blockedOwners := make(map[*localDocResourceOutcome]struct{})
|
|
for i, target := range targets {
|
|
if i > 0 {
|
|
if err := waitLocalDocResourceRequest(runtime.Ctx(), localDocResourceVerifyInterval); err != nil {
|
|
target.Owner.CleanupStatus = "skipped_ambiguous"
|
|
target.Owner.SafeToCleanup = false
|
|
blockedOwners[target.Owner] = struct{}{}
|
|
appendLocalDocResourceOutcomeError(target.Owner, err)
|
|
continue
|
|
}
|
|
}
|
|
data, err := runtime.CallAPITyped("GET", fmt.Sprintf("/open-apis/docx/v1/documents/%s/blocks/%s", validate.EncodePathSegment(documentKey), validate.EncodePathSegment(target.BlockID)), nil, nil)
|
|
if err != nil {
|
|
target.Owner.Status = "bind_ambiguous"
|
|
target.Owner.CleanupStatus = "skipped_ambiguous"
|
|
target.Owner.SafeToCleanup = false
|
|
blockedOwners[target.Owner] = struct{}{}
|
|
appendLocalDocResourceOutcomeError(target.Owner, err)
|
|
continue
|
|
}
|
|
block := common.GetMap(data, "block")
|
|
if !localDocResourceBlockMatchesKind(block, target.Owner.Resource.Kind) {
|
|
blockedOwners[target.Owner] = struct{}{}
|
|
target.Owner.Status = "bind_ambiguous"
|
|
target.Owner.CleanupStatus = "skipped_ambiguous"
|
|
target.Owner.SafeToCleanup = false
|
|
appendLocalDocResourceOutcomeError(target.Owner, errs.NewInternalError(
|
|
errs.SubtypeInvalidResponse,
|
|
"local %s #%d block type could not be verified before cleanup; placeholder was preserved",
|
|
target.Owner.Resource.Kind,
|
|
target.Owner.Resource.Occurrence,
|
|
))
|
|
continue
|
|
}
|
|
|
|
actualTokens := localDocResourceBlockTokens(block)
|
|
if len(actualTokens) == 0 {
|
|
if target.Owner.Resource.Kind == localDocResourceFile {
|
|
// DocX represents a file as a source child inside a figure
|
|
// block. Deleting the source ID is accepted by docs_ai but
|
|
// leaves an empty <figure><source/></figure> behind, so remove
|
|
// the owning figure after the tokenless child is verified.
|
|
parentID := strings.TrimSpace(common.GetString(block, "parent_id"))
|
|
if parentID == "" || parentID == target.BlockID || parentID == documentKey {
|
|
blockedOwners[target.Owner] = struct{}{}
|
|
target.Owner.Status = "bind_ambiguous"
|
|
target.Owner.CleanupStatus = "skipped_ambiguous"
|
|
target.Owner.SafeToCleanup = false
|
|
appendLocalDocResourceOutcomeError(target.Owner, errs.NewInternalError(
|
|
errs.SubtypeInvalidResponse,
|
|
"local file #%d figure parent could not be verified before cleanup; placeholder was preserved",
|
|
target.Owner.Resource.Occurrence,
|
|
))
|
|
continue
|
|
}
|
|
parentData, err := runtime.CallAPITyped("GET", fmt.Sprintf("/open-apis/docx/v1/documents/%s/blocks/%s", validate.EncodePathSegment(documentKey), validate.EncodePathSegment(parentID)), nil, nil)
|
|
if err != nil {
|
|
blockedOwners[target.Owner] = struct{}{}
|
|
target.Owner.Status = "bind_ambiguous"
|
|
target.Owner.CleanupStatus = "skipped_ambiguous"
|
|
target.Owner.SafeToCleanup = false
|
|
appendLocalDocResourceOutcomeError(target.Owner, err)
|
|
continue
|
|
}
|
|
parentBlock := common.GetMap(parentData, "block")
|
|
if !localDocResourceIsSoleFileFigure(parentBlock, parentID, target.BlockID) {
|
|
blockedOwners[target.Owner] = struct{}{}
|
|
target.Owner.Status = "bind_ambiguous"
|
|
target.Owner.CleanupStatus = "skipped_ambiguous"
|
|
target.Owner.SafeToCleanup = false
|
|
appendLocalDocResourceOutcomeError(target.Owner, errs.NewInternalError(
|
|
errs.SubtypeInvalidResponse,
|
|
"local file #%d parent is not a sole-source figure; placeholder was preserved",
|
|
target.Owner.Resource.Occurrence,
|
|
))
|
|
continue
|
|
}
|
|
target.BlockID = parentID
|
|
}
|
|
verifiedTargets = append(verifiedTargets, target)
|
|
continue
|
|
}
|
|
|
|
blockedOwners[target.Owner] = struct{}{}
|
|
target.Owner.SafeToCleanup = false
|
|
if target.Owner.FileToken != "" && len(actualTokens) == 1 && actualTokens[0] == target.Owner.FileToken {
|
|
target.Owner.Status = "bound"
|
|
target.Owner.CleanupStatus = "not_needed"
|
|
if target.Owner.Block != nil {
|
|
target.Owner.Block["block_token"] = target.Owner.FileToken
|
|
}
|
|
continue
|
|
}
|
|
target.Owner.Status = "bind_conflict"
|
|
target.Owner.CleanupStatus = "skipped_conflict"
|
|
appendLocalDocResourceOutcomeError(target.Owner, errs.NewInternalError(
|
|
errs.SubtypeInvalidResponse,
|
|
"local %s #%d block token changed before cleanup; placeholder was preserved",
|
|
target.Owner.Resource.Kind,
|
|
target.Owner.Resource.Occurrence,
|
|
))
|
|
}
|
|
verifiedByBlockID := make(map[string]cleanupTarget, len(verifiedTargets))
|
|
for _, target := range verifiedTargets {
|
|
if previous, duplicate := verifiedByBlockID[target.BlockID]; duplicate {
|
|
blockedOwners[previous.Owner] = struct{}{}
|
|
blockedOwners[target.Owner] = struct{}{}
|
|
for _, owner := range []*localDocResourceOutcome{previous.Owner, target.Owner} {
|
|
owner.Status = "bind_ambiguous"
|
|
owner.CleanupStatus = "skipped_ambiguous"
|
|
owner.SafeToCleanup = false
|
|
appendLocalDocResourceOutcomeError(owner, errs.NewInternalError(
|
|
errs.SubtypeInvalidResponse,
|
|
"multiple local resources resolved to cleanup block %s; placeholders were preserved",
|
|
target.BlockID,
|
|
))
|
|
}
|
|
continue
|
|
}
|
|
verifiedByBlockID[target.BlockID] = target
|
|
}
|
|
deleteTargets := verifiedTargets[:0]
|
|
for _, target := range verifiedTargets {
|
|
if _, blocked := blockedOwners[target.Owner]; !blocked {
|
|
deleteTargets = append(deleteTargets, target)
|
|
}
|
|
}
|
|
if len(deleteTargets) == 0 {
|
|
return nil, false
|
|
}
|
|
|
|
var lastRevision interface{}
|
|
currentRevision := baseRevision
|
|
for start := 0; start < len(deleteTargets); start += localDocResourceCleanupBatchSize {
|
|
end := min(start+localDocResourceCleanupBatchSize, len(deleteTargets))
|
|
chunk := deleteTargets[start:end]
|
|
ids := make([]string, 0, len(chunk))
|
|
for _, target := range chunk {
|
|
ids = append(ids, target.BlockID)
|
|
}
|
|
body := map[string]interface{}{
|
|
"format": "xml",
|
|
"command": "block_delete",
|
|
"block_id": strings.Join(ids, ","),
|
|
"revision_id": currentRevision,
|
|
}
|
|
injectDocsScene(runtime, body)
|
|
data, err := doDocAPI(runtime, "PUT", fmt.Sprintf("/open-apis/docs_ai/v1/documents/%s", validate.EncodePathSegment(documentKey)), body)
|
|
if err != nil {
|
|
for _, target := range chunk {
|
|
target.Owner.CleanupStatus = "failed"
|
|
target.Owner.SafeToCleanup = false
|
|
appendLocalDocResourceOutcomeError(target.Owner, err)
|
|
}
|
|
for _, target := range deleteTargets[end:] {
|
|
target.Owner.CleanupStatus = "skipped_ambiguous"
|
|
target.Owner.SafeToCleanup = false
|
|
appendLocalDocResourceOutcomeError(target.Owner, errs.NewInternalError(errs.SubtypeInvalidResponse, "cleanup revision could not be confirmed; placeholder was preserved"))
|
|
}
|
|
return nil, false
|
|
}
|
|
if docsAPIOperationFailed(data) {
|
|
serviceErr := errs.NewAPIError(errs.SubtypeUnknown, "local resource placeholder cleanup returned result=failed")
|
|
warnings := common.GetSlice(data, "warnings")
|
|
for _, target := range chunk {
|
|
target.Owner.CleanupStatus = "failed"
|
|
target.Owner.SafeToCleanup = false
|
|
target.Owner.ServerWarnings = append(target.Owner.ServerWarnings, warnings...)
|
|
appendLocalDocResourceOutcomeError(target.Owner, serviceErr)
|
|
}
|
|
for _, target := range deleteTargets[end:] {
|
|
target.Owner.CleanupStatus = "skipped_ambiguous"
|
|
target.Owner.SafeToCleanup = false
|
|
appendLocalDocResourceOutcomeError(target.Owner, errs.NewInternalError(errs.SubtypeInvalidResponse, "cleanup revision could not be confirmed; placeholder was preserved"))
|
|
}
|
|
return nil, false
|
|
}
|
|
for _, target := range chunk {
|
|
target.Owner.CleanupStatus = "succeeded"
|
|
target.Owner.SafeToCleanup = false
|
|
}
|
|
revision := localDocResourceRevisionFromDocsAI(data)
|
|
if revision == nil {
|
|
for _, target := range deleteTargets[end:] {
|
|
target.Owner.CleanupStatus = "skipped_ambiguous"
|
|
target.Owner.SafeToCleanup = false
|
|
appendLocalDocResourceOutcomeError(target.Owner, errs.NewInternalError(errs.SubtypeInvalidResponse, "cleanup response is missing a revision; placeholder was preserved"))
|
|
}
|
|
return nil, false
|
|
}
|
|
lastRevision = revision
|
|
currentRevision = revision
|
|
}
|
|
return lastRevision, true
|
|
}
|
|
|
|
func appendLocalDocResourceOutcomeError(outcome *localDocResourceOutcome, err error) {
|
|
if err == nil {
|
|
return
|
|
}
|
|
if outcome.Err == nil {
|
|
outcome.Err = err
|
|
return
|
|
}
|
|
outcome.Err = errors.Join(outcome.Err, err)
|
|
}
|
|
|
|
func localDocResourceBlockToken(block map[string]interface{}, kind localDocResourceKind) string {
|
|
if block == nil {
|
|
return ""
|
|
}
|
|
if token := strings.TrimSpace(common.GetString(block, "token")); token != "" {
|
|
return token
|
|
}
|
|
return strings.TrimSpace(common.GetString(common.GetMap(block, string(kind)), "token"))
|
|
}
|
|
|
|
func localDocResourceBlockMatchesKind(block map[string]interface{}, kind localDocResourceKind) bool {
|
|
if block == nil {
|
|
return false
|
|
}
|
|
rawType, ok := block["block_type"]
|
|
if !ok {
|
|
return false
|
|
}
|
|
if blockType, ok := rawType.(string); ok {
|
|
return strings.TrimSpace(blockType) == string(kind)
|
|
}
|
|
blockType, ok := normalizeLocalDocResourceRevision(rawType).(int64)
|
|
return ok && blockType == int64(blockTypeForMediaType(string(kind)))
|
|
}
|
|
|
|
func localDocResourceIsSoleFileFigure(block map[string]interface{}, expectedBlockID, childBlockID string) bool {
|
|
if block == nil || strings.TrimSpace(common.GetString(block, "block_id")) != expectedBlockID {
|
|
return false
|
|
}
|
|
rawType, ok := block["block_type"]
|
|
if !ok {
|
|
return false
|
|
}
|
|
figureType := false
|
|
if blockType, ok := rawType.(string); ok {
|
|
switch strings.ToLower(strings.TrimSpace(blockType)) {
|
|
case "figure", "view":
|
|
figureType = true
|
|
}
|
|
} else if blockType, ok := normalizeLocalDocResourceRevision(rawType).(int64); ok {
|
|
figureType = blockType == 33
|
|
}
|
|
if !figureType {
|
|
return false
|
|
}
|
|
children := common.GetSlice(block, "children")
|
|
if len(children) != 1 {
|
|
return false
|
|
}
|
|
child, ok := children[0].(string)
|
|
return ok && strings.TrimSpace(child) == childBlockID
|
|
}
|
|
|
|
func localDocResourceBlockTokens(block map[string]interface{}) []string {
|
|
if block == nil {
|
|
return nil
|
|
}
|
|
tokens := make([]string, 0, 3)
|
|
seen := make(map[string]struct{}, 3)
|
|
for _, token := range []string{
|
|
common.GetString(block, "token"),
|
|
common.GetString(common.GetMap(block, string(localDocResourceImage)), "token"),
|
|
common.GetString(common.GetMap(block, string(localDocResourceFile)), "token"),
|
|
} {
|
|
token = strings.TrimSpace(token)
|
|
if token == "" {
|
|
continue
|
|
}
|
|
if _, duplicate := seen[token]; duplicate {
|
|
continue
|
|
}
|
|
seen[token] = struct{}{}
|
|
tokens = append(tokens, token)
|
|
}
|
|
return tokens
|
|
}
|
|
|
|
func localDocResourceRevisionFromBatch(data map[string]interface{}) interface{} {
|
|
if data == nil {
|
|
return nil
|
|
}
|
|
return normalizeLocalDocResourceRevision(data["document_revision_id"])
|
|
}
|
|
|
|
func localDocResourceRevisionFromDocsAI(data map[string]interface{}) interface{} {
|
|
doc, _ := data["document"].(map[string]interface{})
|
|
if doc == nil {
|
|
return nil
|
|
}
|
|
return normalizeLocalDocResourceRevision(doc["revision_id"])
|
|
}
|
|
|
|
func normalizeLocalDocResourceRevision(value interface{}) interface{} {
|
|
var revision int64
|
|
switch number := value.(type) {
|
|
case int:
|
|
revision = int64(number)
|
|
case int8:
|
|
revision = int64(number)
|
|
case int16:
|
|
revision = int64(number)
|
|
case int32:
|
|
revision = int64(number)
|
|
case int64:
|
|
revision = number
|
|
case uint:
|
|
if uint64(number) > math.MaxInt64 {
|
|
return nil
|
|
}
|
|
revision = int64(number)
|
|
case uint8:
|
|
revision = int64(number)
|
|
case uint16:
|
|
revision = int64(number)
|
|
case uint32:
|
|
revision = int64(number)
|
|
case uint64:
|
|
if number > math.MaxInt64 {
|
|
return nil
|
|
}
|
|
revision = int64(number)
|
|
case float32:
|
|
value := float64(number)
|
|
if value > math.MaxInt64 || value < 0 || math.Trunc(value) != value {
|
|
return nil
|
|
}
|
|
revision = int64(value)
|
|
case float64:
|
|
if number > math.MaxInt64 || number < 0 || math.Trunc(number) != number {
|
|
return nil
|
|
}
|
|
revision = int64(number)
|
|
case json.Number:
|
|
parsed, err := strconv.ParseInt(number.String(), 10, 64)
|
|
if err != nil {
|
|
return nil
|
|
}
|
|
revision = parsed
|
|
case string:
|
|
parsed, err := strconv.ParseInt(strings.TrimSpace(number), 10, 64)
|
|
if err != nil {
|
|
return nil
|
|
}
|
|
revision = parsed
|
|
default:
|
|
return nil
|
|
}
|
|
if revision < 0 {
|
|
return nil
|
|
}
|
|
return revision
|
|
}
|
|
|
|
func setLocalDocResourceRevision(data map[string]interface{}, revision interface{}) {
|
|
doc, _ := data["document"].(map[string]interface{})
|
|
if doc != nil {
|
|
if revision = normalizeLocalDocResourceRevision(revision); revision != nil {
|
|
doc["revision_id"] = revision
|
|
}
|
|
}
|
|
}
|
|
|
|
func clearLocalDocResourceRevision(data map[string]interface{}) {
|
|
doc, _ := data["document"].(map[string]interface{})
|
|
if doc != nil {
|
|
delete(doc, "revision_id")
|
|
}
|
|
}
|
|
|
|
func appendLocalDocResourcesDryRun(dry *common.DryRunAPI, documentKey string, resources []localDocResource) *common.DryRunAPI {
|
|
if len(resources) == 0 {
|
|
return dry
|
|
}
|
|
encodedDocumentKey := validate.EncodePathSegment(documentKey)
|
|
routeExtra, _ := buildDriveRouteExtra(documentKey)
|
|
for _, resource := range resources {
|
|
parentType := parentTypeForMediaType(string(resource.Kind))
|
|
body := map[string]interface{}{
|
|
"file_name": fmt.Sprintf("<local_%s_%d_filename>", resource.Kind, resource.Occurrence),
|
|
"parent_type": parentType,
|
|
"parent_node": fmt.Sprintf("<local_%s_%d_block_id>", resource.Kind, resource.Occurrence),
|
|
"size": resource.Size,
|
|
"extra": routeExtra,
|
|
}
|
|
if resource.Size > common.MaxDriveMediaUploadSinglePartSize {
|
|
dry.POST("/open-apis/drive/v1/medias/upload_prepare").
|
|
Desc(fmt.Sprintf("Upload local %s #%d: initialize multipart upload", resource.Kind, resource.Occurrence)).
|
|
Body(body).
|
|
POST("/open-apis/drive/v1/medias/upload_part").
|
|
Desc(fmt.Sprintf("Upload local %s #%d: upload parts (repeated)", resource.Kind, resource.Occurrence)).
|
|
Body(map[string]interface{}{"upload_id": "<upload_id>", "seq": "<chunk_index>", "size": "<chunk_size>", "file": "<local_resource_binary>"}).
|
|
POST("/open-apis/drive/v1/medias/upload_finish").
|
|
Desc(fmt.Sprintf("Upload local %s #%d: finish multipart upload", resource.Kind, resource.Occurrence)).
|
|
Body(map[string]interface{}{"upload_id": "<upload_id>", "block_num": "<block_num>"})
|
|
} else {
|
|
body["file"] = "<local_resource_binary>"
|
|
dry.POST("/open-apis/drive/v1/medias/upload_all").
|
|
Desc(fmt.Sprintf("Upload local %s #%d", resource.Kind, resource.Occurrence)).
|
|
Body(body)
|
|
}
|
|
}
|
|
for start := 0; start < len(resources); start += localDocResourceBindBatchSize {
|
|
end := min(start+localDocResourceBindBatchSize, len(resources))
|
|
requests := make([]interface{}, 0, end-start)
|
|
for _, resource := range resources[start:end] {
|
|
request := map[string]interface{}{"block_id": fmt.Sprintf("<local_%s_%d_block_id>", resource.Kind, resource.Occurrence)}
|
|
if resource.Kind == localDocResourceFile {
|
|
request["replace_file"] = map[string]interface{}{"token": fmt.Sprintf("<uploaded_file_token_%d>", resource.Occurrence)}
|
|
} else {
|
|
replaceImage := map[string]interface{}{"token": fmt.Sprintf("<uploaded_file_token_%d>", resource.Occurrence)}
|
|
if resource.ImageWidth > 0 {
|
|
replaceImage["width"] = resource.ImageWidth
|
|
}
|
|
if resource.ImageHeight > 0 {
|
|
replaceImage["height"] = resource.ImageHeight
|
|
}
|
|
if align, ok := alignMap[resource.ImageAlign]; ok {
|
|
replaceImage["align"] = align
|
|
}
|
|
if resource.HasScale {
|
|
replaceImage["scale"] = resource.ImageScale
|
|
}
|
|
request["replace_image"] = replaceImage
|
|
}
|
|
requests = append(requests, request)
|
|
}
|
|
dry.PATCH(fmt.Sprintf("/open-apis/docx/v1/documents/%s/blocks/batch_update", encodedDocumentKey)).
|
|
Desc(fmt.Sprintf("Bind uploaded local resources (batch %d, max %d requests)", start/localDocResourceBindBatchSize+1, localDocResourceBindBatchSize)).
|
|
Params(map[string]interface{}{"client_token": fmt.Sprintf("<stable_client_token_%d>", start/localDocResourceBindBatchSize+1)}).
|
|
Body(map[string]interface{}{"requests": requests})
|
|
}
|
|
dry.GET(fmt.Sprintf("/open-apis/docx/v1/documents/%s/blocks/%s", encodedDocumentKey, validate.EncodePathSegment("<local_resource_block_id>"))).
|
|
Desc("Conditional: verify block token after an ambiguous bind response or immediately before cleanup")
|
|
dry.PUT(fmt.Sprintf("/open-apis/docs_ai/v1/documents/%s", encodedDocumentKey)).
|
|
Desc("Conditional: delete placeholders whose upload or bind failed; successful resources are preserved").
|
|
Body(map[string]interface{}{"format": "xml", "command": "block_delete", "block_id": "<failed_placeholder_block_ids>", "revision_id": "<known_revision_id>"})
|
|
return dry
|
|
}
|
|
|
|
func appendRemoteDocImageDownloadsDryRun(dry *common.DryRunAPI, resources []localDocResource) *common.DryRunAPI {
|
|
for _, resource := range resources {
|
|
if resource.RemoteURL == "" {
|
|
continue
|
|
}
|
|
dry.GET(redactRemoteDocImageURL(resource.RemoteURL)).
|
|
Desc(fmt.Sprintf("Download remote image #%d in a bounded concurrent upload worker after the document write succeeds (userinfo, query, and fragment are redacted)", resource.Occurrence))
|
|
}
|
|
return dry
|
|
}
|
|
|
|
func redactRemoteDocImageURL(rawURL string) string {
|
|
parsed, err := url.Parse(rawURL)
|
|
if err != nil || parsed == nil {
|
|
return "<invalid-remote-image-url>"
|
|
}
|
|
parsed.User = nil
|
|
parsed.RawQuery = ""
|
|
parsed.ForceQuery = false
|
|
parsed.Fragment = ""
|
|
return parsed.String()
|
|
}
|
|
|
|
func (t localDocResourceTag) attr(name string) (string, bool) {
|
|
for _, attr := range t.Attrs {
|
|
if attr.Name == name {
|
|
return attr.Value, true
|
|
}
|
|
}
|
|
return "", false
|
|
}
|
|
|
|
func (t localDocResourceTag) hasAttr(name string) bool {
|
|
_, ok := t.attr(name)
|
|
return ok
|
|
}
|
|
|
|
func (t *localDocResourceTag) setAttr(name, value string) {
|
|
for i := range t.Attrs {
|
|
if t.Attrs[i].Name == name {
|
|
t.Attrs[i].Value = value
|
|
return
|
|
}
|
|
}
|
|
t.Attrs = append(t.Attrs, html5BlockAttr{Name: name, Value: value})
|
|
}
|
|
|
|
func (t *localDocResourceTag) deleteAttr(name string) {
|
|
attrs := t.Attrs[:0]
|
|
for _, attr := range t.Attrs {
|
|
if attr.Name != name {
|
|
attrs = append(attrs, attr)
|
|
}
|
|
}
|
|
t.Attrs = attrs
|
|
}
|
|
|
|
func (t *localDocResourceTag) renameAttr(oldName, newName string) bool {
|
|
if t.hasAttr(newName) {
|
|
return false
|
|
}
|
|
for i := range t.Attrs {
|
|
if t.Attrs[i].Name == oldName {
|
|
t.Attrs[i].Name = newName
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
type markdownIndentContext struct {
|
|
quoteDepth int
|
|
listContentIndents []int
|
|
}
|
|
|
|
func (c *markdownIndentContext) isIndentedCodeLine(line string) bool {
|
|
line = strings.TrimSuffix(strings.TrimSuffix(line, "\n"), "\r")
|
|
content, quoteDepth := stripMarkdownBlockQuotePrefixes(line)
|
|
if quoteDepth != c.quoteDepth {
|
|
c.quoteDepth = quoteDepth
|
|
c.listContentIndents = nil
|
|
}
|
|
if strings.TrimSpace(content) == "" {
|
|
return false
|
|
}
|
|
|
|
indent, offset := markdownLeadingIndent(content)
|
|
for len(c.listContentIndents) > 0 && indent < c.listContentIndents[len(c.listContentIndents)-1] {
|
|
c.listContentIndents = c.listContentIndents[:len(c.listContentIndents)-1]
|
|
}
|
|
if indent <= 3 && isMarkdownThematicBreak(content[offset:]) {
|
|
return false
|
|
}
|
|
if markerIndent, ok := markdownListItemContentIndent(content[offset:], indent); ok && c.enterListItem(indent, markerIndent) {
|
|
return false
|
|
}
|
|
|
|
containerIndent := 0
|
|
if len(c.listContentIndents) > 0 {
|
|
containerIndent = c.listContentIndents[len(c.listContentIndents)-1]
|
|
}
|
|
return indent >= containerIndent+4
|
|
}
|
|
|
|
func (c *markdownIndentContext) enterListItem(indent, contentIndent int) bool {
|
|
for len(c.listContentIndents) > 0 {
|
|
parentIndent := c.listContentIndents[len(c.listContentIndents)-1]
|
|
if indent >= parentIndent && indent <= parentIndent+3 {
|
|
c.listContentIndents = append(c.listContentIndents, contentIndent)
|
|
return true
|
|
}
|
|
if indent < parentIndent {
|
|
c.listContentIndents = c.listContentIndents[:len(c.listContentIndents)-1]
|
|
continue
|
|
}
|
|
return false
|
|
}
|
|
if indent > 3 {
|
|
return false
|
|
}
|
|
c.listContentIndents = append(c.listContentIndents[:0], contentIndent)
|
|
return true
|
|
}
|
|
|
|
func stripMarkdownBlockQuotePrefixes(line string) (string, int) {
|
|
depth := 0
|
|
for {
|
|
indent := 0
|
|
for indent < len(line) && indent < 3 && line[indent] == ' ' {
|
|
indent++
|
|
}
|
|
if indent >= len(line) || line[indent] != '>' {
|
|
return line, depth
|
|
}
|
|
line = line[indent+1:]
|
|
if strings.HasPrefix(line, " ") || strings.HasPrefix(line, "\t") {
|
|
line = line[1:]
|
|
}
|
|
depth++
|
|
}
|
|
}
|
|
|
|
func markdownLeadingIndent(line string) (columns, offset int) {
|
|
for offset < len(line) {
|
|
switch line[offset] {
|
|
case ' ':
|
|
columns++
|
|
case '\t':
|
|
columns += 4 - columns%4
|
|
default:
|
|
return columns, offset
|
|
}
|
|
offset++
|
|
}
|
|
return columns, offset
|
|
}
|
|
|
|
func markdownListItemContentIndent(line string, markerColumn int) (int, bool) {
|
|
markerWidth := 0
|
|
if len(line) > 0 && (line[0] == '-' || line[0] == '+' || line[0] == '*') {
|
|
markerWidth = 1
|
|
} else {
|
|
for markerWidth < len(line) && markerWidth < 9 && line[markerWidth] >= '0' && line[markerWidth] <= '9' {
|
|
markerWidth++
|
|
}
|
|
if markerWidth == 0 || markerWidth >= len(line) || (line[markerWidth] != '.' && line[markerWidth] != ')') {
|
|
return 0, false
|
|
}
|
|
markerWidth++
|
|
}
|
|
if markerWidth == len(line) {
|
|
return markerColumn + markerWidth + 1, true
|
|
}
|
|
if line[markerWidth] != ' ' && line[markerWidth] != '\t' {
|
|
return 0, false
|
|
}
|
|
|
|
paddingColumns := 0
|
|
column := markerColumn + markerWidth
|
|
for i := markerWidth; i < len(line); i++ {
|
|
switch line[i] {
|
|
case ' ':
|
|
column++
|
|
paddingColumns++
|
|
case '\t':
|
|
width := 4 - column%4
|
|
column += width
|
|
paddingColumns += width
|
|
default:
|
|
if paddingColumns > 4 {
|
|
paddingColumns = 1
|
|
}
|
|
return markerColumn + markerWidth + paddingColumns, true
|
|
}
|
|
}
|
|
if paddingColumns > 4 {
|
|
paddingColumns = 1
|
|
}
|
|
return markerColumn + markerWidth + paddingColumns, true
|
|
}
|
|
|
|
func isMarkdownThematicBreak(line string) bool {
|
|
marker := byte(0)
|
|
count := 0
|
|
for i := 0; i < len(line); i++ {
|
|
switch line[i] {
|
|
case ' ', '\t':
|
|
continue
|
|
case '*', '-', '_':
|
|
if marker == 0 {
|
|
marker = line[i]
|
|
} else if line[i] != marker {
|
|
return false
|
|
}
|
|
count++
|
|
default:
|
|
return false
|
|
}
|
|
}
|
|
return count >= 3
|
|
}
|
|
|
|
func findMarkdownRawHTMLInertEnd(content string, start int) (int, bool) {
|
|
if start < 0 || start >= len(content) || content[start] != '<' || start+2 >= len(content) {
|
|
return 0, false
|
|
}
|
|
for _, name := range []string{"pre", "script", "style", "textarea"} {
|
|
nameEnd := start + 1 + len(name)
|
|
if nameEnd >= len(content) || !strings.EqualFold(content[start+1:nameEnd], name) || !isLocalDocResourceHTMLTagBoundary(content[nameEnd]) {
|
|
continue
|
|
}
|
|
startTagEnd := findXMLStartTagEnd(content, start)
|
|
if startTagEnd < 0 {
|
|
return len(content), true
|
|
}
|
|
if strings.HasSuffix(strings.TrimSpace(content[start:startTagEnd]), "/>") {
|
|
return startTagEnd, true
|
|
}
|
|
|
|
searchFrom := startTagEnd
|
|
lowerRest := strings.ToLower(content[searchFrom:])
|
|
needle := "</" + name
|
|
for {
|
|
offset := strings.Index(lowerRest, needle)
|
|
if offset < 0 {
|
|
return len(content), true
|
|
}
|
|
closeStart := searchFrom + offset
|
|
boundary := closeStart + len(needle)
|
|
if boundary < len(content) && isLocalDocResourceHTMLTagBoundary(content[boundary]) {
|
|
closeEnd := findXMLStartTagEnd(content, closeStart)
|
|
if closeEnd < 0 {
|
|
return len(content), true
|
|
}
|
|
return closeEnd, true
|
|
}
|
|
searchFrom = boundary
|
|
lowerRest = strings.ToLower(content[searchFrom:])
|
|
}
|
|
}
|
|
return 0, false
|
|
}
|
|
|
|
func isLocalDocResourceHTMLTagBoundary(value byte) bool {
|
|
return value == '>' || value == '/' || value == ' ' || value == '\t' || value == '\r' || value == '\n'
|
|
}
|
|
|
|
func (t localDocResourceTag) render() string {
|
|
var out strings.Builder
|
|
out.WriteByte('<')
|
|
out.WriteString(t.Name)
|
|
for _, attr := range t.Attrs {
|
|
out.WriteByte(' ')
|
|
out.WriteString(attr.Name)
|
|
out.WriteString(`="`)
|
|
out.WriteString(escapeXMLAttr(attr.Value))
|
|
out.WriteByte('"')
|
|
}
|
|
if t.SelfClosing {
|
|
out.WriteString("/>")
|
|
} else {
|
|
out.WriteByte('>')
|
|
}
|
|
return out.String()
|
|
}
|
|
|
|
func parseLocalDocResourceTag(raw, expected string) (localDocResourceTag, error) {
|
|
if expected == "img" {
|
|
raw = escapeBareXMLAmpersandsInTagAttr(raw, "href")
|
|
}
|
|
decoder := xml.NewDecoder(strings.NewReader(raw))
|
|
for {
|
|
token, err := decoder.Token()
|
|
if err != nil {
|
|
return localDocResourceTag{}, err
|
|
}
|
|
start, ok := token.(xml.StartElement)
|
|
if !ok {
|
|
continue
|
|
}
|
|
if start.Name.Local != expected {
|
|
return localDocResourceTag{}, fmt.Errorf("expected <%s>, got <%s>", expected, start.Name.Local) //nolint:forbidigo // caller wraps with typed validation error.
|
|
}
|
|
attrs := make([]html5BlockAttr, 0, len(start.Attr))
|
|
seenAttrs := make(map[string]struct{}, len(start.Attr))
|
|
for _, attr := range start.Attr {
|
|
attrName := strings.ToLower(strings.TrimSpace(attr.Name.Local))
|
|
if _, exists := seenAttrs[attrName]; exists {
|
|
return localDocResourceTag{}, fmt.Errorf("duplicate attribute %q", attrName) //nolint:forbidigo // caller wraps with typed validation error.
|
|
}
|
|
seenAttrs[attrName] = struct{}{}
|
|
attrs = append(attrs, html5BlockAttr{Name: attr.Name.Local, Value: attr.Value})
|
|
}
|
|
return localDocResourceTag{Name: expected, Attrs: attrs, SelfClosing: strings.HasSuffix(strings.TrimSpace(raw), "/>")}, nil
|
|
}
|
|
}
|
|
|
|
func escapeBareXMLAmpersandsInTagAttr(raw, targetAttr string) string {
|
|
for i := 1; i < len(raw); {
|
|
for i < len(raw) && isXMLSpace(raw[i]) {
|
|
i++
|
|
}
|
|
nameStart := i
|
|
for i < len(raw) && isLocalDocResourceAttrNameByte(raw[i]) {
|
|
i++
|
|
}
|
|
if nameStart == i {
|
|
i++
|
|
continue
|
|
}
|
|
name := raw[nameStart:i]
|
|
for i < len(raw) && isXMLSpace(raw[i]) {
|
|
i++
|
|
}
|
|
if i >= len(raw) || raw[i] != '=' {
|
|
continue
|
|
}
|
|
i++
|
|
for i < len(raw) && isXMLSpace(raw[i]) {
|
|
i++
|
|
}
|
|
if i >= len(raw) || (raw[i] != '"' && raw[i] != '\'') {
|
|
continue
|
|
}
|
|
quote := raw[i]
|
|
valueStart := i + 1
|
|
valueEnd := strings.IndexByte(raw[valueStart:], quote)
|
|
if valueEnd < 0 {
|
|
return raw
|
|
}
|
|
valueEnd += valueStart
|
|
if strings.EqualFold(name, targetAttr) {
|
|
value := escapeBareXMLAmpersands(raw[valueStart:valueEnd])
|
|
return raw[:valueStart] + value + raw[valueEnd:]
|
|
}
|
|
i = valueEnd + 1
|
|
}
|
|
return raw
|
|
}
|
|
|
|
func isXMLSpace(value byte) bool {
|
|
return value == ' ' || value == '\t' || value == '\r' || value == '\n'
|
|
}
|
|
|
|
func isLocalDocResourceAttrNameByte(value byte) bool {
|
|
return value > ' ' && value != '=' && value != '/' && value != '>'
|
|
}
|
|
|
|
func escapeBareXMLAmpersands(value string) string {
|
|
var out strings.Builder
|
|
for i := 0; i < len(value); {
|
|
if value[i] != '&' {
|
|
out.WriteByte(value[i])
|
|
i++
|
|
continue
|
|
}
|
|
if entityLen := validXMLCharacterEntityLength(value[i:]); entityLen > 0 {
|
|
out.WriteString(value[i : i+entityLen])
|
|
i += entityLen
|
|
continue
|
|
}
|
|
out.WriteString("&")
|
|
i++
|
|
}
|
|
return out.String()
|
|
}
|
|
|
|
func validXMLCharacterEntityLength(value string) int {
|
|
for _, entity := range []string{"&", "<", ">", """, "'"} {
|
|
if strings.HasPrefix(value, entity) {
|
|
return len(entity)
|
|
}
|
|
}
|
|
if !strings.HasPrefix(value, "&#") {
|
|
return 0
|
|
}
|
|
i := 2
|
|
isDigit := func(value byte) bool { return value >= '0' && value <= '9' }
|
|
if i < len(value) && (value[i] == 'x' || value[i] == 'X') {
|
|
i++
|
|
isDigit = func(value byte) bool {
|
|
return (value >= '0' && value <= '9') || (value >= 'a' && value <= 'f') || (value >= 'A' && value <= 'F')
|
|
}
|
|
}
|
|
digitsStart := i
|
|
for i < len(value) && isDigit(value[i]) {
|
|
i++
|
|
}
|
|
if i == digitsStart || i >= len(value) || value[i] != ';' {
|
|
return 0
|
|
}
|
|
return i + 1
|
|
}
|
|
|
|
func localResourceTagNameAt(content string, index int) string {
|
|
for _, name := range []string{"img", "source"} {
|
|
prefix := "<" + name
|
|
if !strings.HasPrefix(content[index:], prefix) {
|
|
continue
|
|
}
|
|
next := index + len(prefix)
|
|
if next >= len(content) || content[next] == '>' || content[next] == '/' || content[next] == ' ' || content[next] == '\t' || content[next] == '\r' || content[next] == '\n' {
|
|
return name
|
|
}
|
|
}
|
|
return ""
|
|
}
|
|
|
|
func findXMLStartTagEnd(content string, start int) int {
|
|
var quote byte
|
|
for i := start + 1; i < len(content); i++ {
|
|
if quote != 0 {
|
|
if content[i] == quote {
|
|
quote = 0
|
|
}
|
|
continue
|
|
}
|
|
switch content[i] {
|
|
case '\'', '"':
|
|
quote = content[i]
|
|
case '>':
|
|
return i + 1
|
|
}
|
|
}
|
|
return -1
|
|
}
|
|
|
|
type parsedMarkdownImage struct {
|
|
Alt string
|
|
Destination string
|
|
Title string
|
|
ReferenceLabel string
|
|
End int
|
|
}
|
|
|
|
func parseMarkdownImageAt(content string, start int) (parsedMarkdownImage, bool) {
|
|
altEnd := findMarkdownClosingBracket(content, start+2)
|
|
if altEnd < 0 {
|
|
return parsedMarkdownImage{}, false
|
|
}
|
|
alt := content[start+2 : altEnd]
|
|
next := altEnd + 1
|
|
if next < len(content) && content[next] == '(' {
|
|
destination, title, end, ok := parseMarkdownImageDestination(content, next)
|
|
if !ok {
|
|
return parsedMarkdownImage{}, false
|
|
}
|
|
return parsedMarkdownImage{Alt: alt, Destination: destination, Title: title, End: end}, true
|
|
}
|
|
if next < len(content) && content[next] == '[' {
|
|
labelEnd := findMarkdownClosingBracket(content, next+1)
|
|
if labelEnd < 0 {
|
|
return parsedMarkdownImage{}, false
|
|
}
|
|
label := content[next+1 : labelEnd]
|
|
if label == "" {
|
|
label = alt
|
|
}
|
|
return parsedMarkdownImage{Alt: alt, ReferenceLabel: label, End: labelEnd + 1}, true
|
|
}
|
|
return parsedMarkdownImage{Alt: alt, ReferenceLabel: alt, End: altEnd + 1}, true
|
|
}
|
|
|
|
func parseMarkdownImageDestination(content string, open int) (string, string, int, bool) {
|
|
i := open + 1
|
|
for i < len(content) && (content[i] == ' ' || content[i] == '\t') {
|
|
i++
|
|
}
|
|
if i >= len(content) {
|
|
return "", "", 0, false
|
|
}
|
|
var destination string
|
|
if content[i] == '<' {
|
|
start := i + 1
|
|
i++
|
|
for i < len(content) {
|
|
if content[i] == '>' && !isEscapedMarkdownByte(content, i) {
|
|
destination = content[start:i]
|
|
i++
|
|
break
|
|
}
|
|
i++
|
|
}
|
|
if destination == "" {
|
|
return "", "", 0, false
|
|
}
|
|
} else {
|
|
start := i
|
|
depth := 0
|
|
for i < len(content) {
|
|
if isEscapedMarkdownByte(content, i) {
|
|
i++
|
|
continue
|
|
}
|
|
switch content[i] {
|
|
case '(':
|
|
depth++
|
|
case ')':
|
|
if depth == 0 {
|
|
destination = content[start:i]
|
|
return unescapeMarkdownText(strings.TrimSpace(destination)), "", i + 1, destination != ""
|
|
}
|
|
depth--
|
|
case ' ', '\t', '\r', '\n':
|
|
if depth == 0 {
|
|
destination = content[start:i]
|
|
goto findClose
|
|
}
|
|
}
|
|
i++
|
|
}
|
|
return "", "", 0, false
|
|
}
|
|
|
|
findClose:
|
|
for i < len(content) && isMarkdownSpace(content[i]) {
|
|
i++
|
|
}
|
|
if i >= len(content) {
|
|
return "", "", 0, false
|
|
}
|
|
if content[i] == ')' {
|
|
return unescapeMarkdownText(strings.TrimSpace(destination)), "", i + 1, destination != ""
|
|
}
|
|
|
|
opener := content[i]
|
|
closer := opener
|
|
if opener == '(' {
|
|
closer = ')'
|
|
} else if opener != '\'' && opener != '"' {
|
|
return "", "", 0, false
|
|
}
|
|
titleStart := i + 1
|
|
i = titleStart
|
|
for i < len(content) {
|
|
if content[i] == closer && !isEscapedMarkdownByte(content, i) {
|
|
title := unescapeMarkdownText(content[titleStart:i])
|
|
i++
|
|
for i < len(content) && isMarkdownSpace(content[i]) {
|
|
i++
|
|
}
|
|
if i >= len(content) || content[i] != ')' {
|
|
return "", "", 0, false
|
|
}
|
|
return unescapeMarkdownText(strings.TrimSpace(destination)), title, i + 1, destination != ""
|
|
}
|
|
i++
|
|
}
|
|
return "", "", 0, false
|
|
}
|
|
|
|
func isMarkdownSpace(char byte) bool {
|
|
return char == ' ' || char == '\t' || char == '\r' || char == '\n'
|
|
}
|
|
|
|
func collectLocalMarkdownImageReferences(content string) map[string]struct{} {
|
|
refs := map[string]struct{}{}
|
|
var fenceChar byte
|
|
var fenceLen int
|
|
var indentCtx markdownIndentContext
|
|
for _, line := range strings.Split(maskLocalDocResourceMarkupInertContexts(content), "\n") {
|
|
indentedCode := false
|
|
if fenceChar == 0 {
|
|
indentedCode = indentCtx.isIndentedCodeLine(line)
|
|
}
|
|
char, run, isFence := markdownFence(line)
|
|
if fenceChar == 0 && isFence {
|
|
fenceChar, fenceLen = char, run
|
|
continue
|
|
}
|
|
if fenceChar != 0 {
|
|
if isFence && char == fenceChar && run >= fenceLen && markdownFenceCloses(line, char, run) {
|
|
fenceChar, fenceLen = 0, 0
|
|
}
|
|
continue
|
|
}
|
|
if indentedCode {
|
|
continue
|
|
}
|
|
label, destination, ok := parseMarkdownReferenceDefinition(line)
|
|
if ok && strings.HasPrefix(destination, "@") {
|
|
refs[normalizeMarkdownReferenceLabel(label)] = struct{}{}
|
|
}
|
|
}
|
|
return refs
|
|
}
|
|
|
|
func maskLocalDocResourceMarkupInertContexts(content string) string {
|
|
var out strings.Builder
|
|
for i := 0; i < len(content); {
|
|
if end, ok := findMarkdownRawHTMLInertEnd(content, i); ok {
|
|
writeLocalDocResourceMaskedSpan(&out, content[i:end])
|
|
i = end
|
|
continue
|
|
}
|
|
terminator := ""
|
|
prefixLen := 0
|
|
switch {
|
|
case strings.HasPrefix(content[i:], "<!--"):
|
|
terminator = "-->"
|
|
prefixLen = 4
|
|
case strings.HasPrefix(content[i:], "<![CDATA["):
|
|
terminator = "]]>"
|
|
prefixLen = 9
|
|
}
|
|
if terminator == "" {
|
|
out.WriteByte(content[i])
|
|
i++
|
|
continue
|
|
}
|
|
end := strings.Index(content[i+prefixLen:], terminator)
|
|
if end < 0 {
|
|
end = len(content)
|
|
} else {
|
|
end += i + prefixLen + len(terminator)
|
|
}
|
|
writeLocalDocResourceMaskedSpan(&out, content[i:end])
|
|
i = end
|
|
}
|
|
return out.String()
|
|
}
|
|
|
|
func writeLocalDocResourceMaskedSpan(out *strings.Builder, content string) {
|
|
for _, char := range content {
|
|
if char == '\n' || char == '\r' {
|
|
out.WriteRune(char)
|
|
} else {
|
|
out.WriteByte(' ')
|
|
}
|
|
}
|
|
}
|
|
|
|
func parseMarkdownReferenceDefinition(line string) (string, string, bool) {
|
|
trimmed := strings.TrimLeft(line, " \t")
|
|
if len(line)-len(trimmed) > 3 || !strings.HasPrefix(trimmed, "[") {
|
|
return "", "", false
|
|
}
|
|
end := findMarkdownClosingBracket(trimmed, 1)
|
|
if end < 0 || end+1 >= len(trimmed) || trimmed[end+1] != ':' {
|
|
return "", "", false
|
|
}
|
|
rest := strings.TrimSpace(trimmed[end+2:])
|
|
if strings.HasPrefix(rest, "<") {
|
|
close := strings.Index(rest, ">")
|
|
if close < 0 {
|
|
return "", "", false
|
|
}
|
|
return trimmed[1:end], rest[1:close], true
|
|
}
|
|
if field := strings.Fields(rest); len(field) > 0 {
|
|
return trimmed[1:end], unescapeMarkdownText(field[0]), true
|
|
}
|
|
return "", "", false
|
|
}
|
|
|
|
func normalizeMarkdownReferenceLabel(label string) string {
|
|
return strings.ToLower(strings.Join(strings.Fields(unescapeMarkdownText(label)), " "))
|
|
}
|
|
|
|
func unescapeMarkdownText(value string) string {
|
|
var out strings.Builder
|
|
for i := 0; i < len(value); i++ {
|
|
if value[i] == '\\' && i+1 < len(value) && isMarkdownEscapablePunctuation(value[i+1]) {
|
|
i++
|
|
}
|
|
out.WriteByte(value[i])
|
|
}
|
|
return out.String()
|
|
}
|
|
|
|
func isMarkdownEscapablePunctuation(value byte) bool {
|
|
return (value >= '!' && value <= '/') || (value >= ':' && value <= '@') ||
|
|
(value >= '[' && value <= '`') || (value >= '{' && value <= '~')
|
|
}
|
|
|
|
func findMarkdownClosingBracket(content string, start int) int {
|
|
depth := 0
|
|
for i := start; i < len(content); i++ {
|
|
if isEscapedMarkdownByte(content, i) {
|
|
continue
|
|
}
|
|
switch content[i] {
|
|
case '[':
|
|
depth++
|
|
case ']':
|
|
if depth == 0 {
|
|
return i
|
|
}
|
|
depth--
|
|
}
|
|
}
|
|
return -1
|
|
}
|
|
|
|
func isEscapedMarkdownByte(content string, index int) bool {
|
|
backslashes := 0
|
|
for i := index - 1; i >= 0 && content[i] == '\\'; i-- {
|
|
backslashes++
|
|
}
|
|
return backslashes%2 == 1
|
|
}
|
|
|
|
func countByteRun(content string, start int, value byte) int {
|
|
i := start
|
|
for i < len(content) && content[i] == value {
|
|
i++
|
|
}
|
|
return i - start
|
|
}
|
|
|
|
func findMatchingBacktickRun(content string, start, run int) int {
|
|
for i := start; i < len(content); {
|
|
if content[i] != '`' {
|
|
i++
|
|
continue
|
|
}
|
|
got := countByteRun(content, i, '`')
|
|
if got == run {
|
|
return i + got
|
|
}
|
|
i += got
|
|
}
|
|
return -1
|
|
}
|
|
|
|
func markdownFence(line string) (byte, int, bool) {
|
|
trimmed, ok := markdownFenceCandidate(line)
|
|
if !ok || len(trimmed) < 3 {
|
|
return 0, 0, false
|
|
}
|
|
char := trimmed[0]
|
|
if char != '`' && char != '~' {
|
|
return 0, 0, false
|
|
}
|
|
run := countByteRun(trimmed, 0, char)
|
|
return char, run, run >= 3
|
|
}
|
|
|
|
func markdownFenceCloses(line string, char byte, run int) bool {
|
|
trimmed, ok := markdownFenceCandidate(line)
|
|
return ok && len(trimmed) >= run && strings.TrimSpace(trimmed[run:]) == "" && trimmed[0] == char
|
|
}
|
|
|
|
func markdownFenceCandidate(line string) (string, bool) {
|
|
rest := strings.TrimSuffix(strings.TrimSuffix(line, "\n"), "\r")
|
|
for {
|
|
indent := 0
|
|
for indent < len(rest) && rest[indent] == ' ' {
|
|
indent++
|
|
}
|
|
if indent > 3 || (indent < len(rest) && rest[indent] == '\t') {
|
|
return "", false
|
|
}
|
|
rest = rest[indent:]
|
|
if strings.HasPrefix(rest, ">") {
|
|
rest = rest[1:]
|
|
if strings.HasPrefix(rest, " ") || strings.HasPrefix(rest, "\t") {
|
|
rest = rest[1:]
|
|
}
|
|
continue
|
|
}
|
|
if prefixLen := markdownListMarkerPrefixLen(rest); prefixLen > 0 {
|
|
rest = rest[prefixLen:]
|
|
continue
|
|
}
|
|
return rest, true
|
|
}
|
|
}
|
|
|
|
func markdownListMarkerPrefixLen(line string) int {
|
|
if len(line) >= 2 && (line[0] == '-' || line[0] == '+' || line[0] == '*') && (line[1] == ' ' || line[1] == '\t') {
|
|
return 2
|
|
}
|
|
digits := 0
|
|
for digits < len(line) && digits < 9 && line[digits] >= '0' && line[digits] <= '9' {
|
|
digits++
|
|
}
|
|
if digits == 0 || digits+1 >= len(line) || (line[digits] != '.' && line[digits] != ')') || (line[digits+1] != ' ' && line[digits+1] != '\t') {
|
|
return 0
|
|
}
|
|
return digits + 2
|
|
}
|