Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 6 additions & 5 deletions api/media.go
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,9 @@ import (
"github.com/gotd/td/tg"
"github.com/krau/ffmpeg-go"
ytdlp "github.com/lrstanley/go-ytdlp"

"github.com/krau/SaveAny-Bot/config"
ytdlptask "github.com/krau/SaveAny-Bot/core/tasks/ytdlp"
)

var (
Expand Down Expand Up @@ -185,15 +188,13 @@ func extractDirectMediaMetadata(ctx context.Context, url string) (*MediaMetadata

func extractYTDLPMediaMetadata(ctx context.Context, url string) (*MediaMetadataResponse, error) {
cmd := ytdlp.New().DumpSingleJSON()
result, err := cmd.Run(ctx, url)
// Use the same proxies as downloads, so a lookup cannot report a URL
// unreachable that a download would fetch.
result, err := ytdlptask.RunWithProxyFailover(ctx, cmd, config.C().Ytdlp, url)
if err != nil {
return nil, fmt.Errorf("failed to inspect media: %w", err)
}

if result.ExitCode != 0 {
return nil, fmt.Errorf("failed to inspect media: %s", result.Stderr)
}

var e *ytdlp.ExtractedInfo
infoList := make([]*ytdlp.ExtractedInfo, 0)

Expand Down
34 changes: 34 additions & 0 deletions config/ytdlp.go
Original file line number Diff line number Diff line change
@@ -1,5 +1,11 @@
package config

import (
"fmt"
"net/url"
"strings"
)

// DefaultYtdlpFilenameTemplate is used when ytdlp.filename_template is not set.
const DefaultYtdlpFilenameTemplate = "%(title)s.%(ext)s"

Expand All @@ -19,4 +25,32 @@ type YtdlpConfig struct {
// RestrictFilenames keeps file names ASCII-only (yt-dlp --restrict-filenames),
// dropping non-latin characters from titles. Disabled by default.
RestrictFilenames bool `toml:"restrict_filenames" mapstructure:"restrict_filenames" json:"restrict_filenames"`
// Proxies are HTTP/HTTPS/SOCKS proxy URLs for yt-dlp's own requests. They
// are tried in order until one succeeds, so put the preferred proxy first.
// Empty means yt-dlp connects directly.
Proxies []string `toml:"proxies" mapstructure:"proxies" json:"proxies"`
}

// ValidateProxies reports the first entry in Proxies that yt-dlp cannot use.
// Blank entries are ignored.
func (c YtdlpConfig) ValidateProxies() error {
for _, raw := range c.Proxies {
raw = strings.TrimSpace(raw)
if raw == "" {
continue
}
u, err := url.Parse(raw)
if err != nil {
return fmt.Errorf("invalid yt-dlp proxy: %w", err)
}
switch u.Scheme {
case "http", "https", "socks4", "socks4a", "socks5", "socks5h":
default:
return fmt.Errorf("invalid yt-dlp proxy %q: scheme must be http, https, socks4, socks4a, socks5 or socks5h", u.Redacted())
}
if u.Host == "" {
return fmt.Errorf("invalid yt-dlp proxy %q: missing host", u.Redacted())
}
}
return nil
}
26 changes: 26 additions & 0 deletions config/ytdlp_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
package config

import "testing"

func TestYtdlpConfigValidateProxies(t *testing.T) {
cases := []struct {
name string
proxies []string
wantErr bool
}{
{name: "empty", proxies: nil},
{name: "supported schemes", proxies: []string{"http://a:1", "https://a:1", "socks4://a:1", "socks4a://a:1", "socks5://u:p@a:1", "socks5h://a:1"}},
{name: "blank entries ignored", proxies: []string{" ", ""}},
{name: "missing scheme", proxies: []string{"127.0.0.1:1080"}, wantErr: true},
{name: "unsupported scheme", proxies: []string{"ftp://a:1"}, wantErr: true},
{name: "missing host", proxies: []string{"socks5://"}, wantErr: true},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
err := YtdlpConfig{Proxies: tc.proxies}.ValidateProxies()
if (err != nil) != tc.wantErr {
t.Fatalf("ValidateProxies() error = %v, wantErr %v", err, tc.wantErr)
}
})
}
}
26 changes: 18 additions & 8 deletions core/tasks/ytdlp/execute.go
Original file line number Diff line number Diff line change
Expand Up @@ -116,7 +116,8 @@ func buildDownloadCommand(cfg config.YtdlpConfig, tempDir string, flags []string
func (t *Task) downloadFiles(ctx context.Context, tempDir string) ([]string, error) {
logger := log.FromContext(ctx)

cmd, flags, err := buildDownloadCommand(config.C().Ytdlp, tempDir, t.Flags)
cfg := config.C().Ytdlp
cmd, flags, err := buildDownloadCommand(cfg, tempDir, t.Flags)
if err != nil {
return nil, err
}
Expand All @@ -130,19 +131,13 @@ func (t *Task) downloadFiles(ctx context.Context, tempDir string) ([]string, err
// yt-dlp accepts: yt-dlp [OPTIONS] URL [URL...]
args := append(flags, t.URLs...)

// Run with context for cancellation support
result, err := cmd.Run(ctx, args...)
if err != nil {
if _, err := RunWithProxyFailover(ctx, cmd, cfg, args...); err != nil {
if errors.Is(err, context.Canceled) {
return nil, err
}
return nil, fmt.Errorf("yt-dlp execution failed: %w", err)
}

if result.ExitCode != 0 {
return nil, fmt.Errorf("yt-dlp exited with code %d: %s", result.ExitCode, result.Stderr)
}

files, err := collectDownloadedFiles(tempDir)
if err != nil {
return nil, err
Expand All @@ -165,6 +160,10 @@ func collectDownloadedFiles(dir string) ([]string, error) {
if entry.IsDir() {
return nil
}
// A failed proxy attempt can leave partial downloads behind.
if isYtdlpTempFile(entry.Name()) {
return nil
}
rel, err := filepath.Rel(dir, path)
if err != nil {
return err
Expand All @@ -177,6 +176,17 @@ func collectDownloadedFiles(dir string) ([]string, error) {
return files, nil
}

// isYtdlpTempFile reports whether name is one of yt-dlp's in-progress files
// rather than a finished download.
func isYtdlpTempFile(name string) bool {
for _, suffix := range []string{".part", ".ytdl", ".temp"} {
if strings.HasSuffix(name, suffix) {
return true
}
}
return false
}

// transferFile transfers the file at relPath below tempDir to storage, keeping
// the directories an output template created.
func (t *Task) transferFile(ctx context.Context, tempDir, relPath string) error {
Expand Down
130 changes: 130 additions & 0 deletions core/tasks/ytdlp/proxy.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,130 @@
package ytdlp

import (
"context"
"errors"
"fmt"
"net/url"
"strings"

"github.com/charmbracelet/log"
ytdlp "github.com/lrstanley/go-ytdlp"

"github.com/krau/SaveAny-Bot/config"
)

// noProxy is the candidate for a direct run without --proxy.
const noProxy = ""

// RunWithProxyFailover runs cmd once per configured proxy, in order, until a
// run succeeds. With no proxies configured it runs once without --proxy.
// Cancellation stops the rotation immediately.
func RunWithProxyFailover(ctx context.Context, cmd *ytdlp.Command, cfg config.YtdlpConfig, args ...string) (*ytdlp.Result, error) {
logger := log.FromContext(ctx)
candidates := proxyCandidates(cfg)

var lastErr error
for attempt, proxy := range candidates {
// Each Run rebuilds the command line, so replacing the proxy on the same
// builder never stacks --proxy flags.
if proxy == noProxy {
cmd = cmd.UnsetProxy()
} else {
cmd = cmd.Proxy(proxy)
}

result, err := cmd.Run(ctx, args...)
switch {
case err == nil && result.ExitCode == 0:
if attempt > 0 {
logger.Infof("yt-dlp succeeded via proxy %d/%d (%s)", attempt+1, len(candidates), redactProxy(proxy))
}
return result, nil
case err != nil:
lastErr = err
default:
lastErr = fmt.Errorf("yt-dlp exited with code %d: %s", result.ExitCode, result.Stderr)
}
lastErr = redactProxiesInError(lastErr, candidates)

if errors.Is(lastErr, context.Canceled) || ctx.Err() != nil {
return nil, lastErr
}
if len(candidates) > 1 {
logger.Warnf("yt-dlp failed via proxy %d/%d (%s): %v", attempt+1, len(candidates), redactProxy(proxy), lastErr)
}
}

if len(candidates) > 1 {
return nil, fmt.Errorf("all %d proxies failed, last error: %w", len(candidates), lastErr)
}
return nil, lastErr
}

// proxyCandidates returns the configured proxies, trimmed and deduplicated in
// their configured order, or a single direct candidate when none are set.
func proxyCandidates(cfg config.YtdlpConfig) []string {
seen := make(map[string]struct{}, len(cfg.Proxies))
candidates := make([]string, 0, len(cfg.Proxies))
for _, proxy := range cfg.Proxies {
proxy = strings.TrimSpace(proxy)
if proxy == "" {
continue
}
if _, dup := seen[proxy]; dup {
continue
}
seen[proxy] = struct{}{}
candidates = append(candidates, proxy)
}
if len(candidates) == 0 {
return []string{noProxy}
}
return candidates
}

// redactProxy hides the password in a proxy URL before it is logged.
func redactProxy(proxy string) string {
if proxy == noProxy {
return "direct"
}
u, err := url.Parse(proxy)
if err != nil || u.Host == "" {
return "<proxy>"
}
if u.User != nil {
if _, ok := u.User.Password(); ok {
u.User = url.UserPassword(u.User.Username(), "xxxxx")
}
}
return u.Redacted()
}

// redactedError keeps the original error for errors.Is/As while replacing its
// message.
type redactedError struct {
msg string
err error
}

func (e *redactedError) Error() string { return e.msg }
func (e *redactedError) Unwrap() error { return e.err }

// redactProxiesInError replaces proxy URLs that yt-dlp echoed into err (for
// example in connection errors) with their redacted form.
func redactProxiesInError(err error, proxies []string) error {
msg := err.Error()
redacted := msg
for _, proxy := range proxies {
if proxy == noProxy {
continue
}
if safe := redactProxy(proxy); safe != proxy {
redacted = strings.ReplaceAll(redacted, proxy, safe)
}
}
if redacted == msg {
return err
}
return &redactedError{msg: redacted, err: err}
}
Loading