transcript.go

// Package yt_transcript fetches YouTube video transcripts via the innertube API.
// Stdlib only, zero dependencies.
package yt_transcript

import (
	"bytes"
	"context"
	"encoding/json"
	"encoding/xml"
	"errors"
	"fmt"
	"io"
	"net/http"
	"net/http/cookiejar"
	"regexp"
	"strconv"
	"strings"
	"sync/atomic"
	"time"
)

const (
	watchURL  = "https://www.youtube.com/watch?v=%s"
	playerURL = "https://www.youtube.com/youtubei/v1/player?key=%s"
	apiKeyRe  = `"INNERTUBE_API_KEY":\s*"([a-zA-Z0-9_-]+)"`
)

// Segment represents a single transcript line with timing.
type Segment struct {
	Text     string  `json:"text"`
	Start    float64 `json:"start"`
	Duration float64 `json:"duration"`
}

// RateLimitError reports that YouTube is throttling or blocking requests from
// this client. RetryAfter is the suggested wait before trying again; zero means
// retry immediately.
type RateLimitError struct {
	RetryAfter time.Duration
}

func (e *RateLimitError) Error() string {
	if e.RetryAfter > 0 {
		return fmt.Sprintf("rate limited by YouTube (429), retry after %s", e.RetryAfter.Round(time.Second))
	}
	return "rate limited by YouTube (429)"
}

// Client fetches YouTube transcripts.
type Client struct {
	http       *http.Client
	breaker    *breaker
	identities []clientIdentity
	rotation   atomic.Uint64
}

// NewClient returns a Client with a default 30s timeout and cookie jar.
func NewClient() *Client {
	jar, _ := cookiejar.New(nil)
	return &Client{
		http:       &http.Client{Timeout: 30 * time.Second, Jar: jar},
		breaker:    newBreaker(3, time.Minute, time.Hour),
		identities: clientIdentities,
	}
}

// WithHTTPClient sets a custom http.Client.
func (c *Client) WithHTTPClient(h *http.Client) *Client {
	c.http = h
	return c
}

// do performs an upstream request, short-circuiting while the breaker is open.
// A 429 trips the breaker and is surfaced as a *RateLimitError.
func (c *Client) do(req *http.Request) (*http.Response, error) {
	if remaining, ok := c.breaker.allow(time.Now()); !ok {
		return nil, &RateLimitError{RetryAfter: remaining}
	}

	resp, err := c.http.Do(req)
	if err != nil {
		return nil, err
	}

	if resp.StatusCode == http.StatusTooManyRequests {
		retryAfter := parseRetryAfter(resp.Header.Get("Retry-After"))
		_ = resp.Body.Close()
		cooldown := c.breaker.fail(retryAfter, time.Now())
		if cooldown > 0 {
			return nil, &RateLimitError{RetryAfter: cooldown}
		}
		return nil, &RateLimitError{RetryAfter: retryAfter}
	}

	c.breaker.succeed()
	return resp, nil
}

func parseRetryAfter(v string) time.Duration {
	if v == "" {
		return 0
	}
	if secs, err := strconv.Atoi(v); err == nil && secs > 0 {
		return time.Duration(secs) * time.Second
	}
	if t, err := http.ParseTime(v); err == nil {
		if d := time.Until(t); d > 0 {
			return d
		}
	}
	return 0
}

// FetchTranscript retrieves the transcript for a YouTube video in the given
// language. It rotates over the known client identities and falls through to
// the next when one fails, so a stale client is not fatal.
func (c *Client) FetchTranscript(ctx context.Context, videoID, lang string) ([]Segment, error) {
	html, err := c.fetchWatchPage(ctx, videoID)
	if err != nil {
		return nil, fmt.Errorf("fetching watch page: %w", err)
	}

	apiKey, err := extractAPIKey(html)
	if err != nil {
		return nil, fmt.Errorf("YouTube did not return expected page content — the video may be unavailable or requests are being blocked: %w", err)
	}

	n := len(c.identities)
	start := int(c.rotation.Add(1)-1) % n
	var lastErr error
	for i := 0; i < n; i++ {
		id := c.identities[(start+i)%n]
		segments, err := c.fetchWithClient(ctx, videoID, lang, apiKey, id)
		if err == nil {
			return segments, nil
		}
		var rateLimited *RateLimitError
		if errors.As(err, &rateLimited) {
			return nil, err
		}
		if strings.Contains(err.Error(), "no transcript found") {
			return nil, err
		}
		lastErr = err
	}
	return nil, fmt.Errorf("all clients failed: %w", lastErr)
}

func (c *Client) fetchWithClient(ctx context.Context, videoID, lang, apiKey string, id clientIdentity) ([]Segment, error) {
	captions, err := c.fetchCaptions(ctx, videoID, apiKey, id)
	if err != nil {
		return nil, fmt.Errorf("fetching captions with %s: %w", id.name, err)
	}

	track, err := findCaptionTrack(captions, lang)
	if err != nil {
		return nil, err
	}

	return c.fetchAndParse(ctx, track.BaseURL, id)
}

// TranscriptLanguage is an available caption track: its language code and
// human-readable name.
type TranscriptLanguage struct {
	Code string `json:"code"`
	Name string `json:"name"`
}

// ListLanguages returns the transcript languages available for a video. A
// language with both a manual and an auto-generated track is listed once,
// preferring the manual name.
func (c *Client) ListLanguages(ctx context.Context, videoID string) ([]TranscriptLanguage, error) {
	html, err := c.fetchWatchPage(ctx, videoID)
	if err != nil {
		return nil, fmt.Errorf("fetching watch page: %w", err)
	}

	apiKey, err := extractAPIKey(html)
	if err != nil {
		return nil, fmt.Errorf("YouTube did not return expected page content — the video may be unavailable or requests are being blocked: %w", err)
	}

	n := len(c.identities)
	start := int(c.rotation.Add(1)-1) % n
	var lastErr error
	for i := 0; i < n; i++ {
		id := c.identities[(start+i)%n]
		tracks, err := c.fetchCaptions(ctx, videoID, apiKey, id)
		if err != nil {
			var rateLimited *RateLimitError
			if errors.As(err, &rateLimited) {
				return nil, err
			}
			lastErr = err
			continue
		}
		return toLanguages(tracks), nil
	}
	return nil, fmt.Errorf("all clients failed: %w", lastErr)
}

func toLanguages(tracks []captionTrack) []TranscriptLanguage {
	order := make([]string, 0, len(tracks))
	best := make(map[string]captionTrack, len(tracks))
	for _, t := range tracks {
		cur, ok := best[t.LanguageCode]
		if !ok {
			order = append(order, t.LanguageCode)
			best[t.LanguageCode] = t
			continue
		}
		if cur.Kind == "asr" && t.Kind != "asr" {
			best[t.LanguageCode] = t
		}
	}

	langs := make([]TranscriptLanguage, 0, len(order))
	for _, code := range order {
		langs = append(langs, TranscriptLanguage{Code: code, Name: trackName(best[code])})
	}
	return langs
}

func trackName(t captionTrack) string {
	var b strings.Builder
	for _, r := range t.Name.Runs {
		b.WriteString(r.Text)
	}
	if name := b.String(); name != "" {
		return name
	}
	return t.LanguageCode
}

func (c *Client) fetchWatchPage(ctx context.Context, videoID string) (string, error) {
	u := fmt.Sprintf(watchURL, videoID)
	req, err := http.NewRequestWithContext(ctx, http.MethodGet, u, nil)
	if err != nil {
		return "", err
	}
	req.Header.Set("Accept-Language", "en-US")

	resp, err := c.do(req)
	if err != nil {
		return "", err
	}
	defer func() { _ = resp.Body.Close() }()

	body, err := io.ReadAll(resp.Body)
	if err != nil {
		return "", err
	}
	html := string(body)

	if strings.Contains(html, `class="g-recaptcha"`) {
		return "", fmt.Errorf("bot detection triggered — YouTube is blocking requests from this IP")
	}

	return html, nil
}

var apiKeyRegex = regexp.MustCompile(apiKeyRe)

func extractAPIKey(html string) (string, error) {
	matches := apiKeyRegex.FindStringSubmatch(html)
	if len(matches) < 2 {
		return "", fmt.Errorf("could not parse YouTube page")
	}
	return matches[1], nil
}

type captionTrack struct {
	BaseURL      string `json:"baseUrl"`
	LanguageCode string `json:"languageCode"`
	Kind         string `json:"kind"`
	Name         struct {
		Runs []struct {
			Text string `json:"text"`
		} `json:"runs"`
	} `json:"name"`
}

func (c *Client) fetchCaptions(ctx context.Context, videoID, apiKey string, id clientIdentity) ([]captionTrack, error) {
	body, err := json.Marshal(map[string]any{
		"context": map[string]any{
			"client": id.context(),
		},
		"videoId": videoID,
	})
	if err != nil {
		return nil, err
	}

	u := fmt.Sprintf(playerURL, apiKey)
	req, err := http.NewRequestWithContext(ctx, http.MethodPost, u, bytes.NewReader(body))
	if err != nil {
		return nil, err
	}
	req.Header.Set("Content-Type", "application/json")
	req.Header.Set("User-Agent", id.userAgent)

	resp, err := c.do(req)
	if err != nil {
		return nil, err
	}
	defer func() { _ = resp.Body.Close() }()

	respBody, err := io.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}

	var data struct {
		Captions struct {
			PlayerCaptionsTracklistRenderer struct {
				CaptionTracks []captionTrack `json:"captionTracks"`
			} `json:"playerCaptionsTracklistRenderer"`
		} `json:"captions"`
	}
	if err := json.Unmarshal(respBody, &data); err != nil {
		return nil, fmt.Errorf("parsing player response: %w", err)
	}

	tracks := data.Captions.PlayerCaptionsTracklistRenderer.CaptionTracks
	if len(tracks) == 0 {
		return nil, fmt.Errorf("no caption tracks available")
	}
	return tracks, nil
}

func findCaptionTrack(tracks []captionTrack, lang string) (captionTrack, error) {
	for _, t := range tracks {
		if t.LanguageCode == lang {
			t.BaseURL = strings.Replace(t.BaseURL, "&fmt=srv3", "", 1)
			return t, nil
		}
	}
	return captionTrack{}, fmt.Errorf("no transcript found for language %q", lang)
}

type transcriptXML struct {
	XMLName xml.Name         `xml:"transcript"`
	Texts   []transcriptText `xml:"text"`
}

type transcriptText struct {
	Start float64 `xml:"start,attr"`
	Dur   float64 `xml:"dur,attr"`
	Value string  `xml:",chardata"`
}

func (c *Client) fetchAndParse(ctx context.Context, baseURL string, id clientIdentity) ([]Segment, error) {
	req, err := http.NewRequestWithContext(ctx, http.MethodGet, baseURL, nil)
	if err != nil {
		return nil, err
	}
	if id.userAgent != "" {
		req.Header.Set("User-Agent", id.userAgent)
	}

	resp, err := c.do(req)
	if err != nil {
		return nil, err
	}
	defer func() { _ = resp.Body.Close() }()

	var x transcriptXML
	if err := xml.NewDecoder(resp.Body).Decode(&x); err != nil {
		return nil, fmt.Errorf("parsing transcript XML: %w", err)
	}

	segments := make([]Segment, len(x.Texts))
	for i, t := range x.Texts {
		segments[i] = Segment{
			Text:     unescapeHTML(t.Value),
			Start:    t.Start,
			Duration: t.Dur,
		}
	}
	return segments, nil
}

var htmlReplacer = strings.NewReplacer(
	"&amp;", "&",
	"&lt;", "<",
	"&gt;", ">",
	"&quot;", `"`,
	"&#39;", "'",
	"&apos;", "'",
)

func unescapeHTML(s string) string {
	return htmlReplacer.Replace(s)
}