playlist.go

package youtube

import (
	"encoding/json"
	"fmt"
	"io"
	"regexp"
	"strconv"
	"strings"
	"time"

	sjson "github.com/bitly/go-simplejson"
	"golang.org/x/net/html"
)

const (
	playlistFetchURL string = "https://www.youtube.com/playlist?list=%s&hl=en"
	// The following are used in tests but also for fetching test data
	testPlaylistResponseDataFile = "./testdata/playlist_test_data.html"
	testPlaylistID               = "PL59FEE129ADFF2B12"
)

var (
	playlistIDRegex    = regexp.MustCompile("^[A-Za-z0-9_-]{18,42}$")
	playlistInURLRegex = regexp.MustCompile("[&?]list=([A-Za-z0-9_-]{18,42})(&.*)?$")
)

type Playlist struct {
	ID     string
	Title  string
	Author string
	Videos []*PlaylistEntry
}

type PlaylistEntry struct {
	ID       string
	Title    string
	Author   string
	Duration time.Duration
}

func extractPlaylistID(url string) (string, error) {
	if playlistIDRegex.Match([]byte(url)) {
		return url, nil
	}

	matches := playlistInURLRegex.FindStringSubmatch(url)

	if matches != nil {
		return matches[1], nil
	}

	return "", ErrInvalidPlaylist
}

func extractPlaylistJSON(r io.Reader) ([]byte, error) {
	const prefix = "var ytInitialData ="

	doc, err := html.Parse(r)
	if err != nil {
		return nil, err
	}
	var data []byte
	var f func(*html.Node)
	f = func(n *html.Node) {
		if n.Type == html.ElementNode && n.Data == "script" && n.FirstChild != nil {
			script := n.FirstChild.Data
			if strings.HasPrefix(script, prefix) {
				script = strings.TrimPrefix(script, prefix)
				data = []byte(strings.Trim(script, ";"))
			}
		}
		for c := n.FirstChild; c != nil; c = c.NextSibling {
			f(c)
		}
	}
	f(doc)
	return data, nil
}

// structs for playlist extraction

// Title: metadata.playlistMetadataRenderer.title | sidebar.playlistSidebarRenderer.items[0].playlistSidebarPrimaryInfoRenderer.title.runs[0].text
// Author: sidebar.playlistSidebarRenderer.items[1].playlistSidebarSecondaryInfoRenderer.videoOwner.videoOwnerRenderer.title.runs[0].text

// Videos: contents.twoColumnBrowseResultsRenderer.tabs[0].tabRenderer.content.sectionListRenderer.contents[0].itemSectionRenderer.contents[0].playlistVideoListRenderer.contents
// ID: .videoId
// Title: title.runs[0].text
// Author: .shortBylineText.runs[0].text
// Duration: .lengthSeconds

// TODO?: Author thumbnails: sidebar.playlistSidebarRenderer.items[0].playlistSidebarPrimaryInfoRenderer.thumbnailRenderer.playlistVideoThumbnailRenderer.thumbnail.thumbnails
// TODO? Video thumbnails: .thumbnail.thumbnails

func (p *Playlist) UnmarshalJSON(b []byte) (err error) {
	var j *sjson.Json
	j, err = sjson.NewJson(b)
	if err != nil {
		return err
	}
	defer func() {
		if r := recover(); r != nil {
			err = fmt.Errorf("JSON parsing error: %v", r)
		}
	}()
	p.Title = j.GetPath("metadata", "playlistMetadataRenderer", "title").MustString()
	p.Author = j.GetPath("sidebar", "playlistSidebarRenderer", "items").GetIndex(1).
		GetPath("playlistSidebarSecondaryInfoRenderer", "videoOwner", "videoOwnerRenderer", "title", "runs").
		GetIndex(0).Get("text").MustString()
	vJSON, err := j.GetPath("contents", "twoColumnBrowseResultsRenderer", "tabs").GetIndex(0).
		GetPath("tabRenderer", "content", "sectionListRenderer", "contents").GetIndex(0).
		GetPath("itemSectionRenderer", "contents").GetIndex(0).
		GetPath("playlistVideoListRenderer", "contents").MarshalJSON()

	var vids []*videosJSONExtractor
	if err := json.Unmarshal(vJSON, &vids); err != nil {
		return err
	}
	p.Videos = make([]*PlaylistEntry, 0, len(vids))
	for _, v := range vids {
		if v.Renderer == nil { // items such as continuationItemRenderer can mess things up in that array
			continue
		}
		p.Videos = append(p.Videos, v.PlaylistEntry())
	}
	return nil
}

type videosJSONExtractor struct {
	Renderer *struct {
		ID       string   `json:"videoId"`
		Title    withRuns `json:"title"`
		Author   withRuns `json:"shortBylineText"`
		Duration string   `json:"lengthSeconds"`
	} `json:"playlistVideoRenderer"`
}

func (vje videosJSONExtractor) PlaylistEntry() *PlaylistEntry {
	ds, err := strconv.Atoi(vje.Renderer.Duration)
	if err != nil {
		panic("invalid video duration: " + vje.Renderer.Duration)
	}
	return &PlaylistEntry{
		ID:       vje.Renderer.ID,
		Title:    vje.Renderer.Title.String(),
		Author:   vje.Renderer.Author.String(),
		Duration: time.Second * time.Duration(ds),
	}
}

type withRuns struct {
	Runs []struct {
		Text string `json:"text"`
	} `json:"runs"`
}

func (wr withRuns) String() string {
	if len(wr.Runs) > 0 {
		return wr.Runs[0].Text
	}
	return ""
}