mirror/mxpv-podsync
1
0
Fork 0
mirror of https://github.com/mxpv/podsync.git synced 2024-05-11 05:55:04 +00:00
Code Issues Releases Activity

Implement initial feed service

Browse Source
This commit is contained in:
Maksym Pavlenko
2017-08-13 14:50:59 -07:00
parent fdfa30454c
commit 535b7dbaa4
15 changed files with 494 additions and 318 deletions
Download Patch File Download Diff File Expand all files Collapse all files
web/pkg/api/api.go
+45 -9
View File
@@ -2,12 +2,32 @@ package api
import "time"
type Provider string
const (
Youtube = Provider("youtube")
Vimeo = Provider("vimeo")
)
type LinkType string
const (
Channel = LinkType("channel")
Playlist = LinkType("playlist")
User = LinkType("user")
Group = LinkType("group")
)
type Quality string
type Format string
const (
HighQuality = Quality("high")
LowQuality = Quality("low")
)
type Format string
const (
AudioFormat = Format("audio")
VideoFormat = Format("video")
)
@@ -19,12 +39,28 @@ const (
)
type Feed struct {
Id int64 `json:"id"`
HashId string `json:"hash_id"`
UserId string `json:"user_id"`
URL string `json:"url"`
PageSize int `json:"page_size"`
Quality Quality `json:"quality"`
Format Format `json:"format"`
LastAccess time.Time `json:"last_access"`
Id int64 `json:"id"`
HashId string `json:"hash_id"` // Short human readable feed id for users
UserId string `json:"user_id"` // Patreon user id
ItemId string `json:"item_id"`
Provider Provider `json:"provider"` // Youtube or Vimeo
LinkType LinkType `json:"link_type"` // Either group, channel or user
PageSize int `json:"page_size"` // The number of episodes to return
Format Format `json:"format"`
Quality Quality `json:"quality"`
FeatureLevel int `json:"feature_level"` // Available features
LastAccess time.Time `json:"last_access"`
}
const (
DefaultFeatures = iota
ExtendedFeatures
PodcasterFeature
)
type CreateFeedRequest struct {
URL string `json:"url"`
PageSize int `json:"page_size"`
Quality Quality `json:"quality"`
Format Format `json:"format"`
}
web/pkg/builders/common.go
-10
View File
@@ -12,16 +12,6 @@ const (
defaultCategory = "TV & Film"
)
type linkType int
const (
_ = iota
linkTypeChannel linkType = iota
linkTypePlaylist
linkTypeUser
linkTypeGroup
)
func makeEnclosure(feed *api.Feed, id string, lengthInBytes int64) (string, itunes.EnclosureType, int64) {
ext := "mp4"
contentType := itunes.MP4
web/pkg/builders/vimeo.go
+32 -78
View File
@@ -3,9 +3,7 @@ package builders
import (
"fmt"
"net/http"
"net/url"
"strconv"
"strings"
itunes "github.com/mxpv/podcast"
"github.com/mxpv/podsync/web/pkg/api"
@@ -23,60 +21,6 @@ type VimeoBuilder struct {
client *vimeo.Client
}
func (v *VimeoBuilder) parseUrl(link string) (kind linkType, id string, err error) {
parsed, err := url.Parse(link)
if err != nil {
err = errors.Wrapf(err, "failed to parse url: %s", link)
return
}
if !strings.HasSuffix(parsed.Host, "vimeo.com") {
err = errors.New("invalid vimeo host")
return
}
parts := strings.Split(parsed.EscapedPath(), "/")
if len(parts) <= 1 {
err = errors.New("invalid vimeo link path")
return
}
if parts[1] == "groups" {
kind = linkTypeGroup
} else if parts[1] == "channels" {
kind = linkTypeChannel
} else {
kind = linkTypeUser
}
if kind == linkTypeGroup || kind == linkTypeChannel {
if len(parts) <= 2 {
err = errors.New("invalid channel link")
return
}
id = parts[2]
if id == "" {
err = errors.New("invalid id")
}
return
}
if kind == linkTypeUser {
id = parts[1]
if id == "" {
err = errors.New("invalid id")
}
return
}
err = errors.New("unsupported link format")
return
}
func (v *VimeoBuilder) selectImage(p *vimeo.Pictures, q api.Quality) string {
if p == nil || len(p.Sizes) < 1 {
return ""
@@ -89,7 +33,9 @@ func (v *VimeoBuilder) selectImage(p *vimeo.Pictures, q api.Quality) string {
}
}
func (v *VimeoBuilder) queryChannel(channelId string, feed *api.Feed) (*itunes.Podcast, error) {
func (v *VimeoBuilder) queryChannel(feed *api.Feed) (*itunes.Podcast, error) {
channelId := feed.ItemId
ch, resp, err := v.client.Channels.Get(channelId)
if err != nil {
return nil, errors.Wrapf(err, "failed to query channel with channelId %s", channelId)
@@ -109,7 +55,9 @@ func (v *VimeoBuilder) queryChannel(channelId string, feed *api.Feed) (*itunes.P
return &podcast, nil
}
func (v *VimeoBuilder) queryGroup(groupId string, feed *api.Feed) (*itunes.Podcast, error) {
func (v *VimeoBuilder) queryGroup(feed *api.Feed) (*itunes.Podcast, error) {
groupId := feed.ItemId
gr, resp, err := v.client.Groups.Get(groupId)
if err != nil {
return nil, errors.Wrapf(err, "failed to query group with id %s", groupId)
@@ -129,7 +77,9 @@ func (v *VimeoBuilder) queryGroup(groupId string, feed *api.Feed) (*itunes.Podca
return &podcast, nil
}
func (v *VimeoBuilder) queryUser(userId string, feed *api.Feed) (*itunes.Podcast, error) {
func (v *VimeoBuilder) queryUser(feed *api.Feed) (*itunes.Podcast, error) {
userId := feed.ItemId
user, resp, err := v.client.Users.Get(userId)
if err != nil {
return nil, errors.Wrapf(err, "failed to query user with id %s", userId)
@@ -154,9 +104,9 @@ func (v *VimeoBuilder) getVideoSize(video *vimeo.Video) int64 {
return int64(float64(video.Duration*video.Width*video.Height) * 0.38848958333)
}
type queryVideosFunc func(id string, opt *vimeo.ListVideoOptions) ([]*vimeo.Video, *vimeo.Response, error)
type getVideosFunc func(id string, opt *vimeo.ListVideoOptions) ([]*vimeo.Video, *vimeo.Response, error)
func (v *VimeoBuilder) queryVideos(queryVideos queryVideosFunc, id string, podcast *itunes.Podcast, feed *api.Feed) error {
func (v *VimeoBuilder) queryVideos(getVideos getVideosFunc, podcast *itunes.Podcast, feed *api.Feed) error {
opt := vimeo.ListVideoOptions{}
opt.Page = 1
opt.PerPage = vimeoDefaultPageSize
@@ -164,7 +114,7 @@ func (v *VimeoBuilder) queryVideos(queryVideos queryVideosFunc, id string, podca
added := 0
for {
videos, response, err := queryVideos(id, &opt)
videos, response, err := getVideos(feed.ItemId, &opt)
if err != nil {
return errors.Wrap(err, "failed to query videos")
}
@@ -208,27 +158,31 @@ func (v *VimeoBuilder) queryVideos(queryVideos queryVideosFunc, id string, podca
}
func (v *VimeoBuilder) Build(feed *api.Feed) (podcast *itunes.Podcast, err error) {
kind, id, err := v.parseUrl(feed.URL)
if err != nil {
return nil, errors.Wrapf(err, "failed to parse link: %s", feed.URL)
if feed.LinkType == api.Channel {
if podcast, err = v.queryChannel(feed); err == nil {
err = v.queryVideos(v.client.Channels.ListVideo, podcast, feed)
}
return
}
if kind == linkTypeChannel {
if podcast, err = v.queryChannel(id, feed); err == nil {
err = v.queryVideos(v.client.Channels.ListVideo, id, podcast, feed)
if feed.LinkType == api.Group {
if podcast, err = v.queryGroup(feed); err == nil {
err = v.queryVideos(v.client.Groups.ListVideo, podcast, feed)
}
} else if kind == linkTypeGroup {
if podcast, err = v.queryGroup(id, feed); err == nil {
err = v.queryVideos(v.client.Groups.ListVideo, id, podcast, feed)
}
} else if kind == linkTypeUser {
if podcast, err = v.queryUser(id, feed); err == nil {
err = v.queryVideos(v.client.Users.ListVideo, id, podcast, feed)
}
} else {
err = errors.New("unsupported feed type")
return
}
if feed.LinkType == api.User {
if podcast, err = v.queryUser(feed); err == nil {
err = v.queryVideos(v.client.Users.ListVideo, podcast, feed)
}
return
}
err = errors.New("unsupported feed type")
return
}
web/pkg/builders/vimeo_test.go
+6 -68
View File
@@ -1,86 +1,24 @@
package builders
import (
"context"
"os"
"testing"
"context"
itunes "github.com/mxpv/podcast"
"github.com/mxpv/podsync/web/pkg/api"
"github.com/stretchr/testify/require"
)
var (
vimeoKey = os.Getenv("VIMEO_TEST_API_KEY")
defaultFeed = &api.Feed{Quality: api.HighQuality}
vimeoKey = os.Getenv("VIMEO_TEST_API_KEY")
)
func TestParseVimeoGroupLink(t *testing.T) {
builder := &VimeoBuilder{}
kind, id, err := builder.parseUrl("https://vimeo.com/groups/109")
require.NoError(t, err)
require.Equal(t, linkTypeGroup, kind)
require.Equal(t, "109", id)
kind, id, err = builder.parseUrl("http://vimeo.com/groups/109")
require.NoError(t, err)
require.Equal(t, linkTypeGroup, kind)
require.Equal(t, "109", id)
kind, id, err = builder.parseUrl("http://www.vimeo.com/groups/109")
require.NoError(t, err)
require.Equal(t, linkTypeGroup, kind)
require.Equal(t, "109", id)
kind, id, err = builder.parseUrl("https://vimeo.com/groups/109/videos/")
require.NoError(t, err)
require.Equal(t, linkTypeGroup, kind)
require.Equal(t, "109", id)
}
func TestParseVimeoChannelLink(t *testing.T) {
builder := &VimeoBuilder{}
kind, id, err := builder.parseUrl("https://vimeo.com/channels/staffpicks")
require.NoError(t, err)
require.Equal(t, linkTypeChannel, kind)
require.Equal(t, "staffpicks", id)
kind, id, err = builder.parseUrl("http://vimeo.com/channels/staffpicks/146224925")
require.NoError(t, err)
require.Equal(t, linkTypeChannel, kind)
require.Equal(t, "staffpicks", id)
}
func TestParseVimeoUserLink(t *testing.T) {
builder := &VimeoBuilder{}
kind, id, err := builder.parseUrl("https://vimeo.com/awhitelabelproduct")
require.NoError(t, err)
require.Equal(t, linkTypeUser, kind)
require.Equal(t, "awhitelabelproduct", id)
}
func TestParseInvalidVimeoLink(t *testing.T) {
builder := &VimeoBuilder{}
_, _, err := builder.parseUrl("")
require.Error(t, err)
_, _, err = builder.parseUrl("http://www.apple.com")
require.Error(t, err)
_, _, err = builder.parseUrl("http://www.vimeo.com")
require.Error(t, err)
}
func TestQueryVimeoChannel(t *testing.T) {
builder, err := NewVimeoBuilder(context.Background(), vimeoKey)
require.NoError(t, err)
podcast, err := builder.queryChannel("staffpicks", defaultFeed)
podcast, err := builder.queryChannel(&api.Feed{ItemId: "staffpicks", Quality: api.HighQuality})
require.NoError(t, err)
require.Equal(t, "https://vimeo.com/channels/staffpicks", podcast.Link)
@@ -95,7 +33,7 @@ func TestQueryVimeoGroup(t *testing.T) {
builder, err := NewVimeoBuilder(context.Background(), vimeoKey)
require.NoError(t, err)
podcast, err := builder.queryGroup("motion", defaultFeed)
podcast, err := builder.queryGroup(&api.Feed{ItemId: "motion", Quality: api.HighQuality})
require.NoError(t, err)
require.Equal(t, "https://vimeo.com/groups/motion", podcast.Link)
@@ -110,7 +48,7 @@ func TestQueryVimeoUser(t *testing.T) {
builder, err := NewVimeoBuilder(context.Background(), vimeoKey)
require.NoError(t, err)
podcast, err := builder.queryUser("motionarray", defaultFeed)
podcast, err := builder.queryUser(&api.Feed{ItemId: "motionarray", Quality: api.HighQuality})
require.NoError(t, err)
require.Equal(t, "https://vimeo.com/motionarray", podcast.Link)
@@ -125,7 +63,7 @@ func TestQueryVimeoVideos(t *testing.T) {
feed := &itunes.Podcast{}
err = builder.queryVideos(builder.client.Channels.ListVideo, "staffpicks", feed, &api.Feed{})
err = builder.queryVideos(builder.client.Channels.ListVideo, feed, &api.Feed{ItemId: "staffpicks"})
require.NoError(t, err)
require.Equal(t, vimeoDefaultPageSize, len(feed.Items))
web/pkg/builders/youtube.go
+10 -85
View File
@@ -3,7 +3,6 @@ package builders
import (
"fmt"
"net/http"
"net/url"
"strings"
"time"
@@ -31,82 +30,12 @@ type YouTubeBuilder struct {
key apiKey
}
func (yt *YouTubeBuilder) parseUrl(link string) (kind linkType, id string, err error) {
parsed, err := url.Parse(link)
if err != nil {
err = errors.Wrapf(err, "failed to parse url: %s", link)
return
}
if !strings.HasSuffix(parsed.Host, "youtube.com") {
err = errors.New("invalid youtube host")
return
}
path := parsed.EscapedPath()
// Parse
// https://www.youtube.com/playlist?list=PLCB9F975ECF01953C
if strings.HasPrefix(path, "/playlist") {
kind = linkTypePlaylist
id = parsed.Query().Get("list")
if id != "" {
return
}
err = errors.New("invalid playlist link")
return
}
// Parse
// - https://www.youtube.com/channel/UC5XPnUk8Vvv_pWslhwom6Og
// - https://www.youtube.com/channel/UCrlakW-ewUT8sOod6Wmzyow/videos
if strings.HasPrefix(path, "/channel") {
kind = linkTypeChannel
parts := strings.Split(parsed.EscapedPath(), "/")
if len(parts) <= 2 {
err = errors.New("invalid youtube channel link")
return
}
id = parts[2]
if id == "" {
err = errors.New("invalid id")
}
return
}
// Parse
// - https://www.youtube.com/user/fxigr1
if strings.HasPrefix(path, "/user") {
kind = linkTypeUser
parts := strings.Split(parsed.EscapedPath(), "/")
if len(parts) <= 2 {
err = errors.New("invalid user link")
return
}
id = parts[2]
if id == "" {
err = errors.New("invalid id")
}
return
}
err = errors.New("unsupported link format")
return
}
func (yt *YouTubeBuilder) listChannels(kind linkType, id string) (*youtube.Channel, error) {
func (yt *YouTubeBuilder) listChannels(linkType api.LinkType, id string) (*youtube.Channel, error) {
req := yt.client.Channels.List("id,snippet,contentDetails")
if kind == linkTypeChannel {
if linkType == api.Channel {
req = req.Id(id)
} else if kind == linkTypeUser {
} else if linkType == api.User {
req = req.ForUsername(id)
} else {
return nil, errors.New("unsupported link type")
@@ -190,11 +119,11 @@ func (yt *YouTubeBuilder) selectThumbnail(snippet *youtube.ThumbnailDetails, qua
return snippet.Default.Url
}
func (yt *YouTubeBuilder) queryFeed(kind linkType, id string, feed *api.Feed) (*itunes.Podcast, string, error) {
func (yt *YouTubeBuilder) queryFeed(feed *api.Feed) (*itunes.Podcast, string, error) {
now := time.Now()
if kind == linkTypeChannel || kind == linkTypeUser {
channel, err := yt.listChannels(kind, id)
if feed.LinkType == api.Channel || feed.LinkType == api.User {
channel, err := yt.listChannels(feed.LinkType, feed.ItemId)
if err != nil {
return nil, "", errors.Wrap(err, "failed to query channel")
}
@@ -202,7 +131,7 @@ func (yt *YouTubeBuilder) queryFeed(kind linkType, id string, feed *api.Feed) (*
itemId := channel.ContentDetails.RelatedPlaylists.Uploads
link := ""
if kind == linkTypeChannel {
if feed.LinkType == api.Channel {
link = fmt.Sprintf("https://youtube.com/channel/%s", itemId)
} else {
link = fmt.Sprintf("https://youtube.com/user/%s", itemId)
@@ -225,8 +154,8 @@ func (yt *YouTubeBuilder) queryFeed(kind linkType, id string, feed *api.Feed) (*
return &podcast, itemId, nil
}
if kind == linkTypePlaylist {
playlist, err := yt.listPlaylists(id, "")
if feed.LinkType == api.Playlist {
playlist, err := yt.listPlaylists(feed.ItemId, "")
if err != nil {
return nil, "", errors.Wrap(err, "failed to query playlist")
}
@@ -362,14 +291,10 @@ func (yt *YouTubeBuilder) queryItems(itemId string, feed *api.Feed, podcast *itu
}
func (yt *YouTubeBuilder) Build(feed *api.Feed) (*itunes.Podcast, error) {
kind, id, err := yt.parseUrl(feed.URL)
if err != nil {
return nil, errors.Wrapf(err, "failed to parse link: %s", feed.URL)
}
// Query general information about feed (title, description, lang, etc)
podcast, itemId, err := yt.queryFeed(kind, id, feed)
podcast, itemId, err := yt.queryFeed(feed)
if err != nil {
return nil, err
}
web/pkg/builders/youtube_test.go
+6 -47
View File
@@ -1,9 +1,8 @@
package builders
import (
"testing"
"os"
"testing"
"github.com/mxpv/podsync/web/pkg/api"
"github.com/stretchr/testify/require"
@@ -11,48 +10,6 @@ import (
var ytKey = os.Getenv("YOUTUBE_TEST_API_KEY")
func TestParseYTPlaylist(t *testing.T) {
builder := &YouTubeBuilder{}
kind, id, err := builder.parseUrl("https://www.youtube.com/playlist?list=PLCB9F975ECF01953C")
require.NoError(t, err)
require.Equal(t, linkTypePlaylist, kind)
require.Equal(t, "PLCB9F975ECF01953C", id)
}
func TestParseYTChannel(t *testing.T) {
builder := &YouTubeBuilder{}
kind, id, err := builder.parseUrl("https://www.youtube.com/channel/UC5XPnUk8Vvv_pWslhwom6Og")
require.NoError(t, err)
require.Equal(t, linkTypeChannel, kind)
require.Equal(t, "UC5XPnUk8Vvv_pWslhwom6Og", id)
kind, id, err = builder.parseUrl("https://www.youtube.com/channel/UCrlakW-ewUT8sOod6Wmzyow/videos")
require.NoError(t, err)
require.Equal(t, linkTypeChannel, kind)
require.Equal(t, "UCrlakW-ewUT8sOod6Wmzyow", id)
}
func TestParseYTUser(t *testing.T) {
builder := &YouTubeBuilder{}
kind, id, err := builder.parseUrl("https://youtube.com/user/fxigr1")
require.NoError(t, err)
require.Equal(t, linkTypeUser, kind)
require.Equal(t, "fxigr1", id)
}
func TestHandleInvalidYTLink(t *testing.T) {
builder := &YouTubeBuilder{}
_, _, err := builder.parseUrl("https://www.youtube.com/user///")
require.Error(t, err)
_, _, err = builder.parseUrl("https://www.youtube.com/channel//videos")
require.Error(t, err)
}
func TestQueryYTChannel(t *testing.T) {
if testing.Short() {
t.Skip("skipping YT test in short mode")
@@ -61,11 +18,11 @@ func TestQueryYTChannel(t *testing.T) {
builder, err := NewYouTubeBuilder(ytKey)
require.NoError(t, err)
channel, err := builder.listChannels(linkTypeChannel, "UC2yTVSttx7lxAOAzx1opjoA")
channel, err := builder.listChannels(api.Channel, "UC2yTVSttx7lxAOAzx1opjoA")
require.NoError(t, err)
require.Equal(t, "UC2yTVSttx7lxAOAzx1opjoA", channel.Id)
channel, err = builder.listChannels(linkTypeUser, "fxigr1")
channel, err = builder.listChannels(api.User, "fxigr1")
require.NoError(t, err)
require.Equal(t, "UCr_fwF-n-2_olTYd-m3n32g", channel.Id)
}
@@ -79,7 +36,9 @@ func TestBuildYTFeed(t *testing.T) {
require.NoError(t, err)
podcast, err := builder.Build(&api.Feed{
URL: "https://youtube.com/channel/UCupvZG-5ko_eiXAupbDfxWw",
Provider: api.Youtube,
LinkType: api.Channel,
ItemId: "UCupvZG-5ko_eiXAupbDfxWw",
PageSize: maxYoutubeResults,
})
require.NoError(t, err)
web/pkg/feeds/feeds.go
+76
View File
@@ -0,0 +1,76 @@
package feeds
import (
"context"
"fmt"
"time"
itunes "github.com/mxpv/podcast"
"github.com/mxpv/podsync/web/pkg/api"
"github.com/pkg/errors"
)
type service struct {
id id
storage storage
parser parser
builders map[api.Provider]builder
}
func (s *service) CreateFeed(ctx context.Context, req *api.CreateFeedRequest) (string, error) {
feed, err := s.parser.ParseURL(req.URL)
if err != nil {
return "", errors.Wrapf(err, "failed to create feed for URL: %s", req.URL)
}
// Make sure builder exists for this provider
_, ok := s.builders[feed.Provider]
if !ok {
return "", fmt.Errorf("failed to get builder for URL: %s", req.URL)
}
// Set default fields
feed.PageSize = api.DefaultPageSize
feed.Format = api.VideoFormat
feed.Quality = api.HighQuality
feed.FeatureLevel = api.DefaultFeatures
feed.LastAccess = time.Now().UTC()
// Generate short id
hashId, err := s.id.Generate(feed)
if err != nil {
return "", errors.Wrap(err, "failed to generate id for feed")
}
feed.HashId = hashId
// Save to database
if err := s.storage.CreateFeed(feed); err != nil {
return "", errors.Wrap(err, "failed to save feed to database")
}
return hashId, nil
}
func (s *service) GetFeed(hashId string) (*itunes.Podcast, error) {
feed, err := s.GetMetadata(hashId)
if err != nil {
return nil, err
}
builder, ok := s.builders[feed.Provider]
if !ok {
return nil, errors.Wrapf(err, "failed to get builder for feed: %s", hashId)
}
return builder.Build(feed)
}
func (s *service) GetMetadata(hashId string) (*api.Feed, error) {
feed, err := s.storage.GetFeed(hashId)
if err != nil {
return nil, errors.Wrapf(err, "failed to query feed: %s", hashId)
}
return feed, nil
}
web/pkg/feeds/interfaces.go
+23
View File
@@ -0,0 +1,23 @@
package feeds
import (
itunes "github.com/mxpv/podcast"
"github.com/mxpv/podsync/web/pkg/api"
)
type id interface {
Generate(feed *api.Feed) (string, error)
}
type storage interface {
CreateFeed(feed *api.Feed) error
GetFeed(hashId string) (*api.Feed, error)
}
type builder interface {
Build(feed *api.Feed) (podcast *itunes.Podcast, err error)
}
type parser interface {
ParseURL(link string) (feed *api.Feed, err error)
}
web/pkg/id/hashids.go
+5 -2
View File
@@ -23,15 +23,18 @@ func hashString(s string) int {
return int(h.Sum32())
}
func (h *hashId) Encode(feed *api.Feed) (string, error) {
func (h *hashId) Generate(feed *api.Feed) (string, error) {
// Don't create duplicate urls for same playlist/settings
// https://github.com/podsync/issues/issues/6
numbers := []int{
hashString(feed.UserId),
hashString(feed.URL),
hashString(string(feed.Provider)),
hashString(string(feed.LinkType)),
hashString(feed.ItemId),
feed.PageSize,
hashString(string(feed.Quality)),
hashString(string(feed.Format)),
feed.FeatureLevel,
}
return h.hid.Encode(numbers)
web/pkg/id/hashids_test.go
+6 -4
View File
@@ -13,23 +13,25 @@ func TestEncode(t *testing.T) {
feed := &api.Feed{
UserId: "1",
URL: "https://www.youtube.com/channel/UC2yTVSttx7lxAOAzx1opjoA",
Provider: api.Youtube,
LinkType: api.Channel,
ItemId: "UC2yTVSttx7lxAOAzx1opjoA",
PageSize: 10,
Quality: api.HighQuality,
Format: api.AudioFormat,
}
hash1, err := hid.Encode(feed)
hash1, err := hid.Generate(feed)
require.NoError(t, err)
require.NotEmpty(t, hash1)
// Ensure we have same hash for same feed/parameters
hash2, err := hid.Encode(feed)
hash2, err := hid.Generate(feed)
require.NoError(t, err)
require.Equal(t, hash1, hash2)
feed.UserId = ""
hash3, err := hid.Encode(feed)
hash3, err := hid.Generate(feed)
require.NoError(t, err)
require.NotEqual(t, hash1, hash3)
}
web/pkg/parsers/parser.go
+146
View File
@@ -0,0 +1,146 @@
package parsers
import (
"net/url"
"strings"
"github.com/mxpv/podsync/web/pkg/api"
"github.com/pkg/errors"
)
func ParseURL(link string) (*api.Feed, error) {
parsed, err := url.Parse(link)
if err != nil {
err = errors.Wrapf(err, "failed to parse url: %s", link)
return nil, err
}
feed := &api.Feed{}
if strings.HasSuffix(parsed.Host, "youtube.com") {
kind, id, err := parseYoutubeURL(parsed)
if err != nil {
return nil, err
}
feed.Provider = api.Youtube
feed.LinkType = kind
feed.ItemId = id
return feed, nil
}
if strings.HasSuffix(parsed.Host, "vimeo.com") {
kind, id, err := parseVimeoURL(parsed)
if err != nil {
return nil, err
}
feed.Provider = api.Vimeo
feed.LinkType = kind
feed.ItemId = id
return feed, nil
}
return nil, errors.New("unsupported URL host")
}
func parseYoutubeURL(parsed *url.URL) (kind api.LinkType, id string, err error) {
path := parsed.EscapedPath()
// https://www.youtube.com/playlist?list=PLCB9F975ECF01953C
if strings.HasPrefix(path, "/playlist") {
kind = api.Playlist
id = parsed.Query().Get("list")
if id != "" {
return
}
err = errors.New("invalid playlist link")
return
}
// - https://www.youtube.com/channel/UC5XPnUk8Vvv_pWslhwom6Og
// - https://www.youtube.com/channel/UCrlakW-ewUT8sOod6Wmzyow/videos
if strings.HasPrefix(path, "/channel") {
kind = api.Channel
parts := strings.Split(parsed.EscapedPath(), "/")
if len(parts) <= 2 {
err = errors.New("invalid youtube channel link")
return
}
id = parts[2]
if id == "" {
err = errors.New("invalid id")
}
return
}
// - https://www.youtube.com/user/fxigr1
if strings.HasPrefix(path, "/user") {
kind = api.User
parts := strings.Split(parsed.EscapedPath(), "/")
if len(parts) <= 2 {
err = errors.New("invalid user link")
return
}
id = parts[2]
if id == "" {
err = errors.New("invalid id")
}
return
}
err = errors.New("unsupported link format")
return
}
func parseVimeoURL(parsed *url.URL) (kind api.LinkType, id string, err error) {
parts := strings.Split(parsed.EscapedPath(), "/")
if len(parts) <= 1 {
err = errors.New("invalid vimeo link path")
return
}
if parts[1] == "groups" {
kind = api.Group
} else if parts[1] == "channels" {
kind = api.Channel
} else {
kind = api.User
}
if kind == api.Group || kind == api.Channel {
if len(parts) <= 2 {
err = errors.New("invalid channel link")
return
}
id = parts[2]
if id == "" {
err = errors.New("invalid id")
}
return
}
if kind == api.User {
id = parts[1]
if id == "" {
err = errors.New("invalid id")
}
return
}
err = errors.New("unsupported link format")
return
}
web/pkg/parsers/parser_test.go
+107
View File
@@ -0,0 +1,107 @@
package parsers
import (
"net/url"
"testing"
"github.com/mxpv/podsync/web/pkg/api"
"github.com/stretchr/testify/require"
)
func TestParseYTPlaylist(t *testing.T) {
link, _ := url.ParseRequestURI("https://www.youtube.com/playlist?list=PLCB9F975ECF01953C")
kind, id, err := parseYoutubeURL(link)
require.NoError(t, err)
require.Equal(t, api.Playlist, kind)
require.Equal(t, "PLCB9F975ECF01953C", id)
}
func TestParseYTChannel(t *testing.T) {
link, _ := url.ParseRequestURI("https://www.youtube.com/channel/UC5XPnUk8Vvv_pWslhwom6Og")
kind, id, err := parseYoutubeURL(link)
require.NoError(t, err)
require.Equal(t, api.Channel, kind)
require.Equal(t, "UC5XPnUk8Vvv_pWslhwom6Og", id)
link, _ = url.ParseRequestURI("https://www.youtube.com/channel/UCrlakW-ewUT8sOod6Wmzyow/videos")
kind, id, err = parseYoutubeURL(link)
require.NoError(t, err)
require.Equal(t, api.Channel, kind)
require.Equal(t, "UCrlakW-ewUT8sOod6Wmzyow", id)
}
func TestParseYTUser(t *testing.T) {
link, _ := url.ParseRequestURI("https://youtube.com/user/fxigr1")
kind, id, err := parseYoutubeURL(link)
require.NoError(t, err)
require.Equal(t, api.User, kind)
require.Equal(t, "fxigr1", id)
}
func TestHandleInvalidYTLink(t *testing.T) {
link, _ := url.ParseRequestURI("https://www.youtube.com/user///")
_, _, err := parseYoutubeURL(link)
require.Error(t, err)
link, _ = url.ParseRequestURI("https://www.youtube.com/channel//videos")
_, _, err = parseYoutubeURL(link)
require.Error(t, err)
}
func TestParseVimeoGroupLink(t *testing.T) {
link, _ := url.ParseRequestURI("https://vimeo.com/groups/109")
kind, id, err := parseVimeoURL(link)
require.NoError(t, err)
require.Equal(t, api.Group, kind)
require.Equal(t, "109", id)
link, _ = url.ParseRequestURI("http://vimeo.com/groups/109")
kind, id, err = parseVimeoURL(link)
require.NoError(t, err)
require.Equal(t, api.Group, kind)
require.Equal(t, "109", id)
link, _ = url.ParseRequestURI("http://www.vimeo.com/groups/109")
kind, id, err = parseVimeoURL(link)
require.NoError(t, err)
require.Equal(t, api.Group, kind)
require.Equal(t, "109", id)
link, _ = url.ParseRequestURI("https://vimeo.com/groups/109/videos/")
kind, id, err = parseVimeoURL(link)
require.NoError(t, err)
require.Equal(t, api.Group, kind)
require.Equal(t, "109", id)
}
func TestParseVimeoChannelLink(t *testing.T) {
link, _ := url.ParseRequestURI("https://vimeo.com/channels/staffpicks")
kind, id, err := parseVimeoURL(link)
require.NoError(t, err)
require.Equal(t, api.Channel, kind)
require.Equal(t, "staffpicks", id)
link, _ = url.ParseRequestURI("http://vimeo.com/channels/staffpicks/146224925")
kind, id, err = parseVimeoURL(link)
require.NoError(t, err)
require.Equal(t, api.Channel, kind)
require.Equal(t, "staffpicks", id)
}
func TestParseVimeoUserLink(t *testing.T) {
link, _ := url.ParseRequestURI("https://vimeo.com/awhitelabelproduct")
kind, id, err := parseVimeoURL(link)
require.NoError(t, err)
require.Equal(t, api.User, kind)
require.Equal(t, "awhitelabelproduct", id)
}
func TestParseInvalidVimeoLink(t *testing.T) {
link, _ := url.ParseRequestURI("http://www.apple.com")
_, _, err := parseVimeoURL(link)
require.Error(t, err)
link, _ = url.ParseRequestURI("http://www.vimeo.com")
_, _, err = parseVimeoURL(link)
require.Error(t, err)
}
web/pkg/storage/pg_sql.go
+11 -2
View File
@@ -5,6 +5,12 @@ BEGIN;
DO $$
BEGIN
IF NOT EXISTS (SELECT 1 FROM pg_type WHERE typname = 'provider') THEN
CREATE TYPE quality AS ENUM ('youtube', 'vimeo');
END IF;
IF NOT EXISTS (SELECT 1 FROM pg_type WHERE typname = 'link_type') THEN
CREATE TYPE quality AS ENUM ('channel', 'playlist', 'user', 'group');
END IF;
IF NOT EXISTS (SELECT 1 FROM pg_type WHERE typname = 'quality') THEN
CREATE TYPE quality AS ENUM ('high', 'low');
END IF;
@@ -18,10 +24,13 @@ CREATE TABLE IF NOT EXISTS feeds (
id BIGSERIAL PRIMARY KEY,
hash_id VARCHAR(12) NOT NULL CHECK (hash_id <> '') UNIQUE,
user_id VARCHAR(32) NULL,
url VARCHAR(64) NOT NULL CHECK (url <> ''),
item_id VARCHAR(32) NOT NULL CHECK (item_id <> ''),
provider provider NOT NULL,
link_type link_type NOT NULL,
page_size INT NOT NULL DEFAULT 50,
quality quality NOT NULL DEFAULT 'high',
format format NOT NULL DEFAULT 'video',
quality quality NOT NULL DEFAULT 'high',
feature_level INT NOT NULL DEFAULT 0,
last_access timestamp WITHOUT TIME ZONE NOT NULL
);
web/pkg/storage/pg_test.go
+21 -11
View File
@@ -9,8 +9,10 @@ import (
func TestCreate(t *testing.T) {
feed := &api.Feed{
HashId: "xyz",
URL: "http://youtube.com",
HashId: "xyz",
Provider: api.Youtube,
LinkType: api.Channel,
ItemId: "123",
}
client := createClient(t)
@@ -21,8 +23,10 @@ func TestCreate(t *testing.T) {
func TestCreateDuplicate(t *testing.T) {
feed := &api.Feed{
HashId: "123",
URL: "http://youtube.com",
HashId: "123",
Provider: api.Youtube,
LinkType: api.Channel,
ItemId: "123",
}
client := createClient(t)
@@ -46,9 +50,11 @@ func TestCreateDuplicate(t *testing.T) {
func TestGetFeed(t *testing.T) {
feed := &api.Feed{
HashId: "xyz",
UserId: "123",
URL: "http://youtube.com",
HashId: "xyz",
UserId: "123",
Provider: api.Youtube,
LinkType: api.Channel,
ItemId: "123",
}
client := createClient(t)
@@ -61,9 +67,11 @@ func TestGetFeed(t *testing.T) {
func TestUpdateLastAccess(t *testing.T) {
feed := &api.Feed{
HashId: "xyz",
UserId: "123",
URL: "http://youtube.com",
HashId: "xyz",
UserId: "123",
Provider: api.Youtube,
LinkType: api.Channel,
ItemId: "123",
}
client := createClient(t)
@@ -78,7 +86,9 @@ func TestUpdateLastAccess(t *testing.T) {
require.NotEmpty(t, last.HashId)
require.NotEmpty(t, last.UserId)
require.NotEmpty(t, last.URL)
require.NotEmpty(t, last.Provider)
require.NotEmpty(t, last.LinkType)
require.NotEmpty(t, last.ItemId)
require.True(t, last.LastAccess.Unix() > lastAccess.Unix())
}
web/pkg/storage/redis.go
-2
View File
@@ -129,8 +129,6 @@ func (r *RedisStorage) GetFeed(hashId string) (*api.Feed, error) {
return nil, err
}
feed.URL = url
// Fetch user id
patreonId, ok := m["patreonid"]
if ok {