From 535b7dbaa4eb2c7ef3019ea355603bff7ea5da5e Mon Sep 17 00:00:00 2001 From: Maksym Pavlenko Date: Sun, 13 Aug 2017 14:50:59 -0700 Subject: [PATCH] Implement initial feed service --- web/pkg/api/api.go | 54 ++++++++++-- web/pkg/builders/common.go | 10 --- web/pkg/builders/vimeo.go | 110 +++++++---------------- web/pkg/builders/vimeo_test.go | 74 ++-------------- web/pkg/builders/youtube.go | 95 +++----------------- web/pkg/builders/youtube_test.go | 53 ++--------- web/pkg/feeds/feeds.go | 76 ++++++++++++++++ web/pkg/feeds/interfaces.go | 23 +++++ web/pkg/id/hashids.go | 7 +- web/pkg/id/hashids_test.go | 10 ++- web/pkg/parsers/parser.go | 146 +++++++++++++++++++++++++++++++ web/pkg/parsers/parser_test.go | 107 ++++++++++++++++++++++ web/pkg/storage/pg_sql.go | 13 ++- web/pkg/storage/pg_test.go | 32 ++++--- web/pkg/storage/redis.go | 2 - 15 files changed, 494 insertions(+), 318 deletions(-) create mode 100644 web/pkg/feeds/feeds.go create mode 100644 web/pkg/feeds/interfaces.go create mode 100644 web/pkg/parsers/parser.go create mode 100644 web/pkg/parsers/parser_test.go diff --git a/web/pkg/api/api.go b/web/pkg/api/api.go index 017ff5d..3da918c 100644 --- a/web/pkg/api/api.go +++ b/web/pkg/api/api.go @@ -2,12 +2,32 @@ package api import "time" +type Provider string + +const ( + Youtube = Provider("youtube") + Vimeo = Provider("vimeo") +) + +type LinkType string + +const ( + Channel = LinkType("channel") + Playlist = LinkType("playlist") + User = LinkType("user") + Group = LinkType("group") +) + type Quality string -type Format string const ( HighQuality = Quality("high") LowQuality = Quality("low") +) + +type Format string + +const ( AudioFormat = Format("audio") VideoFormat = Format("video") ) @@ -19,12 +39,28 @@ const ( ) type Feed struct { - Id int64 `json:"id"` - HashId string `json:"hash_id"` - UserId string `json:"user_id"` - URL string `json:"url"` - PageSize int `json:"page_size"` - Quality Quality `json:"quality"` - Format Format `json:"format"` - LastAccess time.Time `json:"last_access"` + Id int64 `json:"id"` + HashId string `json:"hash_id"` // Short human readable feed id for users + UserId string `json:"user_id"` // Patreon user id + ItemId string `json:"item_id"` + Provider Provider `json:"provider"` // Youtube or Vimeo + LinkType LinkType `json:"link_type"` // Either group, channel or user + PageSize int `json:"page_size"` // The number of episodes to return + Format Format `json:"format"` + Quality Quality `json:"quality"` + FeatureLevel int `json:"feature_level"` // Available features + LastAccess time.Time `json:"last_access"` +} + +const ( + DefaultFeatures = iota + ExtendedFeatures + PodcasterFeature +) + +type CreateFeedRequest struct { + URL string `json:"url"` + PageSize int `json:"page_size"` + Quality Quality `json:"quality"` + Format Format `json:"format"` } diff --git a/web/pkg/builders/common.go b/web/pkg/builders/common.go index fd01499..db9fcb1 100644 --- a/web/pkg/builders/common.go +++ b/web/pkg/builders/common.go @@ -12,16 +12,6 @@ const ( defaultCategory = "TV & Film" ) -type linkType int - -const ( - _ = iota - linkTypeChannel linkType = iota - linkTypePlaylist - linkTypeUser - linkTypeGroup -) - func makeEnclosure(feed *api.Feed, id string, lengthInBytes int64) (string, itunes.EnclosureType, int64) { ext := "mp4" contentType := itunes.MP4 diff --git a/web/pkg/builders/vimeo.go b/web/pkg/builders/vimeo.go index fa168e1..1727fe4 100644 --- a/web/pkg/builders/vimeo.go +++ b/web/pkg/builders/vimeo.go @@ -3,9 +3,7 @@ package builders import ( "fmt" "net/http" - "net/url" "strconv" - "strings" itunes "github.com/mxpv/podcast" "github.com/mxpv/podsync/web/pkg/api" @@ -23,60 +21,6 @@ type VimeoBuilder struct { client *vimeo.Client } -func (v *VimeoBuilder) parseUrl(link string) (kind linkType, id string, err error) { - parsed, err := url.Parse(link) - if err != nil { - err = errors.Wrapf(err, "failed to parse url: %s", link) - return - } - - if !strings.HasSuffix(parsed.Host, "vimeo.com") { - err = errors.New("invalid vimeo host") - return - } - - parts := strings.Split(parsed.EscapedPath(), "/") - - if len(parts) <= 1 { - err = errors.New("invalid vimeo link path") - return - } - - if parts[1] == "groups" { - kind = linkTypeGroup - } else if parts[1] == "channels" { - kind = linkTypeChannel - } else { - kind = linkTypeUser - } - - if kind == linkTypeGroup || kind == linkTypeChannel { - if len(parts) <= 2 { - err = errors.New("invalid channel link") - return - } - - id = parts[2] - if id == "" { - err = errors.New("invalid id") - } - - return - } - - if kind == linkTypeUser { - id = parts[1] - if id == "" { - err = errors.New("invalid id") - } - - return - } - - err = errors.New("unsupported link format") - return -} - func (v *VimeoBuilder) selectImage(p *vimeo.Pictures, q api.Quality) string { if p == nil || len(p.Sizes) < 1 { return "" @@ -89,7 +33,9 @@ func (v *VimeoBuilder) selectImage(p *vimeo.Pictures, q api.Quality) string { } } -func (v *VimeoBuilder) queryChannel(channelId string, feed *api.Feed) (*itunes.Podcast, error) { +func (v *VimeoBuilder) queryChannel(feed *api.Feed) (*itunes.Podcast, error) { + channelId := feed.ItemId + ch, resp, err := v.client.Channels.Get(channelId) if err != nil { return nil, errors.Wrapf(err, "failed to query channel with channelId %s", channelId) @@ -109,7 +55,9 @@ func (v *VimeoBuilder) queryChannel(channelId string, feed *api.Feed) (*itunes.P return &podcast, nil } -func (v *VimeoBuilder) queryGroup(groupId string, feed *api.Feed) (*itunes.Podcast, error) { +func (v *VimeoBuilder) queryGroup(feed *api.Feed) (*itunes.Podcast, error) { + groupId := feed.ItemId + gr, resp, err := v.client.Groups.Get(groupId) if err != nil { return nil, errors.Wrapf(err, "failed to query group with id %s", groupId) @@ -129,7 +77,9 @@ func (v *VimeoBuilder) queryGroup(groupId string, feed *api.Feed) (*itunes.Podca return &podcast, nil } -func (v *VimeoBuilder) queryUser(userId string, feed *api.Feed) (*itunes.Podcast, error) { +func (v *VimeoBuilder) queryUser(feed *api.Feed) (*itunes.Podcast, error) { + userId := feed.ItemId + user, resp, err := v.client.Users.Get(userId) if err != nil { return nil, errors.Wrapf(err, "failed to query user with id %s", userId) @@ -154,9 +104,9 @@ func (v *VimeoBuilder) getVideoSize(video *vimeo.Video) int64 { return int64(float64(video.Duration*video.Width*video.Height) * 0.38848958333) } -type queryVideosFunc func(id string, opt *vimeo.ListVideoOptions) ([]*vimeo.Video, *vimeo.Response, error) +type getVideosFunc func(id string, opt *vimeo.ListVideoOptions) ([]*vimeo.Video, *vimeo.Response, error) -func (v *VimeoBuilder) queryVideos(queryVideos queryVideosFunc, id string, podcast *itunes.Podcast, feed *api.Feed) error { +func (v *VimeoBuilder) queryVideos(getVideos getVideosFunc, podcast *itunes.Podcast, feed *api.Feed) error { opt := vimeo.ListVideoOptions{} opt.Page = 1 opt.PerPage = vimeoDefaultPageSize @@ -164,7 +114,7 @@ func (v *VimeoBuilder) queryVideos(queryVideos queryVideosFunc, id string, podca added := 0 for { - videos, response, err := queryVideos(id, &opt) + videos, response, err := getVideos(feed.ItemId, &opt) if err != nil { return errors.Wrap(err, "failed to query videos") } @@ -208,27 +158,31 @@ func (v *VimeoBuilder) queryVideos(queryVideos queryVideosFunc, id string, podca } func (v *VimeoBuilder) Build(feed *api.Feed) (podcast *itunes.Podcast, err error) { - kind, id, err := v.parseUrl(feed.URL) - if err != nil { - return nil, errors.Wrapf(err, "failed to parse link: %s", feed.URL) + if feed.LinkType == api.Channel { + if podcast, err = v.queryChannel(feed); err == nil { + err = v.queryVideos(v.client.Channels.ListVideo, podcast, feed) + } + + return } - if kind == linkTypeChannel { - if podcast, err = v.queryChannel(id, feed); err == nil { - err = v.queryVideos(v.client.Channels.ListVideo, id, podcast, feed) + if feed.LinkType == api.Group { + if podcast, err = v.queryGroup(feed); err == nil { + err = v.queryVideos(v.client.Groups.ListVideo, podcast, feed) } - } else if kind == linkTypeGroup { - if podcast, err = v.queryGroup(id, feed); err == nil { - err = v.queryVideos(v.client.Groups.ListVideo, id, podcast, feed) - } - } else if kind == linkTypeUser { - if podcast, err = v.queryUser(id, feed); err == nil { - err = v.queryVideos(v.client.Users.ListVideo, id, podcast, feed) - } - } else { - err = errors.New("unsupported feed type") + + return } + if feed.LinkType == api.User { + if podcast, err = v.queryUser(feed); err == nil { + err = v.queryVideos(v.client.Users.ListVideo, podcast, feed) + } + + return + } + + err = errors.New("unsupported feed type") return } diff --git a/web/pkg/builders/vimeo_test.go b/web/pkg/builders/vimeo_test.go index 6394d51..3e50120 100644 --- a/web/pkg/builders/vimeo_test.go +++ b/web/pkg/builders/vimeo_test.go @@ -1,86 +1,24 @@ package builders import ( + "context" "os" "testing" - "context" - itunes "github.com/mxpv/podcast" "github.com/mxpv/podsync/web/pkg/api" "github.com/stretchr/testify/require" ) var ( - vimeoKey = os.Getenv("VIMEO_TEST_API_KEY") - defaultFeed = &api.Feed{Quality: api.HighQuality} + vimeoKey = os.Getenv("VIMEO_TEST_API_KEY") ) -func TestParseVimeoGroupLink(t *testing.T) { - builder := &VimeoBuilder{} - - kind, id, err := builder.parseUrl("https://vimeo.com/groups/109") - require.NoError(t, err) - require.Equal(t, linkTypeGroup, kind) - require.Equal(t, "109", id) - - kind, id, err = builder.parseUrl("http://vimeo.com/groups/109") - require.NoError(t, err) - require.Equal(t, linkTypeGroup, kind) - require.Equal(t, "109", id) - - kind, id, err = builder.parseUrl("http://www.vimeo.com/groups/109") - require.NoError(t, err) - require.Equal(t, linkTypeGroup, kind) - require.Equal(t, "109", id) - - kind, id, err = builder.parseUrl("https://vimeo.com/groups/109/videos/") - require.NoError(t, err) - require.Equal(t, linkTypeGroup, kind) - require.Equal(t, "109", id) -} - -func TestParseVimeoChannelLink(t *testing.T) { - builder := &VimeoBuilder{} - - kind, id, err := builder.parseUrl("https://vimeo.com/channels/staffpicks") - require.NoError(t, err) - require.Equal(t, linkTypeChannel, kind) - require.Equal(t, "staffpicks", id) - - kind, id, err = builder.parseUrl("http://vimeo.com/channels/staffpicks/146224925") - require.NoError(t, err) - require.Equal(t, linkTypeChannel, kind) - require.Equal(t, "staffpicks", id) -} - -func TestParseVimeoUserLink(t *testing.T) { - builder := &VimeoBuilder{} - - kind, id, err := builder.parseUrl("https://vimeo.com/awhitelabelproduct") - require.NoError(t, err) - require.Equal(t, linkTypeUser, kind) - require.Equal(t, "awhitelabelproduct", id) -} - -func TestParseInvalidVimeoLink(t *testing.T) { - builder := &VimeoBuilder{} - - _, _, err := builder.parseUrl("") - require.Error(t, err) - - _, _, err = builder.parseUrl("http://www.apple.com") - require.Error(t, err) - - _, _, err = builder.parseUrl("http://www.vimeo.com") - require.Error(t, err) -} - func TestQueryVimeoChannel(t *testing.T) { builder, err := NewVimeoBuilder(context.Background(), vimeoKey) require.NoError(t, err) - podcast, err := builder.queryChannel("staffpicks", defaultFeed) + podcast, err := builder.queryChannel(&api.Feed{ItemId: "staffpicks", Quality: api.HighQuality}) require.NoError(t, err) require.Equal(t, "https://vimeo.com/channels/staffpicks", podcast.Link) @@ -95,7 +33,7 @@ func TestQueryVimeoGroup(t *testing.T) { builder, err := NewVimeoBuilder(context.Background(), vimeoKey) require.NoError(t, err) - podcast, err := builder.queryGroup("motion", defaultFeed) + podcast, err := builder.queryGroup(&api.Feed{ItemId: "motion", Quality: api.HighQuality}) require.NoError(t, err) require.Equal(t, "https://vimeo.com/groups/motion", podcast.Link) @@ -110,7 +48,7 @@ func TestQueryVimeoUser(t *testing.T) { builder, err := NewVimeoBuilder(context.Background(), vimeoKey) require.NoError(t, err) - podcast, err := builder.queryUser("motionarray", defaultFeed) + podcast, err := builder.queryUser(&api.Feed{ItemId: "motionarray", Quality: api.HighQuality}) require.NoError(t, err) require.Equal(t, "https://vimeo.com/motionarray", podcast.Link) @@ -125,7 +63,7 @@ func TestQueryVimeoVideos(t *testing.T) { feed := &itunes.Podcast{} - err = builder.queryVideos(builder.client.Channels.ListVideo, "staffpicks", feed, &api.Feed{}) + err = builder.queryVideos(builder.client.Channels.ListVideo, feed, &api.Feed{ItemId: "staffpicks"}) require.NoError(t, err) require.Equal(t, vimeoDefaultPageSize, len(feed.Items)) diff --git a/web/pkg/builders/youtube.go b/web/pkg/builders/youtube.go index 06a10e1..5f2e2a8 100644 --- a/web/pkg/builders/youtube.go +++ b/web/pkg/builders/youtube.go @@ -3,7 +3,6 @@ package builders import ( "fmt" "net/http" - "net/url" "strings" "time" @@ -31,82 +30,12 @@ type YouTubeBuilder struct { key apiKey } -func (yt *YouTubeBuilder) parseUrl(link string) (kind linkType, id string, err error) { - parsed, err := url.Parse(link) - if err != nil { - err = errors.Wrapf(err, "failed to parse url: %s", link) - return - } - - if !strings.HasSuffix(parsed.Host, "youtube.com") { - err = errors.New("invalid youtube host") - return - } - - path := parsed.EscapedPath() - - // Parse - // https://www.youtube.com/playlist?list=PLCB9F975ECF01953C - if strings.HasPrefix(path, "/playlist") { - kind = linkTypePlaylist - - id = parsed.Query().Get("list") - if id != "" { - return - } - - err = errors.New("invalid playlist link") - return - } - - // Parse - // - https://www.youtube.com/channel/UC5XPnUk8Vvv_pWslhwom6Og - // - https://www.youtube.com/channel/UCrlakW-ewUT8sOod6Wmzyow/videos - if strings.HasPrefix(path, "/channel") { - kind = linkTypeChannel - parts := strings.Split(parsed.EscapedPath(), "/") - if len(parts) <= 2 { - err = errors.New("invalid youtube channel link") - return - } - - id = parts[2] - if id == "" { - err = errors.New("invalid id") - } - - return - } - - // Parse - // - https://www.youtube.com/user/fxigr1 - if strings.HasPrefix(path, "/user") { - kind = linkTypeUser - - parts := strings.Split(parsed.EscapedPath(), "/") - if len(parts) <= 2 { - err = errors.New("invalid user link") - return - } - - id = parts[2] - if id == "" { - err = errors.New("invalid id") - } - - return - } - - err = errors.New("unsupported link format") - return -} - -func (yt *YouTubeBuilder) listChannels(kind linkType, id string) (*youtube.Channel, error) { +func (yt *YouTubeBuilder) listChannels(linkType api.LinkType, id string) (*youtube.Channel, error) { req := yt.client.Channels.List("id,snippet,contentDetails") - if kind == linkTypeChannel { + if linkType == api.Channel { req = req.Id(id) - } else if kind == linkTypeUser { + } else if linkType == api.User { req = req.ForUsername(id) } else { return nil, errors.New("unsupported link type") @@ -190,11 +119,11 @@ func (yt *YouTubeBuilder) selectThumbnail(snippet *youtube.ThumbnailDetails, qua return snippet.Default.Url } -func (yt *YouTubeBuilder) queryFeed(kind linkType, id string, feed *api.Feed) (*itunes.Podcast, string, error) { +func (yt *YouTubeBuilder) queryFeed(feed *api.Feed) (*itunes.Podcast, string, error) { now := time.Now() - if kind == linkTypeChannel || kind == linkTypeUser { - channel, err := yt.listChannels(kind, id) + if feed.LinkType == api.Channel || feed.LinkType == api.User { + channel, err := yt.listChannels(feed.LinkType, feed.ItemId) if err != nil { return nil, "", errors.Wrap(err, "failed to query channel") } @@ -202,7 +131,7 @@ func (yt *YouTubeBuilder) queryFeed(kind linkType, id string, feed *api.Feed) (* itemId := channel.ContentDetails.RelatedPlaylists.Uploads link := "" - if kind == linkTypeChannel { + if feed.LinkType == api.Channel { link = fmt.Sprintf("https://youtube.com/channel/%s", itemId) } else { link = fmt.Sprintf("https://youtube.com/user/%s", itemId) @@ -225,8 +154,8 @@ func (yt *YouTubeBuilder) queryFeed(kind linkType, id string, feed *api.Feed) (* return &podcast, itemId, nil } - if kind == linkTypePlaylist { - playlist, err := yt.listPlaylists(id, "") + if feed.LinkType == api.Playlist { + playlist, err := yt.listPlaylists(feed.ItemId, "") if err != nil { return nil, "", errors.Wrap(err, "failed to query playlist") } @@ -362,14 +291,10 @@ func (yt *YouTubeBuilder) queryItems(itemId string, feed *api.Feed, podcast *itu } func (yt *YouTubeBuilder) Build(feed *api.Feed) (*itunes.Podcast, error) { - kind, id, err := yt.parseUrl(feed.URL) - if err != nil { - return nil, errors.Wrapf(err, "failed to parse link: %s", feed.URL) - } // Query general information about feed (title, description, lang, etc) - podcast, itemId, err := yt.queryFeed(kind, id, feed) + podcast, itemId, err := yt.queryFeed(feed) if err != nil { return nil, err } diff --git a/web/pkg/builders/youtube_test.go b/web/pkg/builders/youtube_test.go index 0c29dd5..136dc1d 100644 --- a/web/pkg/builders/youtube_test.go +++ b/web/pkg/builders/youtube_test.go @@ -1,9 +1,8 @@ package builders import ( - "testing" - "os" + "testing" "github.com/mxpv/podsync/web/pkg/api" "github.com/stretchr/testify/require" @@ -11,48 +10,6 @@ import ( var ytKey = os.Getenv("YOUTUBE_TEST_API_KEY") -func TestParseYTPlaylist(t *testing.T) { - builder := &YouTubeBuilder{} - - kind, id, err := builder.parseUrl("https://www.youtube.com/playlist?list=PLCB9F975ECF01953C") - require.NoError(t, err) - require.Equal(t, linkTypePlaylist, kind) - require.Equal(t, "PLCB9F975ECF01953C", id) -} - -func TestParseYTChannel(t *testing.T) { - builder := &YouTubeBuilder{} - - kind, id, err := builder.parseUrl("https://www.youtube.com/channel/UC5XPnUk8Vvv_pWslhwom6Og") - require.NoError(t, err) - require.Equal(t, linkTypeChannel, kind) - require.Equal(t, "UC5XPnUk8Vvv_pWslhwom6Og", id) - - kind, id, err = builder.parseUrl("https://www.youtube.com/channel/UCrlakW-ewUT8sOod6Wmzyow/videos") - require.NoError(t, err) - require.Equal(t, linkTypeChannel, kind) - require.Equal(t, "UCrlakW-ewUT8sOod6Wmzyow", id) -} - -func TestParseYTUser(t *testing.T) { - builder := &YouTubeBuilder{} - - kind, id, err := builder.parseUrl("https://youtube.com/user/fxigr1") - require.NoError(t, err) - require.Equal(t, linkTypeUser, kind) - require.Equal(t, "fxigr1", id) -} - -func TestHandleInvalidYTLink(t *testing.T) { - builder := &YouTubeBuilder{} - - _, _, err := builder.parseUrl("https://www.youtube.com/user///") - require.Error(t, err) - - _, _, err = builder.parseUrl("https://www.youtube.com/channel//videos") - require.Error(t, err) -} - func TestQueryYTChannel(t *testing.T) { if testing.Short() { t.Skip("skipping YT test in short mode") @@ -61,11 +18,11 @@ func TestQueryYTChannel(t *testing.T) { builder, err := NewYouTubeBuilder(ytKey) require.NoError(t, err) - channel, err := builder.listChannels(linkTypeChannel, "UC2yTVSttx7lxAOAzx1opjoA") + channel, err := builder.listChannels(api.Channel, "UC2yTVSttx7lxAOAzx1opjoA") require.NoError(t, err) require.Equal(t, "UC2yTVSttx7lxAOAzx1opjoA", channel.Id) - channel, err = builder.listChannels(linkTypeUser, "fxigr1") + channel, err = builder.listChannels(api.User, "fxigr1") require.NoError(t, err) require.Equal(t, "UCr_fwF-n-2_olTYd-m3n32g", channel.Id) } @@ -79,7 +36,9 @@ func TestBuildYTFeed(t *testing.T) { require.NoError(t, err) podcast, err := builder.Build(&api.Feed{ - URL: "https://youtube.com/channel/UCupvZG-5ko_eiXAupbDfxWw", + Provider: api.Youtube, + LinkType: api.Channel, + ItemId: "UCupvZG-5ko_eiXAupbDfxWw", PageSize: maxYoutubeResults, }) require.NoError(t, err) diff --git a/web/pkg/feeds/feeds.go b/web/pkg/feeds/feeds.go new file mode 100644 index 0000000..03e35cb --- /dev/null +++ b/web/pkg/feeds/feeds.go @@ -0,0 +1,76 @@ +package feeds + +import ( + "context" + "fmt" + "time" + + itunes "github.com/mxpv/podcast" + "github.com/mxpv/podsync/web/pkg/api" + "github.com/pkg/errors" +) + +type service struct { + id id + storage storage + parser parser + builders map[api.Provider]builder +} + +func (s *service) CreateFeed(ctx context.Context, req *api.CreateFeedRequest) (string, error) { + feed, err := s.parser.ParseURL(req.URL) + if err != nil { + return "", errors.Wrapf(err, "failed to create feed for URL: %s", req.URL) + } + + // Make sure builder exists for this provider + _, ok := s.builders[feed.Provider] + if !ok { + return "", fmt.Errorf("failed to get builder for URL: %s", req.URL) + } + + // Set default fields + feed.PageSize = api.DefaultPageSize + feed.Format = api.VideoFormat + feed.Quality = api.HighQuality + feed.FeatureLevel = api.DefaultFeatures + feed.LastAccess = time.Now().UTC() + + // Generate short id + hashId, err := s.id.Generate(feed) + if err != nil { + return "", errors.Wrap(err, "failed to generate id for feed") + } + + feed.HashId = hashId + + // Save to database + if err := s.storage.CreateFeed(feed); err != nil { + return "", errors.Wrap(err, "failed to save feed to database") + } + + return hashId, nil +} + +func (s *service) GetFeed(hashId string) (*itunes.Podcast, error) { + feed, err := s.GetMetadata(hashId) + if err != nil { + return nil, err + } + + builder, ok := s.builders[feed.Provider] + if !ok { + return nil, errors.Wrapf(err, "failed to get builder for feed: %s", hashId) + } + + return builder.Build(feed) +} + +func (s *service) GetMetadata(hashId string) (*api.Feed, error) { + feed, err := s.storage.GetFeed(hashId) + if err != nil { + return nil, errors.Wrapf(err, "failed to query feed: %s", hashId) + } + + return feed, nil +} diff --git a/web/pkg/feeds/interfaces.go b/web/pkg/feeds/interfaces.go new file mode 100644 index 0000000..d29998f --- /dev/null +++ b/web/pkg/feeds/interfaces.go @@ -0,0 +1,23 @@ +package feeds + +import ( + itunes "github.com/mxpv/podcast" + "github.com/mxpv/podsync/web/pkg/api" +) + +type id interface { + Generate(feed *api.Feed) (string, error) +} + +type storage interface { + CreateFeed(feed *api.Feed) error + GetFeed(hashId string) (*api.Feed, error) +} + +type builder interface { + Build(feed *api.Feed) (podcast *itunes.Podcast, err error) +} + +type parser interface { + ParseURL(link string) (feed *api.Feed, err error) +} diff --git a/web/pkg/id/hashids.go b/web/pkg/id/hashids.go index 5f0d6e2..d589fc9 100644 --- a/web/pkg/id/hashids.go +++ b/web/pkg/id/hashids.go @@ -23,15 +23,18 @@ func hashString(s string) int { return int(h.Sum32()) } -func (h *hashId) Encode(feed *api.Feed) (string, error) { +func (h *hashId) Generate(feed *api.Feed) (string, error) { // Don't create duplicate urls for same playlist/settings // https://github.com/podsync/issues/issues/6 numbers := []int{ hashString(feed.UserId), - hashString(feed.URL), + hashString(string(feed.Provider)), + hashString(string(feed.LinkType)), + hashString(feed.ItemId), feed.PageSize, hashString(string(feed.Quality)), hashString(string(feed.Format)), + feed.FeatureLevel, } return h.hid.Encode(numbers) diff --git a/web/pkg/id/hashids_test.go b/web/pkg/id/hashids_test.go index a5e4bfb..9b04789 100644 --- a/web/pkg/id/hashids_test.go +++ b/web/pkg/id/hashids_test.go @@ -13,23 +13,25 @@ func TestEncode(t *testing.T) { feed := &api.Feed{ UserId: "1", - URL: "https://www.youtube.com/channel/UC2yTVSttx7lxAOAzx1opjoA", + Provider: api.Youtube, + LinkType: api.Channel, + ItemId: "UC2yTVSttx7lxAOAzx1opjoA", PageSize: 10, Quality: api.HighQuality, Format: api.AudioFormat, } - hash1, err := hid.Encode(feed) + hash1, err := hid.Generate(feed) require.NoError(t, err) require.NotEmpty(t, hash1) // Ensure we have same hash for same feed/parameters - hash2, err := hid.Encode(feed) + hash2, err := hid.Generate(feed) require.NoError(t, err) require.Equal(t, hash1, hash2) feed.UserId = "" - hash3, err := hid.Encode(feed) + hash3, err := hid.Generate(feed) require.NoError(t, err) require.NotEqual(t, hash1, hash3) } diff --git a/web/pkg/parsers/parser.go b/web/pkg/parsers/parser.go new file mode 100644 index 0000000..771f8c6 --- /dev/null +++ b/web/pkg/parsers/parser.go @@ -0,0 +1,146 @@ +package parsers + +import ( + "net/url" + "strings" + + "github.com/mxpv/podsync/web/pkg/api" + "github.com/pkg/errors" +) + +func ParseURL(link string) (*api.Feed, error) { + parsed, err := url.Parse(link) + if err != nil { + err = errors.Wrapf(err, "failed to parse url: %s", link) + return nil, err + } + + feed := &api.Feed{} + + if strings.HasSuffix(parsed.Host, "youtube.com") { + kind, id, err := parseYoutubeURL(parsed) + if err != nil { + return nil, err + } + + feed.Provider = api.Youtube + feed.LinkType = kind + feed.ItemId = id + + return feed, nil + } + + if strings.HasSuffix(parsed.Host, "vimeo.com") { + kind, id, err := parseVimeoURL(parsed) + if err != nil { + return nil, err + } + + feed.Provider = api.Vimeo + feed.LinkType = kind + feed.ItemId = id + + return feed, nil + } + + return nil, errors.New("unsupported URL host") +} + +func parseYoutubeURL(parsed *url.URL) (kind api.LinkType, id string, err error) { + path := parsed.EscapedPath() + + // https://www.youtube.com/playlist?list=PLCB9F975ECF01953C + if strings.HasPrefix(path, "/playlist") { + kind = api.Playlist + + id = parsed.Query().Get("list") + if id != "" { + return + } + + err = errors.New("invalid playlist link") + return + } + + // - https://www.youtube.com/channel/UC5XPnUk8Vvv_pWslhwom6Og + // - https://www.youtube.com/channel/UCrlakW-ewUT8sOod6Wmzyow/videos + if strings.HasPrefix(path, "/channel") { + kind = api.Channel + parts := strings.Split(parsed.EscapedPath(), "/") + if len(parts) <= 2 { + err = errors.New("invalid youtube channel link") + return + } + + id = parts[2] + if id == "" { + err = errors.New("invalid id") + } + + return + } + + // - https://www.youtube.com/user/fxigr1 + if strings.HasPrefix(path, "/user") { + kind = api.User + + parts := strings.Split(parsed.EscapedPath(), "/") + if len(parts) <= 2 { + err = errors.New("invalid user link") + return + } + + id = parts[2] + if id == "" { + err = errors.New("invalid id") + } + + return + } + + err = errors.New("unsupported link format") + return +} + +func parseVimeoURL(parsed *url.URL) (kind api.LinkType, id string, err error) { + parts := strings.Split(parsed.EscapedPath(), "/") + + if len(parts) <= 1 { + err = errors.New("invalid vimeo link path") + return + } + + if parts[1] == "groups" { + kind = api.Group + } else if parts[1] == "channels" { + kind = api.Channel + } else { + kind = api.User + } + + if kind == api.Group || kind == api.Channel { + if len(parts) <= 2 { + err = errors.New("invalid channel link") + return + } + + id = parts[2] + if id == "" { + err = errors.New("invalid id") + } + + return + } + + if kind == api.User { + id = parts[1] + if id == "" { + err = errors.New("invalid id") + } + + return + } + + err = errors.New("unsupported link format") + return +} diff --git a/web/pkg/parsers/parser_test.go b/web/pkg/parsers/parser_test.go new file mode 100644 index 0000000..a9c5f03 --- /dev/null +++ b/web/pkg/parsers/parser_test.go @@ -0,0 +1,107 @@ +package parsers + +import ( + "net/url" + "testing" + + "github.com/mxpv/podsync/web/pkg/api" + "github.com/stretchr/testify/require" +) + +func TestParseYTPlaylist(t *testing.T) { + link, _ := url.ParseRequestURI("https://www.youtube.com/playlist?list=PLCB9F975ECF01953C") + kind, id, err := parseYoutubeURL(link) + require.NoError(t, err) + require.Equal(t, api.Playlist, kind) + require.Equal(t, "PLCB9F975ECF01953C", id) +} + +func TestParseYTChannel(t *testing.T) { + link, _ := url.ParseRequestURI("https://www.youtube.com/channel/UC5XPnUk8Vvv_pWslhwom6Og") + kind, id, err := parseYoutubeURL(link) + require.NoError(t, err) + require.Equal(t, api.Channel, kind) + require.Equal(t, "UC5XPnUk8Vvv_pWslhwom6Og", id) + + link, _ = url.ParseRequestURI("https://www.youtube.com/channel/UCrlakW-ewUT8sOod6Wmzyow/videos") + kind, id, err = parseYoutubeURL(link) + require.NoError(t, err) + require.Equal(t, api.Channel, kind) + require.Equal(t, "UCrlakW-ewUT8sOod6Wmzyow", id) +} + +func TestParseYTUser(t *testing.T) { + link, _ := url.ParseRequestURI("https://youtube.com/user/fxigr1") + kind, id, err := parseYoutubeURL(link) + require.NoError(t, err) + require.Equal(t, api.User, kind) + require.Equal(t, "fxigr1", id) +} + +func TestHandleInvalidYTLink(t *testing.T) { + link, _ := url.ParseRequestURI("https://www.youtube.com/user///") + _, _, err := parseYoutubeURL(link) + require.Error(t, err) + + link, _ = url.ParseRequestURI("https://www.youtube.com/channel//videos") + _, _, err = parseYoutubeURL(link) + require.Error(t, err) +} + +func TestParseVimeoGroupLink(t *testing.T) { + link, _ := url.ParseRequestURI("https://vimeo.com/groups/109") + kind, id, err := parseVimeoURL(link) + require.NoError(t, err) + require.Equal(t, api.Group, kind) + require.Equal(t, "109", id) + + link, _ = url.ParseRequestURI("http://vimeo.com/groups/109") + kind, id, err = parseVimeoURL(link) + require.NoError(t, err) + require.Equal(t, api.Group, kind) + require.Equal(t, "109", id) + + link, _ = url.ParseRequestURI("http://www.vimeo.com/groups/109") + kind, id, err = parseVimeoURL(link) + require.NoError(t, err) + require.Equal(t, api.Group, kind) + require.Equal(t, "109", id) + + link, _ = url.ParseRequestURI("https://vimeo.com/groups/109/videos/") + kind, id, err = parseVimeoURL(link) + require.NoError(t, err) + require.Equal(t, api.Group, kind) + require.Equal(t, "109", id) +} + +func TestParseVimeoChannelLink(t *testing.T) { + link, _ := url.ParseRequestURI("https://vimeo.com/channels/staffpicks") + kind, id, err := parseVimeoURL(link) + require.NoError(t, err) + require.Equal(t, api.Channel, kind) + require.Equal(t, "staffpicks", id) + + link, _ = url.ParseRequestURI("http://vimeo.com/channels/staffpicks/146224925") + kind, id, err = parseVimeoURL(link) + require.NoError(t, err) + require.Equal(t, api.Channel, kind) + require.Equal(t, "staffpicks", id) +} + +func TestParseVimeoUserLink(t *testing.T) { + link, _ := url.ParseRequestURI("https://vimeo.com/awhitelabelproduct") + kind, id, err := parseVimeoURL(link) + require.NoError(t, err) + require.Equal(t, api.User, kind) + require.Equal(t, "awhitelabelproduct", id) +} + +func TestParseInvalidVimeoLink(t *testing.T) { + link, _ := url.ParseRequestURI("http://www.apple.com") + _, _, err := parseVimeoURL(link) + require.Error(t, err) + + link, _ = url.ParseRequestURI("http://www.vimeo.com") + _, _, err = parseVimeoURL(link) + require.Error(t, err) +} diff --git a/web/pkg/storage/pg_sql.go b/web/pkg/storage/pg_sql.go index 72d8543..58a5d2a 100644 --- a/web/pkg/storage/pg_sql.go +++ b/web/pkg/storage/pg_sql.go @@ -5,6 +5,12 @@ BEGIN; DO $$ BEGIN + IF NOT EXISTS (SELECT 1 FROM pg_type WHERE typname = 'provider') THEN + CREATE TYPE quality AS ENUM ('youtube', 'vimeo'); + END IF; + IF NOT EXISTS (SELECT 1 FROM pg_type WHERE typname = 'link_type') THEN + CREATE TYPE quality AS ENUM ('channel', 'playlist', 'user', 'group'); + END IF; IF NOT EXISTS (SELECT 1 FROM pg_type WHERE typname = 'quality') THEN CREATE TYPE quality AS ENUM ('high', 'low'); END IF; @@ -18,10 +24,13 @@ CREATE TABLE IF NOT EXISTS feeds ( id BIGSERIAL PRIMARY KEY, hash_id VARCHAR(12) NOT NULL CHECK (hash_id <> '') UNIQUE, user_id VARCHAR(32) NULL, - url VARCHAR(64) NOT NULL CHECK (url <> ''), + item_id VARCHAR(32) NOT NULL CHECK (item_id <> ''), + provider provider NOT NULL, + link_type link_type NOT NULL, page_size INT NOT NULL DEFAULT 50, - quality quality NOT NULL DEFAULT 'high', format format NOT NULL DEFAULT 'video', + quality quality NOT NULL DEFAULT 'high', + feature_level INT NOT NULL DEFAULT 0, last_access timestamp WITHOUT TIME ZONE NOT NULL ); diff --git a/web/pkg/storage/pg_test.go b/web/pkg/storage/pg_test.go index e2da63e..7d7bd09 100644 --- a/web/pkg/storage/pg_test.go +++ b/web/pkg/storage/pg_test.go @@ -9,8 +9,10 @@ import ( func TestCreate(t *testing.T) { feed := &api.Feed{ - HashId: "xyz", - URL: "http://youtube.com", + HashId: "xyz", + Provider: api.Youtube, + LinkType: api.Channel, + ItemId: "123", } client := createClient(t) @@ -21,8 +23,10 @@ func TestCreate(t *testing.T) { func TestCreateDuplicate(t *testing.T) { feed := &api.Feed{ - HashId: "123", - URL: "http://youtube.com", + HashId: "123", + Provider: api.Youtube, + LinkType: api.Channel, + ItemId: "123", } client := createClient(t) @@ -46,9 +50,11 @@ func TestCreateDuplicate(t *testing.T) { func TestGetFeed(t *testing.T) { feed := &api.Feed{ - HashId: "xyz", - UserId: "123", - URL: "http://youtube.com", + HashId: "xyz", + UserId: "123", + Provider: api.Youtube, + LinkType: api.Channel, + ItemId: "123", } client := createClient(t) @@ -61,9 +67,11 @@ func TestGetFeed(t *testing.T) { func TestUpdateLastAccess(t *testing.T) { feed := &api.Feed{ - HashId: "xyz", - UserId: "123", - URL: "http://youtube.com", + HashId: "xyz", + UserId: "123", + Provider: api.Youtube, + LinkType: api.Channel, + ItemId: "123", } client := createClient(t) @@ -78,7 +86,9 @@ func TestUpdateLastAccess(t *testing.T) { require.NotEmpty(t, last.HashId) require.NotEmpty(t, last.UserId) - require.NotEmpty(t, last.URL) + require.NotEmpty(t, last.Provider) + require.NotEmpty(t, last.LinkType) + require.NotEmpty(t, last.ItemId) require.True(t, last.LastAccess.Unix() > lastAccess.Unix()) } diff --git a/web/pkg/storage/redis.go b/web/pkg/storage/redis.go index c72c700..98fc83c 100644 --- a/web/pkg/storage/redis.go +++ b/web/pkg/storage/redis.go @@ -129,8 +129,6 @@ func (r *RedisStorage) GetFeed(hashId string) (*api.Feed, error) { return nil, err } - feed.URL = url - // Fetch user id patreonId, ok := m["patreonid"] if ok {