pytastic code 3

#293
Raw
Author
Anonymous
Created
Nov. 15, 2020, 8 a.m.
Expires
Never
Size
879.5 KB
Hits
56
Syntax
Python
from __future__ import unicode_literals

import base64
import json
import re

from .common import InfoExtractor
from .theplatform import ThePlatformIE
from .adobepass import AdobePassIE
from ..compat import compat_urllib_parse_unquote
from ..utils import (
    int_or_none,
    js_to_json,
    parse_duration,
    smuggle_url,
    try_get,
    unified_timestamp,
    update_url_query,
)


class NBCIE(AdobePassIE):
    _VALID_URL = r'https?(?P<permalink>://(?:www\.)?nbc\.com/(?:classic-tv/)?[^/]+/video/[^/]+/(?P<id>n?\d+))'

    _TESTS = [
        {
            'url': 'http://www.nbc.com/the-tonight-show/video/jimmy-fallon-surprises-fans-at-ben-jerrys/2848237',
            'info_dict': {
                'id': '2848237',
                'ext': 'mp4',
                'title': 'Jimmy Fallon Surprises Fans at Ben & Jerry\'s',
                'description': 'Jimmy gives out free scoops of his new "Tonight Dough" ice cream flavor by surprising customers at the Ben & Jerry\'s scoop shop.',
                'timestamp': 1424246400,
                'upload_date': '20150218',
                'uploader': 'NBCU-COM',
            },
            'params': {
                # m3u8 download
                'skip_download': True,
            },
        },
        {
            'url': 'http://www.nbc.com/saturday-night-live/video/star-wars-teaser/2832821',
            'info_dict': {
                'id': '2832821',
                'ext': 'mp4',
                'title': 'Star Wars Teaser',
                'description': 'md5:0b40f9cbde5b671a7ff62fceccc4f442',
                'timestamp': 1417852800,
                'upload_date': '20141206',
                'uploader': 'NBCU-COM',
            },
            'params': {
                # m3u8 download
                'skip_download': True,
            },
            'skip': 'Only works from US',
        },
        {
            # HLS streams requires the 'hdnea3' cookie
            'url': 'http://www.nbc.com/Kings/video/goliath/n1806',
            'info_dict': {
                'id': '101528f5a9e8127b107e98c5e6ce4638',
                'ext': 'mp4',
                'title': 'Goliath',
                'description': 'When an unknown soldier saves the life of the King\'s son in battle, he\'s thrust into the limelight and politics of the kingdom.',
                'timestamp': 1237100400,
                'upload_date': '20090315',
                'uploader': 'NBCU-COM',
            },
            'params': {
                'skip_download': True,
            },
            'skip': 'Only works from US',
        },
        {
            'url': 'https://www.nbc.com/classic-tv/charles-in-charge/video/charles-in-charge-pilot/n3310',
            'only_matching': True,
        },
        {
            # Percent escaped url
            'url': 'https://www.nbc.com/up-all-night/video/day-after-valentine%27s-day/n2189',
            'only_matching': True,
        }
    ]

    def _real_extract(self, url):
        permalink, video_id = re.match(self._VALID_URL, url).groups()
        permalink = 'http' + compat_urllib_parse_unquote(permalink)
        video_data = self._download_json(
            'https://friendship.nbc.co/v2/graphql', video_id, query={
                'query': '''query bonanzaPage(
  $app: NBCUBrands! = nbc
  $name: String!
  $oneApp: Boolean
  $platform: SupportedPlatforms! = web
  $type: EntityPageType! = VIDEO
  $userId: String!
) {
  bonanzaPage(
    app: $app
    name: $name
    oneApp: $oneApp
    platform: $platform
    type: $type
    userId: $userId
  ) {
    metadata {
      ... on VideoPageData {
        description
        episodeNumber
        keywords
        locked
        mpxAccountId
        mpxGuid
        rating
        resourceId
        seasonNumber
        secondaryTitle
        seriesShortTitle
      }
    }
  }
}''',
                'variables': json.dumps({
                    'name': permalink,
                    'oneApp': True,
                    'userId': '0',
                }),
            })['data']['bonanzaPage']['metadata']
        query = {
            'mbr': 'true',
            'manifest': 'm3u',
        }
        video_id = video_data['mpxGuid']
        title = video_data['secondaryTitle']
        if video_data.get('locked'):
            resource = self._get_mvpd_resource(
                video_data.get('resourceId') or 'nbcentertainment',
                title, video_id, video_data.get('rating'))
            query['auth'] = self._extract_mvpd_auth(
                url, video_id, 'nbcentertainment', resource)
        theplatform_url = smuggle_url(update_url_query(
            'http://link.theplatform.com/s/NnzsPC/media/guid/%s/%s' % (video_data.get('mpxAccountId') or '2410887629', video_id),
            query), {'force_smil_url': True})
        return {
            '_type': 'url_transparent',
            'id': video_id,
            'title': title,
            'url': theplatform_url,
            'description': video_data.get('description'),
            'tags': video_data.get('keywords'),
            'season_number': int_or_none(video_data.get('seasonNumber')),
            'episode_number': int_or_none(video_data.get('episodeNumber')),
            'episode': title,
            'series': video_data.get('seriesShortTitle'),
            'ie_key': 'ThePlatform',
        }


class NBCSportsVPlayerIE(InfoExtractor):
    _VALID_URL = r'https?://vplayer\.nbcsports\.com/(?:[^/]+/)+(?P<id>[0-9a-zA-Z_]+)'

    _TESTS = [{
        'url': 'https://vplayer.nbcsports.com/p/BxmELC/nbcsports_embed/select/9CsDKds0kvHI',
        'info_dict': {
            'id': '9CsDKds0kvHI',
            'ext': 'mp4',
            'description': 'md5:df390f70a9ba7c95ff1daace988f0d8d',
            'title': 'Tyler Kalinoski hits buzzer-beater to lift Davidson',
            'timestamp': 1426270238,
            'upload_date': '20150313',
            'uploader': 'NBCU-SPORTS',
        }
    }, {
        'url': 'https://vplayer.nbcsports.com/p/BxmELC/nbcsports_embed/select/media/_hqLjQ95yx8Z',
        'only_matching': True,
    }]

    @staticmethod
    def _extract_url(webpage):
        iframe_m = re.search(
            r'<iframe[^>]+src="(?P<url>https?://vplayer\.nbcsports\.com/[^"]+)"', webpage)
        if iframe_m:
            return iframe_m.group('url')

    def _real_extract(self, url):
        video_id = self._match_id(url)
        webpage = self._download_webpage(url, video_id)
        theplatform_url = self._og_search_video_url(webpage).replace(
            'vplayer.nbcsports.com', 'player.theplatform.com')
        return self.url_result(theplatform_url, 'ThePlatform')


class NBCSportsIE(InfoExtractor):
    # Does not include https because its certificate is invalid
    _VALID_URL = r'https?://(?:www\.)?nbcsports\.com//?(?:[^/]+/)+(?P<id>[0-9a-z-]+)'

    _TEST = {
        'url': 'http://www.nbcsports.com//college-basketball/ncaab/tom-izzo-michigan-st-has-so-much-respect-duke',
        'info_dict': {
            'id': 'PHJSaFWbrTY9',
            'ext': 'flv',
            'title': 'Tom Izzo, Michigan St. has \'so much respect\' for Duke',
            'description': 'md5:ecb459c9d59e0766ac9c7d5d0eda8113',
            'uploader': 'NBCU-SPORTS',
            'upload_date': '20150330',
            'timestamp': 1427726529,
        }
    }

    def _real_extract(self, url):
        video_id = self._match_id(url)
        webpage = self._download_webpage(url, video_id)
        return self.url_result(
            NBCSportsVPlayerIE._extract_url(webpage), 'NBCSportsVPlayer')


class NBCSportsStreamIE(AdobePassIE):
    _VALID_URL = r'https?://stream\.nbcsports\.com/.+?\bpid=(?P<id>\d+)'
    _TEST = {
        'url': 'http://stream.nbcsports.com/nbcsn/generic?pid=206559',
        'info_dict': {
            'id': '206559',
            'ext': 'mp4',
            'title': 'Amgen Tour of California Women\'s Recap',
            'description': 'md5:66520066b3b5281ada7698d0ea2aa894',
        },
        'params': {
            # m3u8 download
            'skip_download': True,
        },
        'skip': 'Requires Adobe Pass Authentication',
    }

    def _real_extract(self, url):
        video_id = self._match_id(url)
        live_source = self._download_json(
            'http://stream.nbcsports.com/data/live_sources_%s.json' % video_id,
            video_id)
        video_source = live_source['videoSources'][0]
        title = video_source['title']
        source_url = None
        for k in ('source', 'msl4source', 'iossource', 'hlsv4'):
            sk = k + 'Url'
            source_url = video_source.get(sk) or video_source.get(sk + 'Alt')
            if source_url:
                break
        else:
            source_url = video_source['ottStreamUrl']
        is_live = video_source.get('type') == 'live' or video_source.get('status') == 'Live'
        resource = self._get_mvpd_resource('nbcsports', title, video_id, '')
        token = self._extract_mvpd_auth(url, video_id, 'nbcsports', resource)
        tokenized_url = self._download_json(
            'https://token.playmakerservices.com/cdn',
            video_id, data=json.dumps({
                'requestorId': 'nbcsports',
                'pid': video_id,
                'application': 'NBCSports',
                'version': 'v1',
                'platform': 'desktop',
                'cdn': 'akamai',
                'url': video_source['sourceUrl'],
                'token': base64.b64encode(token.encode()).decode(),
                'resourceId': base64.b64encode(resource.encode()).decode(),
            }).encode())['tokenizedUrl']
        formats = self._extract_m3u8_formats(tokenized_url, video_id, 'mp4')
        self._sort_formats(formats)
        return {
            'id': video_id,
            'title': self._live_title(title) if is_live else title,
            'description': live_source.get('description'),
            'formats': formats,
            'is_live': is_live,
        }


class CSNNEIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?csnne\.com/video/(?P<id>[0-9a-z-]+)'

    _TEST = {
        'url': 'http://www.csnne.com/video/snc-evening-update-wright-named-red-sox-no-5-starter',
        'info_dict': {
            'id': 'yvBLLUgQ8WU0',
            'ext': 'mp4',
            'title': 'SNC evening update: Wright named Red Sox\' No. 5 starter.',
            'description': 'md5:1753cfee40d9352b19b4c9b3e589b9e3',
            'timestamp': 1459369979,
            'upload_date': '20160330',
            'uploader': 'NBCU-SPORTS',
        }
    }

    def _real_extract(self, url):
        display_id = self._match_id(url)
        webpage = self._download_webpage(url, display_id)
        return {
            '_type': 'url_transparent',
            'ie_key': 'ThePlatform',
            'url': self._html_search_meta('twitter:player:stream', webpage),
            'display_id': display_id,
        }


class NBCNewsIE(ThePlatformIE):
    _VALID_URL = r'(?x)https?://(?:www\.)?(?:nbcnews|today|msnbc)\.com/([^/]+/)*(?:.*-)?(?P<id>[^/?]+)'

    _TESTS = [
        {
            'url': 'http://www.nbcnews.com/watch/nbcnews-com/how-twitter-reacted-to-the-snowden-interview-269389891880',
            'md5': 'cf4bc9e6ce0130f00f545d80ecedd4bf',
            'info_dict': {
                'id': '269389891880',
                'ext': 'mp4',
                'title': 'How Twitter Reacted To The Snowden Interview',
                'description': 'md5:65a0bd5d76fe114f3c2727aa3a81fe64',
                'timestamp': 1401363060,
                'upload_date': '20140529',
            },
        },
        {
            'url': 'http://www.nbcnews.com/feature/dateline-full-episodes/full-episode-family-business-n285156',
            'md5': 'fdbf39ab73a72df5896b6234ff98518a',
            'info_dict': {
                'id': '529953347624',
                'ext': 'mp4',
                'title': 'FULL EPISODE: Family Business',
                'description': 'md5:757988edbaae9d7be1d585eb5d55cc04',
            },
            'skip': 'This page is unavailable.',
        },
        {
            'url': 'http://www.nbcnews.com/nightly-news/video/nightly-news-with-brian-williams-full-broadcast-february-4-394064451844',
            'md5': '8eb831eca25bfa7d25ddd83e85946548',
            'info_dict': {
                'id': '394064451844',
                'ext': 'mp4',
                'title': 'Nightly News with Brian Williams Full Broadcast (February 4)',
                'description': 'md5:1c10c1eccbe84a26e5debb4381e2d3c5',
                'timestamp': 1423104900,
                'upload_date': '20150205',
            },
        },
        {
            'url': 'http://www.nbcnews.com/business/autos/volkswagen-11-million-vehicles-could-have-suspect-software-emissions-scandal-n431456',
            'md5': '4a8c4cec9e1ded51060bdda36ff0a5c0',
            'info_dict': {
                'id': 'n431456',
                'ext': 'mp4',
                'title': "Volkswagen U.S. Chief:  We 'Totally Screwed Up'",
                'description': 'md5:d22d1281a24f22ea0880741bb4dd6301',
                'upload_date': '20150922',
                'timestamp': 1442917800,
            },
        },
        {
            'url': 'http://www.today.com/video/see-the-aurora-borealis-from-space-in-stunning-new-nasa-video-669831235788',
            'md5': '118d7ca3f0bea6534f119c68ef539f71',
            'info_dict': {
                'id': '669831235788',
                'ext': 'mp4',
                'title': 'See the aurora borealis from space in stunning new NASA video',
                'description': 'md5:74752b7358afb99939c5f8bb2d1d04b1',
                'upload_date': '20160420',
                'timestamp': 1461152093,
            },
        },
        {
            'url': 'http://www.msnbc.com/all-in-with-chris-hayes/watch/the-chaotic-gop-immigration-vote-314487875924',
            'md5': '6d236bf4f3dddc226633ce6e2c3f814d',
            'info_dict': {
                'id': '314487875924',
                'ext': 'mp4',
                'title': 'The chaotic GOP immigration vote',
                'description': 'The Republican House votes on a border bill that has no chance of getting through the Senate or signed by the President and is drawing criticism from all sides.',
                'thumbnail': r're:^https?://.*\.jpg$',
                'timestamp': 1406937606,
                'upload_date': '20140802',
            },
        },
        {
            'url': 'http://www.nbcnews.com/watch/dateline/full-episode--deadly-betrayal-386250819952',
            'only_matching': True,
        },
        {
            # From http://www.vulture.com/2016/06/letterman-couldnt-care-less-about-late-night.html
            'url': 'http://www.nbcnews.com/widget/video-embed/701714499682',
            'only_matching': True,
        },
    ]

    def _real_extract(self, url):
        video_id = self._match_id(url)
        webpage = self._download_webpage(url, video_id)

        data = self._parse_json(self._search_regex(
            r'window\.__data\s*=\s*({.+});', webpage,
            'bootstrap json'), video_id, js_to_json)
        video_data = try_get(data, lambda x: x['video']['current'], dict)
        if not video_data:
            video_data = data['article']['content'][0]['primaryMedia']['video']
        title = video_data['headline']['primary']

        formats = []
        for va in video_data.get('videoAssets', []):
            public_url = va.get('publicUrl')
            if not public_url:
                continue
            if '://link.theplatform.com/' in public_url:
                public_url = update_url_query(public_url, {'format': 'redirect'})
            format_id = va.get('format')
            if format_id == 'M3U':
                formats.extend(self._extract_m3u8_formats(
                    public_url, video_id, 'mp4', 'm3u8_native',
                    m3u8_id=format_id, fatal=False))
                continue
            tbr = int_or_none(va.get('bitrate'), 1000)
            if tbr:
                format_id += '-%d' % tbr
            formats.append({
                'format_id': format_id,
                'url': public_url,
                'width': int_or_none(va.get('width')),
                'height': int_or_none(va.get('height')),
                'tbr': tbr,
                'ext': 'mp4',
            })
        self._sort_formats(formats)

        subtitles = {}
        closed_captioning = video_data.get('closedCaptioning')
        if closed_captioning:
            for cc_url in closed_captioning.values():
                if not cc_url:
                    continue
                subtitles.setdefault('en', []).append({
                    'url': cc_url,
                })

        return {
            'id': video_id,
            'title': title,
            'description': try_get(video_data, lambda x: x['description']['primary']),
            'thumbnail': try_get(video_data, lambda x: x['primaryImage']['url']['primary']),
            'duration': parse_duration(video_data.get('duration')),
            'timestamp': unified_timestamp(video_data.get('datePublished')),
            'formats': formats,
            'subtitles': subtitles,
        }


class NBCOlympicsIE(InfoExtractor):
    IE_NAME = 'nbcolympics'
    _VALID_URL = r'https?://www\.nbcolympics\.com/video/(?P<id>[a-z-]+)'

    _TEST = {
        # Geo-restricted to US
        'url': 'http://www.nbcolympics.com/video/justin-roses-son-leo-was-tears-after-his-dad-won-gold',
        'md5': '54fecf846d05429fbaa18af557ee523a',
        'info_dict': {
            'id': 'WjTBzDXx5AUq',
            'display_id': 'justin-roses-son-leo-was-tears-after-his-dad-won-gold',
            'ext': 'mp4',
            'title': 'Rose\'s son Leo was in tears after his dad won gold',
            'description': 'Olympic gold medalist Justin Rose gets emotional talking to the impact his win in men\'s golf has already had on his children.',
            'timestamp': 1471274964,
            'upload_date': '20160815',
            'uploader': 'NBCU-SPORTS',
        },
    }

    def _real_extract(self, url):
        display_id = self._match_id(url)

        webpage = self._download_webpage(url, display_id)

        drupal_settings = self._parse_json(self._search_regex(
            r'jQuery\.extend\(Drupal\.settings\s*,\s*({.+?})\);',
            webpage, 'drupal settings'), display_id)

        iframe_url = drupal_settings['vod']['iframe_url']
        theplatform_url = iframe_url.replace(
            'vplayer.nbcolympics.com', 'player.theplatform.com')

        return {
            '_type': 'url_transparent',
            'url': theplatform_url,
            'ie_key': ThePlatformIE.ie_key(),
            'display_id': display_id,
        }


class NBCOlympicsStreamIE(AdobePassIE):
    IE_NAME = 'nbcolympics:stream'
    _VALID_URL = r'https?://stream\.nbcolympics\.com/(?P<id>[0-9a-z-]+)'
    _TEST = {
        'url': 'http://stream.nbcolympics.com/2018-winter-olympics-nbcsn-evening-feb-8',
        'info_dict': {
            'id': '203493',
            'ext': 'mp4',
            'title': 're:Curling, Alpine, Luge [0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}$',
        },
        'params': {
            # m3u8 download
            'skip_download': True,
        },
    }
    _DATA_URL_TEMPLATE = 'http://stream.nbcolympics.com/data/%s_%s.json'

    def _real_extract(self, url):
        display_id = self._match_id(url)
        webpage = self._download_webpage(url, display_id)
        pid = self._search_regex(r'pid\s*=\s*(\d+);', webpage, 'pid')
        resource = self._search_regex(
            r"resource\s*=\s*'(.+)';", webpage,
            'resource').replace("' + pid + '", pid)
        event_config = self._download_json(
            self._DATA_URL_TEMPLATE % ('event_config', pid),
            pid)['eventConfig']
        title = self._live_title(event_config['eventTitle'])
        source_url = self._download_json(
            self._DATA_URL_TEMPLATE % ('live_sources', pid),
            pid)['videoSources'][0]['sourceUrl']
        media_token = self._extract_mvpd_auth(
            url, pid, event_config.get('requestorId', 'NBCOlympics'), resource)
        formats = self._extract_m3u8_formats(self._download_webpage(
            'http://sp.auth.adobe.com/tvs/v1/sign', pid, query={
                'cdn': 'akamai',
                'mediaToken': base64.b64encode(media_token.encode()),
                'resource': base64.b64encode(resource.encode()),
                'url': source_url,
            }), pid, 'mp4')
        self._sort_formats(formats)

        return {
            'id': pid,
            'display_id': display_id,
            'title': title,
            'formats': formats,
            'is_live': True,
        }
# coding: utf-8
from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..utils import (
    determine_ext,
    int_or_none,
    merge_dicts,
    parse_iso8601,
    qualities,
    try_get,
    urljoin,
)


class NDRBaseIE(InfoExtractor):
    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        display_id = next(group for group in mobj.groups() if group)
        webpage = self._download_webpage(url, display_id)
        return self._extract_embed(webpage, display_id)


class NDRIE(NDRBaseIE):
    IE_NAME = 'ndr'
    IE_DESC = 'NDR.de - Norddeutscher Rundfunk'
    _VALID_URL = r'https?://(?:www\.)?ndr\.de/(?:[^/]+/)*(?P<id>[^/?#]+),[\da-z]+\.html'
    _TESTS = [{
        # httpVideo, same content id
        'url': 'http://www.ndr.de/fernsehen/Party-Poette-und-Parade,hafengeburtstag988.html',
        'md5': '6515bc255dc5c5f8c85bbc38e035a659',
        'info_dict': {
            'id': 'hafengeburtstag988',
            'display_id': 'Party-Poette-und-Parade',
            'ext': 'mp4',
            'title': 'Party, Pötte und Parade',
            'description': 'md5:ad14f9d2f91d3040b6930c697e5f6b4c',
            'uploader': 'ndrtv',
            'timestamp': 1431108900,
            'upload_date': '20150510',
            'duration': 3498,
        },
        'params': {
            'skip_download': True,
        },
    }, {
        # httpVideo, different content id
        'url': 'http://www.ndr.de/sport/fussball/40-Osnabrueck-spielt-sich-in-einen-Rausch,osna270.html',
        'md5': '1043ff203eab307f0c51702ec49e9a71',
        'info_dict': {
            'id': 'osna272',
            'display_id': '40-Osnabrueck-spielt-sich-in-einen-Rausch',
            'ext': 'mp4',
            'title': 'Osnabrück - Wehen Wiesbaden: Die Highlights',
            'description': 'md5:32e9b800b3d2d4008103752682d5dc01',
            'uploader': 'ndrtv',
            'timestamp': 1442059200,
            'upload_date': '20150912',
            'duration': 510,
        },
        'params': {
            'skip_download': True,
        },
    }, {
        # httpAudio, same content id
        'url': 'http://www.ndr.de/info/La-Valette-entgeht-der-Hinrichtung,audio51535.html',
        'md5': 'bb3cd38e24fbcc866d13b50ca59307b8',
        'info_dict': {
            'id': 'audio51535',
            'display_id': 'La-Valette-entgeht-der-Hinrichtung',
            'ext': 'mp3',
            'title': 'La Valette entgeht der Hinrichtung',
            'description': 'md5:22f9541913a40fe50091d5cdd7c9f536',
            'uploader': 'ndrinfo',
            'timestamp': 1290626100,
            'upload_date': '20140729',
            'duration': 884,
        },
        'params': {
            'skip_download': True,
        },
    }, {
        'url': 'https://www.ndr.de/Fettes-Brot-Ferris-MC-und-Thees-Uhlmann-live-on-stage,festivalsommer116.html',
        'only_matching': True,
    }]

    def _extract_embed(self, webpage, display_id):
        embed_url = self._html_search_meta(
            'embedURL', webpage, 'embed URL',
            default=None) or self._search_regex(
            r'\bembedUrl["\']\s*:\s*(["\'])(?P<url>(?:(?!\1).)+)\1', webpage,
            'embed URL', group='url')
        description = self._search_regex(
            r'<p[^>]+itemprop="description">([^<]+)</p>',
            webpage, 'description', default=None) or self._og_search_description(webpage)
        timestamp = parse_iso8601(
            self._search_regex(
                r'<span[^>]+itemprop="(?:datePublished|uploadDate)"[^>]+content="([^"]+)"',
                webpage, 'upload date', default=None))
        info = self._search_json_ld(webpage, display_id, default={})
        return merge_dicts({
            '_type': 'url_transparent',
            'url': embed_url,
            'display_id': display_id,
            'description': description,
            'timestamp': timestamp,
        }, info)


class NJoyIE(NDRBaseIE):
    IE_NAME = 'njoy'
    IE_DESC = 'N-JOY'
    _VALID_URL = r'https?://(?:www\.)?n-joy\.de/(?:[^/]+/)*(?:(?P<display_id>[^/?#]+),)?(?P<id>[\da-z]+)\.html'
    _TESTS = [{
        # httpVideo, same content id
        'url': 'http://www.n-joy.de/entertainment/comedy/comedy_contest/Benaissa-beim-NDR-Comedy-Contest,comedycontest2480.html',
        'md5': 'cb63be60cd6f9dd75218803146d8dc67',
        'info_dict': {
            'id': 'comedycontest2480',
            'display_id': 'Benaissa-beim-NDR-Comedy-Contest',
            'ext': 'mp4',
            'title': 'Benaissa beim NDR Comedy Contest',
            'description': 'md5:f057a6c4e1c728b10d33b5ffd36ddc39',
            'uploader': 'ndrtv',
            'upload_date': '20141129',
            'duration': 654,
        },
        'params': {
            'skip_download': True,
        },
    }, {
        # httpVideo, different content id
        'url': 'http://www.n-joy.de/musik/Das-frueheste-DJ-Set-des-Nordens-live-mit-Felix-Jaehn-,felixjaehn168.html',
        'md5': '417660fffa90e6df2fda19f1b40a64d8',
        'info_dict': {
            'id': 'dockville882',
            'display_id': 'Das-frueheste-DJ-Set-des-Nordens-live-mit-Felix-Jaehn-',
            'ext': 'mp4',
            'title': '"Ich hab noch nie" mit Felix Jaehn',
            'description': 'md5:85dd312d53be1b99e1f998a16452a2f3',
            'uploader': 'njoy',
            'upload_date': '20150822',
            'duration': 211,
        },
        'params': {
            'skip_download': True,
        },
    }, {
        'url': 'http://www.n-joy.de/radio/webradio/morningshow209.html',
        'only_matching': True,
    }]

    def _extract_embed(self, webpage, display_id):
        video_id = self._search_regex(
            r'<iframe[^>]+id="pp_([\da-z]+)"', webpage, 'embed id')
        description = self._search_regex(
            r'<div[^>]+class="subline"[^>]*>[^<]+</div>\s*<p>([^<]+)</p>',
            webpage, 'description', fatal=False)
        return {
            '_type': 'url_transparent',
            'ie_key': 'NDREmbedBase',
            'url': 'ndr:%s' % video_id,
            'display_id': display_id,
            'description': description,
        }


class NDREmbedBaseIE(InfoExtractor):
    IE_NAME = 'ndr:embed:base'
    _VALID_URL = r'(?:ndr:(?P<id_s>[\da-z]+)|https?://www\.ndr\.de/(?P<id>[\da-z]+)-ppjson\.json)'
    _TESTS = [{
        'url': 'ndr:soundcheck3366',
        'only_matching': True,
    }, {
        'url': 'http://www.ndr.de/soundcheck3366-ppjson.json',
        'only_matching': True,
    }]

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        video_id = mobj.group('id') or mobj.group('id_s')

        ppjson = self._download_json(
            'http://www.ndr.de/%s-ppjson.json' % video_id, video_id)

        playlist = ppjson['playlist']

        formats = []
        quality_key = qualities(('xs', 's', 'm', 'l', 'xl'))

        for format_id, f in playlist.items():
            src = f.get('src')
            if not src:
                continue
            ext = determine_ext(src, None)
            if ext == 'f4m':
                formats.extend(self._extract_f4m_formats(
                    src + '?hdcore=3.7.0&plugin=aasp-3.7.0.39.44', video_id,
                    f4m_id='hds', fatal=False))
            elif ext == 'm3u8':
                formats.extend(self._extract_m3u8_formats(
                    src, video_id, 'mp4', m3u8_id='hls',
                    entry_protocol='m3u8_native', fatal=False))
            else:
                quality = f.get('quality')
                ff = {
                    'url': src,
                    'format_id': quality or format_id,
                    'quality': quality_key(quality),
                }
                type_ = f.get('type')
                if type_ and type_.split('/')[0] == 'audio':
                    ff['vcodec'] = 'none'
                    ff['ext'] = ext or 'mp3'
                formats.append(ff)
        self._sort_formats(formats)

        config = playlist['config']

        live = playlist.get('config', {}).get('streamType') in ['httpVideoLive', 'httpAudioLive']
        title = config['title']
        if live:
            title = self._live_title(title)
        uploader = ppjson.get('config', {}).get('branding')
        upload_date = ppjson.get('config', {}).get('publicationDate')
        duration = int_or_none(config.get('duration'))

        thumbnails = []
        poster = try_get(config, lambda x: x['poster'], dict) or {}
        for thumbnail_id, thumbnail in poster.items():
            thumbnail_url = urljoin(url, thumbnail.get('src'))
            if not thumbnail_url:
                continue
            thumbnails.append({
                'id': thumbnail.get('quality') or thumbnail_id,
                'url': thumbnail_url,
                'preference': quality_key(thumbnail.get('quality')),
            })

        return {
            'id': video_id,
            'title': title,
            'is_live': live,
            'uploader': uploader if uploader != '-' else None,
            'upload_date': upload_date[0:8] if upload_date else None,
            'duration': duration,
            'thumbnails': thumbnails,
            'formats': formats,
        }


class NDREmbedIE(NDREmbedBaseIE):
    IE_NAME = 'ndr:embed'
    _VALID_URL = r'https?://(?:www\.)?ndr\.de/(?:[^/]+/)*(?P<id>[\da-z]+)-(?:player|externalPlayer)\.html'
    _TESTS = [{
        'url': 'http://www.ndr.de/fernsehen/sendungen/ndr_aktuell/ndraktuell28488-player.html',
        'md5': '8b9306142fe65bbdefb5ce24edb6b0a9',
        'info_dict': {
            'id': 'ndraktuell28488',
            'ext': 'mp4',
            'title': 'Norddeutschland begrüßt Flüchtlinge',
            'is_live': False,
            'uploader': 'ndrtv',
            'upload_date': '20150907',
            'duration': 132,
        },
    }, {
        'url': 'http://www.ndr.de/ndr2/events/soundcheck/soundcheck3366-player.html',
        'md5': '002085c44bae38802d94ae5802a36e78',
        'info_dict': {
            'id': 'soundcheck3366',
            'ext': 'mp4',
            'title': 'Ella Henderson braucht Vergleiche nicht zu scheuen',
            'is_live': False,
            'uploader': 'ndr2',
            'upload_date': '20150912',
            'duration': 3554,
        },
        'params': {
            'skip_download': True,
        },
    }, {
        'url': 'http://www.ndr.de/info/audio51535-player.html',
        'md5': 'bb3cd38e24fbcc866d13b50ca59307b8',
        'info_dict': {
            'id': 'audio51535',
            'ext': 'mp3',
            'title': 'La Valette entgeht der Hinrichtung',
            'is_live': False,
            'uploader': 'ndrinfo',
            'upload_date': '20140729',
            'duration': 884,
        },
        'params': {
            'skip_download': True,
        },
    }, {
        'url': 'http://www.ndr.de/fernsehen/sendungen/visite/visite11010-externalPlayer.html',
        'md5': 'ae57f80511c1e1f2fd0d0d3d31aeae7c',
        'info_dict': {
            'id': 'visite11010',
            'ext': 'mp4',
            'title': 'Visite - die ganze Sendung',
            'is_live': False,
            'uploader': 'ndrtv',
            'upload_date': '20150902',
            'duration': 3525,
        },
        'params': {
            'skip_download': True,
        },
    }, {
        # httpVideoLive
        'url': 'http://www.ndr.de/fernsehen/livestream/livestream217-externalPlayer.html',
        'info_dict': {
            'id': 'livestream217',
            'ext': 'flv',
            'title': r're:^NDR Fernsehen Niedersachsen \d{4}-\d{2}-\d{2} \d{2}:\d{2}$',
            'is_live': True,
            'upload_date': '20150910',
        },
        'params': {
            'skip_download': True,
        },
    }, {
        'url': 'http://www.ndr.de/ndrkultur/audio255020-player.html',
        'only_matching': True,
    }, {
        'url': 'http://www.ndr.de/fernsehen/sendungen/nordtour/nordtour7124-player.html',
        'only_matching': True,
    }, {
        'url': 'http://www.ndr.de/kultur/film/videos/videoimport10424-player.html',
        'only_matching': True,
    }, {
        'url': 'http://www.ndr.de/fernsehen/sendungen/hamburg_journal/hamj43006-player.html',
        'only_matching': True,
    }, {
        'url': 'http://www.ndr.de/fernsehen/sendungen/weltbilder/weltbilder4518-player.html',
        'only_matching': True,
    }, {
        'url': 'http://www.ndr.de/fernsehen/doku952-player.html',
        'only_matching': True,
    }]


class NJoyEmbedIE(NDREmbedBaseIE):
    IE_NAME = 'njoy:embed'
    _VALID_URL = r'https?://(?:www\.)?n-joy\.de/(?:[^/]+/)*(?P<id>[\da-z]+)-(?:player|externalPlayer)_[^/]+\.html'
    _TESTS = [{
        # httpVideo
        'url': 'http://www.n-joy.de/events/reeperbahnfestival/doku948-player_image-bc168e87-5263-4d6d-bd27-bb643005a6de_theme-n-joy.html',
        'md5': '8483cbfe2320bd4d28a349d62d88bd74',
        'info_dict': {
            'id': 'doku948',
            'ext': 'mp4',
            'title': 'Zehn Jahre Reeperbahn Festival - die Doku',
            'is_live': False,
            'upload_date': '20150807',
            'duration': 1011,
        },
    }, {
        # httpAudio
        'url': 'http://www.n-joy.de/news_wissen/stefanrichter100-player_image-d5e938b1-f21a-4b9a-86b8-aaba8bca3a13_theme-n-joy.html',
        'md5': 'd989f80f28ac954430f7b8a48197188a',
        'info_dict': {
            'id': 'stefanrichter100',
            'ext': 'mp3',
            'title': 'Interview mit einem Augenzeugen',
            'is_live': False,
            'uploader': 'njoy',
            'upload_date': '20150909',
            'duration': 140,
        },
        'params': {
            'skip_download': True,
        },
    }, {
        # httpAudioLive, no explicit ext
        'url': 'http://www.n-joy.de/news_wissen/webradioweltweit100-player_image-3fec0484-2244-4565-8fb8-ed25fd28b173_theme-n-joy.html',
        'info_dict': {
            'id': 'webradioweltweit100',
            'ext': 'mp3',
            'title': r're:^N-JOY Weltweit \d{4}-\d{2}-\d{2} \d{2}:\d{2}$',
            'is_live': True,
            'uploader': 'njoy',
            'upload_date': '20150810',
        },
        'params': {
            'skip_download': True,
        },
    }, {
        'url': 'http://www.n-joy.de/musik/dockville882-player_image-3905259e-0803-4764-ac72-8b7de077d80a_theme-n-joy.html',
        'only_matching': True,
    }, {
        'url': 'http://www.n-joy.de/radio/sendungen/morningshow/urlaubsfotos190-player_image-066a5df1-5c95-49ec-a323-941d848718db_theme-n-joy.html',
        'only_matching': True,
    }, {
        'url': 'http://www.n-joy.de/entertainment/comedy/krudetv290-player_image-ab261bfe-51bf-4bf3-87ba-c5122ee35b3d_theme-n-joy.html',
        'only_matching': True,
    }]
# coding: utf-8
from __future__ import unicode_literals

from .common import InfoExtractor
from ..compat import (
    compat_urllib_parse_unquote_plus
)
from ..utils import (
    parse_duration,
    remove_end,
    unified_strdate,
    urljoin
)


class NDTVIE(InfoExtractor):
    _VALID_URL = r'https?://(?:[^/]+\.)?ndtv\.com/(?:[^/]+/)*videos?/?(?:[^/]+/)*[^/?^&]+-(?P<id>\d+)'

    _TESTS = [
        {
            'url': 'https://khabar.ndtv.com/video/show/prime-time/prime-time-ill-system-and-poor-education-468818',
            'md5': '78efcf3880ef3fd9b83d405ca94a38eb',
            'info_dict': {
                'id': '468818',
                'ext': 'mp4',
                'title': "प्राइम टाइम: सिस्टम बीमार, स्कूल बदहाल",
                'description': 'md5:f410512f1b49672e5695dea16ef2731d',
                'upload_date': '20170928',
                'duration': 2218,
                'thumbnail': r're:https?://.*\.jpg',
            }
        },
        {
            # __filename is url
            'url': 'http://movies.ndtv.com/videos/cracker-free-diwali-wishes-from-karan-johar-kriti-sanon-other-stars-470304',
            'md5': 'f1d709352305b44443515ac56b45aa46',
            'info_dict': {
                'id': '470304',
                'ext': 'mp4',
                'title': "Cracker-Free Diwali Wishes From Karan Johar, Kriti Sanon & Other Stars",
                'description': 'md5:f115bba1adf2f6433fa7c1ade5feb465',
                'upload_date': '20171019',
                'duration': 137,
                'thumbnail': r're:https?://.*\.jpg',
            }
        },
        {
            'url': 'https://www.ndtv.com/video/news/news/delhi-s-air-quality-status-report-after-diwali-is-very-poor-470372',
            'only_matching': True
        },
        {
            'url': 'https://auto.ndtv.com/videos/the-cnb-daily-october-13-2017-469935',
            'only_matching': True
        },
        {
            'url': 'https://sports.ndtv.com/cricket/videos/2nd-t20i-rock-thrown-at-australia-cricket-team-bus-after-win-over-india-469764',
            'only_matching': True
        },
        {
            'url': 'http://gadgets.ndtv.com/videos/uncharted-the-lost-legacy-review-465568',
            'only_matching': True
        },
        {
            'url': 'http://profit.ndtv.com/videos/news/video-indian-economy-on-very-solid-track-international-monetary-fund-chief-470040',
            'only_matching': True
        },
        {
            'url': 'http://food.ndtv.com/video-basil-seeds-coconut-porridge-419083',
            'only_matching': True
        },
        {
            'url': 'https://doctor.ndtv.com/videos/top-health-stories-of-the-week-467396',
            'only_matching': True
        },
        {
            'url': 'https://swirlster.ndtv.com/video/how-to-make-friends-at-work-469324',
            'only_matching': True
        }
    ]

    def _real_extract(self, url):
        video_id = self._match_id(url)
        webpage = self._download_webpage(url, video_id)

        # '__title' does not contain extra words such as sub-site name, "Video" etc.
        title = compat_urllib_parse_unquote_plus(
            self._search_regex(r"__title\s*=\s*'([^']+)'", webpage, 'title', default=None)
            or self._og_search_title(webpage))

        filename = self._search_regex(
            r"(?:__)?filename\s*[:=]\s*'([^']+)'", webpage, 'video filename')
        # in "movies" sub-site pages, filename is URL
        video_url = urljoin('https://ndtvod.bc-ssl.cdn.bitgravity.com/23372/ndtv/', filename.lstrip('/'))

        # "doctor" sub-site has MM:SS format
        duration = parse_duration(self._search_regex(
            r"(?:__)?duration\s*[:=]\s*'([^']+)'", webpage, 'duration', fatal=False))

        # "sports", "doctor", "swirlster" sub-sites don't have 'publish-date'
        upload_date = unified_strdate(self._html_search_meta(
            'publish-date', webpage, 'upload date', default=None) or self._html_search_meta(
            'uploadDate', webpage, 'upload date', default=None) or self._search_regex(
            r'datePublished"\s*:\s*"([^"]+)"', webpage, 'upload date', fatal=False))

        description = remove_end(self._og_search_description(webpage), ' (Read more)')

        return {
            'id': video_id,
            'url': video_url,
            'title': title,
            'description': description,
            'thumbnail': self._og_search_thumbnail(webpage),
            'duration': duration,
            'upload_date': upload_date,
        }
# coding: utf-8
from __future__ import unicode_literals

import datetime

from .common import InfoExtractor


class NerdCubedFeedIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?nerdcubed\.co\.uk/feed\.json'
    _TEST = {
        'url': 'http://www.nerdcubed.co.uk/feed.json',
        'info_dict': {
            'id': 'nerdcubed-feed',
            'title': 'nerdcubed.co.uk feed',
        },
        'playlist_mincount': 1300,
    }

    def _real_extract(self, url):
        feed = self._download_json(url, url, 'Downloading NerdCubed JSON feed')

        entries = [{
            '_type': 'url',
            'title': feed_entry['title'],
            'uploader': feed_entry['source']['name'] if feed_entry['source'] else None,
            'upload_date': datetime.datetime.strptime(feed_entry['date'], '%Y-%m-%d').strftime('%Y%m%d'),
            'url': 'http://www.youtube.com/watch?v=' + feed_entry['youtube_id'],
        } for feed_entry in feed]

        return {
            '_type': 'playlist',
            'title': 'nerdcubed.co.uk feed',
            'id': 'nerdcubed-feed',
            'entries': entries,
        }
# coding: utf-8
from __future__ import unicode_literals

from hashlib import md5
from base64 import b64encode
from datetime import datetime
import re

from .common import InfoExtractor
from ..compat import (
    compat_urllib_parse_urlencode,
    compat_str,
    compat_itertools_count,
)
from ..utils import (
    sanitized_Request,
    float_or_none,
)


class NetEaseMusicBaseIE(InfoExtractor):
    _FORMATS = ['bMusic', 'mMusic', 'hMusic']
    _NETEASE_SALT = '3go8&$8*3*3h0k(2)2'
    _API_BASE = 'http://music.163.com/api/'

    @classmethod
    def _encrypt(cls, dfsid):
        salt_bytes = bytearray(cls._NETEASE_SALT.encode('utf-8'))
        string_bytes = bytearray(compat_str(dfsid).encode('ascii'))
        salt_len = len(salt_bytes)
        for i in range(len(string_bytes)):
            string_bytes[i] = string_bytes[i] ^ salt_bytes[i % salt_len]
        m = md5()
        m.update(bytes(string_bytes))
        result = b64encode(m.digest()).decode('ascii')
        return result.replace('/', '_').replace('+', '-')

    def extract_formats(self, info):
        formats = []
        for song_format in self._FORMATS:
            details = info.get(song_format)
            if not details:
                continue
            song_file_path = '/%s/%s.%s' % (
                self._encrypt(details['dfsId']), details['dfsId'], details['extension'])

            # 203.130.59.9, 124.40.233.182, 115.231.74.139, etc is a reverse proxy-like feature
            # from NetEase's CDN provider that can be used if m5.music.126.net does not
            # work, especially for users outside of Mainland China
            # via: https://github.com/JixunMoe/unblock-163/issues/3#issuecomment-163115880
            for host in ('http://m5.music.126.net', 'http://115.231.74.139/m1.music.126.net',
                         'http://124.40.233.182/m1.music.126.net', 'http://203.130.59.9/m1.music.126.net'):
                song_url = host + song_file_path
                if self._is_valid_url(song_url, info['id'], 'song'):
                    formats.append({
                        'url': song_url,
                        'ext': details.get('extension'),
                        'abr': float_or_none(details.get('bitrate'), scale=1000),
                        'format_id': song_format,
                        'filesize': details.get('size'),
                        'asr': details.get('sr')
                    })
                    break
        return formats

    @classmethod
    def convert_milliseconds(cls, ms):
        return int(round(ms / 1000.0))

    def query_api(self, endpoint, video_id, note):
        req = sanitized_Request('%s%s' % (self._API_BASE, endpoint))
        req.add_header('Referer', self._API_BASE)
        return self._download_json(req, video_id, note)


class NetEaseMusicIE(NetEaseMusicBaseIE):
    IE_NAME = 'netease:song'
    IE_DESC = '网易云音乐'
    _VALID_URL = r'https?://music\.163\.com/(#/)?song\?id=(?P<id>[0-9]+)'
    _TESTS = [{
        'url': 'http://music.163.com/#/song?id=32102397',
        'md5': 'f2e97280e6345c74ba9d5677dd5dcb45',
        'info_dict': {
            'id': '32102397',
            'ext': 'mp3',
            'title': 'Bad Blood (feat. Kendrick Lamar)',
            'creator': 'Taylor Swift / Kendrick Lamar',
            'upload_date': '20150517',
            'timestamp': 1431878400,
            'description': 'md5:a10a54589c2860300d02e1de821eb2ef',
        },
        'skip': 'Blocked outside Mainland China',
    }, {
        'note': 'No lyrics translation.',
        'url': 'http://music.163.com/#/song?id=29822014',
        'info_dict': {
            'id': '29822014',
            'ext': 'mp3',
            'title': '听见下雨的声音',
            'creator': '周杰伦',
            'upload_date': '20141225',
            'timestamp': 1419523200,
            'description': 'md5:a4d8d89f44656af206b7b2555c0bce6c',
        },
        'skip': 'Blocked outside Mainland China',
    }, {
        'note': 'No lyrics.',
        'url': 'http://music.163.com/song?id=17241424',
        'info_dict': {
            'id': '17241424',
            'ext': 'mp3',
            'title': 'Opus 28',
            'creator': 'Dustin O\'Halloran',
            'upload_date': '20080211',
            'timestamp': 1202745600,
        },
        'skip': 'Blocked outside Mainland China',
    }, {
        'note': 'Has translated name.',
        'url': 'http://music.163.com/#/song?id=22735043',
        'info_dict': {
            'id': '22735043',
            'ext': 'mp3',
            'title': '소원을 말해봐 (Genie)',
            'creator': '少女时代',
            'description': 'md5:79d99cc560e4ca97e0c4d86800ee4184',
            'upload_date': '20100127',
            'timestamp': 1264608000,
            'alt_title': '说出愿望吧(Genie)',
        },
        'skip': 'Blocked outside Mainland China',
    }]

    def _process_lyrics(self, lyrics_info):
        original = lyrics_info.get('lrc', {}).get('lyric')
        translated = lyrics_info.get('tlyric', {}).get('lyric')

        if not translated:
            return original

        lyrics_expr = r'(\[[0-9]{2}:[0-9]{2}\.[0-9]{2,}\])([^\n]+)'
        original_ts_texts = re.findall(lyrics_expr, original)
        translation_ts_dict = dict(
            (time_stamp, text) for time_stamp, text in re.findall(lyrics_expr, translated)
        )
        lyrics = '\n'.join([
            '%s%s / %s' % (time_stamp, text, translation_ts_dict.get(time_stamp, ''))
            for time_stamp, text in original_ts_texts
        ])
        return lyrics

    def _real_extract(self, url):
        song_id = self._match_id(url)

        params = {
            'id': song_id,
            'ids': '[%s]' % song_id
        }
        info = self.query_api(
            'song/detail?' + compat_urllib_parse_urlencode(params),
            song_id, 'Downloading song info')['songs'][0]

        formats = self.extract_formats(info)
        self._sort_formats(formats)

        lyrics_info = self.query_api(
            'song/lyric?id=%s&lv=-1&tv=-1' % song_id,
            song_id, 'Downloading lyrics data')
        lyrics = self._process_lyrics(lyrics_info)

        alt_title = None
        if info.get('transNames'):
            alt_title = '/'.join(info.get('transNames'))

        return {
            'id': song_id,
            'title': info['name'],
            'alt_title': alt_title,
            'creator': ' / '.join([artist['name'] for artist in info.get('artists', [])]),
            'timestamp': self.convert_milliseconds(info.get('album', {}).get('publishTime')),
            'thumbnail': info.get('album', {}).get('picUrl'),
            'duration': self.convert_milliseconds(info.get('duration', 0)),
            'description': lyrics,
            'formats': formats,
        }


class NetEaseMusicAlbumIE(NetEaseMusicBaseIE):
    IE_NAME = 'netease:album'
    IE_DESC = '网易云音乐 - 专辑'
    _VALID_URL = r'https?://music\.163\.com/(#/)?album\?id=(?P<id>[0-9]+)'
    _TEST = {
        'url': 'http://music.163.com/#/album?id=220780',
        'info_dict': {
            'id': '220780',
            'title': 'B\'day',
        },
        'playlist_count': 23,
        'skip': 'Blocked outside Mainland China',
    }

    def _real_extract(self, url):
        album_id = self._match_id(url)

        info = self.query_api(
            'album/%s?id=%s' % (album_id, album_id),
            album_id, 'Downloading album data')['album']

        name = info['name']
        desc = info.get('description')
        entries = [
            self.url_result('http://music.163.com/#/song?id=%s' % song['id'],
                            'NetEaseMusic', song['id'])
            for song in info['songs']
        ]
        return self.playlist_result(entries, album_id, name, desc)


class NetEaseMusicSingerIE(NetEaseMusicBaseIE):
    IE_NAME = 'netease:singer'
    IE_DESC = '网易云音乐 - 歌手'
    _VALID_URL = r'https?://music\.163\.com/(#/)?artist\?id=(?P<id>[0-9]+)'
    _TESTS = [{
        'note': 'Singer has aliases.',
        'url': 'http://music.163.com/#/artist?id=10559',
        'info_dict': {
            'id': '10559',
            'title': '张惠妹 - aMEI;阿密特',
        },
        'playlist_count': 50,
        'skip': 'Blocked outside Mainland China',
    }, {
        'note': 'Singer has translated name.',
        'url': 'http://music.163.com/#/artist?id=124098',
        'info_dict': {
            'id': '124098',
            'title': '李昇基 - 이승기',
        },
        'playlist_count': 50,
        'skip': 'Blocked outside Mainland China',
    }]

    def _real_extract(self, url):
        singer_id = self._match_id(url)

        info = self.query_api(
            'artist/%s?id=%s' % (singer_id, singer_id),
            singer_id, 'Downloading singer data')

        name = info['artist']['name']
        if info['artist']['trans']:
            name = '%s - %s' % (name, info['artist']['trans'])
        if info['artist']['alias']:
            name = '%s - %s' % (name, ';'.join(info['artist']['alias']))

        entries = [
            self.url_result('http://music.163.com/#/song?id=%s' % song['id'],
                            'NetEaseMusic', song['id'])
            for song in info['hotSongs']
        ]
        return self.playlist_result(entries, singer_id, name)


class NetEaseMusicListIE(NetEaseMusicBaseIE):
    IE_NAME = 'netease:playlist'
    IE_DESC = '网易云音乐 - 歌单'
    _VALID_URL = r'https?://music\.163\.com/(#/)?(playlist|discover/toplist)\?id=(?P<id>[0-9]+)'
    _TESTS = [{
        'url': 'http://music.163.com/#/playlist?id=79177352',
        'info_dict': {
            'id': '79177352',
            'title': 'Billboard 2007 Top 100',
            'description': 'md5:12fd0819cab2965b9583ace0f8b7b022'
        },
        'playlist_count': 99,
        'skip': 'Blocked outside Mainland China',
    }, {
        'note': 'Toplist/Charts sample',
        'url': 'http://music.163.com/#/discover/toplist?id=3733003',
        'info_dict': {
            'id': '3733003',
            'title': 're:韩国Melon排行榜周榜 [0-9]{4}-[0-9]{2}-[0-9]{2}',
            'description': 'md5:73ec782a612711cadc7872d9c1e134fc',
        },
        'playlist_count': 50,
        'skip': 'Blocked outside Mainland China',
    }]

    def _real_extract(self, url):
        list_id = self._match_id(url)

        info = self.query_api(
            'playlist/detail?id=%s&lv=-1&tv=-1' % list_id,
            list_id, 'Downloading playlist data')['result']

        name = info['name']
        desc = info.get('description')

        if info.get('specialType') == 10:  # is a chart/toplist
            datestamp = datetime.fromtimestamp(
                self.convert_milliseconds(info['updateTime'])).strftime('%Y-%m-%d')
            name = '%s %s' % (name, datestamp)

        entries = [
            self.url_result('http://music.163.com/#/song?id=%s' % song['id'],
                            'NetEaseMusic', song['id'])
            for song in info['tracks']
        ]
        return self.playlist_result(entries, list_id, name, desc)


class NetEaseMusicMvIE(NetEaseMusicBaseIE):
    IE_NAME = 'netease:mv'
    IE_DESC = '网易云音乐 - MV'
    _VALID_URL = r'https?://music\.163\.com/(#/)?mv\?id=(?P<id>[0-9]+)'
    _TEST = {
        'url': 'http://music.163.com/#/mv?id=415350',
        'info_dict': {
            'id': '415350',
            'ext': 'mp4',
            'title': '이럴거면 그러지말지',
            'description': '白雅言自作曲唱甜蜜爱情',
            'creator': '白雅言',
            'upload_date': '20150520',
        },
        'skip': 'Blocked outside Mainland China',
    }

    def _real_extract(self, url):
        mv_id = self._match_id(url)

        info = self.query_api(
            'mv/detail?id=%s&type=mp4' % mv_id,
            mv_id, 'Downloading mv info')['data']

        formats = [
            {'url': mv_url, 'ext': 'mp4', 'format_id': '%sp' % brs, 'height': int(brs)}
            for brs, mv_url in info['brs'].items()
        ]
        self._sort_formats(formats)

        return {
            'id': mv_id,
            'title': info['name'],
            'description': info.get('desc') or info.get('briefDesc'),
            'creator': info['artistName'],
            'upload_date': info['publishTime'].replace('-', ''),
            'formats': formats,
            'thumbnail': info.get('cover'),
            'duration': self.convert_milliseconds(info.get('duration', 0)),
        }


class NetEaseMusicProgramIE(NetEaseMusicBaseIE):
    IE_NAME = 'netease:program'
    IE_DESC = '网易云音乐 - 电台节目'
    _VALID_URL = r'https?://music\.163\.com/(#/?)program\?id=(?P<id>[0-9]+)'
    _TESTS = [{
        'url': 'http://music.163.com/#/program?id=10109055',
        'info_dict': {
            'id': '10109055',
            'ext': 'mp3',
            'title': '不丹足球背后的故事',
            'description': '喜马拉雅人的足球梦 ...',
            'creator': '大话西藏',
            'timestamp': 1434179342,
            'upload_date': '20150613',
            'duration': 900,
        },
        'skip': 'Blocked outside Mainland China',
    }, {
        'note': 'This program has accompanying songs.',
        'url': 'http://music.163.com/#/program?id=10141022',
        'info_dict': {
            'id': '10141022',
            'title': '25岁,你是自在如风的少年<27°C>',
            'description': 'md5:8d594db46cc3e6509107ede70a4aaa3b',
        },
        'playlist_count': 4,
        'skip': 'Blocked outside Mainland China',
    }, {
        'note': 'This program has accompanying songs.',
        'url': 'http://music.163.com/#/program?id=10141022',
        'info_dict': {
            'id': '10141022',
            'ext': 'mp3',
            'title': '25岁,你是自在如风的少年<27°C>',
            'description': 'md5:8d594db46cc3e6509107ede70a4aaa3b',
            'timestamp': 1434450841,
            'upload_date': '20150616',
        },
        'params': {
            'noplaylist': True
        },
        'skip': 'Blocked outside Mainland China',
    }]

    def _real_extract(self, url):
        program_id = self._match_id(url)

        info = self.query_api(
            'dj/program/detail?id=%s' % program_id,
            program_id, 'Downloading program info')['program']

        name = info['name']
        description = info['description']

        if not info['songs'] or self._downloader.params.get('noplaylist'):
            if info['songs']:
                self.to_screen(
                    'Downloading just the main audio %s because of --no-playlist'
                    % info['mainSong']['id'])

            formats = self.extract_formats(info['mainSong'])
            self._sort_formats(formats)

            return {
                'id': program_id,
                'title': name,
                'description': description,
                'creator': info['dj']['brand'],
                'timestamp': self.convert_milliseconds(info['createTime']),
                'thumbnail': info['coverUrl'],
                'duration': self.convert_milliseconds(info.get('duration', 0)),
                'formats': formats,
            }

        self.to_screen(
            'Downloading playlist %s - add --no-playlist to just download the main audio %s'
            % (program_id, info['mainSong']['id']))

        song_ids = [info['mainSong']['id']]
        song_ids.extend([song['id'] for song in info['songs']])
        entries = [
            self.url_result('http://music.163.com/#/song?id=%s' % song_id,
                            'NetEaseMusic', song_id)
            for song_id in song_ids
        ]
        return self.playlist_result(entries, program_id, name, description)


class NetEaseMusicDjRadioIE(NetEaseMusicBaseIE):
    IE_NAME = 'netease:djradio'
    IE_DESC = '网易云音乐 - 电台'
    _VALID_URL = r'https?://music\.163\.com/(#/)?djradio\?id=(?P<id>[0-9]+)'
    _TEST = {
        'url': 'http://music.163.com/#/djradio?id=42',
        'info_dict': {
            'id': '42',
            'title': '声音蔓延',
            'description': 'md5:766220985cbd16fdd552f64c578a6b15'
        },
        'playlist_mincount': 40,
        'skip': 'Blocked outside Mainland China',
    }
    _PAGE_SIZE = 1000

    def _real_extract(self, url):
        dj_id = self._match_id(url)

        name = None
        desc = None
        entries = []
        for offset in compat_itertools_count(start=0, step=self._PAGE_SIZE):
            info = self.query_api(
                'dj/program/byradio?asc=false&limit=%d&radioId=%s&offset=%d'
                % (self._PAGE_SIZE, dj_id, offset),
                dj_id, 'Downloading dj programs - %d' % offset)

            entries.extend([
                self.url_result(
                    'http://music.163.com/#/program?id=%s' % program['id'],
                    'NetEaseMusicProgram', program['id'])
                for program in info['programs']
            ])

            if name is None:
                radio = info['programs'][0]['radio']
                name = radio['name']
                desc = radio['desc']

            if not info['more']:
                break

        return self.playlist_result(entries, dj_id, name, desc)
# coding: utf-8
from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..utils import (
    clean_html,
    int_or_none,
    js_to_json,
    parse_iso8601,
)


class NetzkinoIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?netzkino\.de/\#!/(?P<category>[^/]+)/(?P<id>[^/]+)'

    _TEST = {
        'url': 'http://www.netzkino.de/#!/scifikino/rakete-zum-mond',
        'md5': '92a3f8b76f8d7220acce5377ea5d4873',
        'info_dict': {
            'id': 'rakete-zum-mond',
            'ext': 'mp4',
            'title': 'Rakete zum Mond (Endstation Mond, Destination Moon)',
            'comments': 'mincount:3',
            'description': 'md5:1eddeacc7e62d5a25a2d1a7290c64a28',
            'upload_date': '20120813',
            'thumbnail': r're:https?://.*\.jpg$',
            'timestamp': 1344858571,
            'age_limit': 12,
        },
        'params': {
            'skip_download': 'Download only works from Germany',
        }
    }

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        category_id = mobj.group('category')
        video_id = mobj.group('id')

        api_url = 'http://api.netzkino.de.simplecache.net/capi-2.0a/categories/%s.json?d=www' % category_id
        api_info = self._download_json(api_url, video_id)
        info = next(
            p for p in api_info['posts'] if p['slug'] == video_id)
        custom_fields = info['custom_fields']

        production_js = self._download_webpage(
            'http://www.netzkino.de/beta/dist/production.min.js', video_id,
            note='Downloading player code')
        avo_js = self._search_regex(
            r'var urlTemplate=(\{.*?"\})',
            production_js, 'URL templates')
        templates = self._parse_json(
            avo_js, video_id, transform_source=js_to_json)

        suffix = {
            'hds': '.mp4/manifest.f4m',
            'hls': '.mp4/master.m3u8',
            'pmd': '.mp4',
        }
        film_fn = custom_fields['Streaming'][0]
        formats = [{
            'format_id': key,
            'ext': 'mp4',
            'url': tpl.replace('{}', film_fn) + suffix[key],
        } for key, tpl in templates.items()]
        self._sort_formats(formats)

        comments = [{
            'timestamp': parse_iso8601(c.get('date'), delimiter=' '),
            'id': c['id'],
            'author': c['name'],
            'html': c['content'],
            'parent': 'root' if c.get('parent', 0) == 0 else c['parent'],
        } for c in info.get('comments', [])]

        return {
            'id': video_id,
            'formats': formats,
            'comments': comments,
            'title': info['title'],
            'age_limit': int_or_none(custom_fields.get('FSK')[0]),
            'timestamp': parse_iso8601(info.get('date'), delimiter=' '),
            'description': clean_html(info.get('content')),
            'thumbnail': info.get('thumbnail'),
            'playlist_title': api_info.get('title'),
            'playlist_id': category_id,
        }
from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..utils import (
    extract_attributes,
    int_or_none,
    parse_duration,
    parse_filesize,
    unified_timestamp,
)


class NewgroundsIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?newgrounds\.com/(?:audio/listen|portal/view)/(?P<id>[0-9]+)'
    _TESTS = [{
        'url': 'https://www.newgrounds.com/audio/listen/549479',
        'md5': 'fe6033d297591288fa1c1f780386f07a',
        'info_dict': {
            'id': '549479',
            'ext': 'mp3',
            'title': 'B7 - BusMode',
            'uploader': 'Burn7',
            'timestamp': 1378878540,
            'upload_date': '20130911',
            'duration': 143,
        },
    }, {
        'url': 'https://www.newgrounds.com/portal/view/673111',
        'md5': '3394735822aab2478c31b1004fe5e5bc',
        'info_dict': {
            'id': '673111',
            'ext': 'mp4',
            'title': 'Dancin',
            'uploader': 'Squirrelman82',
            'timestamp': 1460256780,
            'upload_date': '20160410',
        },
    }, {
        # source format unavailable, additional mp4 formats
        'url': 'http://www.newgrounds.com/portal/view/689400',
        'info_dict': {
            'id': '689400',
            'ext': 'mp4',
            'title': 'ZTV News Episode 8',
            'uploader': 'BennettTheSage',
            'timestamp': 1487965140,
            'upload_date': '20170224',
        },
        'params': {
            'skip_download': True,
        },
    }]

    def _real_extract(self, url):
        media_id = self._match_id(url)

        webpage = self._download_webpage(url, media_id)

        title = self._html_search_regex(
            r'<title>([^>]+)</title>', webpage, 'title')

        media_url = self._parse_json(self._search_regex(
            r'"url"\s*:\s*("[^"]+"),', webpage, ''), media_id)

        formats = [{
            'url': media_url,
            'format_id': 'source',
            'quality': 1,
        }]

        max_resolution = int_or_none(self._search_regex(
            r'max_resolution["\']\s*:\s*(\d+)', webpage, 'max resolution',
            default=None))
        if max_resolution:
            url_base = media_url.rpartition('.')[0]
            for resolution in (360, 720, 1080):
                if resolution > max_resolution:
                    break
                formats.append({
                    'url': '%s.%dp.mp4' % (url_base, resolution),
                    'format_id': '%dp' % resolution,
                    'height': resolution,
                })

        self._check_formats(formats, media_id)
        self._sort_formats(formats)

        uploader = self._html_search_regex(
            (r'(?s)<h4[^>]*>(.+?)</h4>.*?<em>\s*Author\s*</em>',
             r'(?:Author|Writer)\s*<a[^>]+>([^<]+)'), webpage, 'uploader',
            fatal=False)

        timestamp = unified_timestamp(self._html_search_regex(
            (r'<dt>\s*Uploaded\s*</dt>\s*<dd>([^<]+</dd>\s*<dd>[^<]+)',
             r'<dt>\s*Uploaded\s*</dt>\s*<dd>([^<]+)'), webpage, 'timestamp',
            default=None))
        duration = parse_duration(self._search_regex(
            r'(?s)<dd>\s*Song\s*</dd>\s*<dd>.+?</dd>\s*<dd>([^<]+)', webpage,
            'duration', default=None))

        filesize_approx = parse_filesize(self._html_search_regex(
            r'(?s)<dd>\s*Song\s*</dd>\s*<dd>(.+?)</dd>', webpage, 'filesize',
            default=None))
        if len(formats) == 1:
            formats[0]['filesize_approx'] = filesize_approx

        if '<dd>Song' in webpage:
            formats[0]['vcodec'] = 'none'

        return {
            'id': media_id,
            'title': title,
            'uploader': uploader,
            'timestamp': timestamp,
            'duration': duration,
            'formats': formats,
        }


class NewgroundsPlaylistIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?newgrounds\.com/(?:collection|[^/]+/search/[^/]+)/(?P<id>[^/?#&]+)'
    _TESTS = [{
        'url': 'https://www.newgrounds.com/collection/cats',
        'info_dict': {
            'id': 'cats',
            'title': 'Cats',
        },
        'playlist_mincount': 46,
    }, {
        'url': 'http://www.newgrounds.com/portal/search/author/ZONE-SAMA',
        'info_dict': {
            'id': 'ZONE-SAMA',
            'title': 'Portal Search: ZONE-SAMA',
        },
        'playlist_mincount': 47,
    }, {
        'url': 'http://www.newgrounds.com/audio/search/title/cats',
        'only_matching': True,
    }]

    def _real_extract(self, url):
        playlist_id = self._match_id(url)

        webpage = self._download_webpage(url, playlist_id)

        title = self._search_regex(
            r'<title>([^>]+)</title>', webpage, 'title', default=None)

        # cut left menu
        webpage = self._search_regex(
            r'(?s)<div[^>]+\bclass=["\']column wide(.+)',
            webpage, 'wide column', default=webpage)

        entries = []
        for a, path, media_id in re.findall(
                r'(<a[^>]+\bhref=["\']/?((?:portal/view|audio/listen)/(\d+))[^>]+>)',
                webpage):
            a_class = extract_attributes(a).get('class')
            if a_class not in ('item-portalsubmission', 'item-audiosubmission'):
                continue
            entries.append(
                self.url_result(
                    'https://www.newgrounds.com/%s' % path,
                    ie=NewgroundsIE.ie_key(), video_id=media_id))

        return self.playlist_result(entries, playlist_id, title)
# coding: utf-8
from __future__ import unicode_literals

import base64
import hashlib

from .common import InfoExtractor
from ..aes import aes_cbc_decrypt
from ..utils import (
    bytes_to_intlist,
    int_or_none,
    intlist_to_bytes,
    parse_codecs,
    parse_duration,
)


class NewstubeIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?newstube\.ru/media/(?P<id>.+)'
    _TEST = {
        'url': 'http://www.newstube.ru/media/telekanal-cnn-peremestil-gorod-slavyansk-v-krym',
        'md5': '9d10320ad473444352f72f746ccb8b8c',
        'info_dict': {
            'id': '728e0ef2-e187-4012-bac0-5a081fdcb1f6',
            'ext': 'mp4',
            'title': 'Телеканал CNN переместил город Славянск в Крым',
            'description': 'md5:419a8c9f03442bc0b0a794d689360335',
            'duration': 31.05,
        },
    }

    def _real_extract(self, url):
        video_id = self._match_id(url)

        page = self._download_webpage(url, video_id)
        title = self._html_search_meta(['og:title', 'twitter:title'], page, fatal=True)

        video_guid = self._html_search_regex(
            r'<meta\s+property="og:video(?::(?:(?:secure_)?url|iframe))?"\s+content="https?://(?:www\.)?newstube\.ru/embed/(?P<guid>[\da-f]{8}-[\da-f]{4}-[\da-f]{4}-[\da-f]{4}-[\da-f]{12})',
            page, 'video GUID')

        enc_data = base64.b64decode(self._download_webpage(
            'https://www.newstube.ru/embed/api/player/getsources2',
            video_guid, query={
                'guid': video_guid,
                'ff': 3,
            }))
        key = hashlib.pbkdf2_hmac(
            'sha1', video_guid.replace('-', '').encode(), enc_data[:16], 1)[:16]
        dec_data = aes_cbc_decrypt(
            bytes_to_intlist(enc_data[32:]), bytes_to_intlist(key),
            bytes_to_intlist(enc_data[16:32]))
        sources = self._parse_json(intlist_to_bytes(dec_data[:-dec_data[-1]]), video_guid)

        formats = []
        for source in sources:
            source_url = source.get('Src')
            if not source_url:
                continue
            height = int_or_none(source.get('Height'))
            f = {
                'format_id': 'http' + ('-%dp' % height if height else ''),
                'url': source_url,
                'width': int_or_none(source.get('Width')),
                'height': height,
            }
            source_type = source.get('Type')
            if source_type:
                f.update(parse_codecs(self._search_regex(
                    r'codecs="([^"]+)"', source_type, 'codecs', fatal=False)))
            formats.append(f)

        self._check_formats(formats, video_guid)
        self._sort_formats(formats)

        return {
            'id': video_guid,
            'title': title,
            'description': self._html_search_meta(['description', 'og:description'], page),
            'thumbnail': self._html_search_meta(['og:image:secure_url', 'og:image', 'twitter:image'], page),
            'duration': parse_duration(self._html_search_meta('duration', page)),
            'formats': formats,
        }
# coding: utf-8
from __future__ import unicode_literals

from .common import InfoExtractor
from ..compat import compat_urlparse
from ..utils import (
    clean_html,
    get_element_by_class,
    int_or_none,
    parse_iso8601,
    remove_start,
    unified_timestamp,
)


class NextMediaIE(InfoExtractor):
    IE_DESC = '蘋果日報'
    _VALID_URL = r'https?://hk\.apple\.nextmedia\.com/[^/]+/[^/]+/(?P<date>\d+)/(?P<id>\d+)'
    _TESTS = [{
        'url': 'http://hk.apple.nextmedia.com/realtime/news/20141108/53109199',
        'md5': 'dff9fad7009311c421176d1ac90bfe4f',
        'info_dict': {
            'id': '53109199',
            'ext': 'mp4',
            'title': '【佔領金鐘】50外國領事議員撐場 讚學生勇敢香港有希望',
            'thumbnail': r're:^https?://.*\.jpg$',
            'description': 'md5:28222b9912b6665a21011b034c70fcc7',
            'timestamp': 1415456273,
            'upload_date': '20141108',
        }
    }]

    _URL_PATTERN = r'\{ url: \'(.+)\' \}'

    def _real_extract(self, url):
        news_id = self._match_id(url)
        page = self._download_webpage(url, news_id)
        return self._extract_from_nextmedia_page(news_id, url, page)

    def _extract_from_nextmedia_page(self, news_id, url, page):
        redirection_url = self._search_regex(
            r'window\.location\.href\s*=\s*([\'"])(?P<url>(?!\1).+)\1',
            page, 'redirection URL', default=None, group='url')
        if redirection_url:
            return self.url_result(compat_urlparse.urljoin(url, redirection_url))

        title = self._fetch_title(page)
        video_url = self._search_regex(self._URL_PATTERN, page, 'video url')

        attrs = {
            'id': news_id,
            'title': title,
            'url': video_url,  # ext can be inferred from url
            'thumbnail': self._fetch_thumbnail(page),
            'description': self._fetch_description(page),
        }

        timestamp = self._fetch_timestamp(page)
        if timestamp:
            attrs['timestamp'] = timestamp
        else:
            attrs['upload_date'] = self._fetch_upload_date(url)

        return attrs

    def _fetch_title(self, page):
        return self._og_search_title(page)

    def _fetch_thumbnail(self, page):
        return self._og_search_thumbnail(page)

    def _fetch_timestamp(self, page):
        dateCreated = self._search_regex('"dateCreated":"([^"]+)"', page, 'created time')
        return parse_iso8601(dateCreated)

    def _fetch_upload_date(self, url):
        return self._search_regex(self._VALID_URL, url, 'upload date', group='date')

    def _fetch_description(self, page):
        return self._og_search_property('description', page)


class NextMediaActionNewsIE(NextMediaIE):
    IE_DESC = '蘋果日報 - 動新聞'
    _VALID_URL = r'https?://hk\.dv\.nextmedia\.com/actionnews/[^/]+/(?P<date>\d+)/(?P<id>\d+)/\d+'
    _TESTS = [{
        'url': 'http://hk.dv.nextmedia.com/actionnews/hit/20150121/19009428/20061460',
        'md5': '05fce8ffeed7a5e00665d4b7cf0f9201',
        'info_dict': {
            'id': '19009428',
            'ext': 'mp4',
            'title': '【壹週刊】細10年男友偷食 50歲邵美琪再失戀',
            'thumbnail': r're:^https?://.*\.jpg$',
            'description': 'md5:cd802fad1f40fd9ea178c1e2af02d659',
            'timestamp': 1421791200,
            'upload_date': '20150120',
        }
    }]

    def _real_extract(self, url):
        news_id = self._match_id(url)
        actionnews_page = self._download_webpage(url, news_id)
        article_url = self._og_search_url(actionnews_page)
        article_page = self._download_webpage(article_url, news_id)
        return self._extract_from_nextmedia_page(news_id, url, article_page)


class AppleDailyIE(NextMediaIE):
    IE_DESC = '臺灣蘋果日報'
    _VALID_URL = r'https?://(www|ent)\.appledaily\.com\.tw/[^/]+/[^/]+/[^/]+/(?P<date>\d+)/(?P<id>\d+)(/.*)?'
    _TESTS = [{
        'url': 'http://ent.appledaily.com.tw/enews/article/entertainment/20150128/36354694',
        'md5': 'a843ab23d150977cc55ef94f1e2c1e4d',
        'info_dict': {
            'id': '36354694',
            'ext': 'mp4',
            'title': '周亭羽走過摩鐵陰霾2男陪吃 九把刀孤寒看醫生',
            'thumbnail': r're:^https?://.*\.jpg$',
            'description': 'md5:2acd430e59956dc47cd7f67cb3c003f4',
            'upload_date': '20150128',
        }
    }, {
        'url': 'http://www.appledaily.com.tw/realtimenews/article/strange/20150128/550549/%E4%B8%8D%E6%BB%BF%E8%A2%AB%E8%B8%A9%E8%85%B3%E3%80%80%E5%B1%B1%E6%9D%B1%E5%85%A9%E5%A4%A7%E5%AA%BD%E4%B8%80%E8%B7%AF%E6%89%93%E4%B8%8B%E8%BB%8A',
        'md5': '86b4e9132d158279c7883822d94ccc49',
        'info_dict': {
            'id': '550549',
            'ext': 'mp4',
            'title': '不滿被踩腳 山東兩大媽一路打下車',
            'thumbnail': r're:^https?://.*\.jpg$',
            'description': 'md5:175b4260c1d7c085993474217e4ab1b4',
            'upload_date': '20150128',
        }
    }, {
        'url': 'http://www.appledaily.com.tw/animation/realtimenews/new/20150128/5003671',
        'md5': '03df296d95dedc2d5886debbb80cb43f',
        'info_dict': {
            'id': '5003671',
            'ext': 'mp4',
            'title': '20正妹熱舞 《刀龍傳說Online》火辣上市',
            'thumbnail': r're:^https?://.*\.jpg$',
            'description': 'md5:23c0aac567dc08c9c16a3161a2c2e3cd',
            'upload_date': '20150128',
        },
        'skip': 'redirect to http://www.appledaily.com.tw/animation/',
    }, {
        # No thumbnail
        'url': 'http://www.appledaily.com.tw/animation/realtimenews/new/20150128/5003673/',
        'md5': 'b06182cd386ea7bc6115ec7ff0f72aeb',
        'info_dict': {
            'id': '5003673',
            'ext': 'mp4',
            'title': '半夜尿尿 好像會看到___',
            'description': 'md5:61d2da7fe117fede148706cdb85ac066',
            'upload_date': '20150128',
        },
        'expected_warnings': [
            'video thumbnail',
        ],
        'skip': 'redirect to http://www.appledaily.com.tw/animation/',
    }, {
        'url': 'http://www.appledaily.com.tw/appledaily/article/supplement/20140417/35770334/',
        'md5': 'eaa20e6b9df418c912d7f5dec2ba734d',
        'info_dict': {
            'id': '35770334',
            'ext': 'mp4',
            'title': '咖啡占卜測 XU裝熟指數',
            'thumbnail': r're:^https?://.*\.jpg$',
            'description': 'md5:7b859991a6a4fedbdf3dd3b66545c748',
            'upload_date': '20140417',
        },
    }, {
        'url': 'http://www.appledaily.com.tw/actionnews/appledaily/7/20161003/960588/',
        'only_matching': True,
    }, {
        # Redirected from http://ent.appledaily.com.tw/enews/article/entertainment/20150128/36354694
        'url': 'http://ent.appledaily.com.tw/section/article/headline/20150128/36354694',
        'only_matching': True,
    }]

    _URL_PATTERN = r'\{url: \'(.+)\'\}'

    def _fetch_title(self, page):
        return (self._html_search_regex(r'<h1 id="h1">([^<>]+)</h1>', page, 'news title', default=None)
                or self._html_search_meta('description', page, 'news title'))

    def _fetch_thumbnail(self, page):
        return self._html_search_regex(r"setInitialImage\(\'([^']+)'\)", page, 'video thumbnail', fatal=False)

    def _fetch_timestamp(self, page):
        return None

    def _fetch_description(self, page):
        return self._html_search_meta('description', page, 'news description')


class NextTVIE(InfoExtractor):
    IE_DESC = '壹電視'
    _VALID_URL = r'https?://(?:www\.)?nexttv\.com\.tw/(?:[^/]+/)+(?P<id>\d+)'

    _TEST = {
        'url': 'http://www.nexttv.com.tw/news/realtime/politics/11779671',
        'info_dict': {
            'id': '11779671',
            'ext': 'mp4',
            'title': '「超收稅」近4千億! 藍議員籲發消費券',
            'thumbnail': r're:^https?://.*\.jpg$',
            'timestamp': 1484825400,
            'upload_date': '20170119',
            'view_count': int,
        },
    }

    def _real_extract(self, url):
        video_id = self._match_id(url)

        webpage = self._download_webpage(url, video_id)

        title = self._html_search_regex(
            r'<h1[^>]*>([^<]+)</h1>', webpage, 'title')

        data = self._hidden_inputs(webpage)

        video_url = data['ntt-vod-src-detailview']

        date_str = get_element_by_class('date', webpage)
        timestamp = unified_timestamp(date_str + '+0800') if date_str else None

        view_count = int_or_none(remove_start(
            clean_html(get_element_by_class('click', webpage)), '點閱:'))

        return {
            'id': video_id,
            'title': title,
            'url': video_url,
            'thumbnail': data.get('ntt-vod-img-src'),
            'timestamp': timestamp,
            'view_count': view_count,
        }
# coding: utf-8
from __future__ import unicode_literals

import hashlib
import random
import re
import time

from .common import InfoExtractor
from ..compat import compat_str
from ..utils import (
    ExtractorError,
    int_or_none,
    parse_duration,
    try_get,
    urlencode_postdata,
)


class NexxIE(InfoExtractor):
    _VALID_URL = r'''(?x)
                        (?:
                            https?://api\.nexx(?:\.cloud|cdn\.com)/v3/(?P<domain_id>\d+)/videos/byid/|
                            nexx:(?:(?P<domain_id_s>\d+):)?|
                            https?://arc\.nexx\.cloud/api/video/
                        )
                        (?P<id>\d+)
                    '''
    _TESTS = [{
        # movie
        'url': 'https://api.nexx.cloud/v3/748/videos/byid/128907',
        'md5': '31899fd683de49ad46f4ee67e53e83fe',
        'info_dict': {
            'id': '128907',
            'ext': 'mp4',
            'title': 'Stiftung Warentest',
            'alt_title': 'Wie ein Test abläuft',
            'description': 'md5:d1ddb1ef63de721132abd38639cc2fd2',
            'creator': 'SPIEGEL TV',
            'thumbnail': r're:^https?://.*\.jpg$',
            'duration': 2509,
            'timestamp': 1384264416,
            'upload_date': '20131112',
        },
    }, {
        # episode
        'url': 'https://api.nexx.cloud/v3/741/videos/byid/247858',
        'info_dict': {
            'id': '247858',
            'ext': 'mp4',
            'title': 'Return of the Golden Child (OV)',
            'description': 'md5:5d969537509a92b733de21bae249dc63',
            'release_year': 2017,
            'thumbnail': r're:^https?://.*\.jpg$',
            'duration': 1397,
            'timestamp': 1495033267,
            'upload_date': '20170517',
            'episode_number': 2,
            'season_number': 2,
        },
        'params': {
            'skip_download': True,
        },
        'skip': 'HTTP Error 404: Not Found',
    }, {
        # does not work via arc
        'url': 'nexx:741:1269984',
        'md5': 'c714b5b238b2958dc8d5642addba6886',
        'info_dict': {
            'id': '1269984',
            'ext': 'mp4',
            'title': '1 TAG ohne KLO... wortwörtlich! 😑',
            'alt_title': '1 TAG ohne KLO... wortwörtlich! 😑',
            'thumbnail': r're:^https?://.*\.jpg$',
            'duration': 607,
            'timestamp': 1518614955,
            'upload_date': '20180214',
        },
    }, {
        # free cdn from http://www.spiegel.de/video/eifel-zoo-aufregung-um-ausgebrochene-raubtiere-video-99018031.html
        'url': 'nexx:747:1533779',
        'md5': '6bf6883912b82b7069fb86c2297e9893',
        'info_dict': {
            'id': '1533779',
            'ext': 'mp4',
            'title': 'Aufregung um ausgebrochene Raubtiere',
            'alt_title': 'Eifel-Zoo',
            'description': 'md5:f21375c91c74ad741dcb164c427999d2',
            'thumbnail': r're:^https?://.*\.jpg$',
            'duration': 111,
            'timestamp': 1527874460,
            'upload_date': '20180601',
        },
    }, {
        'url': 'https://api.nexxcdn.com/v3/748/videos/byid/128907',
        'only_matching': True,
    }, {
        'url': 'nexx:748:128907',
        'only_matching': True,
    }, {
        'url': 'nexx:128907',
        'only_matching': True,
    }, {
        'url': 'https://arc.nexx.cloud/api/video/128907.json',
        'only_matching': True,
    }]

    @staticmethod
    def _extract_domain_id(webpage):
        mobj = re.search(
            r'<script\b[^>]+\bsrc=["\'](?:https?:)?//(?:require|arc)\.nexx(?:\.cloud|cdn\.com)/(?:sdk/)?(?P<id>\d+)',
            webpage)
        return mobj.group('id') if mobj else None

    @staticmethod
    def _extract_urls(webpage):
        # Reference:
        # 1. https://nx-s.akamaized.net/files/201510/44.pdf

        entries = []

        # JavaScript Integration
        domain_id = NexxIE._extract_domain_id(webpage)
        if domain_id:
            for video_id in re.findall(
                    r'(?is)onPLAYReady.+?_play\.(?:init|(?:control\.)?addPlayer)\s*\(.+?\s*,\s*["\']?(\d+)',
                    webpage):
                entries.append(
                    'https://api.nexx.cloud/v3/%s/videos/byid/%s'
                    % (domain_id, video_id))

        # TODO: support more embed formats

        return entries

    @staticmethod
    def _extract_url(webpage):
        return NexxIE._extract_urls(webpage)[0]

    def _handle_error(self, response):
        status = int_or_none(try_get(
            response, lambda x: x['metadata']['status']) or 200)
        if 200 <= status < 300:
            return
        raise ExtractorError(
            '%s said: %s' % (self.IE_NAME, response['metadata']['errorhint']),
            expected=True)

    def _call_api(self, domain_id, path, video_id, data=None, headers={}):
        headers['Content-Type'] = 'application/x-www-form-urlencoded; charset=UTF-8'
        result = self._download_json(
            'https://api.nexx.cloud/v3/%s/%s' % (domain_id, path), video_id,
            'Downloading %s JSON' % path, data=urlencode_postdata(data),
            headers=headers)
        self._handle_error(result)
        return result['result']

    def _extract_free_formats(self, video, video_id):
        stream_data = video['streamdata']
        cdn = stream_data['cdnType']
        assert cdn == 'free'

        hash = video['general']['hash']

        ps = compat_str(stream_data['originalDomain'])
        if stream_data['applyFolderHierarchy'] == 1:
            s = ('%04d' % int(video_id))[::-1]
            ps += '/%s/%s' % (s[0:2], s[2:4])
        ps += '/%s/%s_' % (video_id, hash)

        t = 'http://%s' + ps
        fd = stream_data['azureFileDistribution'].split(',')
        cdn_provider = stream_data['cdnProvider']

        def p0(p):
            return '_%s' % p if stream_data['applyAzureStructure'] == 1 else ''

        formats = []
        if cdn_provider == 'ak':
            t += ','
            for i in fd:
                p = i.split(':')
                t += p[1] + p0(int(p[0])) + ','
            t += '.mp4.csmil/master.%s'
        elif cdn_provider == 'ce':
            k = t.split('/')
            h = k.pop()
            http_base = t = '/'.join(k)
            http_base = http_base % stream_data['cdnPathHTTP']
            t += '/asset.ism/manifest.%s?dcp_ver=aos4&videostream='
            for i in fd:
                p = i.split(':')
                tbr = int(p[0])
                filename = '%s%s%s.mp4' % (h, p[1], p0(tbr))
                f = {
                    'url': http_base + '/' + filename,
                    'format_id': '%s-http-%d' % (cdn, tbr),
                    'tbr': tbr,
                }
                width_height = p[1].split('x')
                if len(width_height) == 2:
                    f.update({
                        'width': int_or_none(width_height[0]),
                        'height': int_or_none(width_height[1]),
                    })
                formats.append(f)
                a = filename + ':%s' % (tbr * 1000)
                t += a + ','
            t = t[:-1] + '&audiostream=' + a.split(':')[0]
        else:
            assert False

        if cdn_provider == 'ce':
            formats.extend(self._extract_mpd_formats(
                t % (stream_data['cdnPathDASH'], 'mpd'), video_id,
                mpd_id='%s-dash' % cdn, fatal=False))
        formats.extend(self._extract_m3u8_formats(
            t % (stream_data['cdnPathHLS'], 'm3u8'), video_id, 'mp4',
            entry_protocol='m3u8_native', m3u8_id='%s-hls' % cdn, fatal=False))

        return formats

    def _extract_azure_formats(self, video, video_id):
        stream_data = video['streamdata']
        cdn = stream_data['cdnType']
        assert cdn == 'azure'

        azure_locator = stream_data['azureLocator']

        def get_cdn_shield_base(shield_type='', static=False):
            for secure in ('', 's'):
                cdn_shield = stream_data.get('cdnShield%sHTTP%s' % (shield_type, secure.upper()))
                if cdn_shield:
                    return 'http%s://%s' % (secure, cdn_shield)
            else:
                if 'fb' in stream_data['azureAccount']:
                    prefix = 'df' if static else 'f'
                else:
                    prefix = 'd' if static else 'p'
                account = int(stream_data['azureAccount'].replace('nexxplayplus', '').replace('nexxplayfb', ''))
                return 'http://nx-%s%02d.akamaized.net/' % (prefix, account)

        language = video['general'].get('language_raw') or ''

        azure_stream_base = get_cdn_shield_base()
        is_ml = ',' in language
        azure_manifest_url = '%s%s/%s_src%s.ism/Manifest' % (
            azure_stream_base, azure_locator, video_id, ('_manifest' if is_ml else '')) + '%s'

        protection_token = try_get(
            video, lambda x: x['protectiondata']['token'], compat_str)
        if protection_token:
            azure_manifest_url += '?hdnts=%s' % protection_token

        formats = self._extract_m3u8_formats(
            azure_manifest_url % '(format=m3u8-aapl)',
            video_id, 'mp4', 'm3u8_native',
            m3u8_id='%s-hls' % cdn, fatal=False)
        formats.extend(self._extract_mpd_formats(
            azure_manifest_url % '(format=mpd-time-csf)',
            video_id, mpd_id='%s-dash' % cdn, fatal=False))
        formats.extend(self._extract_ism_formats(
            azure_manifest_url % '', video_id, ism_id='%s-mss' % cdn, fatal=False))

        azure_progressive_base = get_cdn_shield_base('Prog', True)
        azure_file_distribution = stream_data.get('azureFileDistribution')
        if azure_file_distribution:
            fds = azure_file_distribution.split(',')
            if fds:
                for fd in fds:
                    ss = fd.split(':')
                    if len(ss) == 2:
                        tbr = int_or_none(ss[0])
                        if tbr:
                            f = {
                                'url': '%s%s/%s_src_%s_%d.mp4' % (
                                    azure_progressive_base, azure_locator, video_id, ss[1], tbr),
                                'format_id': '%s-http-%d' % (cdn, tbr),
                                'tbr': tbr,
                            }
                            width_height = ss[1].split('x')
                            if len(width_height) == 2:
                                f.update({
                                    'width': int_or_none(width_height[0]),
                                    'height': int_or_none(width_height[1]),
                                })
                            formats.append(f)

        return formats

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        domain_id = mobj.group('domain_id') or mobj.group('domain_id_s')
        video_id = mobj.group('id')

        video = None

        def find_video(result):
            if isinstance(result, dict):
                return result
            elif isinstance(result, list):
                vid = int(video_id)
                for v in result:
                    if try_get(v, lambda x: x['general']['ID'], int) == vid:
                        return v
            return None

        response = self._download_json(
            'https://arc.nexx.cloud/api/video/%s.json' % video_id,
            video_id, fatal=False)
        if response and isinstance(response, dict):
            result = response.get('result')
            if result:
                video = find_video(result)

        # not all videos work via arc, e.g. nexx:741:1269984
        if not video:
            # Reverse engineered from JS code (see getDeviceID function)
            device_id = '%d:%d:%d%d' % (
                random.randint(1, 4), int(time.time()),
                random.randint(1e4, 99999), random.randint(1, 9))

            result = self._call_api(domain_id, 'session/init', video_id, data={
                'nxp_devh': device_id,
                'nxp_userh': '',
                'precid': '0',
                'playlicense': '0',
                'screenx': '1920',
                'screeny': '1080',
                'playerversion': '6.0.00',
                'gateway': 'html5',
                'adGateway': '',
                'explicitlanguage': 'en-US',
                'addTextTemplates': '1',
                'addDomainData': '1',
                'addAdModel': '1',
            }, headers={
                'X-Request-Enable-Auth-Fallback': '1',
            })

            cid = result['general']['cid']

            # As described in [1] X-Request-Token generation algorithm is
            # as follows:
            #   md5( operation + domain_id + domain_secret )
            # where domain_secret is a static value that will be given by nexx.tv
            # as per [1]. Here is how this "secret" is generated (reversed
            # from _play.api.init function, search for clienttoken). So it's
            # actually not static and not that much of a secret.
            # 1. https://nexxtvstorage.blob.core.windows.net/files/201610/27.pdf
            secret = result['device']['clienttoken'][int(device_id[0]):]
            secret = secret[0:len(secret) - int(device_id[-1])]

            op = 'byid'

            # Reversed from JS code for _play.api.call function (search for
            # X-Request-Token)
            request_token = hashlib.md5(
                ''.join((op, domain_id, secret)).encode('utf-8')).hexdigest()

            result = self._call_api(
                domain_id, 'videos/%s/%s' % (op, video_id), video_id, data={
                    'additionalfields': 'language,channel,actors,studio,licenseby,slug,subtitle,teaser,description',
                    'addInteractionOptions': '1',
                    'addStatusDetails': '1',
                    'addStreamDetails': '1',
                    'addCaptions': '1',
                    'addScenes': '1',
                    'addHotSpots': '1',
                    'addBumpers': '1',
                    'captionFormat': 'data',
                }, headers={
                    'X-Request-CID': cid,
                    'X-Request-Token': request_token,
                })
            video = find_video(result)

        general = video['general']
        title = general['title']

        cdn = video['streamdata']['cdnType']

        if cdn == 'azure':
            formats = self._extract_azure_formats(video, video_id)
        elif cdn == 'free':
            formats = self._extract_free_formats(video, video_id)
        else:
            # TODO: reverse more cdns
            assert False

        self._sort_formats(formats)

        return {
            'id': video_id,
            'title': title,
            'alt_title': general.get('subtitle'),
            'description': general.get('description'),
            'release_year': int_or_none(general.get('year')),
            'creator': general.get('studio') or general.get('studio_adref'),
            'thumbnail': try_get(
                video, lambda x: x['imagedata']['thumb'], compat_str),
            'duration': parse_duration(general.get('runtime')),
            'timestamp': int_or_none(general.get('uploaded')),
            'episode_number': int_or_none(try_get(
                video, lambda x: x['episodedata']['episode'])),
            'season_number': int_or_none(try_get(
                video, lambda x: x['episodedata']['season'])),
            'formats': formats,
        }


class NexxEmbedIE(InfoExtractor):
    _VALID_URL = r'https?://embed\.nexx(?:\.cloud|cdn\.com)/\d+/(?:video/)?(?P<id>[^/?#&]+)'
    _TESTS = [{
        'url': 'http://embed.nexx.cloud/748/KC1614647Z27Y7T?autoplay=1',
        'md5': '16746bfc28c42049492385c989b26c4a',
        'info_dict': {
            'id': '161464',
            'ext': 'mp4',
            'title': 'Nervenkitzel Achterbahn',
            'alt_title': 'Karussellbauer in Deutschland',
            'description': 'md5:ffe7b1cc59a01f585e0569949aef73cc',
            'creator': 'SPIEGEL TV',
            'thumbnail': r're:^https?://.*\.jpg$',
            'duration': 2761,
            'timestamp': 1394021479,
            'upload_date': '20140305',
        },
        'params': {
            'format': 'bestvideo',
            'skip_download': True,
        },
    }, {
        'url': 'https://embed.nexx.cloud/11888/video/DSRTO7UVOX06S7',
        'only_matching': True,
    }]

    @staticmethod
    def _extract_urls(webpage):
        # Reference:
        # 1. https://nx-s.akamaized.net/files/201510/44.pdf

        # iFrame Embed Integration
        return [mobj.group('url') for mobj in re.finditer(
            r'<iframe[^>]+\bsrc=(["\'])(?P<url>(?:https?:)?//embed\.nexx(?:\.cloud|cdn\.com)/\d+/(?:(?!\1).)+)\1',
            webpage)]

    def _real_extract(self, url):
        embed_id = self._match_id(url)

        webpage = self._download_webpage(url, embed_id)

        return self.url_result(NexxIE._extract_url(webpage), ie=NexxIE.ie_key())
# coding: utf-8
from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..compat import (
    compat_urllib_parse_urlparse,
)
from ..utils import (
    ExtractorError,
    int_or_none,
    remove_end,
)


class NFLIE(InfoExtractor):
    IE_NAME = 'nfl.com'
    _VALID_URL = r'''(?x)
                    https?://
                        (?P<host>
                            (?:www\.)?
                            (?:
                                (?:
                                    nfl|
                                    buffalobills|
                                    miamidolphins|
                                    patriots|
                                    newyorkjets|
                                    baltimoreravens|
                                    bengals|
                                    clevelandbrowns|
                                    steelers|
                                    houstontexans|
                                    colts|
                                    jaguars|
                                    titansonline|
                                    denverbroncos|
                                    kcchiefs|
                                    raiders|
                                    chargers|
                                    dallascowboys|
                                    giants|
                                    philadelphiaeagles|
                                    redskins|
                                    chicagobears|
                                    detroitlions|
                                    packers|
                                    vikings|
                                    atlantafalcons|
                                    panthers|
                                    neworleanssaints|
                                    buccaneers|
                                    azcardinals|
                                    stlouisrams|
                                    49ers|
                                    seahawks
                                )\.com|
                                .+?\.clubs\.nfl\.com
                            )
                        )/
                        (?:.+?/)*
                        (?P<id>[^/#?&]+)
                    '''
    _TESTS = [{
        'url': 'http://www.nfl.com/videos/nfl-game-highlights/0ap3000000398478/Week-3-Redskins-vs-Eagles-highlights',
        'md5': '394ef771ddcd1354f665b471d78ec4c6',
        'info_dict': {
            'id': '0ap3000000398478',
            'ext': 'mp4',
            'title': 'Week 3: Redskins vs. Eagles highlights',
            'description': 'md5:56323bfb0ac4ee5ab24bd05fdf3bf478',
            'upload_date': '20140921',
            'timestamp': 1411337580,
            'thumbnail': r're:^https?://.*\.jpg$',
        }
    }, {
        'url': 'http://prod.www.steelers.clubs.nfl.com/video-and-audio/videos/LIVE_Post_Game_vs_Browns/9d72f26a-9e2b-4718-84d3-09fb4046c266',
        'md5': 'cf85bdb4bc49f6e9d3816d130c78279c',
        'info_dict': {
            'id': '9d72f26a-9e2b-4718-84d3-09fb4046c266',
            'ext': 'mp4',
            'title': 'LIVE: Post Game vs. Browns',
            'description': 'md5:6a97f7e5ebeb4c0e69a418a89e0636e8',
            'upload_date': '20131229',
            'timestamp': 1388354455,
            'thumbnail': r're:^https?://.*\.jpg$',
        }
    }, {
        'url': 'http://www.nfl.com/news/story/0ap3000000467586/article/patriots-seahawks-involved-in-lategame-skirmish',
        'info_dict': {
            'id': '0ap3000000467607',
            'ext': 'mp4',
            'title': 'Frustrations flare on the field',
            'description': 'Emotions ran high at the end of the Super Bowl on both sides of the ball after a dramatic finish.',
            'timestamp': 1422850320,
            'upload_date': '20150202',
        },
    }, {
        'url': 'http://www.patriots.com/video/2015/09/18/10-days-gillette',
        'md5': '4c319e2f625ffd0b481b4382c6fc124c',
        'info_dict': {
            'id': 'n-238346',
            'ext': 'mp4',
            'title': '10 Days at Gillette',
            'description': 'md5:8cd9cd48fac16de596eadc0b24add951',
            'timestamp': 1442618809,
            'upload_date': '20150918',
        },
    }, {
        # lowercase data-contentid
        'url': 'http://www.steelers.com/news/article-1/Tomlin-on-Ben-getting-Vick-ready/56399c96-4160-48cf-a7ad-1d17d4a3aef7',
        'info_dict': {
            'id': '12693586-6ea9-4743-9c1c-02c59e4a5ef2',
            'ext': 'mp4',
            'title': 'Tomlin looks ahead to Ravens on a short week',
            'description': 'md5:32f3f7b139f43913181d5cbb24ecad75',
            'timestamp': 1443459651,
            'upload_date': '20150928',
        },
        'params': {
            'skip_download': True,
        },
    }, {
        'url': 'http://www.nfl.com/videos/nfl-network-top-ten/09000d5d810a6bd4/Top-10-Gutsiest-Performances-Jack-Youngblood',
        'only_matching': True,
    }, {
        'url': 'http://www.buffalobills.com/video/videos/Rex_Ryan_Show_World_Wide_Rex/b1dcfab2-3190-4bb1-bfc0-d6e603d6601a',
        'only_matching': True,
    }]

    @staticmethod
    def prepend_host(host, url):
        if not url.startswith('http'):
            if not url.startswith('/'):
                url = '/%s' % url
            url = 'http://{0:}{1:}'.format(host, url)
        return url

    @staticmethod
    def format_from_stream(stream, protocol, host, path_prefix='',
                           preference=0, note=None):
        url = '{protocol:}://{host:}/{prefix:}{path:}'.format(
            protocol=protocol,
            host=host,
            prefix=path_prefix,
            path=stream.get('path'),
        )
        return {
            'url': url,
            'vbr': int_or_none(stream.get('rate', 0), 1000),
            'preference': preference,
            'format_note': note,
        }

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        video_id, host = mobj.group('id'), mobj.group('host')

        webpage = self._download_webpage(url, video_id)

        config_url = NFLIE.prepend_host(host, self._search_regex(
            r'(?:(?:config|configURL)\s*:\s*|<nflcs:avplayer[^>]+data-config\s*=\s*)(["\'])(?P<config>.+?)\1',
            webpage, 'config URL', default='static/content/static/config/video/config.json',
            group='config'))
        # For articles, the id in the url is not the video id
        video_id = self._search_regex(
            r'(?:<nflcs:avplayer[^>]+data-content[Ii]d\s*=\s*|content[Ii]d\s*:\s*)(["\'])(?P<id>(?:(?!\1).)+)\1',
            webpage, 'video id', default=video_id, group='id')
        config = self._download_json(config_url, video_id, 'Downloading player config')
        url_template = NFLIE.prepend_host(
            host, '{contentURLTemplate:}'.format(**config))
        video_data = self._download_json(
            url_template.format(id=video_id), video_id)

        formats = []
        cdn_data = video_data.get('cdnData', {})
        streams = cdn_data.get('bitrateInfo', [])
        if cdn_data.get('format') == 'EXTERNAL_HTTP_STREAM':
            parts = compat_urllib_parse_urlparse(cdn_data.get('uri'))
            protocol, host = parts.scheme, parts.netloc
            for stream in streams:
                formats.append(
                    NFLIE.format_from_stream(stream, protocol, host))
        else:
            cdns = config.get('cdns')
            if not cdns:
                raise ExtractorError('Failed to get CDN data', expected=True)

            for name, cdn in cdns.items():
                # LimeLight streams don't seem to work
                if cdn.get('name') == 'LIMELIGHT':
                    continue

                protocol = cdn.get('protocol')
                host = remove_end(cdn.get('host', ''), '/')
                if not (protocol and host):
                    continue

                prefix = cdn.get('pathprefix', '')
                if prefix and not prefix.endswith('/'):
                    prefix = '%s/' % prefix

                preference = 0
                if protocol == 'rtmp':
                    preference = -2
                elif 'prog' in name.lower():
                    preference = 1

                for stream in streams:
                    formats.append(
                        NFLIE.format_from_stream(stream, protocol, host,
                                                 prefix, preference, name))

        self._sort_formats(formats)

        thumbnail = None
        for q in ('xl', 'l', 'm', 's', 'xs'):
            thumbnail = video_data.get('imagePaths', {}).get(q)
            if thumbnail:
                break

        return {
            'id': video_id,
            'title': video_data.get('headline'),
            'formats': formats,
            'description': video_data.get('caption'),
            'duration': video_data.get('duration'),
            'thumbnail': thumbnail,
            'timestamp': int_or_none(video_data.get('posted'), 1000),
        }
from __future__ import unicode_literals

import re

from .common import InfoExtractor


class NhkVodIE(InfoExtractor):
    _VALID_URL = r'https?://www3\.nhk\.or\.jp/nhkworld/(?P<lang>[a-z]{2})/ondemand/(?P<type>video|audio)/(?P<id>\d{7}|[^/]+?-\d{8}-\d+)'
    # Content available only for a limited period of time. Visit
    # https://www3.nhk.or.jp/nhkworld/en/ondemand/ for working samples.
    _TESTS = [{
        # clip
        'url': 'https://www3.nhk.or.jp/nhkworld/en/ondemand/video/9999011/',
        'md5': '256a1be14f48d960a7e61e2532d95ec3',
        'info_dict': {
            'id': 'a95j5iza',
            'ext': 'mp4',
            'title': "Dining with the Chef - Chef Saito's Family recipe: MENCHI-KATSU",
            'description': 'md5:5aee4a9f9d81c26281862382103b0ea5',
            'timestamp': 1565965194,
            'upload_date': '20190816',
        },
    }, {
        'url': 'https://www3.nhk.or.jp/nhkworld/en/ondemand/video/2015173/',
        'only_matching': True,
    }, {
        'url': 'https://www3.nhk.or.jp/nhkworld/en/ondemand/audio/plugin-20190404-1/',
        'only_matching': True,
    }, {
        'url': 'https://www3.nhk.or.jp/nhkworld/fr/ondemand/audio/plugin-20190404-1/',
        'only_matching': True,
    }, {
        'url': 'https://www3.nhk.or.jp/nhkworld/en/ondemand/audio/j_art-20150903-1/',
        'only_matching': True,
    }]
    _API_URL_TEMPLATE = 'https://api.nhk.or.jp/nhkworld/%sod%slist/v7a/episode/%s/%s/all%s.json'

    def _real_extract(self, url):
        lang, m_type, episode_id = re.match(self._VALID_URL, url).groups()
        if episode_id.isdigit():
            episode_id = episode_id[:4] + '-' + episode_id[4:]

        is_video = m_type == 'video'
        episode = self._download_json(
            self._API_URL_TEMPLATE % (
                'v' if is_video else 'r',
                'clip' if episode_id[:4] == '9999' else 'esd',
                episode_id, lang, '/all' if is_video else ''),
            episode_id, query={'apikey': 'EJfK8jdS57GqlupFgAfAAwr573q01y6k'})['data']['episodes'][0]
        title = episode.get('sub_title_clean') or episode['sub_title']

        def get_clean_field(key):
            return episode.get(key + '_clean') or episode.get(key)

        series = get_clean_field('title')

        thumbnails = []
        for s, w, h in [('', 640, 360), ('_l', 1280, 720)]:
            img_path = episode.get('image' + s)
            if not img_path:
                continue
            thumbnails.append({
                'id': '%dp' % h,
                'height': h,
                'width': w,
                'url': 'https://www3.nhk.or.jp' + img_path,
            })

        info = {
            'id': episode_id + '-' + lang,
            'title': '%s - %s' % (series, title) if series and title else title,
            'description': get_clean_field('description'),
            'thumbnails': thumbnails,
            'series': series,
            'episode': title,
        }
        if is_video:
            info.update({
                '_type': 'url_transparent',
                'ie_key': 'Piksel',
                'url': 'https://player.piksel.com/v/refid/nhkworld/prefid/' + episode['vod_id'],
            })
        else:
            audio = episode['audio']
            audio_path = audio['audio']
            info['formats'] = self._extract_m3u8_formats(
                'https://nhkworld-vh.akamaihd.net/i%s/master.m3u8' % audio_path,
                episode_id, 'm4a', entry_protocol='m3u8_native',
                m3u8_id='hls', fatal=False)
            for f in info['formats']:
                f['language'] = lang
        return info
from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..compat import compat_str
from ..utils import (
    determine_ext,
    int_or_none,
    parse_iso8601,
    parse_duration,
)


class NHLBaseIE(InfoExtractor):
    def _real_extract(self, url):
        site, tmp_id = re.match(self._VALID_URL, url).groups()
        video_data = self._download_json(
            'https://%s/%s/%sid/v1/%s/details/web-v1.json'
            % (self._CONTENT_DOMAIN, site[:3], 'item/' if site == 'mlb' else '', tmp_id), tmp_id)
        if video_data.get('type') != 'video':
            video_data = video_data['media']
            video = video_data.get('video')
            if video:
                video_data = video
            else:
                videos = video_data.get('videos')
                if videos:
                    video_data = videos[0]

        video_id = compat_str(video_data['id'])
        title = video_data['title']

        formats = []
        for playback in video_data.get('playbacks', []):
            playback_url = playback.get('url')
            if not playback_url:
                continue
            ext = determine_ext(playback_url)
            if ext == 'm3u8':
                m3u8_formats = self._extract_m3u8_formats(
                    playback_url, video_id, 'mp4', 'm3u8_native',
                    m3u8_id=playback.get('name', 'hls'), fatal=False)
                self._check_formats(m3u8_formats, video_id)
                formats.extend(m3u8_formats)
            else:
                height = int_or_none(playback.get('height'))
                formats.append({
                    'format_id': playback.get('name', 'http' + ('-%dp' % height if height else '')),
                    'url': playback_url,
                    'width': int_or_none(playback.get('width')),
                    'height': height,
                    'tbr': int_or_none(self._search_regex(r'_(\d+)[kK]', playback_url, 'bitrate', default=None)),
                })
        self._sort_formats(formats)

        thumbnails = []
        cuts = video_data.get('image', {}).get('cuts') or []
        if isinstance(cuts, dict):
            cuts = cuts.values()
        for thumbnail_data in cuts:
            thumbnail_url = thumbnail_data.get('src')
            if not thumbnail_url:
                continue
            thumbnails.append({
                'url': thumbnail_url,
                'width': int_or_none(thumbnail_data.get('width')),
                'height': int_or_none(thumbnail_data.get('height')),
            })

        return {
            'id': video_id,
            'title': title,
            'description': video_data.get('description'),
            'timestamp': parse_iso8601(video_data.get('date')),
            'duration': parse_duration(video_data.get('duration')),
            'thumbnails': thumbnails,
            'formats': formats,
        }


class NHLIE(NHLBaseIE):
    IE_NAME = 'nhl.com'
    _VALID_URL = r'https?://(?:www\.)?(?P<site>nhl|wch2016)\.com/(?:[^/]+/)*c-(?P<id>\d+)'
    _CONTENT_DOMAIN = 'nhl.bamcontent.com'
    _TESTS = [{
        # type=video
        'url': 'https://www.nhl.com/video/anisimov-cleans-up-mess/t-277752844/c-43663503',
        'md5': '0f7b9a8f986fb4b4eeeece9a56416eaf',
        'info_dict': {
            'id': '43663503',
            'ext': 'mp4',
            'title': 'Anisimov cleans up mess',
            'description': 'md5:a02354acdfe900e940ce40706939ca63',
            'timestamp': 1461288600,
            'upload_date': '20160422',
        },
    }, {
        # type=article
        'url': 'https://www.nhl.com/news/dennis-wideman-suspended/c-278258934',
        'md5': '1f39f4ea74c1394dea110699a25b366c',
        'info_dict': {
            'id': '40784403',
            'ext': 'mp4',
            'title': 'Wideman suspended by NHL',
            'description': 'Flames defenseman Dennis Wideman was banned 20 games for violation of Rule 40 (Physical Abuse of Officials)',
            'upload_date': '20160204',
            'timestamp': 1454544904,
        },
    }, {
        # Some m3u8 URLs are invalid (https://github.com/ytdl-org/youtube-dl/issues/10713)
        'url': 'https://www.nhl.com/predators/video/poile-laviolette-on-subban-trade/t-277437416/c-44315003',
        'md5': '50b2bb47f405121484dda3ccbea25459',
        'info_dict': {
            'id': '44315003',
            'ext': 'mp4',
            'title': 'Poile, Laviolette on Subban trade',
            'description': 'General manager David Poile and head coach Peter Laviolette share their thoughts on acquiring P.K. Subban from Montreal (06/29/16)',
            'timestamp': 1467242866,
            'upload_date': '20160629',
        },
    }, {
        'url': 'https://www.wch2016.com/video/caneur-best-of-game-2-micd-up/t-281230378/c-44983703',
        'only_matching': True,
    }, {
        'url': 'https://www.wch2016.com/news/3-stars-team-europe-vs-team-canada/c-282195068',
        'only_matching': True,
    }]
# coding: utf-8
from __future__ import unicode_literals

import re

from .mtv import MTVServicesInfoExtractor
from ..utils import update_url_query


class NickIE(MTVServicesInfoExtractor):
    # None of videos on the website are still alive?
    IE_NAME = 'nick.com'
    _VALID_URL = r'https?://(?P<domain>(?:(?:www|beta)\.)?nick(?:jr)?\.com)/(?:[^/]+/)?(?:videos/clip|[^/]+/videos)/(?P<id>[^/?#.]+)'
    _FEED_URL = 'http://udat.mtvnservices.com/service1/dispatch.htm'
    _GEO_COUNTRIES = ['US']
    _TESTS = [{
        'url': 'http://www.nick.com/videos/clip/alvinnn-and-the-chipmunks-112-full-episode.html',
        'playlist': [
            {
                'md5': '6e5adc1e28253bbb1b28ab05403dd4d4',
                'info_dict': {
                    'id': 'be6a17b0-412d-11e5-8ff7-0026b9414f30',
                    'ext': 'mp4',
                    'title': 'ALVINNN!!! and The Chipmunks: "Mojo Missing/Who\'s The Animal" S1',
                    'description': 'Alvin is convinced his mojo was in a cap he gave to a fan, and must find a way to get his hat back before the Chipmunks’ big concert.\nDuring a costume visit to the zoo, Alvin finds himself mistaken for the real Tasmanian devil.',

                }
            },
            {
                'md5': 'd7be441fc53a1d4882fa9508a1e5b3ce',
                'info_dict': {
                    'id': 'be6b8f96-412d-11e5-8ff7-0026b9414f30',
                    'ext': 'mp4',
                    'title': 'ALVINNN!!! and The Chipmunks: "Mojo Missing/Who\'s The Animal" S2',
                    'description': 'Alvin is convinced his mojo was in a cap he gave to a fan, and must find a way to get his hat back before the Chipmunks’ big concert.\nDuring a costume visit to the zoo, Alvin finds himself mistaken for the real Tasmanian devil.',

                }
            },
            {
                'md5': 'efffe1728a234b2b0d2f2b343dd1946f',
                'info_dict': {
                    'id': 'be6cf7e6-412d-11e5-8ff7-0026b9414f30',
                    'ext': 'mp4',
                    'title': 'ALVINNN!!! and The Chipmunks: "Mojo Missing/Who\'s The Animal" S3',
                    'description': 'Alvin is convinced his mojo was in a cap he gave to a fan, and must find a way to get his hat back before the Chipmunks’ big concert.\nDuring a costume visit to the zoo, Alvin finds himself mistaken for the real Tasmanian devil.',
                }
            },
            {
                'md5': '1ec6690733ab9f41709e274a1d5c7556',
                'info_dict': {
                    'id': 'be6e3354-412d-11e5-8ff7-0026b9414f30',
                    'ext': 'mp4',
                    'title': 'ALVINNN!!! and The Chipmunks: "Mojo Missing/Who\'s The Animal" S4',
                    'description': 'Alvin is convinced his mojo was in a cap he gave to a fan, and must find a way to get his hat back before the Chipmunks’ big concert.\nDuring a costume visit to the zoo, Alvin finds himself mistaken for the real Tasmanian devil.',
                }
            },
        ],
    }, {
        'url': 'http://www.nickjr.com/paw-patrol/videos/pups-save-a-goldrush-s3-ep302-full-episode/',
        'only_matching': True,
    }, {
        'url': 'http://beta.nick.com/nicky-ricky-dicky-and-dawn/videos/nicky-ricky-dicky-dawn-301-full-episode/',
        'only_matching': True,
    }]

    def _get_feed_query(self, uri):
        return {
            'feed': 'nick_arc_player_prime',
            'mgid': uri,
        }

    def _real_extract(self, url):
        domain, display_id = re.match(self._VALID_URL, url).groups()
        video_data = self._download_json(
            'http://%s/data/video.endLevel.json' % domain,
            display_id, query={
                'urlKey': display_id,
            })
        return self._get_videos_info(video_data['player'] + video_data['id'])


class NickBrIE(MTVServicesInfoExtractor):
    IE_NAME = 'nickelodeon:br'
    _VALID_URL = r'''(?x)
                    https?://
                        (?:
                            (?P<domain>(?:www\.)?nickjr|mundonick\.uol)\.com\.br|
                            (?:www\.)?nickjr\.[a-z]{2}|
                            (?:www\.)?nickelodeonjunior\.fr
                        )
                        /(?:programas/)?[^/]+/videos/(?:episodios/)?(?P<id>[^/?\#.]+)
                    '''
    _TESTS = [{
        'url': 'http://www.nickjr.com.br/patrulha-canina/videos/210-labirinto-de-pipoca/',
        'only_matching': True,
    }, {
        'url': 'http://mundonick.uol.com.br/programas/the-loud-house/videos/muitas-irmas/7ljo9j',
        'only_matching': True,
    }, {
        'url': 'http://www.nickjr.nl/paw-patrol/videos/311-ge-wol-dig-om-terug-te-zijn/',
        'only_matching': True,
    }, {
        'url': 'http://www.nickjr.de/blaze-und-die-monster-maschinen/videos/f6caaf8f-e4e8-4cc1-b489-9380d6dcd059/',
        'only_matching': True,
    }, {
        'url': 'http://www.nickelodeonjunior.fr/paw-patrol-la-pat-patrouille/videos/episode-401-entier-paw-patrol/',
        'only_matching': True,
    }]

    def _real_extract(self, url):
        domain, display_id = re.match(self._VALID_URL, url).groups()
        webpage = self._download_webpage(url, display_id)
        uri = self._search_regex(
            r'data-(?:contenturi|mgid)="([^"]+)', webpage, 'mgid')
        video_id = self._id_from_uri(uri)
        config = self._download_json(
            'http://media.mtvnservices.com/pmt/e1/access/index.html',
            video_id, query={
                'uri': uri,
                'configtype': 'edge',
            }, headers={
                'Referer': url,
            })
        info_url = self._remove_template_parameter(config['feedWithQueryParams'])
        if info_url == 'None':
            if domain.startswith('www.'):
                domain = domain[4:]
            content_domain = {
                'mundonick.uol': 'mundonick.com.br',
                'nickjr': 'br.nickelodeonjunior.tv',
            }[domain]
            query = {
                'mgid': uri,
                'imageEp': content_domain,
                'arcEp': content_domain,
            }
            if domain == 'nickjr.com.br':
                query['ep'] = 'c4b16088'
            info_url = update_url_query(
                'http://feeds.mtvnservices.com/od/feed/intl-mrss-player-feed', query)
        return self._get_videos_info_from_url(info_url, video_id)


class NickDeIE(MTVServicesInfoExtractor):
    IE_NAME = 'nick.de'
    _VALID_URL = r'https?://(?:www\.)?(?P<host>nick\.(?:de|com\.pl|ch)|nickelodeon\.(?:nl|be|at|dk|no|se))/[^/]+/(?:[^/]+/)*(?P<id>[^/?#&]+)'
    _TESTS = [{
        'url': 'http://www.nick.de/playlist/3773-top-videos/videos/episode/17306-zu-wasser-und-zu-land-rauchende-erdnusse',
        'only_matching': True,
    }, {
        'url': 'http://www.nick.de/shows/342-icarly',
        'only_matching': True,
    }, {
        'url': 'http://www.nickelodeon.nl/shows/474-spongebob/videos/17403-een-kijkje-in-de-keuken-met-sandy-van-binnenuit',
        'only_matching': True,
    }, {
        'url': 'http://www.nickelodeon.at/playlist/3773-top-videos/videos/episode/77993-das-letzte-gefecht',
        'only_matching': True,
    }, {
        'url': 'http://www.nick.com.pl/seriale/474-spongebob-kanciastoporty/wideo/17412-teatr-to-jest-to-rodeo-oszolom',
        'only_matching': True,
    }, {
        'url': 'http://www.nickelodeon.no/program/2626-bulderhuset/videoer/90947-femteklasse-veronica-vs-vanzilla',
        'only_matching': True,
    }, {
        'url': 'http://www.nickelodeon.dk/serier/2626-hojs-hus/videoer/761-tissepause',
        'only_matching': True,
    }, {
        'url': 'http://www.nickelodeon.se/serier/2626-lugn-i-stormen/videos/998-',
        'only_matching': True,
    }, {
        'url': 'http://www.nick.ch/shows/2304-adventure-time-abenteuerzeit-mit-finn-und-jake',
        'only_matching': True,
    }, {
        'url': 'http://www.nickelodeon.be/afspeellijst/4530-top-videos/videos/episode/73917-inval-broodschapper-lariekoek-arie',
        'only_matching': True,
    }]

    def _extract_mrss_url(self, webpage, host):
        return update_url_query(self._search_regex(
            r'data-mrss=(["\'])(?P<url>http.+?)\1', webpage, 'mrss url', group='url'),
            {'siteKey': host})

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        video_id = mobj.group('id')
        host = mobj.group('host')

        webpage = self._download_webpage(url, video_id)

        mrss_url = self._extract_mrss_url(webpage, host)

        return self._get_videos_info_from_url(mrss_url, video_id)


class NickNightIE(NickDeIE):
    IE_NAME = 'nicknight'
    _VALID_URL = r'https?://(?:www\.)(?P<host>nicknight\.(?:de|at|tv))/(?:playlist|shows)/(?:[^/]+/)*(?P<id>[^/?#&]+)'
    _TESTS = [{
        'url': 'http://www.nicknight.at/shows/977-awkward/videos/85987-nimmer-beste-freunde',
        'only_matching': True,
    }, {
        'url': 'http://www.nicknight.at/shows/977-awkward',
        'only_matching': True,
    }, {
        'url': 'http://www.nicknight.at/shows/1900-faking-it',
        'only_matching': True,
    }]

    def _extract_mrss_url(self, webpage, *args):
        return self._search_regex(
            r'mrss\s*:\s*(["\'])(?P<url>http.+?)\1', webpage,
            'mrss url', group='url')


class NickRuIE(MTVServicesInfoExtractor):
    IE_NAME = 'nickelodeonru'
    _VALID_URL = r'https?://(?:www\.)nickelodeon\.(?:ru|fr|es|pt|ro|hu|com\.tr)/[^/]+/(?:[^/]+/)*(?P<id>[^/?#&]+)'
    _TESTS = [{
        'url': 'http://www.nickelodeon.ru/shows/henrydanger/videos/episodes/3-sezon-15-seriya-licenziya-na-polyot/pmomfb#playlist/7airc6',
        'only_matching': True,
    }, {
        'url': 'http://www.nickelodeon.ru/videos/smotri-na-nickelodeon-v-iyule/g9hvh7',
        'only_matching': True,
    }, {
        'url': 'http://www.nickelodeon.fr/programmes/bob-l-eponge/videos/le-marathon-de-booh-kini-bottom-mardi-31-octobre/nfn7z0',
        'only_matching': True,
    }, {
        'url': 'http://www.nickelodeon.es/videos/nickelodeon-consejos-tortitas/f7w7xy',
        'only_matching': True,
    }, {
        'url': 'http://www.nickelodeon.pt/series/spongebob-squarepants/videos/a-bolha-de-tinta-gigante/xutq1b',
        'only_matching': True,
    }, {
        'url': 'http://www.nickelodeon.ro/emisiuni/shimmer-si-shine/video/nahal-din-bomboane/uw5u2k',
        'only_matching': True,
    }, {
        'url': 'http://www.nickelodeon.hu/musorok/spongyabob-kockanadrag/videok/episodes/buborekfujas-az-elszakadt-nadrag/q57iob#playlist/k6te4y',
        'only_matching': True,
    }, {
        'url': 'http://www.nickelodeon.com.tr/programlar/sunger-bob/videolar/kayip-yatak/mgqbjy',
        'only_matching': True,
    }]

    def _real_extract(self, url):
        video_id = self._match_id(url)
        webpage = self._download_webpage(url, video_id)
        mgid = self._extract_mgid(webpage)
        return self.url_result('http://media.mtvnservices.com/embed/%s' % mgid)
# coding: utf-8
from __future__ import unicode_literals

import json
import datetime

from .common import InfoExtractor
from ..compat import (
    compat_parse_qs,
    compat_urlparse,
)
from ..utils import (
    determine_ext,
    dict_get,
    ExtractorError,
    int_or_none,
    float_or_none,
    parse_duration,
    parse_iso8601,
    remove_start,
    try_get,
    unified_timestamp,
    urlencode_postdata,
    xpath_text,
)


class NiconicoIE(InfoExtractor):
    IE_NAME = 'niconico'
    IE_DESC = 'ニコニコ動画'

    _TESTS = [{
        'url': 'http://www.nicovideo.jp/watch/sm22312215',
        'md5': 'd1a75c0823e2f629128c43e1212760f9',
        'info_dict': {
            'id': 'sm22312215',
            'ext': 'mp4',
            'title': 'Big Buck Bunny',
            'thumbnail': r're:https?://.*',
            'uploader': 'takuya0301',
            'uploader_id': '2698420',
            'upload_date': '20131123',
            'timestamp': int,  # timestamp is unstable
            'description': '(c) copyright 2008, Blender Foundation / www.bigbuckbunny.org',
            'duration': 33,
            'view_count': int,
            'comment_count': int,
        },
        'skip': 'Requires an account',
    }, {
        # File downloaded with and without credentials are different, so omit
        # the md5 field
        'url': 'http://www.nicovideo.jp/watch/nm14296458',
        'info_dict': {
            'id': 'nm14296458',
            'ext': 'swf',
            'title': '【鏡音リン】Dance on media【オリジナル】take2!',
            'description': 'md5:689f066d74610b3b22e0f1739add0f58',
            'thumbnail': r're:https?://.*',
            'uploader': 'りょうた',
            'uploader_id': '18822557',
            'upload_date': '20110429',
            'timestamp': 1304065916,
            'duration': 209,
        },
        'skip': 'Requires an account',
    }, {
        # 'video exists but is marked as "deleted"
        # md5 is unstable
        'url': 'http://www.nicovideo.jp/watch/sm10000',
        'info_dict': {
            'id': 'sm10000',
            'ext': 'unknown_video',
            'description': 'deleted',
            'title': 'ドラえもんエターナル第3話「決戦第3新東京市」<前編>',
            'thumbnail': r're:https?://.*',
            'upload_date': '20071224',
            'timestamp': int,  # timestamp field has different value if logged in
            'duration': 304,
            'view_count': int,
        },
        'skip': 'Requires an account',
    }, {
        'url': 'http://www.nicovideo.jp/watch/so22543406',
        'info_dict': {
            'id': '1388129933',
            'ext': 'mp4',
            'title': '【第1回】RADIOアニメロミックス ラブライブ!~のぞえりRadio Garden~',
            'description': 'md5:b27d224bb0ff53d3c8269e9f8b561cf1',
            'thumbnail': r're:https?://.*',
            'timestamp': 1388851200,
            'upload_date': '20140104',
            'uploader': 'アニメロチャンネル',
            'uploader_id': '312',
        },
        'skip': 'The viewing period of the video you were searching for has expired.',
    }, {
        # video not available via `getflv`; "old" HTML5 video
        'url': 'http://www.nicovideo.jp/watch/sm1151009',
        'md5': '8fa81c364eb619d4085354eab075598a',
        'info_dict': {
            'id': 'sm1151009',
            'ext': 'mp4',
            'title': 'マスターシステム本体内蔵のスペハリのメインテーマ(PSG版)',
            'description': 'md5:6ee077e0581ff5019773e2e714cdd0b7',
            'thumbnail': r're:https?://.*',
            'duration': 184,
            'timestamp': 1190868283,
            'upload_date': '20070927',
            'uploader': 'denden2',
            'uploader_id': '1392194',
            'view_count': int,
            'comment_count': int,
        },
        'skip': 'Requires an account',
    }, {
        # "New" HTML5 video
        # md5 is unstable
        'url': 'http://www.nicovideo.jp/watch/sm31464864',
        'info_dict': {
            'id': 'sm31464864',
            'ext': 'mp4',
            'title': '新作TVアニメ「戦姫絶唱シンフォギアAXZ」PV 最高画質',
            'description': 'md5:e52974af9a96e739196b2c1ca72b5feb',
            'timestamp': 1498514060,
            'upload_date': '20170626',
            'uploader': 'ゲスト',
            'uploader_id': '40826363',
            'thumbnail': r're:https?://.*',
            'duration': 198,
            'view_count': int,
            'comment_count': int,
        },
        'skip': 'Requires an account',
    }, {
        # Video without owner
        'url': 'http://www.nicovideo.jp/watch/sm18238488',
        'md5': 'd265680a1f92bdcbbd2a507fc9e78a9e',
        'info_dict': {
            'id': 'sm18238488',
            'ext': 'mp4',
            'title': '【実写版】ミュータントタートルズ',
            'description': 'md5:15df8988e47a86f9e978af2064bf6d8e',
            'timestamp': 1341160408,
            'upload_date': '20120701',
            'uploader': None,
            'uploader_id': None,
            'thumbnail': r're:https?://.*',
            'duration': 5271,
            'view_count': int,
            'comment_count': int,
        },
        'skip': 'Requires an account',
    }, {
        'url': 'http://sp.nicovideo.jp/watch/sm28964488?ss_pos=1&cp_in=wt_tg',
        'only_matching': True,
    }]

    _VALID_URL = r'https?://(?:www\.|secure\.|sp\.)?nicovideo\.jp/watch/(?P<id>(?:[a-z]{2})?[0-9]+)'
    _NETRC_MACHINE = 'niconico'

    def _real_initialize(self):
        self._login()

    def _login(self):
        username, password = self._get_login_info()
        # No authentication to be performed
        if not username:
            return True

        # Log in
        login_ok = True
        login_form_strs = {
            'mail_tel': username,
            'password': password,
        }
        urlh = self._request_webpage(
            'https://account.nicovideo.jp/api/v1/login', None,
            note='Logging in', errnote='Unable to log in',
            data=urlencode_postdata(login_form_strs))
        if urlh is False:
            login_ok = False
        else:
            parts = compat_urlparse.urlparse(urlh.geturl())
            if compat_parse_qs(parts.query).get('message', [None])[0] == 'cant_login':
                login_ok = False
        if not login_ok:
            self._downloader.report_warning('unable to log in: bad username or password')
        return login_ok

    def _extract_format_for_quality(self, api_data, video_id, audio_quality, video_quality):
        def yesno(boolean):
            return 'yes' if boolean else 'no'

        session_api_data = api_data['video']['dmcInfo']['session_api']
        session_api_endpoint = session_api_data['urls'][0]

        format_id = '-'.join(map(lambda s: remove_start(s['id'], 'archive_'), [video_quality, audio_quality]))

        session_response = self._download_json(
            session_api_endpoint['url'], video_id,
            query={'_format': 'json'},
            headers={'Content-Type': 'application/json'},
            note='Downloading JSON metadata for %s' % format_id,
            data=json.dumps({
                'session': {
                    'client_info': {
                        'player_id': session_api_data['player_id'],
                    },
                    'content_auth': {
                        'auth_type': session_api_data['auth_types'][session_api_data['protocols'][0]],
                        'content_key_timeout': session_api_data['content_key_timeout'],
                        'service_id': 'nicovideo',
                        'service_user_id': session_api_data['service_user_id']
                    },
                    'content_id': session_api_data['content_id'],
                    'content_src_id_sets': [{
                        'content_src_ids': [{
                            'src_id_to_mux': {
                                'audio_src_ids': [audio_quality['id']],
                                'video_src_ids': [video_quality['id']],
                            }
                        }]
                    }],
                    'content_type': 'movie',
                    'content_uri': '',
                    'keep_method': {
                        'heartbeat': {
                            'lifetime': session_api_data['heartbeat_lifetime']
                        }
                    },
                    'priority': session_api_data['priority'],
                    'protocol': {
                        'name': 'http',
                        'parameters': {
                            'http_parameters': {
                                'parameters': {
                                    'http_output_download_parameters': {
                                        'use_ssl': yesno(session_api_endpoint['is_ssl']),
                                        'use_well_known_port': yesno(session_api_endpoint['is_well_known_port']),
                                    }
                                }
                            }
                        }
                    },
                    'recipe_id': session_api_data['recipe_id'],
                    'session_operation_auth': {
                        'session_operation_auth_by_signature': {
                            'signature': session_api_data['signature'],
                            'token': session_api_data['token'],
                        }
                    },
                    'timing_constraint': 'unlimited'
                }
            }).encode())

        resolution = video_quality.get('resolution', {})

        return {
            'url': session_response['data']['session']['content_uri'],
            'format_id': format_id,
            'ext': 'mp4',  # Session API are used in HTML5, which always serves mp4
            'abr': float_or_none(audio_quality.get('bitrate'), 1000),
            'vbr': float_or_none(video_quality.get('bitrate'), 1000),
            'height': resolution.get('height'),
            'width': resolution.get('width'),
        }

    def _real_extract(self, url):
        video_id = self._match_id(url)

        # Get video webpage. We are not actually interested in it for normal
        # cases, but need the cookies in order to be able to download the
        # info webpage
        webpage, handle = self._download_webpage_handle(
            'http://www.nicovideo.jp/watch/' + video_id, video_id)
        if video_id.startswith('so'):
            video_id = self._match_id(handle.geturl())

        api_data = self._parse_json(self._html_search_regex(
            'data-api-data="([^"]+)"', webpage,
            'API data', default='{}'), video_id)

        def _format_id_from_url(video_url):
            return 'economy' if video_real_url.endswith('low') else 'normal'

        try:
            video_real_url = api_data['video']['smileInfo']['url']
        except KeyError:  # Flash videos
            # Get flv info
            flv_info_webpage = self._download_webpage(
                'http://flapi.nicovideo.jp/api/getflv/' + video_id + '?as3=1',
                video_id, 'Downloading flv info')

            flv_info = compat_urlparse.parse_qs(flv_info_webpage)
            if 'url' not in flv_info:
                if 'deleted' in flv_info:
                    raise ExtractorError('The video has been deleted.',
                                         expected=True)
                elif 'closed' in flv_info:
                    raise ExtractorError('Niconico videos now require logging in',
                                         expected=True)
                elif 'error' in flv_info:
                    raise ExtractorError('%s reports error: %s' % (
                        self.IE_NAME, flv_info['error'][0]), expected=True)
                else:
                    raise ExtractorError('Unable to find video URL')

            video_info_xml = self._download_xml(
                'http://ext.nicovideo.jp/api/getthumbinfo/' + video_id,
                video_id, note='Downloading video info page')

            def get_video_info(items):
                if not isinstance(items, list):
                    items = [items]
                for item in items:
                    ret = xpath_text(video_info_xml, './/' + item)
                    if ret:
                        return ret

            video_real_url = flv_info['url'][0]

            extension = get_video_info('movie_type')
            if not extension:
                extension = determine_ext(video_real_url)

            formats = [{
                'url': video_real_url,
                'ext': extension,
                'format_id': _format_id_from_url(video_real_url),
            }]
        else:
            formats = []

            dmc_info = api_data['video'].get('dmcInfo')
            if dmc_info:  # "New" HTML5 videos
                quality_info = dmc_info['quality']
                for audio_quality in quality_info['audios']:
                    for video_quality in quality_info['videos']:
                        if not audio_quality['available'] or not video_quality['available']:
                            continue
                        formats.append(self._extract_format_for_quality(
                            api_data, video_id, audio_quality, video_quality))

                self._sort_formats(formats)
            else:  # "Old" HTML5 videos
                formats = [{
                    'url': video_real_url,
                    'ext': 'mp4',
                    'format_id': _format_id_from_url(video_real_url),
                }]

            def get_video_info(items):
                return dict_get(api_data['video'], items)

        # Start extracting information
        title = get_video_info('title')
        if not title:
            title = self._og_search_title(webpage, default=None)
        if not title:
            title = self._html_search_regex(
                r'<span[^>]+class="videoHeaderTitle"[^>]*>([^<]+)</span>',
                webpage, 'video title')

        watch_api_data_string = self._html_search_regex(
            r'<div[^>]+id="watchAPIDataContainer"[^>]+>([^<]+)</div>',
            webpage, 'watch api data', default=None)
        watch_api_data = self._parse_json(watch_api_data_string, video_id) if watch_api_data_string else {}
        video_detail = watch_api_data.get('videoDetail', {})

        thumbnail = (
            get_video_info(['thumbnail_url', 'thumbnailURL'])
            or self._html_search_meta('image', webpage, 'thumbnail', default=None)
            or video_detail.get('thumbnail'))

        description = get_video_info('description')

        timestamp = (parse_iso8601(get_video_info('first_retrieve'))
                     or unified_timestamp(get_video_info('postedDateTime')))
        if not timestamp:
            match = self._html_search_meta('datePublished', webpage, 'date published', default=None)
            if match:
                timestamp = parse_iso8601(match.replace('+', ':00+'))
        if not timestamp and video_detail.get('postedAt'):
            timestamp = parse_iso8601(
                video_detail['postedAt'].replace('/', '-'),
                delimiter=' ', timezone=datetime.timedelta(hours=9))

        view_count = int_or_none(get_video_info(['view_counter', 'viewCount']))
        if not view_count:
            match = self._html_search_regex(
                r'>Views: <strong[^>]*>([^<]+)</strong>',
                webpage, 'view count', default=None)
            if match:
                view_count = int_or_none(match.replace(',', ''))
        view_count = view_count or video_detail.get('viewCount')

        comment_count = (int_or_none(get_video_info('comment_num'))
                         or video_detail.get('commentCount')
                         or try_get(api_data, lambda x: x['thread']['commentCount']))
        if not comment_count:
            match = self._html_search_regex(
                r'>Comments: <strong[^>]*>([^<]+)</strong>',
                webpage, 'comment count', default=None)
            if match:
                comment_count = int_or_none(match.replace(',', ''))

        duration = (parse_duration(
            get_video_info('length')
            or self._html_search_meta(
                'video:duration', webpage, 'video duration', default=None))
            or video_detail.get('length')
            or get_video_info('duration'))

        webpage_url = get_video_info('watch_url') or url

        # Note: cannot use api_data.get('owner', {}) because owner may be set to "null"
        # in the JSON, which will cause None to be returned instead of {}.
        owner = try_get(api_data, lambda x: x.get('owner'), dict) or {}
        uploader_id = get_video_info(['ch_id', 'user_id']) or owner.get('id')
        uploader = get_video_info(['ch_name', 'user_nickname']) or owner.get('nickname')

        return {
            'id': video_id,
            'title': title,
            'formats': formats,
            'thumbnail': thumbnail,
            'description': description,
            'uploader': uploader,
            'timestamp': timestamp,
            'uploader_id': uploader_id,
            'view_count': view_count,
            'comment_count': comment_count,
            'duration': duration,
            'webpage_url': webpage_url,
        }


class NiconicoPlaylistIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?nicovideo\.jp/mylist/(?P<id>\d+)'

    _TEST = {
        'url': 'http://www.nicovideo.jp/mylist/27411728',
        'info_dict': {
            'id': '27411728',
            'title': 'AKB48のオールナイトニッポン',
        },
        'playlist_mincount': 225,
    }

    def _real_extract(self, url):
        list_id = self._match_id(url)
        webpage = self._download_webpage(url, list_id)

        entries_json = self._search_regex(r'Mylist\.preload\(\d+, (\[.*\])\);',
                                          webpage, 'entries')
        entries = json.loads(entries_json)
        entries = [{
            '_type': 'url',
            'ie_key': NiconicoIE.ie_key(),
            'url': ('http://www.nicovideo.jp/watch/%s' %
                    entry['item_data']['video_id']),
        } for entry in entries]

        return {
            '_type': 'playlist',
            'title': self._search_regex(r'\s+name: "(.*?)"', webpage, 'title'),
            'id': list_id,
            'entries': entries,
        }
# coding: utf-8
from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..utils import (
    parse_iso8601,
    float_or_none,
    ExtractorError,
    int_or_none,
)


class NineCNineMediaIE(InfoExtractor):
    IE_NAME = '9c9media'
    _GEO_COUNTRIES = ['CA']
    _VALID_URL = r'9c9media:(?P<destination_code>[^:]+):(?P<id>\d+)'
    _API_BASE_TEMPLATE = 'http://capi.9c9media.com/destinations/%s/platforms/desktop/contents/%s/'

    def _real_extract(self, url):
        destination_code, content_id = re.match(self._VALID_URL, url).groups()
        api_base_url = self._API_BASE_TEMPLATE % (destination_code, content_id)
        content = self._download_json(api_base_url, content_id, query={
            '$include': '[Media,Season,ContentPackages]',
        })
        title = content['Name']
        if len(content['ContentPackages']) > 1:
            raise ExtractorError('multiple content packages')
        content_package = content['ContentPackages'][0]
        package_id = content_package['Id']
        content_package_url = api_base_url + 'contentpackages/%s/' % package_id
        content_package = self._download_json(
            content_package_url, content_id, query={
                '$include': '[HasClosedCaptions]',
            })

        if content_package.get('Constraints', {}).get('Security', {}).get('Type'):
            raise ExtractorError('This video is DRM protected.', expected=True)

        manifest_base_url = content_package_url + 'manifest.'
        formats = []
        formats.extend(self._extract_m3u8_formats(
            manifest_base_url + 'm3u8', content_id, 'mp4',
            'm3u8_native', m3u8_id='hls', fatal=False))
        formats.extend(self._extract_f4m_formats(
            manifest_base_url + 'f4m', content_id,
            f4m_id='hds', fatal=False))
        formats.extend(self._extract_mpd_formats(
            manifest_base_url + 'mpd', content_id,
            mpd_id='dash', fatal=False))
        self._sort_formats(formats)

        thumbnails = []
        for image in content.get('Images', []):
            image_url = image.get('Url')
            if not image_url:
                continue
            thumbnails.append({
                'url': image_url,
                'width': int_or_none(image.get('Width')),
                'height': int_or_none(image.get('Height')),
            })

        tags, categories = [], []
        for source_name, container in (('Tags', tags), ('Genres', categories)):
            for e in content.get(source_name, []):
                e_name = e.get('Name')
                if not e_name:
                    continue
                container.append(e_name)

        season = content.get('Season', {})

        info = {
            'id': content_id,
            'title': title,
            'description': content.get('Desc') or content.get('ShortDesc'),
            'timestamp': parse_iso8601(content.get('BroadcastDateTime')),
            'episode_number': int_or_none(content.get('Episode')),
            'season': season.get('Name'),
            'season_number': season.get('Number'),
            'season_id': season.get('Id'),
            'series': content.get('Media', {}).get('Name'),
            'tags': tags,
            'categories': categories,
            'duration': float_or_none(content_package.get('Duration')),
            'formats': formats,
        }

        if content_package.get('HasClosedCaptions'):
            info['subtitles'] = {
                'en': [{
                    'url': manifest_base_url + 'vtt',
                    'ext': 'vtt',
                }, {
                    'url': manifest_base_url + 'srt',
                    'ext': 'srt',
                }]
            }

        return info
from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..utils import str_to_int


class NineGagIE(InfoExtractor):
    IE_NAME = '9gag'
    _VALID_URL = r'https?://(?:www\.)?9gag(?:\.com/tv|\.tv)/(?:p|embed)/(?P<id>[a-zA-Z0-9]+)(?:/(?P<display_id>[^?#/]+))?'

    _TESTS = [{
        'url': 'http://9gag.com/tv/p/Kk2X5/people-are-awesome-2013-is-absolutely-awesome',
        'info_dict': {
            'id': 'kXzwOKyGlSA',
            'ext': 'mp4',
            'description': 'This 3-minute video will make you smile and then make you feel untalented and insignificant. Anyway, you should share this awesomeness. (Thanks, Dino!)',
            'title': '\"People Are Awesome 2013\" Is Absolutely Awesome',
            'uploader_id': 'UCdEH6EjDKwtTe-sO2f0_1XA',
            'uploader': 'CompilationChannel',
            'upload_date': '20131110',
            'view_count': int,
        },
        'add_ie': ['Youtube'],
    }, {
        'url': 'http://9gag.com/tv/p/aKolP3',
        'info_dict': {
            'id': 'aKolP3',
            'ext': 'mp4',
            'title': 'This Guy Travelled 11 countries In 44 days Just To Make This Amazing Video',
            'description': "I just saw more in 1 minute than I've seen in 1 year. This guy's video is epic!!",
            'uploader_id': 'rickmereki',
            'uploader': 'Rick Mereki',
            'upload_date': '20110803',
            'view_count': int,
        },
        'add_ie': ['Vimeo'],
    }, {
        'url': 'http://9gag.com/tv/p/KklwM',
        'only_matching': True,
    }, {
        'url': 'http://9gag.tv/p/Kk2X5',
        'only_matching': True,
    }, {
        'url': 'http://9gag.com/tv/embed/a5Dmvl',
        'only_matching': True,
    }]

    _EXTERNAL_VIDEO_PROVIDER = {
        '1': {
            'url': '%s',
            'ie_key': 'Youtube',
        },
        '2': {
            'url': 'http://player.vimeo.com/video/%s',
            'ie_key': 'Vimeo',
        },
        '3': {
            'url': 'http://instagram.com/p/%s',
            'ie_key': 'Instagram',
        },
        '4': {
            'url': 'http://vine.co/v/%s',
            'ie_key': 'Vine',
        },
    }

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        video_id = mobj.group('id')
        display_id = mobj.group('display_id') or video_id

        webpage = self._download_webpage(url, display_id)

        post_view = self._parse_json(
            self._search_regex(
                r'var\s+postView\s*=\s*new\s+app\.PostView\({\s*post:\s*({.+?})\s*,\s*posts:\s*prefetchedCurrentPost',
                webpage, 'post view'),
            display_id)

        ie_key = None
        source_url = post_view.get('sourceUrl')
        if not source_url:
            external_video_id = post_view['videoExternalId']
            external_video_provider = post_view['videoExternalProvider']
            source_url = self._EXTERNAL_VIDEO_PROVIDER[external_video_provider]['url'] % external_video_id
            ie_key = self._EXTERNAL_VIDEO_PROVIDER[external_video_provider]['ie_key']
        title = post_view['title']
        description = post_view.get('description')
        view_count = str_to_int(post_view.get('externalView'))
        thumbnail = post_view.get('thumbnail_700w') or post_view.get('ogImageUrl') or post_view.get('thumbnail_300w')

        return {
            '_type': 'url_transparent',
            'url': source_url,
            'ie_key': ie_key,
            'id': video_id,
            'display_id': display_id,
            'title': title,
            'description': description,
            'view_count': view_count,
            'thumbnail': thumbnail,
        }
# coding: utf-8
from __future__ import unicode_literals

from .common import InfoExtractor
from ..compat import compat_str
from ..utils import (
    ExtractorError,
    int_or_none,
    float_or_none,
    smuggle_url,
)


class NineNowIE(InfoExtractor):
    IE_NAME = '9now.com.au'
    _VALID_URL = r'https?://(?:www\.)?9now\.com\.au/(?:[^/]+/){2}(?P<id>[^/?#]+)'
    _GEO_COUNTRIES = ['AU']
    _TESTS = [{
        # clip
        'url': 'https://www.9now.com.au/afl-footy-show/2016/clip-ciql02091000g0hp5oktrnytc',
        'md5': '17cf47d63ec9323e562c9957a968b565',
        'info_dict': {
            'id': '16801',
            'ext': 'mp4',
            'title': 'St. Kilda\'s Joey Montagna on the potential for a player\'s strike',
            'description': 'Is a boycott of the NAB Cup "on the table"?',
            'uploader_id': '4460760524001',
            'upload_date': '20160713',
            'timestamp': 1468421266,
        },
        'skip': 'Only available in Australia',
    }, {
        # episode
        'url': 'https://www.9now.com.au/afl-footy-show/2016/episode-19',
        'only_matching': True,
    }, {
        # DRM protected
        'url': 'https://www.9now.com.au/andrew-marrs-history-of-the-world/season-1/episode-1',
        'only_matching': True,
    }]
    BRIGHTCOVE_URL_TEMPLATE = 'http://players.brightcove.net/4460760524001/default_default/index.html?videoId=%s'

    def _real_extract(self, url):
        display_id = self._match_id(url)
        webpage = self._download_webpage(url, display_id)
        page_data = self._parse_json(self._search_regex(
            r'window\.__data\s*=\s*({.*?});', webpage,
            'page data', default='{}'), display_id, fatal=False)
        if not page_data:
            page_data = self._parse_json(self._parse_json(self._search_regex(
                r'window\.__data\s*=\s*JSON\.parse\s*\(\s*(".+?")\s*\)\s*;',
                webpage, 'page data'), display_id), display_id)

        for kind in ('episode', 'clip'):
            current_key = page_data.get(kind, {}).get(
                'current%sKey' % kind.capitalize())
            if not current_key:
                continue
            cache = page_data.get(kind, {}).get('%sCache' % kind, {})
            if not cache:
                continue
            common_data = (cache.get(current_key) or list(cache.values())[0])[kind]
            break
        else:
            raise ExtractorError('Unable to find video data')

        video_data = common_data['video']

        if video_data.get('drm'):
            raise ExtractorError('This video is DRM protected.', expected=True)

        brightcove_id = video_data.get('brightcoveId') or 'ref:' + video_data['referenceId']
        video_id = compat_str(video_data.get('id') or brightcove_id)
        title = common_data['name']

        thumbnails = [{
            'id': thumbnail_id,
            'url': thumbnail_url,
            'width': int_or_none(thumbnail_id[1:])
        } for thumbnail_id, thumbnail_url in common_data.get('image', {}).get('sizes', {}).items()]

        return {
            '_type': 'url_transparent',
            'url': smuggle_url(
                self.BRIGHTCOVE_URL_TEMPLATE % brightcove_id,
                {'geo_countries': self._GEO_COUNTRIES}),
            'id': video_id,
            'title': title,
            'description': common_data.get('description'),
            'duration': float_or_none(video_data.get('duration'), 1000),
            'thumbnails': thumbnails,
            'ie_key': 'BrightcoveNew',
        }
# coding: utf-8
from __future__ import unicode_literals

import re

from .common import InfoExtractor
from .ooyala import OoyalaIE


class NintendoIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?nintendo\.com/(?:games/detail|nintendo-direct)/(?P<id>[^/?#&]+)'
    _TESTS = [{
        'url': 'https://www.nintendo.com/games/detail/duck-hunt-wii-u/',
        'info_dict': {
            'id': 'MzMmticjp0VPzO3CCj4rmFOuohEuEWoW',
            'ext': 'flv',
            'title': 'Duck Hunt Wii U VC NES - Trailer',
            'duration': 60.326,
        },
        'params': {
            'skip_download': True,
        },
        'add_ie': ['Ooyala'],
    }, {
        'url': 'http://www.nintendo.com/games/detail/tokyo-mirage-sessions-fe-wii-u',
        'info_dict': {
            'id': 'tokyo-mirage-sessions-fe-wii-u',
            'title': 'Tokyo Mirage Sessions ♯FE',
        },
        'playlist_count': 4,
    }, {
        'url': 'https://www.nintendo.com/nintendo-direct/09-04-2019/',
        'info_dict': {
            'id': 'J2bXdmaTE6fe3dWJTPcc7m23FNbc_A1V',
            'ext': 'mp4',
            'title': 'Switch_ROS_ND0904-H264.mov',
            'duration': 2324.758,
        },
        'params': {
            'skip_download': True,
        },
        'add_ie': ['Ooyala'],
    }]

    def _real_extract(self, url):
        page_id = self._match_id(url)

        webpage = self._download_webpage(url, page_id)

        entries = [
            OoyalaIE._build_url_result(m.group('code'))
            for m in re.finditer(
                r'data-(?:video-id|directVideoId)=(["\'])(?P<code>(?:(?!\1).)+)\1', webpage)]

        title = self._html_search_regex(
            r'(?s)<(?:span|div)[^>]+class="(?:title|wrapper)"[^>]*>.*?<h1>(.+?)</h1>',
            webpage, 'title', fatal=False)

        return self.playlist_result(
            entries, page_id, title)
# coding: utf-8
from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..compat import compat_urlparse
from ..utils import (
    extract_attributes,
    get_element_by_class,
    urlencode_postdata,
)


class NJPWWorldIE(InfoExtractor):
    _VALID_URL = r'https?://njpwworld\.com/p/(?P<id>[a-z0-9_]+)'
    IE_DESC = '新日本プロレスワールド'
    _NETRC_MACHINE = 'njpwworld'

    _TEST = {
        'url': 'http://njpwworld.com/p/s_series_00155_1_9/',
        'info_dict': {
            'id': 's_series_00155_1_9',
            'ext': 'mp4',
            'title': '第9試合 ランディ・サベージ vs リック・スタイナー',
            'tags': list,
        },
        'params': {
            'skip_download': True,  # AES-encrypted m3u8
        },
        'skip': 'Requires login',
    }

    _LOGIN_URL = 'https://front.njpwworld.com/auth/login'

    def _real_initialize(self):
        self._login()

    def _login(self):
        username, password = self._get_login_info()
        # No authentication to be performed
        if not username:
            return True

        # Setup session (will set necessary cookies)
        self._request_webpage(
            'https://njpwworld.com/', None, note='Setting up session')

        webpage, urlh = self._download_webpage_handle(
            self._LOGIN_URL, None,
            note='Logging in', errnote='Unable to login',
            data=urlencode_postdata({'login_id': username, 'pw': password}),
            headers={'Referer': 'https://front.njpwworld.com/auth'})
        # /auth/login will return 302 for successful logins
        if urlh.geturl() == self._LOGIN_URL:
            self.report_warning('unable to login')
            return False

        return True

    def _real_extract(self, url):
        video_id = self._match_id(url)

        webpage = self._download_webpage(url, video_id)

        formats = []
        for mobj in re.finditer(r'<a[^>]+\bhref=(["\'])/player.+?[^>]*>', webpage):
            player = extract_attributes(mobj.group(0))
            player_path = player.get('href')
            if not player_path:
                continue
            kind = self._search_regex(
                r'(low|high)$', player.get('class') or '', 'kind',
                default='low')
            player_url = compat_urlparse.urljoin(url, player_path)
            player_page = self._download_webpage(
                player_url, video_id, note='Downloading player page')
            entries = self._parse_html5_media_entries(
                player_url, player_page, video_id, m3u8_id='hls-%s' % kind,
                m3u8_entry_protocol='m3u8_native')
            kind_formats = entries[0]['formats']
            for f in kind_formats:
                f['quality'] = 2 if kind == 'high' else 1
            formats.extend(kind_formats)

        self._sort_formats(formats)

        post_content = get_element_by_class('post-content', webpage)
        tags = re.findall(
            r'<li[^>]+class="tag-[^"]+"><a[^>]*>([^<]+)</a></li>', post_content
        ) if post_content else None

        return {
            'id': video_id,
            'title': self._og_search_title(webpage),
            'formats': formats,
            'tags': tags,
        }
# coding: utf-8
from __future__ import unicode_literals

from .common import InfoExtractor
from ..utils import (
    js_to_json,
    mimetype2ext,
    determine_ext,
    update_url_query,
    get_element_by_attribute,
    int_or_none,
)


class NobelPrizeIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?nobelprize\.org/mediaplayer.*?\bid=(?P<id>\d+)'
    _TEST = {
        'url': 'http://www.nobelprize.org/mediaplayer/?id=2636',
        'md5': '04c81e5714bb36cc4e2232fee1d8157f',
        'info_dict': {
            'id': '2636',
            'ext': 'mp4',
            'title': 'Announcement of the 2016 Nobel Prize in Physics',
            'description': 'md5:05beba57f4f5a4bbd4cf2ef28fcff739',
        }
    }

    def _real_extract(self, url):
        video_id = self._match_id(url)
        webpage = self._download_webpage(url, video_id)
        media = self._parse_json(self._search_regex(
            r'(?s)var\s*config\s*=\s*({.+?});', webpage,
            'config'), video_id, js_to_json)['media']
        title = media['title']

        formats = []
        for source in media.get('source', []):
            source_src = source.get('src')
            if not source_src:
                continue
            ext = mimetype2ext(source.get('type')) or determine_ext(source_src)
            if ext == 'm3u8':
                formats.extend(self._extract_m3u8_formats(
                    source_src, video_id, 'mp4', 'm3u8_native',
                    m3u8_id='hls', fatal=False))
            elif ext == 'f4m':
                formats.extend(self._extract_f4m_formats(
                    update_url_query(source_src, {'hdcore': '3.7.0'}),
                    video_id, f4m_id='hds', fatal=False))
            else:
                formats.append({
                    'url': source_src,
                })
        self._sort_formats(formats)

        return {
            'id': video_id,
            'title': title,
            'description': get_element_by_attribute('itemprop', 'description', webpage),
            'duration': int_or_none(media.get('duration')),
            'formats': formats,
        }
# coding: utf-8
from __future__ import unicode_literals

import re
import time
import hashlib

from .common import InfoExtractor
from ..compat import (
    compat_str,
    compat_urlparse,
)
from ..utils import (
    clean_html,
    ExtractorError,
    int_or_none,
    float_or_none,
    parse_iso8601,
    sanitized_Request,
    urlencode_postdata,
)


class NocoIE(InfoExtractor):
    _VALID_URL = r'https?://(?:(?:www\.)?noco\.tv/emission/|player\.noco\.tv/\?idvideo=)(?P<id>\d+)'
    _LOGIN_URL = 'https://noco.tv/do.php'
    _API_URL_TEMPLATE = 'https://api.noco.tv/1.1/%s?ts=%s&tk=%s'
    _SUB_LANG_TEMPLATE = '&sub_lang=%s'
    _NETRC_MACHINE = 'noco'

    _TESTS = [
        {
            'url': 'http://noco.tv/emission/11538/nolife/ami-ami-idol-hello-france/',
            'md5': '0a993f0058ddbcd902630b2047ef710e',
            'info_dict': {
                'id': '11538',
                'ext': 'mp4',
                'title': 'Ami Ami Idol - Hello! France',
                'description': 'md5:4eaab46ab68fa4197a317a88a53d3b86',
                'upload_date': '20140412',
                'uploader': 'Nolife',
                'uploader_id': 'NOL',
                'duration': 2851.2,
            },
            'skip': 'Requires noco account',
        },
        {
            'url': 'http://noco.tv/emission/12610/lbl42/the-guild/s01e01-wake-up-call',
            'md5': 'c190f1f48e313c55838f1f412225934d',
            'info_dict': {
                'id': '12610',
                'ext': 'mp4',
                'title': 'The Guild #1 - Wake-Up Call',
                'timestamp': 1403863200,
                'upload_date': '20140627',
                'uploader': 'LBL42',
                'uploader_id': 'LBL',
                'duration': 233.023,
            },
            'skip': 'Requires noco account',
        }
    ]

    def _real_initialize(self):
        self._login()

    def _login(self):
        username, password = self._get_login_info()
        if username is None:
            return

        login = self._download_json(
            self._LOGIN_URL, None, 'Logging in',
            data=urlencode_postdata({
                'a': 'login',
                'cookie': '1',
                'username': username,
                'password': password,
            }),
            headers={
                'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8',
            })

        if 'erreur' in login:
            raise ExtractorError('Unable to login: %s' % clean_html(login['erreur']), expected=True)

    @staticmethod
    def _ts():
        return int(time.time() * 1000)

    def _call_api(self, path, video_id, note, sub_lang=None):
        ts = compat_str(self._ts() + self._ts_offset)
        tk = hashlib.md5((hashlib.md5(ts.encode('ascii')).hexdigest() + '#8S?uCraTedap6a').encode('ascii')).hexdigest()
        url = self._API_URL_TEMPLATE % (path, ts, tk)
        if sub_lang:
            url += self._SUB_LANG_TEMPLATE % sub_lang

        request = sanitized_Request(url)
        request.add_header('Referer', self._referer)

        resp = self._download_json(request, video_id, note)

        if isinstance(resp, dict) and resp.get('error'):
            self._raise_error(resp['error'], resp['description'])

        return resp

    def _raise_error(self, error, description):
        raise ExtractorError(
            '%s returned error: %s - %s' % (self.IE_NAME, error, description),
            expected=True)

    def _real_extract(self, url):
        video_id = self._match_id(url)

        # Timestamp adjustment offset between server time and local time
        # must be calculated in order to use timestamps closest to server's
        # in all API requests (see https://github.com/ytdl-org/youtube-dl/issues/7864)
        webpage = self._download_webpage(url, video_id)

        player_url = self._search_regex(
            r'(["\'])(?P<player>https?://noco\.tv/(?:[^/]+/)+NocoPlayer.+?\.swf.*?)\1',
            webpage, 'noco player', group='player',
            default='http://noco.tv/cdata/js/player/NocoPlayer-v1.2.40.swf')

        qs = compat_urlparse.parse_qs(compat_urlparse.urlparse(player_url).query)
        ts = int_or_none(qs.get('ts', [None])[0])
        self._ts_offset = ts - self._ts() if ts else 0
        self._referer = player_url

        medias = self._call_api(
            'shows/%s/medias' % video_id,
            video_id, 'Downloading video JSON')

        show = self._call_api(
            'shows/by_id/%s' % video_id,
            video_id, 'Downloading show JSON')[0]

        options = self._call_api(
            'users/init', video_id,
            'Downloading user options JSON')['options']
        audio_lang_pref = options.get('audio_language') or options.get('language', 'fr')

        if audio_lang_pref == 'original':
            audio_lang_pref = show['original_lang']
        if len(medias) == 1:
            audio_lang_pref = list(medias.keys())[0]
        elif audio_lang_pref not in medias:
            audio_lang_pref = 'fr'

        qualities = self._call_api(
            'qualities',
            video_id, 'Downloading qualities JSON')

        formats = []

        for audio_lang, audio_lang_dict in medias.items():
            preference = 1 if audio_lang == audio_lang_pref else 0
            for sub_lang, lang_dict in audio_lang_dict['video_list'].items():
                for format_id, fmt in lang_dict['quality_list'].items():
                    format_id_extended = 'audio-%s_sub-%s_%s' % (audio_lang, sub_lang, format_id)

                    video = self._call_api(
                        'shows/%s/video/%s/%s' % (video_id, format_id.lower(), audio_lang),
                        video_id, 'Downloading %s video JSON' % format_id_extended,
                        sub_lang if sub_lang != 'none' else None)

                    file_url = video['file']
                    if not file_url:
                        continue

                    if file_url in ['forbidden', 'not found']:
                        popmessage = video['popmessage']
                        self._raise_error(popmessage['title'], popmessage['message'])

                    formats.append({
                        'url': file_url,
                        'format_id': format_id_extended,
                        'width': int_or_none(fmt.get('res_width')),
                        'height': int_or_none(fmt.get('res_lines')),
                        'abr': int_or_none(fmt.get('audiobitrate'), 1000),
                        'vbr': int_or_none(fmt.get('videobitrate'), 1000),
                        'filesize': int_or_none(fmt.get('filesize')),
                        'format_note': qualities[format_id].get('quality_name'),
                        'quality': qualities[format_id].get('priority'),
                        'preference': preference,
                    })

        self._sort_formats(formats)

        timestamp = parse_iso8601(show.get('online_date_start_utc'), ' ')

        if timestamp is not None and timestamp < 0:
            timestamp = None

        uploader = show.get('partner_name')
        uploader_id = show.get('partner_key')
        duration = float_or_none(show.get('duration_ms'), 1000)

        thumbnails = []
        for thumbnail_key, thumbnail_url in show.items():
            m = re.search(r'^screenshot_(?P<width>\d+)x(?P<height>\d+)$', thumbnail_key)
            if not m:
                continue
            thumbnails.append({
                'url': thumbnail_url,
                'width': int(m.group('width')),
                'height': int(m.group('height')),
            })

        episode = show.get('show_TT') or show.get('show_OT')
        family = show.get('family_TT') or show.get('family_OT')
        episode_number = show.get('episode_number')

        title = ''
        if family:
            title += family
        if episode_number:
            title += ' #' + compat_str(episode_number)
        if episode:
            title += ' - ' + compat_str(episode)

        description = show.get('show_resume') or show.get('family_resume')

        return {
            'id': video_id,
            'title': title,
            'description': description,
            'thumbnails': thumbnails,
            'timestamp': timestamp,
            'uploader': uploader,
            'uploader_id': uploader_id,
            'duration': duration,
            'formats': formats,
        }
from __future__ import unicode_literals

from .nuevo import NuevoBaseIE


class NonkTubeIE(NuevoBaseIE):
    _VALID_URL = r'https?://(?:www\.)?nonktube\.com/(?:(?:video|embed)/|media/nuevo/embed\.php\?.*?\bid=)(?P<id>\d+)'
    _TESTS = [{
        'url': 'https://www.nonktube.com/video/118636/sensual-wife-uncensored-fucked-in-hairy-pussy-and-facialized',
        'info_dict': {
            'id': '118636',
            'ext': 'mp4',
            'title': 'Sensual Wife Uncensored Fucked In Hairy Pussy And Facialized',
            'age_limit': 18,
            'duration': 1150.98,
        },
        'params': {
            'skip_download': True,
        }
    }, {
        'url': 'https://www.nonktube.com/embed/118636',
        'only_matching': True,
    }]

    def _real_extract(self, url):
        video_id = self._match_id(url)

        webpage = self._download_webpage(url, video_id)

        title = self._og_search_title(webpage)
        info = self._parse_html5_media_entries(url, webpage, video_id)[0]

        info.update({
            'id': video_id,
            'title': title,
            'age_limit': 18,
        })
        return info
# coding: utf-8
from __future__ import unicode_literals

from .brightcove import BrightcoveNewIE
from .common import InfoExtractor
from ..compat import compat_str
from ..utils import (
    int_or_none,
    js_to_json,
    smuggle_url,
    try_get,
)


class NoovoIE(InfoExtractor):
    _VALID_URL = r'https?://(?:[^/]+\.)?noovo\.ca/videos/(?P<id>[^/]+/[^/?#&]+)'
    _TESTS = [{
        # clip
        'url': 'http://noovo.ca/videos/rpm-plus/chrysler-imperial',
        'info_dict': {
            'id': '5386045029001',
            'ext': 'mp4',
            'title': 'Chrysler Imperial',
            'description': 'md5:de3c898d1eb810f3e6243e08c8b4a056',
            'timestamp': 1491399228,
            'upload_date': '20170405',
            'uploader_id': '618566855001',
            'series': 'RPM+',
        },
        'params': {
            'skip_download': True,
        },
    }, {
        # episode
        'url': 'http://noovo.ca/videos/l-amour-est-dans-le-pre/episode-13-8',
        'info_dict': {
            'id': '5395865725001',
            'title': 'Épisode 13 : Les retrouvailles',
            'description': 'md5:888c3330f0c1b4476c5bc99a1c040473',
            'ext': 'mp4',
            'timestamp': 1492019320,
            'upload_date': '20170412',
            'uploader_id': '618566855001',
            'series': "L'amour est dans le pré",
            'season_number': 5,
            'episode': 'Épisode 13',
            'episode_number': 13,
        },
        'params': {
            'skip_download': True,
        },
    }]
    BRIGHTCOVE_URL_TEMPLATE = 'http://players.brightcove.net/618566855001/default_default/index.html?videoId=%s'

    def _real_extract(self, url):
        video_id = self._match_id(url)

        webpage = self._download_webpage(url, video_id)

        brightcove_id = self._search_regex(
            r'data-video-id=["\'](\d+)', webpage, 'brightcove id')

        data = self._parse_json(
            self._search_regex(
                r'(?s)dataLayer\.push\(\s*({.+?})\s*\);', webpage, 'data',
                default='{}'),
            video_id, transform_source=js_to_json, fatal=False)

        title = try_get(
            data, lambda x: x['video']['nom'],
            compat_str) or self._html_search_meta(
            'dcterms.Title', webpage, 'title', fatal=True)

        description = self._html_search_meta(
            ('dcterms.Description', 'description'), webpage, 'description')

        series = try_get(
            data, lambda x: x['emission']['nom']) or self._search_regex(
            r'<div[^>]+class="banner-card__subtitle h4"[^>]*>([^<]+)',
            webpage, 'series', default=None)

        season_el = try_get(data, lambda x: x['emission']['saison'], dict) or {}
        season = try_get(season_el, lambda x: x['nom'], compat_str)
        season_number = int_or_none(try_get(season_el, lambda x: x['numero']))

        episode_el = try_get(season_el, lambda x: x['episode'], dict) or {}
        episode = try_get(episode_el, lambda x: x['nom'], compat_str)
        episode_number = int_or_none(try_get(episode_el, lambda x: x['numero']))

        return {
            '_type': 'url_transparent',
            'ie_key': BrightcoveNewIE.ie_key(),
            'url': smuggle_url(
                self.BRIGHTCOVE_URL_TEMPLATE % brightcove_id,
                {'geo_countries': ['CA']}),
            'id': brightcove_id,
            'title': title,
            'description': description,
            'series': series,
            'season': season,
            'season_number': season_number,
            'episode': episode,
            'episode_number': episode_number,
        }
# coding: utf-8
from __future__ import unicode_literals

from .common import InfoExtractor
from .jwplatform import JWPlatformIE

from ..utils import (
    unified_strdate,
)


class NormalbootsIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?normalboots\.com/video/(?P<id>[0-9a-z-]*)/?$'
    _TEST = {
        'url': 'http://normalboots.com/video/home-alone-games-jontron/',
        'info_dict': {
            'id': 'home-alone-games-jontron',
            'ext': 'mp4',
            'title': 'Home Alone Games - JonTron - NormalBoots',
            'description': 'Jon is late for Christmas. Typical. Thanks to: Paul Ritchey for Co-Writing/Filming: http://www.youtube.com/user/ContinueShow Michael Azzi for Christmas Intro Animation: http://michafrar.tumblr.com/ Jerrod Waters for Christmas Intro Music: http://www.youtube.com/user/xXJerryTerryXx Casey Ormond for ‘Tense Battle Theme’:\xa0http://www.youtube.com/Kiamet/',
            'uploader': 'JonTron',
            'upload_date': '20140125',
        },
        'params': {
            # m3u8 download
            'skip_download': True,
        },
        'add_ie': ['JWPlatform'],
    }

    def _real_extract(self, url):
        video_id = self._match_id(url)
        webpage = self._download_webpage(url, video_id)

        video_uploader = self._html_search_regex(
            r'Posted\sby\s<a\shref="[A-Za-z0-9/]*">(?P<uploader>[A-Za-z]*)\s</a>',
            webpage, 'uploader', fatal=False)
        video_upload_date = unified_strdate(self._html_search_regex(
            r'<span style="text-transform:uppercase; font-size:inherit;">[A-Za-z]+, (?P<date>.*)</span>',
            webpage, 'date', fatal=False))

        jwplatform_url = JWPlatformIE._extract_url(webpage)

        return {
            '_type': 'url_transparent',
            'id': video_id,
            'url': jwplatform_url,
            'ie_key': JWPlatformIE.ie_key(),
            'title': self._og_search_title(webpage),
            'description': self._og_search_description(webpage),
            'thumbnail': self._og_search_thumbnail(webpage),
            'uploader': video_uploader,
            'upload_date': video_upload_date,
        }
# coding: utf-8
from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..utils import (
    ExtractorError,
    sanitized_Request,
    urlencode_postdata,
    xpath_text,
    xpath_with_ns,
)

_x = lambda p: xpath_with_ns(p, {'xspf': 'http://xspf.org/ns/0/'})


class NosVideoIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?nosvideo\.com/' + \
                 r'(?:embed/|\?v=)(?P<id>[A-Za-z0-9]{12})/?'
    _PLAYLIST_URL = 'http://nosvideo.com/xml/{xml_id:s}.xml'
    _FILE_DELETED_REGEX = r'<b>File Not Found</b>'
    _TEST = {
        'url': 'http://nosvideo.com/?v=mu8fle7g7rpq',
        'md5': '6124ed47130d8be3eacae635b071e6b6',
        'info_dict': {
            'id': 'mu8fle7g7rpq',
            'ext': 'mp4',
            'title': 'big_buck_bunny_480p_surround-fix.avi.mp4',
            'thumbnail': r're:^https?://.*\.jpg$',
        }
    }

    def _real_extract(self, url):
        video_id = self._match_id(url)

        fields = {
            'id': video_id,
            'op': 'download1',
            'method_free': 'Continue to Video',
        }
        req = sanitized_Request(url, urlencode_postdata(fields))
        req.add_header('Content-type', 'application/x-www-form-urlencoded')
        webpage = self._download_webpage(req, video_id,
                                         'Downloading download page')
        if re.search(self._FILE_DELETED_REGEX, webpage) is not None:
            raise ExtractorError('Video %s does not exist' % video_id,
                                 expected=True)

        xml_id = self._search_regex(r'php\|([^\|]+)\|', webpage, 'XML ID')
        playlist_url = self._PLAYLIST_URL.format(xml_id=xml_id)
        playlist = self._download_xml(playlist_url, video_id)

        track = playlist.find(_x('.//xspf:track'))
        if track is None:
            raise ExtractorError(
                'XML playlist is missing the \'track\' element',
                expected=True)
        title = xpath_text(track, _x('./xspf:title'), 'title')
        url = xpath_text(track, _x('./xspf:file'), 'URL', fatal=True)
        thumbnail = xpath_text(track, _x('./xspf:image'), 'thumbnail')
        if title is not None:
            title = title.strip()

        formats = [{
            'format_id': 'sd',
            'url': url,
        }]

        return {
            'id': video_id,
            'title': title,
            'thumbnail': thumbnail,
            'formats': formats,
        }
# coding: utf-8
from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..utils import (
    clean_html,
    determine_ext,
    int_or_none,
    js_to_json,
    qualities,
    unified_strdate,
    url_or_none,
)


class NovaEmbedIE(InfoExtractor):
    _VALID_URL = r'https?://media\.cms\.nova\.cz/embed/(?P<id>[^/?#&]+)'
    _TEST = {
        'url': 'https://media.cms.nova.cz/embed/8o0n0r?autoplay=1',
        'md5': 'ee009bafcc794541570edd44b71cbea3',
        'info_dict': {
            'id': '8o0n0r',
            'ext': 'mp4',
            'title': '2180. díl',
            'thumbnail': r're:^https?://.*\.jpg',
            'duration': 2578,
        },
    }

    def _real_extract(self, url):
        video_id = self._match_id(url)

        webpage = self._download_webpage(url, video_id)

        duration = None
        formats = []

        player = self._parse_json(
            self._search_regex(
                r'Player\.init\s*\([^,]+,\s*({.+?})\s*,\s*{.+?}\s*\)\s*;',
                webpage, 'player', default='{}'), video_id, fatal=False)
        if player:
            for format_id, format_list in player['tracks'].items():
                if not isinstance(format_list, list):
                    format_list = [format_list]
                for format_dict in format_list:
                    if not isinstance(format_dict, dict):
                        continue
                    format_url = url_or_none(format_dict.get('src'))
                    format_type = format_dict.get('type')
                    ext = determine_ext(format_url)
                    if (format_type == 'application/x-mpegURL'
                            or format_id == 'HLS' or ext == 'm3u8'):
                        formats.extend(self._extract_m3u8_formats(
                            format_url, video_id, 'mp4',
                            entry_protocol='m3u8_native', m3u8_id='hls',
                            fatal=False))
                    elif (format_type == 'application/dash+xml'
                          or format_id == 'DASH' or ext == 'mpd'):
                        formats.extend(self._extract_mpd_formats(
                            format_url, video_id, mpd_id='dash', fatal=False))
                    else:
                        formats.append({
                            'url': format_url,
                        })
            duration = int_or_none(player.get('duration'))
        else:
            # Old path, not actual as of 08.04.2020
            bitrates = self._parse_json(
                self._search_regex(
                    r'(?s)(?:src|bitrates)\s*=\s*({.+?})\s*;', webpage, 'formats'),
                video_id, transform_source=js_to_json)

            QUALITIES = ('lq', 'mq', 'hq', 'hd')
            quality_key = qualities(QUALITIES)

            for format_id, format_list in bitrates.items():
                if not isinstance(format_list, list):
                    format_list = [format_list]
                for format_url in format_list:
                    format_url = url_or_none(format_url)
                    if not format_url:
                        continue
                    if format_id == 'hls':
                        formats.extend(self._extract_m3u8_formats(
                            format_url, video_id, ext='mp4',
                            entry_protocol='m3u8_native', m3u8_id='hls',
                            fatal=False))
                        continue
                    f = {
                        'url': format_url,
                    }
                    f_id = format_id
                    for quality in QUALITIES:
                        if '%s.mp4' % quality in format_url:
                            f_id += '-%s' % quality
                            f.update({
                                'quality': quality_key(quality),
                                'format_note': quality.upper(),
                            })
                            break
                    f['format_id'] = f_id
                    formats.append(f)

        self._sort_formats(formats)

        title = self._og_search_title(
            webpage, default=None) or self._search_regex(
            (r'<value>(?P<title>[^<]+)',
             r'videoTitle\s*:\s*(["\'])(?P<value>(?:(?!\1).)+)\1'), webpage,
            'title', group='value')
        thumbnail = self._og_search_thumbnail(
            webpage, default=None) or self._search_regex(
            r'poster\s*:\s*(["\'])(?P<value>(?:(?!\1).)+)\1', webpage,
            'thumbnail', fatal=False, group='value')
        duration = int_or_none(self._search_regex(
            r'videoDuration\s*:\s*(\d+)', webpage, 'duration',
            default=duration))

        return {
            'id': video_id,
            'title': title,
            'thumbnail': thumbnail,
            'duration': duration,
            'formats': formats,
        }


class NovaIE(InfoExtractor):
    IE_DESC = 'TN.cz, Prásk.tv, Nova.cz, Novaplus.cz, FANDA.tv, Krásná.cz and Doma.cz'
    _VALID_URL = r'https?://(?:[^.]+\.)?(?P<site>tv(?:noviny)?|tn|novaplus|vymena|fanda|krasna|doma|prask)\.nova\.cz/(?:[^/]+/)+(?P<id>[^/]+?)(?:\.html|/|$)'
    _TESTS = [{
        'url': 'http://tn.nova.cz/clanek/tajemstvi-ukryte-v-podzemi-specialni-nemocnice-v-prazske-krci.html#player_13260',
        'md5': '249baab7d0104e186e78b0899c7d5f28',
        'info_dict': {
            'id': '1757139',
            'display_id': 'tajemstvi-ukryte-v-podzemi-specialni-nemocnice-v-prazske-krci',
            'ext': 'mp4',
            'title': 'Podzemní nemocnice v pražské Krči',
            'description': 'md5:f0a42dd239c26f61c28f19e62d20ef53',
            'thumbnail': r're:^https?://.*\.(?:jpg)',
        }
    }, {
        'url': 'http://fanda.nova.cz/clanek/fun-and-games/krvavy-epos-zaklinac-3-divoky-hon-vychazi-vyhrajte-ho-pro-sebe.html',
        'info_dict': {
            'id': '1753621',
            'ext': 'mp4',
            'title': 'Zaklínač 3: Divoký hon',
            'description': 're:.*Pokud se stejně jako my nemůžete.*',
            'thumbnail': r're:https?://.*\.jpg(\?.*)?',
            'upload_date': '20150521',
        },
        'params': {
            # rtmp download
            'skip_download': True,
        },
        'skip': 'gone',
    }, {
        # media.cms.nova.cz embed
        'url': 'https://novaplus.nova.cz/porad/ulice/epizoda/18760-2180-dil',
        'info_dict': {
            'id': '8o0n0r',
            'ext': 'mp4',
            'title': '2180. díl',
            'thumbnail': r're:^https?://.*\.jpg',
            'duration': 2578,
        },
        'params': {
            'skip_download': True,
        },
        'add_ie': [NovaEmbedIE.ie_key()],
        'skip': 'CHYBA 404: STRÁNKA NENALEZENA',
    }, {
        'url': 'http://sport.tn.nova.cz/clanek/sport/hokej/nhl/zivot-jde-dal-hodnotil-po-vyrazeni-z-playoff-jiri-sekac.html',
        'only_matching': True,
    }, {
        'url': 'http://fanda.nova.cz/clanek/fun-and-games/krvavy-epos-zaklinac-3-divoky-hon-vychazi-vyhrajte-ho-pro-sebe.html',
        'only_matching': True,
    }, {
        'url': 'http://doma.nova.cz/clanek/zdravi/prijdte-se-zapsat-do-registru-kostni-drene-jiz-ve-stredu-3-cervna.html',
        'only_matching': True,
    }, {
        'url': 'http://prask.nova.cz/clanek/novinky/co-si-na-sobe-nase-hvezdy-nechaly-pojistit.html',
        'only_matching': True,
    }, {
        'url': 'http://tv.nova.cz/clanek/novinky/zivot-je-zivot-bondovsky-trailer.html',
        'only_matching': True,
    }]

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        display_id = mobj.group('id')
        site = mobj.group('site')

        webpage = self._download_webpage(url, display_id)

        description = clean_html(self._og_search_description(webpage, default=None))
        if site == 'novaplus':
            upload_date = unified_strdate(self._search_regex(
                r'(\d{1,2}-\d{1,2}-\d{4})$', display_id, 'upload date', default=None))
        elif site == 'fanda':
            upload_date = unified_strdate(self._search_regex(
                r'<span class="date_time">(\d{1,2}\.\d{1,2}\.\d{4})', webpage, 'upload date', default=None))
        else:
            upload_date = None

        # novaplus
        embed_id = self._search_regex(
            r'<iframe[^>]+\bsrc=["\'](?:https?:)?//media\.cms\.nova\.cz/embed/([^/?#&]+)',
            webpage, 'embed url', default=None)
        if embed_id:
            return {
                '_type': 'url_transparent',
                'url': 'https://media.cms.nova.cz/embed/%s' % embed_id,
                'ie_key': NovaEmbedIE.ie_key(),
                'id': embed_id,
                'description': description,
                'upload_date': upload_date
            }

        video_id = self._search_regex(
            [r"(?:media|video_id)\s*:\s*'(\d+)'",
             r'media=(\d+)',
             r'id="article_video_(\d+)"',
             r'id="player_(\d+)"'],
            webpage, 'video id')

        config_url = self._search_regex(
            r'src="(https?://(?:tn|api)\.nova\.cz/bin/player/videojs/config\.php\?[^"]+)"',
            webpage, 'config url', default=None)
        config_params = {}

        if not config_url:
            player = self._parse_json(
                self._search_regex(
                    r'(?s)Player\s*\(.+?\s*,\s*({.+?\bmedia\b["\']?\s*:\s*["\']?\d+.+?})\s*\)', webpage,
                    'player', default='{}'),
                video_id, transform_source=js_to_json, fatal=False)
            if player:
                config_url = url_or_none(player.get('configUrl'))
                params = player.get('configParams')
                if isinstance(params, dict):
                    config_params = params

        if not config_url:
            DEFAULT_SITE_ID = '23000'
            SITES = {
                'tvnoviny': DEFAULT_SITE_ID,
                'novaplus': DEFAULT_SITE_ID,
                'vymena': DEFAULT_SITE_ID,
                'krasna': DEFAULT_SITE_ID,
                'fanda': '30',
                'tn': '30',
                'doma': '30',
            }

            site_id = self._search_regex(
                r'site=(\d+)', webpage, 'site id', default=None) or SITES.get(
                site, DEFAULT_SITE_ID)

            config_url = 'https://api.nova.cz/bin/player/videojs/config.php'
            config_params = {
                'site': site_id,
                'media': video_id,
                'quality': 3,
                'version': 1,
            }

        config = self._download_json(
            config_url, display_id,
            'Downloading config JSON', query=config_params,
            transform_source=lambda s: s[s.index('{'):s.rindex('}') + 1])

        mediafile = config['mediafile']
        video_url = mediafile['src']

        m = re.search(r'^(?P<url>rtmpe?://[^/]+/(?P<app>[^/]+?))/&*(?P<playpath>.+)$', video_url)
        if m:
            formats = [{
                'url': m.group('url'),
                'app': m.group('app'),
                'play_path': m.group('playpath'),
                'player_path': 'http://tvnoviny.nova.cz/static/shared/app/videojs/video-js.swf',
                'ext': 'flv',
            }]
        else:
            formats = [{
                'url': video_url,
            }]
        self._sort_formats(formats)

        title = mediafile.get('meta', {}).get('title') or self._og_search_title(webpage)
        thumbnail = config.get('poster')

        return {
            'id': video_id,
            'display_id': display_id,
            'title': title,
            'description': description,
            'upload_date': upload_date,
            'thumbnail': thumbnail,
            'formats': formats,
        }
# coding: utf-8
from __future__ import unicode_literals

from .brightcove import (
    BrightcoveLegacyIE,
    BrightcoveNewIE,
)
from .common import InfoExtractor
from ..compat import compat_str
from ..utils import (
    ExtractorError,
    sanitized_Request,
)


class NownessBaseIE(InfoExtractor):
    def _extract_url_result(self, post):
        if post['type'] == 'video':
            for media in post['media']:
                if media['type'] == 'video':
                    video_id = media['content']
                    source = media['source']
                    if source == 'brightcove':
                        player_code = self._download_webpage(
                            'http://www.nowness.com/iframe?id=%s' % video_id, video_id,
                            note='Downloading player JavaScript',
                            errnote='Unable to download player JavaScript')
                        bc_url = BrightcoveLegacyIE._extract_brightcove_url(player_code)
                        if bc_url:
                            return self.url_result(bc_url, BrightcoveLegacyIE.ie_key())
                        bc_url = BrightcoveNewIE._extract_url(self, player_code)
                        if bc_url:
                            return self.url_result(bc_url, BrightcoveNewIE.ie_key())
                        raise ExtractorError('Could not find player definition')
                    elif source == 'vimeo':
                        return self.url_result('http://vimeo.com/%s' % video_id, 'Vimeo')
                    elif source == 'youtube':
                        return self.url_result(video_id, 'Youtube')
                    elif source == 'cinematique':
                        # youtube-dl currently doesn't support cinematique
                        # return self.url_result('http://cinematique.com/embed/%s' % video_id, 'Cinematique')
                        pass

    def _api_request(self, url, request_path):
        display_id = self._match_id(url)
        request = sanitized_Request(
            'http://api.nowness.com/api/' + request_path % display_id,
            headers={
                'X-Nowness-Language': 'zh-cn' if 'cn.nowness.com' in url else 'en-us',
            })
        return display_id, self._download_json(request, display_id)


class NownessIE(NownessBaseIE):
    IE_NAME = 'nowness'
    _VALID_URL = r'https?://(?:(?:www|cn)\.)?nowness\.com/(?:story|(?:series|category)/[^/]+)/(?P<id>[^/]+?)(?:$|[?#])'
    _TESTS = [{
        'url': 'https://www.nowness.com/story/candor-the-art-of-gesticulation',
        'md5': '068bc0202558c2e391924cb8cc470676',
        'info_dict': {
            'id': '2520295746001',
            'ext': 'mp4',
            'title': 'Candor: The Art of Gesticulation',
            'description': 'Candor: The Art of Gesticulation',
            'thumbnail': r're:^https?://.*\.jpg',
            'timestamp': 1446745676,
            'upload_date': '20151105',
            'uploader_id': '2385340575001',
        },
        'add_ie': ['BrightcoveNew'],
    }, {
        'url': 'https://cn.nowness.com/story/kasper-bjorke-ft-jaakko-eino-kalevi-tnr',
        'md5': 'e79cf125e387216f86b2e0a5b5c63aa3',
        'info_dict': {
            'id': '3716354522001',
            'ext': 'mp4',
            'title': 'Kasper Bjørke ft. Jaakko Eino Kalevi: TNR',
            'description': 'Kasper Bjørke ft. Jaakko Eino Kalevi: TNR',
            'thumbnail': r're:^https?://.*\.jpg',
            'timestamp': 1407315371,
            'upload_date': '20140806',
            'uploader_id': '2385340575001',
        },
        'add_ie': ['BrightcoveNew'],
    }, {
        # vimeo
        'url': 'https://www.nowness.com/series/nowness-picks/jean-luc-godard-supercut',
        'md5': '9a5a6a8edf806407e411296ab6bc2a49',
        'info_dict': {
            'id': '130020913',
            'ext': 'mp4',
            'title': 'Bleu, Blanc, Rouge - A Godard Supercut',
            'description': 'md5:f0ea5f1857dffca02dbd37875d742cec',
            'thumbnail': r're:^https?://.*\.jpg',
            'upload_date': '20150607',
            'uploader': 'Cinema Sem Lei',
            'uploader_id': 'cinemasemlei',
        },
        'add_ie': ['Vimeo'],
    }]

    def _real_extract(self, url):
        _, post = self._api_request(url, 'post/getBySlug/%s')
        return self._extract_url_result(post)


class NownessPlaylistIE(NownessBaseIE):
    IE_NAME = 'nowness:playlist'
    _VALID_URL = r'https?://(?:(?:www|cn)\.)?nowness\.com/playlist/(?P<id>\d+)'
    _TEST = {
        'url': 'https://www.nowness.com/playlist/3286/i-guess-thats-why-they-call-it-the-blues',
        'info_dict': {
            'id': '3286',
        },
        'playlist_mincount': 8,
    }

    def _real_extract(self, url):
        playlist_id, playlist = self._api_request(url, 'post?PlaylistId=%s')
        entries = [self._extract_url_result(item) for item in playlist['items']]
        return self.playlist_result(entries, playlist_id)


class NownessSeriesIE(NownessBaseIE):
    IE_NAME = 'nowness:series'
    _VALID_URL = r'https?://(?:(?:www|cn)\.)?nowness\.com/series/(?P<id>[^/]+?)(?:$|[?#])'
    _TEST = {
        'url': 'https://www.nowness.com/series/60-seconds',
        'info_dict': {
            'id': '60',
            'title': '60 Seconds',
            'description': 'One-minute wisdom in a new NOWNESS series',
        },
        'playlist_mincount': 4,
    }

    def _real_extract(self, url):
        display_id, series = self._api_request(url, 'series/getBySlug/%s')
        entries = [self._extract_url_result(post) for post in series['posts']]
        series_title = None
        series_description = None
        translations = series.get('translations', [])
        if translations:
            series_title = translations[0].get('title') or translations[0]['seoTitle']
            series_description = translations[0].get('seoDescription')
        return self.playlist_result(
            entries, compat_str(series['id']), series_title, series_description)
# coding: utf-8
from __future__ import unicode_literals

from .common import InfoExtractor
from ..compat import (
    compat_urllib_parse_unquote,
    compat_xpath,
)
from ..utils import (
    int_or_none,
    find_xpath_attr,
    xpath_text,
    update_url_query,
)


class NozIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?noz\.de/video/(?P<id>[0-9]+)/'
    _TESTS = [{
        'url': 'http://www.noz.de/video/25151/32-Deutschland-gewinnt-Badminton-Lnderspiel-in-Melle',
        'info_dict': {
            'id': '25151',
            'ext': 'mp4',
            'duration': 215,
            'title': '3:2 - Deutschland gewinnt Badminton-Länderspiel in Melle',
            'description': 'Vor rund 370 Zuschauern gewinnt die deutsche Badminton-Nationalmannschaft am Donnerstag ein EM-Vorbereitungsspiel gegen Frankreich in Melle. Video Moritz Frankenberg.',
            'thumbnail': r're:^http://.*\.jpg',
        },
    }]

    def _real_extract(self, url):
        video_id = self._match_id(url)
        webpage = self._download_webpage(url, video_id)
        description = self._og_search_description(webpage)

        edge_url = self._html_search_regex(
            r'<script\s+(?:type="text/javascript"\s+)?src="(.*?/videojs_.*?)"',
            webpage, 'edge URL')
        edge_content = self._download_webpage(edge_url, 'meta configuration')

        config_url_encoded = self._search_regex(
            r'so\.addVariable\("config_url","[^,]*,(.*?)"',
            edge_content, 'config URL'
        )
        config_url = compat_urllib_parse_unquote(config_url_encoded)

        doc = self._download_xml(config_url, 'video configuration')
        title = xpath_text(doc, './/title')
        thumbnail = xpath_text(doc, './/article/thumbnail/url')
        duration = int_or_none(xpath_text(
            doc, './/article/movie/file/duration'))
        formats = []
        for qnode in doc.findall(compat_xpath('.//article/movie/file/qualities/qual')):
            http_url_ele = find_xpath_attr(
                qnode, './html_urls/video_url', 'format', 'video/mp4')
            http_url = http_url_ele.text if http_url_ele is not None else None
            if http_url:
                formats.append({
                    'url': http_url,
                    'format_name': xpath_text(qnode, './name'),
                    'format_id': '%s-%s' % ('http', xpath_text(qnode, './id')),
                    'height': int_or_none(xpath_text(qnode, './height')),
                    'width': int_or_none(xpath_text(qnode, './width')),
                    'tbr': int_or_none(xpath_text(qnode, './bitrate'), scale=1000),
                })
            else:
                f4m_url = xpath_text(qnode, 'url_hd2')
                if f4m_url:
                    formats.extend(self._extract_f4m_formats(
                        update_url_query(f4m_url, {'hdcore': '3.4.0'}),
                        video_id, f4m_id='hds', fatal=False))
                m3u8_url_ele = find_xpath_attr(
                    qnode, './html_urls/video_url',
                    'format', 'application/vnd.apple.mpegurl')
                m3u8_url = m3u8_url_ele.text if m3u8_url_ele is not None else None
                if m3u8_url:
                    formats.extend(self._extract_m3u8_formats(
                        m3u8_url, video_id, 'mp4', 'm3u8_native',
                        m3u8_id='hls', fatal=False))
        self._sort_formats(formats)

        return {
            'id': video_id,
            'formats': formats,
            'title': title,
            'duration': duration,
            'description': description,
            'thumbnail': thumbnail,
        }
from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..compat import (
    compat_HTTPError,
    compat_str,
)
from ..utils import (
    determine_ext,
    ExtractorError,
    fix_xml_ampersands,
    int_or_none,
    merge_dicts,
    orderedSet,
    parse_duration,
    qualities,
    str_or_none,
    strip_jsonp,
    unified_strdate,
    unified_timestamp,
    url_or_none,
    urlencode_postdata,
)


class NPOBaseIE(InfoExtractor):
    def _get_token(self, video_id):
        return self._download_json(
            'http://ida.omroep.nl/app.php/auth', video_id,
            note='Downloading token')['token']


class NPOIE(NPOBaseIE):
    IE_NAME = 'npo'
    IE_DESC = 'npo.nl, ntr.nl, omroepwnl.nl, zapp.nl and npo3.nl'
    _VALID_URL = r'''(?x)
                    (?:
                        npo:|
                        https?://
                            (?:www\.)?
                            (?:
                                npo\.nl/(?:[^/]+/)*|
                                (?:ntr|npostart)\.nl/(?:[^/]+/){2,}|
                                omroepwnl\.nl/video/fragment/[^/]+__|
                                (?:zapp|npo3)\.nl/(?:[^/]+/){2,}
                            )
                        )
                        (?P<id>[^/?#]+)
                '''

    _TESTS = [{
        'url': 'http://www.npo.nl/nieuwsuur/22-06-2014/VPWON_1220719',
        'md5': '4b3f9c429157ec4775f2c9cb7b911016',
        'info_dict': {
            'id': 'VPWON_1220719',
            'ext': 'm4v',
            'title': 'Nieuwsuur',
            'description': 'Dagelijks tussen tien en elf: nieuws, sport en achtergronden.',
            'upload_date': '20140622',
        },
    }, {
        'url': 'http://www.npo.nl/de-mega-mike-mega-thomas-show/27-02-2009/VARA_101191800',
        'md5': 'da50a5787dbfc1603c4ad80f31c5120b',
        'info_dict': {
            'id': 'VARA_101191800',
            'ext': 'm4v',
            'title': 'De Mega Mike & Mega Thomas show: The best of.',
            'description': 'md5:3b74c97fc9d6901d5a665aac0e5400f4',
            'upload_date': '20090227',
            'duration': 2400,
        },
    }, {
        'url': 'http://www.npo.nl/tegenlicht/25-02-2013/VPWON_1169289',
        'md5': 'f8065e4e5a7824068ed3c7e783178f2c',
        'info_dict': {
            'id': 'VPWON_1169289',
            'ext': 'm4v',
            'title': 'Tegenlicht: Zwart geld. De toekomst komt uit Afrika',
            'description': 'md5:52cf4eefbc96fffcbdc06d024147abea',
            'upload_date': '20130225',
            'duration': 3000,
        },
    }, {
        'url': 'http://www.npo.nl/de-nieuwe-mens-deel-1/21-07-2010/WO_VPRO_043706',
        'info_dict': {
            'id': 'WO_VPRO_043706',
            'ext': 'm4v',
            'title': 'De nieuwe mens - Deel 1',
            'description': 'md5:518ae51ba1293ffb80d8d8ce90b74e4b',
            'duration': 4680,
        },
        'params': {
            'skip_download': True,
        }
    }, {
        # non asf in streams
        'url': 'http://www.npo.nl/hoe-gaat-europa-verder-na-parijs/10-01-2015/WO_NOS_762771',
        'info_dict': {
            'id': 'WO_NOS_762771',
            'ext': 'mp4',
            'title': 'Hoe gaat Europa verder na Parijs?',
        },
        'params': {
            'skip_download': True,
        }
    }, {
        'url': 'http://www.ntr.nl/Aap-Poot-Pies/27/detail/Aap-poot-pies/VPWON_1233944#content',
        'info_dict': {
            'id': 'VPWON_1233944',
            'ext': 'm4v',
            'title': 'Aap, poot, pies',
            'description': 'md5:c9c8005d1869ae65b858e82c01a91fde',
            'upload_date': '20150508',
            'duration': 599,
        },
        'params': {
            'skip_download': True,
        }
    }, {
        'url': 'http://www.omroepwnl.nl/video/fragment/vandaag-de-dag-verkiezingen__POMS_WNL_853698',
        'info_dict': {
            'id': 'POW_00996502',
            'ext': 'm4v',
            'title': '''"Dit is wel een 'landslide'..."''',
            'description': 'md5:f8d66d537dfb641380226e31ca57b8e8',
            'upload_date': '20150508',
            'duration': 462,
        },
        'params': {
            'skip_download': True,
        }
    }, {
        # audio
        'url': 'http://www.npo.nl/jouw-stad-rotterdam/29-01-2017/RBX_FUNX_6683215/RBX_FUNX_7601437',
        'info_dict': {
            'id': 'RBX_FUNX_6683215',
            'ext': 'mp3',
            'title': 'Jouw Stad Rotterdam',
            'description': 'md5:db251505244f097717ec59fabc372d9f',
        },
        'params': {
            'skip_download': True,
        }
    }, {
        'url': 'http://www.zapp.nl/de-bzt-show/gemist/KN_1687547',
        'only_matching': True,
    }, {
        'url': 'http://www.zapp.nl/de-bzt-show/filmpjes/POMS_KN_7315118',
        'only_matching': True,
    }, {
        'url': 'http://www.zapp.nl/beste-vrienden-quiz/extra-video-s/WO_NTR_1067990',
        'only_matching': True,
    }, {
        'url': 'https://www.npo3.nl/3onderzoekt/16-09-2015/VPWON_1239870',
        'only_matching': True,
    }, {
        # live stream
        'url': 'npo:LI_NL1_4188102',
        'only_matching': True,
    }, {
        'url': 'http://www.npo.nl/radio-gaga/13-06-2017/BNN_101383373',
        'only_matching': True,
    }, {
        'url': 'https://www.zapp.nl/1803-skelterlab/instructie-video-s/740-instructievideo-s/POMS_AT_11736927',
        'only_matching': True,
    }, {
        'url': 'https://www.npostart.nl/broodje-gezond-ei/28-05-2018/KN_1698996',
        'only_matching': True,
    }, {
        'url': 'https://npo.nl/KN_1698996',
        'only_matching': True,
    }]

    @classmethod
    def suitable(cls, url):
        return (False if any(ie.suitable(url)
                for ie in (NPOLiveIE, NPORadioIE, NPORadioFragmentIE))
                else super(NPOIE, cls).suitable(url))

    def _real_extract(self, url):
        video_id = self._match_id(url)
        return self._get_info(url, video_id) or self._get_old_info(video_id)

    def _get_info(self, url, video_id):
        token = self._download_json(
            'https://www.npostart.nl/api/token', video_id,
            'Downloading token', headers={
                'Referer': url,
                'X-Requested-With': 'XMLHttpRequest',
            })['token']

        player = self._download_json(
            'https://www.npostart.nl/player/%s' % video_id, video_id,
            'Downloading player JSON', data=urlencode_postdata({
                'autoplay': 0,
                'share': 1,
                'pageUrl': url,
                'hasAdConsent': 0,
                '_token': token,
            }))

        player_token = player['token']

        drm = False
        format_urls = set()
        formats = []
        for profile in ('hls', 'dash-widevine', 'dash-playready', 'smooth'):
            streams = self._download_json(
                'https://start-player.npo.nl/video/%s/streams' % video_id,
                video_id, 'Downloading %s profile JSON' % profile, fatal=False,
                query={
                    'profile': profile,
                    'quality': 'npo',
                    'tokenId': player_token,
                    'streamType': 'broadcast',
                })
            if not streams:
                continue
            stream = streams.get('stream')
            if not isinstance(stream, dict):
                continue
            stream_url = url_or_none(stream.get('src'))
            if not stream_url or stream_url in format_urls:
                continue
            format_urls.add(stream_url)
            if stream.get('protection') is not None or stream.get('keySystemOptions') is not None:
                drm = True
                continue
            stream_type = stream.get('type')
            stream_ext = determine_ext(stream_url)
            if stream_type == 'application/dash+xml' or stream_ext == 'mpd':
                formats.extend(self._extract_mpd_formats(
                    stream_url, video_id, mpd_id='dash', fatal=False))
            elif stream_type == 'application/vnd.apple.mpegurl' or stream_ext == 'm3u8':
                formats.extend(self._extract_m3u8_formats(
                    stream_url, video_id, ext='mp4',
                    entry_protocol='m3u8_native', m3u8_id='hls', fatal=False))
            elif re.search(r'\.isml?/Manifest', stream_url):
                formats.extend(self._extract_ism_formats(
                    stream_url, video_id, ism_id='mss', fatal=False))
            else:
                formats.append({
                    'url': stream_url,
                })

        if not formats:
            if drm:
                raise ExtractorError('This video is DRM protected.', expected=True)
            return

        self._sort_formats(formats)

        info = {
            'id': video_id,
            'title': video_id,
            'formats': formats,
        }

        embed_url = url_or_none(player.get('embedUrl'))
        if embed_url:
            webpage = self._download_webpage(
                embed_url, video_id, 'Downloading embed page', fatal=False)
            if webpage:
                video = self._parse_json(
                    self._search_regex(
                        r'\bvideo\s*=\s*({.+?})\s*;', webpage, 'video',
                        default='{}'), video_id)
                if video:
                    title = video.get('episodeTitle')
                    subtitles = {}
                    subtitles_list = video.get('subtitles')
                    if isinstance(subtitles_list, list):
                        for cc in subtitles_list:
                            cc_url = url_or_none(cc.get('src'))
                            if not cc_url:
                                continue
                            lang = str_or_none(cc.get('language')) or 'nl'
                            subtitles.setdefault(lang, []).append({
                                'url': cc_url,
                            })
                    return merge_dicts({
                        'title': title,
                        'description': video.get('description'),
                        'thumbnail': url_or_none(
                            video.get('still_image_url') or video.get('orig_image_url')),
                        'duration': int_or_none(video.get('duration')),
                        'timestamp': unified_timestamp(video.get('broadcastDate')),
                        'creator': video.get('channel'),
                        'series': video.get('title'),
                        'episode': title,
                        'episode_number': int_or_none(video.get('episodeNumber')),
                        'subtitles': subtitles,
                    }, info)

        return info

    def _get_old_info(self, video_id):
        metadata = self._download_json(
            'http://e.omroep.nl/metadata/%s' % video_id,
            video_id,
            # We have to remove the javascript callback
            transform_source=strip_jsonp,
        )

        error = metadata.get('error')
        if error:
            raise ExtractorError(error, expected=True)

        # For some videos actual video id (prid) is different (e.g. for
        # http://www.omroepwnl.nl/video/fragment/vandaag-de-dag-verkiezingen__POMS_WNL_853698
        # video id is POMS_WNL_853698 but prid is POW_00996502)
        video_id = metadata.get('prid') or video_id

        # titel is too generic in some cases so utilize aflevering_titel as well
        # when available (e.g. http://tegenlicht.vpro.nl/afleveringen/2014-2015/access-to-africa.html)
        title = metadata['titel']
        sub_title = metadata.get('aflevering_titel')
        if sub_title and sub_title != title:
            title += ': %s' % sub_title

        token = self._get_token(video_id)

        formats = []
        urls = set()

        def is_legal_url(format_url):
            return format_url and format_url not in urls and re.match(
                r'^(?:https?:)?//', format_url)

        QUALITY_LABELS = ('Laag', 'Normaal', 'Hoog')
        QUALITY_FORMATS = ('adaptive', 'wmv_sb', 'h264_sb', 'wmv_bb', 'h264_bb', 'wvc1_std', 'h264_std')

        quality_from_label = qualities(QUALITY_LABELS)
        quality_from_format_id = qualities(QUALITY_FORMATS)
        items = self._download_json(
            'http://ida.omroep.nl/app.php/%s' % video_id, video_id,
            'Downloading formats JSON', query={
                'adaptive': 'yes',
                'token': token,
            })['items'][0]
        for num, item in enumerate(items):
            item_url = item.get('url')
            if not is_legal_url(item_url):
                continue
            urls.add(item_url)
            format_id = self._search_regex(
                r'video/ida/([^/]+)', item_url, 'format id',
                default=None)

            item_label = item.get('label')

            def add_format_url(format_url):
                width = int_or_none(self._search_regex(
                    r'(\d+)[xX]\d+', format_url, 'width', default=None))
                height = int_or_none(self._search_regex(
                    r'\d+[xX](\d+)', format_url, 'height', default=None))
                if item_label in QUALITY_LABELS:
                    quality = quality_from_label(item_label)
                    f_id = item_label
                elif item_label in QUALITY_FORMATS:
                    quality = quality_from_format_id(format_id)
                    f_id = format_id
                else:
                    quality, f_id = [None] * 2
                formats.append({
                    'url': format_url,
                    'format_id': f_id,
                    'width': width,
                    'height': height,
                    'quality': quality,
                })

            # Example: http://www.npo.nl/de-nieuwe-mens-deel-1/21-07-2010/WO_VPRO_043706
            if item.get('contentType') in ('url', 'audio'):
                add_format_url(item_url)
                continue

            try:
                stream_info = self._download_json(
                    item_url + '&type=json', video_id,
                    'Downloading %s stream JSON'
                    % item_label or item.get('format') or format_id or num)
            except ExtractorError as ee:
                if isinstance(ee.cause, compat_HTTPError) and ee.cause.code == 404:
                    error = (self._parse_json(
                        ee.cause.read().decode(), video_id,
                        fatal=False) or {}).get('errorstring')
                    if error:
                        raise ExtractorError(error, expected=True)
                raise
            # Stream URL instead of JSON, example: npo:LI_NL1_4188102
            if isinstance(stream_info, compat_str):
                if not stream_info.startswith('http'):
                    continue
                video_url = stream_info
            # JSON
            else:
                video_url = stream_info.get('url')
            if not video_url or 'vodnotavailable.' in video_url or video_url in urls:
                continue
            urls.add(video_url)
            if determine_ext(video_url) == 'm3u8':
                formats.extend(self._extract_m3u8_formats(
                    video_url, video_id, ext='mp4',
                    entry_protocol='m3u8_native', m3u8_id='hls', fatal=False))
            else:
                add_format_url(video_url)

        is_live = metadata.get('medium') == 'live'

        if not is_live:
            for num, stream in enumerate(metadata.get('streams', [])):
                stream_url = stream.get('url')
                if not is_legal_url(stream_url):
                    continue
                urls.add(stream_url)
                # smooth streaming is not supported
                stream_type = stream.get('type', '').lower()
                if stream_type in ['ss', 'ms']:
                    continue
                if stream_type == 'hds':
                    f4m_formats = self._extract_f4m_formats(
                        stream_url, video_id, fatal=False)
                    # f4m downloader downloads only piece of live stream
                    for f4m_format in f4m_formats:
                        f4m_format['preference'] = -1
                    formats.extend(f4m_formats)
                elif stream_type == 'hls':
                    formats.extend(self._extract_m3u8_formats(
                        stream_url, video_id, ext='mp4', fatal=False))
                # Example: http://www.npo.nl/de-nieuwe-mens-deel-1/21-07-2010/WO_VPRO_043706
                elif '.asf' in stream_url:
                    asx = self._download_xml(
                        stream_url, video_id,
                        'Downloading stream %d ASX playlist' % num,
                        transform_source=fix_xml_ampersands, fatal=False)
                    if not asx:
                        continue
                    ref = asx.find('./ENTRY/Ref')
                    if ref is None:
                        continue
                    video_url = ref.get('href')
                    if not video_url or video_url in urls:
                        continue
                    urls.add(video_url)
                    formats.append({
                        'url': video_url,
                        'ext': stream.get('formaat', 'asf'),
                        'quality': stream.get('kwaliteit'),
                        'preference': -10,
                    })
                else:
                    formats.append({
                        'url': stream_url,
                        'quality': stream.get('kwaliteit'),
                    })

        self._sort_formats(formats)

        subtitles = {}
        if metadata.get('tt888') == 'ja':
            subtitles['nl'] = [{
                'ext': 'vtt',
                'url': 'http://tt888.omroep.nl/tt888/%s' % video_id,
            }]

        return {
            'id': video_id,
            'title': self._live_title(title) if is_live else title,
            'description': metadata.get('info'),
            'thumbnail': metadata.get('images', [{'url': None}])[-1]['url'],
            'upload_date': unified_strdate(metadata.get('gidsdatum')),
            'duration': parse_duration(metadata.get('tijdsduur')),
            'formats': formats,
            'subtitles': subtitles,
            'is_live': is_live,
        }


class NPOLiveIE(NPOBaseIE):
    IE_NAME = 'npo.nl:live'
    _VALID_URL = r'https?://(?:www\.)?npo(?:start)?\.nl/live(?:/(?P<id>[^/?#&]+))?'

    _TESTS = [{
        'url': 'http://www.npo.nl/live/npo-1',
        'info_dict': {
            'id': 'LI_NL1_4188102',
            'display_id': 'npo-1',
            'ext': 'mp4',
            'title': 're:^NPO 1 [0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}$',
            'is_live': True,
        },
        'params': {
            'skip_download': True,
        }
    }, {
        'url': 'http://www.npo.nl/live',
        'only_matching': True,
    }, {
        'url': 'https://www.npostart.nl/live/npo-1',
        'only_matching': True,
    }]

    def _real_extract(self, url):
        display_id = self._match_id(url) or 'npo-1'

        webpage = self._download_webpage(url, display_id)

        live_id = self._search_regex(
            [r'media-id="([^"]+)"', r'data-prid="([^"]+)"'], webpage, 'live id')

        return {
            '_type': 'url_transparent',
            'url': 'npo:%s' % live_id,
            'ie_key': NPOIE.ie_key(),
            'id': live_id,
            'display_id': display_id,
        }


class NPORadioIE(InfoExtractor):
    IE_NAME = 'npo.nl:radio'
    _VALID_URL = r'https?://(?:www\.)?npo\.nl/radio/(?P<id>[^/]+)'

    _TEST = {
        'url': 'http://www.npo.nl/radio/radio-1',
        'info_dict': {
            'id': 'radio-1',
            'ext': 'mp3',
            'title': 're:^NPO Radio 1 [0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}$',
            'is_live': True,
        },
        'params': {
            'skip_download': True,
        }
    }

    @classmethod
    def suitable(cls, url):
        return False if NPORadioFragmentIE.suitable(url) else super(NPORadioIE, cls).suitable(url)

    @staticmethod
    def _html_get_attribute_regex(attribute):
        return r'{0}\s*=\s*\'([^\']+)\''.format(attribute)

    def _real_extract(self, url):
        video_id = self._match_id(url)

        webpage = self._download_webpage(url, video_id)

        title = self._html_search_regex(
            self._html_get_attribute_regex('data-channel'), webpage, 'title')

        stream = self._parse_json(
            self._html_search_regex(self._html_get_attribute_regex('data-streams'), webpage, 'data-streams'),
            video_id)

        codec = stream.get('codec')

        return {
            'id': video_id,
            'url': stream['url'],
            'title': self._live_title(title),
            'acodec': codec,
            'ext': codec,
            'is_live': True,
        }


class NPORadioFragmentIE(InfoExtractor):
    IE_NAME = 'npo.nl:radio:fragment'
    _VALID_URL = r'https?://(?:www\.)?npo\.nl/radio/[^/]+/fragment/(?P<id>\d+)'

    _TEST = {
        'url': 'http://www.npo.nl/radio/radio-5/fragment/174356',
        'md5': 'dd8cc470dad764d0fdc70a9a1e2d18c2',
        'info_dict': {
            'id': '174356',
            'ext': 'mp3',
            'title': 'Jubileumconcert Willeke Alberti',
        },
    }

    def _real_extract(self, url):
        audio_id = self._match_id(url)

        webpage = self._download_webpage(url, audio_id)

        title = self._html_search_regex(
            r'href="/radio/[^/]+/fragment/%s" title="([^"]+)"' % audio_id,
            webpage, 'title')

        audio_url = self._search_regex(
            r"data-streams='([^']+)'", webpage, 'audio url')

        return {
            'id': audio_id,
            'url': audio_url,
            'title': title,
        }


class NPODataMidEmbedIE(InfoExtractor):
    def _real_extract(self, url):
        display_id = self._match_id(url)
        webpage = self._download_webpage(url, display_id)
        video_id = self._search_regex(
            r'data-mid=(["\'])(?P<id>(?:(?!\1).)+)\1', webpage, 'video_id', group='id')
        return {
            '_type': 'url_transparent',
            'ie_key': 'NPO',
            'url': 'npo:%s' % video_id,
            'display_id': display_id
        }


class SchoolTVIE(NPODataMidEmbedIE):
    IE_NAME = 'schooltv'
    _VALID_URL = r'https?://(?:www\.)?schooltv\.nl/video/(?P<id>[^/?#&]+)'

    _TEST = {
        'url': 'http://www.schooltv.nl/video/ademhaling-de-hele-dag-haal-je-adem-maar-wat-gebeurt-er-dan-eigenlijk-in-je-lichaam/',
        'info_dict': {
            'id': 'WO_NTR_429477',
            'display_id': 'ademhaling-de-hele-dag-haal-je-adem-maar-wat-gebeurt-er-dan-eigenlijk-in-je-lichaam',
            'title': 'Ademhaling: De hele dag haal je adem. Maar wat gebeurt er dan eigenlijk in je lichaam?',
            'ext': 'mp4',
            'description': 'md5:abfa0ff690adb73fd0297fd033aaa631'
        },
        'params': {
            # Skip because of m3u8 download
            'skip_download': True
        }
    }


class HetKlokhuisIE(NPODataMidEmbedIE):
    IE_NAME = 'hetklokhuis'
    _VALID_URL = r'https?://(?:www\.)?hetklokhuis\.nl/[^/]+/\d+/(?P<id>[^/?#&]+)'

    _TEST = {
        'url': 'http://hetklokhuis.nl/tv-uitzending/3471/Zwaartekrachtsgolven',
        'info_dict': {
            'id': 'VPWON_1260528',
            'display_id': 'Zwaartekrachtsgolven',
            'ext': 'm4v',
            'title': 'Het Klokhuis: Zwaartekrachtsgolven',
            'description': 'md5:c94f31fb930d76c2efa4a4a71651dd48',
            'upload_date': '20170223',
        },
        'params': {
            'skip_download': True
        }
    }


class NPOPlaylistBaseIE(NPOIE):
    def _real_extract(self, url):
        playlist_id = self._match_id(url)

        webpage = self._download_webpage(url, playlist_id)

        entries = [
            self.url_result('npo:%s' % video_id if not video_id.startswith('http') else video_id)
            for video_id in orderedSet(re.findall(self._PLAYLIST_ENTRY_RE, webpage))
        ]

        playlist_title = self._html_search_regex(
            self._PLAYLIST_TITLE_RE, webpage, 'playlist title',
            default=None) or self._og_search_title(webpage)

        return self.playlist_result(entries, playlist_id, playlist_title)


class VPROIE(NPOPlaylistBaseIE):
    IE_NAME = 'vpro'
    _VALID_URL = r'https?://(?:www\.)?(?:(?:tegenlicht\.)?vpro|2doc)\.nl/(?:[^/]+/)*(?P<id>[^/]+)\.html'
    _PLAYLIST_TITLE_RE = (r'<h1[^>]+class=["\'].*?\bmedia-platform-title\b.*?["\'][^>]*>([^<]+)',
                          r'<h5[^>]+class=["\'].*?\bmedia-platform-subtitle\b.*?["\'][^>]*>([^<]+)')
    _PLAYLIST_ENTRY_RE = r'data-media-id="([^"]+)"'

    _TESTS = [
        {
            'url': 'http://tegenlicht.vpro.nl/afleveringen/2012-2013/de-toekomst-komt-uit-afrika.html',
            'md5': 'f8065e4e5a7824068ed3c7e783178f2c',
            'info_dict': {
                'id': 'VPWON_1169289',
                'ext': 'm4v',
                'title': 'De toekomst komt uit Afrika',
                'description': 'md5:52cf4eefbc96fffcbdc06d024147abea',
                'upload_date': '20130225',
            },
            'skip': 'Video gone',
        },
        {
            'url': 'http://www.vpro.nl/programmas/2doc/2015/sergio-herman.html',
            'info_dict': {
                'id': 'sergio-herman',
                'title': 'sergio herman: fucking perfect',
            },
            'playlist_count': 2,
        },
        {
            # playlist with youtube embed
            'url': 'http://www.vpro.nl/programmas/2doc/2015/education-education.html',
            'info_dict': {
                'id': 'education-education',
                'title': 'education education',
            },
            'playlist_count': 2,
        },
        {
            'url': 'http://www.2doc.nl/documentaires/series/2doc/2015/oktober/de-tegenprestatie.html',
            'info_dict': {
                'id': 'de-tegenprestatie',
                'title': 'De Tegenprestatie',
            },
            'playlist_count': 2,
        }, {
            'url': 'http://www.2doc.nl/speel~VARA_101375237~mh17-het-verdriet-van-nederland~.html',
            'info_dict': {
                'id': 'VARA_101375237',
                'ext': 'm4v',
                'title': 'MH17: Het verdriet van Nederland',
                'description': 'md5:09e1a37c1fdb144621e22479691a9f18',
                'upload_date': '20150716',
            },
            'params': {
                # Skip because of m3u8 download
                'skip_download': True
            },
        }
    ]


class WNLIE(NPOPlaylistBaseIE):
    IE_NAME = 'wnl'
    _VALID_URL = r'https?://(?:www\.)?omroepwnl\.nl/video/detail/(?P<id>[^/]+)__\d+'
    _PLAYLIST_TITLE_RE = r'(?s)<h1[^>]+class="subject"[^>]*>(.+?)</h1>'
    _PLAYLIST_ENTRY_RE = r'<a[^>]+href="([^"]+)"[^>]+class="js-mid"[^>]*>Deel \d+'

    _TESTS = [{
        'url': 'http://www.omroepwnl.nl/video/detail/vandaag-de-dag-6-mei__060515',
        'info_dict': {
            'id': 'vandaag-de-dag-6-mei',
            'title': 'Vandaag de Dag 6 mei',
        },
        'playlist_count': 4,
    }]


class AndereTijdenIE(NPOPlaylistBaseIE):
    IE_NAME = 'anderetijden'
    _VALID_URL = r'https?://(?:www\.)?anderetijden\.nl/programma/(?:[^/]+/)+(?P<id>[^/?#&]+)'
    _PLAYLIST_TITLE_RE = r'(?s)<h1[^>]+class=["\'].*?\bpage-title\b.*?["\'][^>]*>(.+?)</h1>'
    _PLAYLIST_ENTRY_RE = r'<figure[^>]+class=["\']episode-container episode-page["\'][^>]+data-prid=["\'](.+?)["\']'

    _TESTS = [{
        'url': 'http://anderetijden.nl/programma/1/Andere-Tijden/aflevering/676/Duitse-soldaten-over-de-Slag-bij-Arnhem',
        'info_dict': {
            'id': 'Duitse-soldaten-over-de-Slag-bij-Arnhem',
            'title': 'Duitse soldaten over de Slag bij Arnhem',
        },
        'playlist_count': 3,
    }]
from __future__ import unicode_literals

from .common import InfoExtractor
from ..utils import (
    int_or_none,
    qualities,
    url_or_none,
)


class NprIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?npr\.org/(?:sections/[^/]+/)?\d{4}/\d{2}/\d{2}/(?P<id>\d+)'
    _TESTS = [{
        'url': 'https://www.npr.org/sections/allsongs/2015/10/21/449974205/new-music-from-beach-house-chairlift-cmj-discoveries-and-more',
        'info_dict': {
            'id': '449974205',
            'title': 'New Music From Beach House, Chairlift, CMJ Discoveries And More'
        },
        'playlist_count': 7,
    }, {
        'url': 'https://www.npr.org/sections/deceptivecadence/2015/10/09/446928052/music-from-the-shadows-ancient-armenian-hymns-and-piano-jazz',
        'info_dict': {
            'id': '446928052',
            'title': "Songs We Love: Tigran Hamasyan, 'Your Mercy is Boundless'"
        },
        'playlist': [{
            'md5': '12fa60cb2d3ed932f53609d4aeceabf1',
            'info_dict': {
                'id': '446929930',
                'ext': 'mp3',
                'title': 'Your Mercy is Boundless (Bazum en Qo gtutyunqd)',
                'duration': 402,
            },
        }],
    }, {
        # mutlimedia, not media title
        'url': 'https://www.npr.org/2017/06/19/533198237/tigers-jaw-tiny-desk-concert',
        'info_dict': {
            'id': '533198237',
            'title': 'Tigers Jaw: Tiny Desk Concert',
        },
        'playlist': [{
            'md5': '12fa60cb2d3ed932f53609d4aeceabf1',
            'info_dict': {
                'id': '533201718',
                'ext': 'mp4',
                'title': 'Tigers Jaw: Tiny Desk Concert',
                'duration': 402,
            },
        }],
        'expected_warnings': ['Failed to download m3u8 information'],
    }, {
        # multimedia, no formats, stream
        'url': 'https://www.npr.org/2020/02/14/805476846/laura-stevenson-tiny-desk-concert',
        'only_matching': True,
    }]

    def _real_extract(self, url):
        playlist_id = self._match_id(url)

        story = self._download_json(
            'http://api.npr.org/query', playlist_id, query={
                'id': playlist_id,
                'fields': 'audio,multimedia,title',
                'format': 'json',
                'apiKey': 'MDAzMzQ2MjAyMDEyMzk4MTU1MDg3ZmM3MQ010',
            })['list']['story'][0]
        playlist_title = story.get('title', {}).get('$text')

        KNOWN_FORMATS = ('threegp', 'm3u8', 'smil', 'mp4', 'mp3')
        quality = qualities(KNOWN_FORMATS)

        entries = []
        for media in story.get('audio', []) + story.get('multimedia', []):
            media_id = media['id']

            formats = []
            for format_id, formats_entry in media.get('format', {}).items():
                if not formats_entry:
                    continue
                if isinstance(formats_entry, list):
                    formats_entry = formats_entry[0]
                format_url = formats_entry.get('$text')
                if not format_url:
                    continue
                if format_id in KNOWN_FORMATS:
                    if format_id == 'm3u8':
                        formats.extend(self._extract_m3u8_formats(
                            format_url, media_id, 'mp4', 'm3u8_native',
                            m3u8_id='hls', fatal=False))
                    elif format_id == 'smil':
                        smil_formats = self._extract_smil_formats(
                            format_url, media_id, transform_source=lambda s: s.replace(
                                'rtmp://flash.npr.org/ondemand/', 'https://ondemand.npr.org/'))
                        self._check_formats(smil_formats, media_id)
                        formats.extend(smil_formats)
                    else:
                        formats.append({
                            'url': format_url,
                            'format_id': format_id,
                            'quality': quality(format_id),
                        })
            for stream_id, stream_entry in media.get('stream', {}).items():
                if not isinstance(stream_entry, dict):
                    continue
                if stream_id != 'hlsUrl':
                    continue
                stream_url = url_or_none(stream_entry.get('$text'))
                if not stream_url:
                    continue
                formats.extend(self._extract_m3u8_formats(
                    stream_url, stream_id, 'mp4', 'm3u8_native',
                    m3u8_id='hls', fatal=False))
            self._sort_formats(formats)

            entries.append({
                'id': media_id,
                'title': media.get('title', {}).get('$text') or playlist_title,
                'thumbnail': media.get('altImageUrl', {}).get('$text'),
                'duration': int_or_none(media.get('duration', {}).get('$text')),
                'formats': formats,
            })

        return self.playlist_result(entries, playlist_id, playlist_title)
# coding: utf-8
from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..compat import (
    compat_str,
    compat_urllib_parse_unquote,
)
from ..utils import (
    ExtractorError,
    int_or_none,
    js_to_json,
    NO_DEFAULT,
    parse_age_limit,
    parse_duration,
    try_get,
)


class NRKBaseIE(InfoExtractor):
    _GEO_COUNTRIES = ['NO']

    _api_host = None

    def _real_extract(self, url):
        video_id = self._match_id(url)

        api_hosts = (self._api_host, ) if self._api_host else self._API_HOSTS

        for api_host in api_hosts:
            data = self._download_json(
                'http://%s/mediaelement/%s' % (api_host, video_id),
                video_id, 'Downloading mediaelement JSON',
                fatal=api_host == api_hosts[-1])
            if not data:
                continue
            self._api_host = api_host
            break

        title = data.get('fullTitle') or data.get('mainTitle') or data['title']
        video_id = data.get('id') or video_id

        entries = []

        conviva = data.get('convivaStatistics') or {}
        live = (data.get('mediaElementType') == 'Live'
                or data.get('isLive') is True or conviva.get('isLive'))

        def make_title(t):
            return self._live_title(t) if live else t

        media_assets = data.get('mediaAssets')
        if media_assets and isinstance(media_assets, list):
            def video_id_and_title(idx):
                return ((video_id, title) if len(media_assets) == 1
                        else ('%s-%d' % (video_id, idx), '%s (Part %d)' % (title, idx)))
            for num, asset in enumerate(media_assets, 1):
                asset_url = asset.get('url')
                if not asset_url:
                    continue
                formats = self._extract_akamai_formats(asset_url, video_id)
                if not formats:
                    continue
                self._sort_formats(formats)

                # Some f4m streams may not work with hdcore in fragments' URLs
                for f in formats:
                    extra_param = f.get('extra_param_to_segment_url')
                    if extra_param and 'hdcore' in extra_param:
                        del f['extra_param_to_segment_url']

                entry_id, entry_title = video_id_and_title(num)
                duration = parse_duration(asset.get('duration'))
                subtitles = {}
                for subtitle in ('webVtt', 'timedText'):
                    subtitle_url = asset.get('%sSubtitlesUrl' % subtitle)
                    if subtitle_url:
                        subtitles.setdefault('no', []).append({
                            'url': compat_urllib_parse_unquote(subtitle_url)
                        })
                entries.append({
                    'id': asset.get('carrierId') or entry_id,
                    'title': make_title(entry_title),
                    'duration': duration,
                    'subtitles': subtitles,
                    'formats': formats,
                })

        if not entries:
            media_url = data.get('mediaUrl')
            if media_url:
                formats = self._extract_akamai_formats(media_url, video_id)
                self._sort_formats(formats)
                duration = parse_duration(data.get('duration'))
                entries = [{
                    'id': video_id,
                    'title': make_title(title),
                    'duration': duration,
                    'formats': formats,
                }]

        if not entries:
            MESSAGES = {
                'ProgramRightsAreNotReady': 'Du kan dessverre ikke se eller høre programmet',
                'ProgramRightsHasExpired': 'Programmet har gått ut',
                'NoProgramRights': 'Ikke tilgjengelig',
                'ProgramIsGeoBlocked': 'NRK har ikke rettigheter til å vise dette programmet utenfor Norge',
            }
            message_type = data.get('messageType', '')
            # Can be ProgramIsGeoBlocked or ChannelIsGeoBlocked*
            if 'IsGeoBlocked' in message_type:
                self.raise_geo_restricted(
                    msg=MESSAGES.get('ProgramIsGeoBlocked'),
                    countries=self._GEO_COUNTRIES)
            raise ExtractorError(
                '%s said: %s' % (self.IE_NAME, MESSAGES.get(
                    message_type, message_type)),
                expected=True)

        series = conviva.get('seriesName') or data.get('seriesTitle')
        episode = conviva.get('episodeName') or data.get('episodeNumberOrDate')

        season_number = None
        episode_number = None
        if data.get('mediaElementType') == 'Episode':
            _season_episode = data.get('scoresStatistics', {}).get('springStreamStream') or \
                data.get('relativeOriginUrl', '')
            EPISODENUM_RE = [
                r'/s(?P<season>\d{,2})e(?P<episode>\d{,2})\.',
                r'/sesong-(?P<season>\d{,2})/episode-(?P<episode>\d{,2})',
            ]
            season_number = int_or_none(self._search_regex(
                EPISODENUM_RE, _season_episode, 'season number',
                default=None, group='season'))
            episode_number = int_or_none(self._search_regex(
                EPISODENUM_RE, _season_episode, 'episode number',
                default=None, group='episode'))

        thumbnails = None
        images = data.get('images')
        if images and isinstance(images, dict):
            web_images = images.get('webImages')
            if isinstance(web_images, list):
                thumbnails = [{
                    'url': image['imageUrl'],
                    'width': int_or_none(image.get('width')),
                    'height': int_or_none(image.get('height')),
                } for image in web_images if image.get('imageUrl')]

        description = data.get('description')
        category = data.get('mediaAnalytics', {}).get('category')

        common_info = {
            'description': description,
            'series': series,
            'episode': episode,
            'season_number': season_number,
            'episode_number': episode_number,
            'categories': [category] if category else None,
            'age_limit': parse_age_limit(data.get('legalAge')),
            'thumbnails': thumbnails,
        }

        vcodec = 'none' if data.get('mediaType') == 'Audio' else None

        for entry in entries:
            entry.update(common_info)
            for f in entry['formats']:
                f['vcodec'] = vcodec

        points = data.get('shortIndexPoints')
        if isinstance(points, list):
            chapters = []
            for next_num, point in enumerate(points, start=1):
                if not isinstance(point, dict):
                    continue
                start_time = parse_duration(point.get('startPoint'))
                if start_time is None:
                    continue
                end_time = parse_duration(
                    data.get('duration')
                    if next_num == len(points)
                    else points[next_num].get('startPoint'))
                if end_time is None:
                    continue
                chapters.append({
                    'start_time': start_time,
                    'end_time': end_time,
                    'title': point.get('title'),
                })
            if chapters and len(entries) == 1:
                entries[0]['chapters'] = chapters

        return self.playlist_result(entries, video_id, title, description)


class NRKIE(NRKBaseIE):
    _VALID_URL = r'''(?x)
                        (?:
                            nrk:|
                            https?://
                                (?:
                                    (?:www\.)?nrk\.no/video/PS\*|
                                    v8[-.]psapi\.nrk\.no/mediaelement/
                                )
                            )
                            (?P<id>[^?#&]+)
                        '''
    _API_HOSTS = ('psapi.nrk.no', 'v8-psapi.nrk.no')
    _TESTS = [{
        # video
        'url': 'http://www.nrk.no/video/PS*150533',
        'md5': '706f34cdf1322577589e369e522b50ef',
        'info_dict': {
            'id': '150533',
            'ext': 'mp4',
            'title': 'Dompap og andre fugler i Piip-Show',
            'description': 'md5:d9261ba34c43b61c812cb6b0269a5c8f',
            'duration': 262,
        }
    }, {
        # audio
        'url': 'http://www.nrk.no/video/PS*154915',
        # MD5 is unstable
        'info_dict': {
            'id': '154915',
            'ext': 'flv',
            'title': 'Slik høres internett ut når du er blind',
            'description': 'md5:a621f5cc1bd75c8d5104cb048c6b8568',
            'duration': 20,
        }
    }, {
        'url': 'nrk:ecc1b952-96dc-4a98-81b9-5296dc7a98d9',
        'only_matching': True,
    }, {
        'url': 'nrk:clip/7707d5a3-ebe7-434a-87d5-a3ebe7a34a70',
        'only_matching': True,
    }, {
        'url': 'https://v8-psapi.nrk.no/mediaelement/ecc1b952-96dc-4a98-81b9-5296dc7a98d9',
        'only_matching': True,
    }]


class NRKTVIE(NRKBaseIE):
    IE_DESC = 'NRK TV and NRK Radio'
    _EPISODE_RE = r'(?P<id>[a-zA-Z]{4}\d{8})'
    _VALID_URL = r'''(?x)
                        https?://
                            (?:tv|radio)\.nrk(?:super)?\.no/
                            (?:serie(?:/[^/]+){1,2}|program)/
                            (?![Ee]pisodes)%s
                            (?:/\d{2}-\d{2}-\d{4})?
                            (?:\#del=(?P<part_id>\d+))?
                    ''' % _EPISODE_RE
    _API_HOSTS = ('psapi-ne.nrk.no', 'psapi-we.nrk.no')
    _TESTS = [{
        'url': 'https://tv.nrk.no/program/MDDP12000117',
        'md5': '8270824df46ec629b66aeaa5796b36fb',
        'info_dict': {
            'id': 'MDDP12000117AA',
            'ext': 'mp4',
            'title': 'Alarm Trolltunga',
            'description': 'md5:46923a6e6510eefcce23d5ef2a58f2ce',
            'duration': 2223,
            'age_limit': 6,
        },
    }, {
        'url': 'https://tv.nrk.no/serie/20-spoersmaal-tv/MUHH48000314/23-05-2014',
        'md5': '9a167e54d04671eb6317a37b7bc8a280',
        'info_dict': {
            'id': 'MUHH48000314AA',
            'ext': 'mp4',
            'title': '20 spørsmål 23.05.2014',
            'description': 'md5:bdea103bc35494c143c6a9acdd84887a',
            'duration': 1741,
            'series': '20 spørsmål',
            'episode': '23.05.2014',
        },
        'skip': 'NoProgramRights',
    }, {
        'url': 'https://tv.nrk.no/program/mdfp15000514',
        'info_dict': {
            'id': 'MDFP15000514CA',
            'ext': 'mp4',
            'title': 'Grunnlovsjubiléet - Stor ståhei for ingenting 24.05.2014',
            'description': 'md5:89290c5ccde1b3a24bb8050ab67fe1db',
            'duration': 4605,
            'series': 'Kunnskapskanalen',
            'episode': '24.05.2014',
        },
        'params': {
            'skip_download': True,
        },
    }, {
        # single playlist video
        'url': 'https://tv.nrk.no/serie/tour-de-ski/MSPO40010515/06-01-2015#del=2',
        'info_dict': {
            'id': 'MSPO40010515-part2',
            'ext': 'flv',
            'title': 'Tour de Ski: Sprint fri teknikk, kvinner og menn 06.01.2015 (del 2:2)',
            'description': 'md5:238b67b97a4ac7d7b4bf0edf8cc57d26',
        },
        'params': {
            'skip_download': True,
        },
        'expected_warnings': ['Video is geo restricted'],
        'skip': 'particular part is not supported currently',
    }, {
        'url': 'https://tv.nrk.no/serie/tour-de-ski/MSPO40010515/06-01-2015',
        'playlist': [{
            'info_dict': {
                'id': 'MSPO40010515AH',
                'ext': 'mp4',
                'title': 'Sprint fri teknikk, kvinner og menn 06.01.2015 (Part 1)',
                'description': 'md5:1f97a41f05a9486ee00c56f35f82993d',
                'duration': 772,
                'series': 'Tour de Ski',
                'episode': '06.01.2015',
            },
            'params': {
                'skip_download': True,
            },
        }, {
            'info_dict': {
                'id': 'MSPO40010515BH',
                'ext': 'mp4',
                'title': 'Sprint fri teknikk, kvinner og menn 06.01.2015 (Part 2)',
                'description': 'md5:1f97a41f05a9486ee00c56f35f82993d',
                'duration': 6175,
                'series': 'Tour de Ski',
                'episode': '06.01.2015',
            },
            'params': {
                'skip_download': True,
            },
        }],
        'info_dict': {
            'id': 'MSPO40010515',
            'title': 'Sprint fri teknikk, kvinner og menn 06.01.2015',
            'description': 'md5:1f97a41f05a9486ee00c56f35f82993d',
        },
        'expected_warnings': ['Video is geo restricted'],
    }, {
        'url': 'https://tv.nrk.no/serie/anno/KMTE50001317/sesong-3/episode-13',
        'info_dict': {
            'id': 'KMTE50001317AA',
            'ext': 'mp4',
            'title': 'Anno 13:30',
            'description': 'md5:11d9613661a8dbe6f9bef54e3a4cbbfa',
            'duration': 2340,
            'series': 'Anno',
            'episode': '13:30',
            'season_number': 3,
            'episode_number': 13,
        },
        'params': {
            'skip_download': True,
        },
    }, {
        'url': 'https://tv.nrk.no/serie/nytt-paa-nytt/MUHH46000317/27-01-2017',
        'info_dict': {
            'id': 'MUHH46000317AA',
            'ext': 'mp4',
            'title': 'Nytt på Nytt 27.01.2017',
            'description': 'md5:5358d6388fba0ea6f0b6d11c48b9eb4b',
            'duration': 1796,
            'series': 'Nytt på nytt',
            'episode': '27.01.2017',
        },
        'params': {
            'skip_download': True,
        },
    }, {
        'url': 'https://radio.nrk.no/serie/dagsnytt/NPUB21019315/12-07-2015#',
        'only_matching': True,
    }, {
        'url': 'https://tv.nrk.no/serie/lindmo/2018/MUHU11006318/avspiller',
        'only_matching': True,
    }]


class NRKTVEpisodeIE(InfoExtractor):
    _VALID_URL = r'https?://tv\.nrk\.no/serie/(?P<id>[^/]+/sesong/\d+/episode/\d+)'
    _TESTS = [{
        'url': 'https://tv.nrk.no/serie/hellums-kro/sesong/1/episode/2',
        'info_dict': {
            'id': 'MUHH36005220BA',
            'ext': 'mp4',
            'title': 'Kro, krig og kjærlighet 2:6',
            'description': 'md5:b32a7dc0b1ed27c8064f58b97bda4350',
            'duration': 1563,
            'series': 'Hellums kro',
            'season_number': 1,
            'episode_number': 2,
            'episode': '2:6',
            'age_limit': 6,
        },
        'params': {
            'skip_download': True,
        },
    }, {
        'url': 'https://tv.nrk.no/serie/backstage/sesong/1/episode/8',
        'info_dict': {
            'id': 'MSUI14000816AA',
            'ext': 'mp4',
            'title': 'Backstage 8:30',
            'description': 'md5:de6ca5d5a2d56849e4021f2bf2850df4',
            'duration': 1320,
            'series': 'Backstage',
            'season_number': 1,
            'episode_number': 8,
            'episode': '8:30',
        },
        'params': {
            'skip_download': True,
        },
        'skip': 'ProgramRightsHasExpired',
    }]

    def _real_extract(self, url):
        display_id = self._match_id(url)

        webpage = self._download_webpage(url, display_id)

        info = self._search_json_ld(webpage, display_id, default={})
        nrk_id = info.get('@id') or self._html_search_meta(
            'nrk:program-id', webpage, default=None) or self._search_regex(
            r'data-program-id=["\'](%s)' % NRKTVIE._EPISODE_RE, webpage,
            'nrk id')
        assert re.match(NRKTVIE._EPISODE_RE, nrk_id)

        info.update({
            '_type': 'url_transparent',
            'id': nrk_id,
            'url': 'nrk:%s' % nrk_id,
            'ie_key': NRKIE.ie_key(),
        })
        return info


class NRKTVSerieBaseIE(InfoExtractor):
    def _extract_series(self, webpage, display_id, fatal=True):
        config = self._parse_json(
            self._search_regex(
                (r'INITIAL_DATA(?:_V\d)?_*\s*=\s*({.+?})\s*;',
                 r'({.+?})\s*,\s*"[^"]+"\s*\)\s*</script>'),
                webpage, 'config', default='{}' if not fatal else NO_DEFAULT),
            display_id, fatal=False, transform_source=js_to_json)
        if not config:
            return
        return try_get(
            config,
            (lambda x: x['initialState']['series'], lambda x: x['series']),
            dict)

    def _extract_seasons(self, seasons):
        if not isinstance(seasons, list):
            return []
        entries = []
        for season in seasons:
            entries.extend(self._extract_episodes(season))
        return entries

    def _extract_episodes(self, season):
        if not isinstance(season, dict):
            return []
        return self._extract_entries(season.get('episodes'))

    def _extract_entries(self, entry_list):
        if not isinstance(entry_list, list):
            return []
        entries = []
        for episode in entry_list:
            nrk_id = episode.get('prfId')
            if not nrk_id or not isinstance(nrk_id, compat_str):
                continue
            entries.append(self.url_result(
                'nrk:%s' % nrk_id, ie=NRKIE.ie_key(), video_id=nrk_id))
        return entries


class NRKTVSeasonIE(NRKTVSerieBaseIE):
    _VALID_URL = r'https?://tv\.nrk\.no/serie/[^/]+/sesong/(?P<id>\d+)'
    _TEST = {
        'url': 'https://tv.nrk.no/serie/backstage/sesong/1',
        'info_dict': {
            'id': '1',
            'title': 'Sesong 1',
        },
        'playlist_mincount': 30,
    }

    @classmethod
    def suitable(cls, url):
        return (False if NRKTVIE.suitable(url) or NRKTVEpisodeIE.suitable(url)
                else super(NRKTVSeasonIE, cls).suitable(url))

    def _real_extract(self, url):
        display_id = self._match_id(url)

        webpage = self._download_webpage(url, display_id)

        series = self._extract_series(webpage, display_id)

        season = next(
            s for s in series['seasons']
            if int(display_id) == s.get('seasonNumber'))

        title = try_get(season, lambda x: x['titles']['title'], compat_str)
        return self.playlist_result(
            self._extract_episodes(season), display_id, title)


class NRKTVSeriesIE(NRKTVSerieBaseIE):
    _VALID_URL = r'https?://(?:tv|radio)\.nrk(?:super)?\.no/serie/(?P<id>[^/]+)'
    _ITEM_RE = r'(?:data-season=["\']|id=["\']season-)(?P<id>\d+)'
    _TESTS = [{
        'url': 'https://tv.nrk.no/serie/blank',
        'info_dict': {
            'id': 'blank',
            'title': 'Blank',
            'description': 'md5:7664b4e7e77dc6810cd3bca367c25b6e',
        },
        'playlist_mincount': 30,
    }, {
        # new layout, seasons
        'url': 'https://tv.nrk.no/serie/backstage',
        'info_dict': {
            'id': 'backstage',
            'title': 'Backstage',
            'description': 'md5:c3ec3a35736fca0f9e1207b5511143d3',
        },
        'playlist_mincount': 60,
    }, {
        # new layout, instalments
        'url': 'https://tv.nrk.no/serie/groenn-glede',
        'info_dict': {
            'id': 'groenn-glede',
            'title': 'Grønn glede',
            'description': 'md5:7576e92ae7f65da6993cf90ee29e4608',
        },
        'playlist_mincount': 10,
    }, {
        # old layout
        'url': 'https://tv.nrksuper.no/serie/labyrint',
        'info_dict': {
            'id': 'labyrint',
            'title': 'Labyrint',
            'description': 'md5:318b597330fdac5959247c9b69fdb1ec',
        },
        'playlist_mincount': 3,
    }, {
        'url': 'https://tv.nrk.no/serie/broedrene-dal-og-spektralsteinene',
        'only_matching': True,
    }, {
        'url': 'https://tv.nrk.no/serie/saving-the-human-race',
        'only_matching': True,
    }, {
        'url': 'https://tv.nrk.no/serie/postmann-pat',
        'only_matching': True,
    }]

    @classmethod
    def suitable(cls, url):
        return (
            False if any(ie.suitable(url)
                         for ie in (NRKTVIE, NRKTVEpisodeIE, NRKTVSeasonIE))
            else super(NRKTVSeriesIE, cls).suitable(url))

    def _real_extract(self, url):
        series_id = self._match_id(url)

        webpage = self._download_webpage(url, series_id)

        # New layout (e.g. https://tv.nrk.no/serie/backstage)
        series = self._extract_series(webpage, series_id, fatal=False)
        if series:
            title = try_get(series, lambda x: x['titles']['title'], compat_str)
            description = try_get(
                series, lambda x: x['titles']['subtitle'], compat_str)
            entries = []
            entries.extend(self._extract_seasons(series.get('seasons')))
            entries.extend(self._extract_entries(series.get('instalments')))
            entries.extend(self._extract_episodes(series.get('extraMaterial')))
            return self.playlist_result(entries, series_id, title, description)

        # Old layout (e.g. https://tv.nrksuper.no/serie/labyrint)
        entries = [
            self.url_result(
                'https://tv.nrk.no/program/Episodes/{series}/{season}'.format(
                    series=series_id, season=season_id))
            for season_id in re.findall(self._ITEM_RE, webpage)
        ]

        title = self._html_search_meta(
            'seriestitle', webpage,
            'title', default=None) or self._og_search_title(
            webpage, fatal=False)
        if title:
            title = self._search_regex(
                r'NRK (?:Super )?TV\s*[-–]\s*(.+)', title, 'title', default=title)

        description = self._html_search_meta(
            'series_description', webpage,
            'description', default=None) or self._og_search_description(webpage)

        return self.playlist_result(entries, series_id, title, description)


class NRKTVDirekteIE(NRKTVIE):
    IE_DESC = 'NRK TV Direkte and NRK Radio Direkte'
    _VALID_URL = r'https?://(?:tv|radio)\.nrk\.no/direkte/(?P<id>[^/?#&]+)'

    _TESTS = [{
        'url': 'https://tv.nrk.no/direkte/nrk1',
        'only_matching': True,
    }, {
        'url': 'https://radio.nrk.no/direkte/p1_oslo_akershus',
        'only_matching': True,
    }]


class NRKPlaylistBaseIE(InfoExtractor):
    def _extract_description(self, webpage):
        pass

    def _real_extract(self, url):
        playlist_id = self._match_id(url)

        webpage = self._download_webpage(url, playlist_id)

        entries = [
            self.url_result('nrk:%s' % video_id, NRKIE.ie_key())
            for video_id in re.findall(self._ITEM_RE, webpage)
        ]

        playlist_title = self. _extract_title(webpage)
        playlist_description = self._extract_description(webpage)

        return self.playlist_result(
            entries, playlist_id, playlist_title, playlist_description)


class NRKPlaylistIE(NRKPlaylistBaseIE):
    _VALID_URL = r'https?://(?:www\.)?nrk\.no/(?!video|skole)(?:[^/]+/)+(?P<id>[^/]+)'
    _ITEM_RE = r'class="[^"]*\brich\b[^"]*"[^>]+data-video-id="([^"]+)"'
    _TESTS = [{
        'url': 'http://www.nrk.no/troms/gjenopplev-den-historiske-solformorkelsen-1.12270763',
        'info_dict': {
            'id': 'gjenopplev-den-historiske-solformorkelsen-1.12270763',
            'title': 'Gjenopplev den historiske solformørkelsen',
            'description': 'md5:c2df8ea3bac5654a26fc2834a542feed',
        },
        'playlist_count': 2,
    }, {
        'url': 'http://www.nrk.no/kultur/bok/rivertonprisen-til-karin-fossum-1.12266449',
        'info_dict': {
            'id': 'rivertonprisen-til-karin-fossum-1.12266449',
            'title': 'Rivertonprisen til Karin Fossum',
            'description': 'Første kvinne på 15 år til å vinne krimlitteraturprisen.',
        },
        'playlist_count': 2,
    }]

    def _extract_title(self, webpage):
        return self._og_search_title(webpage, fatal=False)

    def _extract_description(self, webpage):
        return self._og_search_description(webpage)


class NRKTVEpisodesIE(NRKPlaylistBaseIE):
    _VALID_URL = r'https?://tv\.nrk\.no/program/[Ee]pisodes/[^/]+/(?P<id>\d+)'
    _ITEM_RE = r'data-episode=["\']%s' % NRKTVIE._EPISODE_RE
    _TESTS = [{
        'url': 'https://tv.nrk.no/program/episodes/nytt-paa-nytt/69031',
        'info_dict': {
            'id': '69031',
            'title': 'Nytt på nytt, sesong: 201210',
        },
        'playlist_count': 4,
    }]

    def _extract_title(self, webpage):
        return self._html_search_regex(
            r'<h1>([^<]+)</h1>', webpage, 'title', fatal=False)


class NRKSkoleIE(InfoExtractor):
    IE_DESC = 'NRK Skole'
    _VALID_URL = r'https?://(?:www\.)?nrk\.no/skole/?\?.*\bmediaId=(?P<id>\d+)'

    _TESTS = [{
        'url': 'https://www.nrk.no/skole/?page=search&q=&mediaId=14099',
        'md5': '18c12c3d071953c3bf8d54ef6b2587b7',
        'info_dict': {
            'id': '6021',
            'ext': 'mp4',
            'title': 'Genetikk og eneggede tvillinger',
            'description': 'md5:3aca25dcf38ec30f0363428d2b265f8d',
            'duration': 399,
        },
    }, {
        'url': 'https://www.nrk.no/skole/?page=objectives&subject=naturfag&objective=K15114&mediaId=19355',
        'only_matching': True,
    }]

    def _real_extract(self, url):
        video_id = self._match_id(url)

        webpage = self._download_webpage(
            'https://mimir.nrk.no/plugin/1.0/static?mediaId=%s' % video_id,
            video_id)

        nrk_id = self._parse_json(
            self._search_regex(
                r'<script[^>]+type=["\']application/json["\'][^>]*>({.+?})</script>',
                webpage, 'application json'),
            video_id)['activeMedia']['psId']

        return self.url_result('nrk:%s' % nrk_id)
# coding: utf-8
from __future__ import unicode_literals

from .common import InfoExtractor


class NRLTVIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?nrl\.com/tv(/[^/]+)*/(?P<id>[^/?&#]+)'
    _TEST = {
        'url': 'https://www.nrl.com/tv/news/match-highlights-titans-v-knights-862805/',
        'info_dict': {
            'id': 'YyNnFuaDE6kPJqlDhG4CGQ_w89mKTau4',
            'ext': 'mp4',
            'title': 'Match Highlights: Titans v Knights',
        },
        'params': {
            # m3u8 download
            'skip_download': True,
            'format': 'bestvideo',
        },
    }

    def _real_extract(self, url):
        display_id = self._match_id(url)
        webpage = self._download_webpage(url, display_id)
        q_data = self._parse_json(self._html_search_regex(
            r'(?s)q-data="({.+?})"', webpage, 'player data'), display_id)
        ooyala_id = q_data['videoId']
        return self.url_result(
            'ooyala:' + ooyala_id, 'Ooyala', ooyala_id, q_data.get('title'))
# coding: utf-8
from __future__ import unicode_literals

from .common import InfoExtractor
from ..utils import (
    js_to_json,
    smuggle_url,
)


class NTVCoJpCUIE(InfoExtractor):
    IE_NAME = 'cu.ntv.co.jp'
    IE_DESC = 'Nippon Television Network'
    _VALID_URL = r'https?://cu\.ntv\.co\.jp/(?!program)(?P<id>[^/?&#]+)'
    _TEST = {
        'url': 'https://cu.ntv.co.jp/televiva-chill-gohan_181031/',
        'info_dict': {
            'id': '5978891207001',
            'ext': 'mp4',
            'title': '桜エビと炒り卵がポイント! 「中華風 エビチリおにぎり」──『美虎』五十嵐美幸',
            'upload_date': '20181213',
            'description': 'md5:211b52f4fd60f3e0e72b68b0c6ba52a9',
            'uploader_id': '3855502814001',
            'timestamp': 1544669941,
        },
        'params': {
            # m3u8 download
            'skip_download': True,
        },
    }
    BRIGHTCOVE_URL_TEMPLATE = 'http://players.brightcove.net/%s/default_default/index.html?videoId=%s'

    def _real_extract(self, url):
        display_id = self._match_id(url)
        webpage = self._download_webpage(url, display_id)
        player_config = self._parse_json(self._search_regex(
            r'(?s)PLAYER_CONFIG\s*=\s*({.+?})',
            webpage, 'player config'), display_id, js_to_json)
        video_id = player_config['videoId']
        account_id = player_config.get('account') or '3855502814001'
        return {
            '_type': 'url_transparent',
            'id': video_id,
            'display_id': display_id,
            'title': self._search_regex(r'<h1[^>]+class="title"[^>]*>([^<]+)', webpage, 'title').strip(),
            'description': self._html_search_meta(['description', 'og:description'], webpage),
            'url': smuggle_url(self.BRIGHTCOVE_URL_TEMPLATE % (account_id, video_id), {'geo_countries': ['JP']}),
            'ie_key': 'BrightcoveNew',
        }
# coding: utf-8
from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..compat import compat_urlparse
from ..utils import (
    int_or_none,
    js_to_json,
    parse_duration,
)


class NTVDeIE(InfoExtractor):
    IE_NAME = 'n-tv.de'
    _VALID_URL = r'https?://(?:www\.)?n-tv\.de/mediathek/videos/[^/?#]+/[^/?#]+-article(?P<id>.+)\.html'

    _TESTS = [{
        'url': 'http://www.n-tv.de/mediathek/videos/panorama/Schnee-und-Glaette-fuehren-zu-zahlreichen-Unfaellen-und-Staus-article14438086.html',
        'md5': '6ef2514d4b1e8e03ca24b49e2f167153',
        'info_dict': {
            'id': '14438086',
            'ext': 'mp4',
            'thumbnail': r're:^https?://.*\.jpg$',
            'title': 'Schnee und Glätte führen zu zahlreichen Unfällen und Staus',
            'alt_title': 'Winterchaos auf deutschen Straßen',
            'description': 'Schnee und Glätte sorgen deutschlandweit für einen chaotischen Start in die Woche: Auf den Straßen kommt es zu kilometerlangen Staus und Dutzenden Glätteunfällen. In Düsseldorf und München wirbelt der Schnee zudem den Flugplan durcheinander. Dutzende Flüge landen zu spät, einige fallen ganz aus.',
            'duration': 4020,
            'timestamp': 1422892797,
            'upload_date': '20150202',
        },
    }]

    def _real_extract(self, url):
        video_id = self._match_id(url)
        webpage = self._download_webpage(url, video_id)

        info = self._parse_json(self._search_regex(
            r'(?s)ntv\.pageInfo\.article\s*=\s*(\{.*?\});', webpage, 'info'),
            video_id, transform_source=js_to_json)
        timestamp = int_or_none(info.get('publishedDateAsUnixTimeStamp'))
        vdata = self._parse_json(self._search_regex(
            r'(?s)\$\(\s*"\#player"\s*\)\s*\.data\(\s*"player",\s*(\{.*?\})\);',
            webpage, 'player data'), video_id,
            transform_source=lambda s: js_to_json(re.sub(r'advertising:\s*{[^}]+},', '', s)))
        duration = parse_duration(vdata.get('duration'))

        formats = []
        if vdata.get('video'):
            formats.append({
                'format_id': 'flash',
                'url': 'rtmp://fms.n-tv.de/%s' % vdata['video'],
            })
        if vdata.get('videoMp4'):
            formats.append({
                'format_id': 'mobile',
                'url': compat_urlparse.urljoin('http://video.n-tv.de', vdata['videoMp4']),
                'tbr': 400,  # estimation
            })
        if vdata.get('videoM3u8'):
            m3u8_url = compat_urlparse.urljoin('http://video.n-tv.de', vdata['videoM3u8'])
            formats.extend(self._extract_m3u8_formats(
                m3u8_url, video_id, ext='mp4', entry_protocol='m3u8_native',
                preference=0, m3u8_id='hls', fatal=False))
        self._sort_formats(formats)

        return {
            'id': video_id,
            'title': info['headline'],
            'description': info.get('intro'),
            'alt_title': info.get('kicker'),
            'timestamp': timestamp,
            'thumbnail': vdata.get('html5VideoPoster'),
            'duration': duration,
            'formats': formats,
        }
# coding: utf-8
from __future__ import unicode_literals

from .common import InfoExtractor
from ..utils import (
    int_or_none,
    strip_or_none,
    unescapeHTML,
    xpath_text,
)


class NTVRuIE(InfoExtractor):
    IE_NAME = 'ntv.ru'
    _VALID_URL = r'https?://(?:www\.)?ntv\.ru/(?:[^/]+/)*(?P<id>[^/?#&]+)'

    _TESTS = [{
        'url': 'http://www.ntv.ru/novosti/863142/',
        'md5': 'ba7ea172a91cb83eb734cad18c10e723',
        'info_dict': {
            'id': '746000',
            'ext': 'mp4',
            'title': 'Командующий Черноморским флотом провел переговоры в штабе ВМС Украины',
            'description': 'Командующий Черноморским флотом провел переговоры в штабе ВМС Украины',
            'thumbnail': r're:^http://.*\.jpg',
            'duration': 136,
        },
    }, {
        'url': 'http://www.ntv.ru/video/novosti/750370/',
        'md5': 'adecff79691b4d71e25220a191477124',
        'info_dict': {
            'id': '750370',
            'ext': 'mp4',
            'title': 'Родные пассажиров пропавшего Boeing не верят в трагический исход',
            'description': 'Родные пассажиров пропавшего Boeing не верят в трагический исход',
            'thumbnail': r're:^http://.*\.jpg',
            'duration': 172,
        },
    }, {
        'url': 'http://www.ntv.ru/peredacha/segodnya/m23700/o232416',
        'md5': '82dbd49b38e3af1d00df16acbeab260c',
        'info_dict': {
            'id': '747480',
            'ext': 'mp4',
            'title': '«Сегодня». 21 марта 2014 года. 16:00',
            'description': '«Сегодня». 21 марта 2014 года. 16:00',
            'thumbnail': r're:^http://.*\.jpg',
            'duration': 1496,
        },
    }, {
        'url': 'https://www.ntv.ru/kino/Koma_film/m70281/o336036/video/',
        'md5': 'e9c7cde24d9d3eaed545911a04e6d4f4',
        'info_dict': {
            'id': '1126480',
            'ext': 'mp4',
            'title': 'Остросюжетный фильм «Кома»',
            'description': 'Остросюжетный фильм «Кома»',
            'thumbnail': r're:^http://.*\.jpg',
            'duration': 5592,
        },
    }, {
        'url': 'http://www.ntv.ru/serial/Delo_vrachey/m31760/o233916/',
        'md5': '9320cd0e23f3ea59c330dc744e06ff3b',
        'info_dict': {
            'id': '751482',
            'ext': 'mp4',
            'title': '«Дело врачей»: «Деревце жизни»',
            'description': '«Дело врачей»: «Деревце жизни»',
            'thumbnail': r're:^http://.*\.jpg',
            'duration': 2590,
        },
    }, {
        # Schemeless file URL
        'url': 'https://www.ntv.ru/video/1797442',
        'only_matching': True,
    }]

    _VIDEO_ID_REGEXES = [
        r'<meta property="og:url" content="http://www\.ntv\.ru/video/(\d+)',
        r'<video embed=[^>]+><id>(\d+)</id>',
        r'<video restriction[^>]+><key>(\d+)</key>',
    ]

    def _real_extract(self, url):
        video_id = self._match_id(url)

        webpage = self._download_webpage(url, video_id)

        video_url = self._og_search_property(
            ('video', 'video:iframe'), webpage, default=None)
        if video_url:
            video_id = self._search_regex(
                r'https?://(?:www\.)?ntv\.ru/video/(?:embed/)?(\d+)',
                video_url, 'video id', default=None)

        if not video_id:
            video_id = self._html_search_regex(
                self._VIDEO_ID_REGEXES, webpage, 'video id')

        player = self._download_xml(
            'http://www.ntv.ru/vi%s/' % video_id,
            video_id, 'Downloading video XML')

        title = strip_or_none(unescapeHTML(xpath_text(player, './data/title', 'title', fatal=