Merge remote-tracking branch 'upstream/master'

youtube_dl/FileDownloader.py

@@ -79,9 +79,13 @@ def calc_eta(start, now, total, current):
         rate = float(current) / dif
         eta = int((float(total) - float(current)) / rate)
         (eta_mins, eta_secs) = divmod(eta, 60)
-        if eta_mins > 99:
+        (eta_hours, eta_mins) = divmod(eta_mins, 60)
-            return '--:--'
+        if eta_hours > 99:
-        return '%02d:%02d' % (eta_mins, eta_secs)
+            return '--:--:--'
+        if eta_hours == 0:
+            return '%02d:%02d' % (eta_mins, eta_secs)
+        else:
+            return '%02d:%02d:%02d' % (eta_hours, eta_mins, eta_secs)
 
     @staticmethod
     def calc_speed(start, now, bytes):

youtube_dl/extractor/collegehumor.py

@@ -4,6 +4,7 @@
 from .common import InfoExtractor
 from ..utils import (
     compat_urllib_parse_urlparse,
+    determine_ext,
 
     ExtractorError,
 )
@@ -12,7 +13,7 @@
 class CollegeHumorIE(InfoExtractor):
     _VALID_URL = r'^(?:https?://)?(?:www\.)?collegehumor\.com/(video|embed|e)/(?P<videoid>[0-9]+)/?(?P<shorttitle>.*)$'
 
-    _TEST = {
+    _TESTS = [{
         u'url': u'http://www.collegehumor.com/video/6902724/comic-con-cosplay-catastrophe',
         u'file': u'6902724.mp4',
         u'md5': u'1264c12ad95dca142a9f0bf7968105a0',
@@ -20,7 +21,16 @@ class CollegeHumorIE(InfoExtractor):
             u'title': u'Comic-Con Cosplay Catastrophe',
             u'description': u'Fans get creative this year at San Diego.  Too creative.  And yes, that\'s really Joss Whedon.',
         },
-    }
+    },
+    {
+        u'url': u'http://www.collegehumor.com/video/3505939/font-conference',
+        u'file': u'3505939.mp4',
+        u'md5': u'c51ca16b82bb456a4397987791a835f5',
+        u'info_dict': {
+            u'title': u'Font Conference',
+            u'description': u'This video wasn\'t long enough, so we made it double-spaced.',
+        },
+    }]
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
@@ -49,25 +59,29 @@ def _real_extract(self, url):
             info['description'] = videoNode.findall('./description')[0].text
             info['title'] = videoNode.findall('./caption')[0].text
             info['thumbnail'] = videoNode.findall('./thumbnail')[0].text
-            manifest_url = videoNode.findall('./file')[0].text
+            next_url = videoNode.findall('./file')[0].text
         except IndexError:
             raise ExtractorError(u'Invalid metadata XML file')
 
-        manifest_url += '?hdcore=2.10.3'
+        if next_url.endswith(u'manifest.f4m'):
-        manifestXml = self._download_webpage(manifest_url, video_id,
+            manifest_url = next_url + '?hdcore=2.10.3'
-                                             u'Downloading XML manifest',
+            manifestXml = self._download_webpage(manifest_url, video_id,
-                                             u'Unable to download video info XML')
+                                         u'Downloading XML manifest',
+                                         u'Unable to download video info XML')
 
-        adoc = xml.etree.ElementTree.fromstring(manifestXml)
+            adoc = xml.etree.ElementTree.fromstring(manifestXml)
-        try:
+            try:
-            media_node = adoc.findall('./{http://ns.adobe.com/f4m/1.0}media')[0]
+                media_node = adoc.findall('./{http://ns.adobe.com/f4m/1.0}media')[0]
-            node_id = media_node.attrib['url']
+                node_id = media_node.attrib['url']
-            video_id = adoc.findall('./{http://ns.adobe.com/f4m/1.0}id')[0].text
+                video_id = adoc.findall('./{http://ns.adobe.com/f4m/1.0}id')[0].text
-        except IndexError as err:
+            except IndexError as err:
-            raise ExtractorError(u'Invalid manifest file')
+                raise ExtractorError(u'Invalid manifest file')
+            url_pr = compat_urllib_parse_urlparse(info['thumbnail'])
+            info['url'] = url_pr.scheme + '://' + url_pr.netloc + video_id[:-2].replace('.csmil','').replace(',','')
+            info['ext'] = 'mp4'
+        else:
+            # Old-style direct links
+            info['url'] = next_url
+            info['ext'] = determine_ext(info['url'])
 
-        url_pr = compat_urllib_parse_urlparse(info['thumbnail'])
+        return info
-
-        info['url'] = url_pr.scheme + '://' + url_pr.netloc + video_id[:-2].replace('.csmil','').replace(',','')
-        info['ext'] = 'mp4'
-        return [info]

youtube_dl/extractor/common.py

@@ -77,7 +77,13 @@ def __init__(self, downloader=None):
     @classmethod
     def suitable(cls, url):
         """Receives a URL and returns True if suitable for this IE."""
-        return re.match(cls._VALID_URL, url) is not None
+
+        # This does not use has/getattr intentionally - we want to know whether
+        # we have cached the regexp for *this* class, whereas getattr would also
+        # match the superclass
+        if '_VALID_URL_RE' not in cls.__dict__:
+            cls._VALID_URL_RE = re.compile(cls._VALID_URL)
+        return cls._VALID_URL_RE.match(url) is not None
 
     @classmethod
     def working(cls):

youtube_dl/extractor/generic.py

@@ -107,8 +107,13 @@ def http_error_405(self, req, fp, code, msg, headers):
         return new_url
 
     def _real_extract(self, url):
-        new_url = self._test_redirect(url)
+        try:
-        if new_url: return [self.url_result(new_url)]
+            new_url = self._test_redirect(url)
+            if new_url:
+                return [self.url_result(new_url)]
+        except compat_urllib_error.HTTPError:
+            # This may be a stupid server that doesn't like HEAD, our UA, or so
+            pass
 
         video_id = url.split('/')[-1]
         try:
@@ -144,6 +149,9 @@ def _real_extract(self, url):
             # We only look in og:video if the MIME type is a video, don't try if it's a Flash player:
             if m_video_type is not None:
                 mobj = re.search(r'<meta.*?property="og:video".*?content="(.*?)"', webpage)
+        if mobj is None:
+            # HTML5 video
+            mobj = re.search(r'<video[^<]*>.*?<source .*?src="([^"]+)"', webpage, flags=re.DOTALL)
         if mobj is None:
             raise ExtractorError(u'Invalid URL: %s' % url)
 

youtube_dl/extractor/rtlnow.py

@@ -2,7 +2,10 @@
 import re
 
 from .common import InfoExtractor
-from ..utils import ExtractorError
+from ..utils import (
+    clean_html,
+    ExtractorError,
+)
 
 class RTLnowIE(InfoExtractor):
     """Information Extractor for RTLnow, RTL2now and VOXnow"""
@@ -18,6 +21,7 @@ class RTLnowIE(InfoExtractor):
         u'params': {
             u'skip_download': True,
         },
+        u'skip': u'Only works from Germany',
     },
     {
         u'url': u'http://rtl2now.rtl2.de/aerger-im-revier/episode-15-teil-1.php?film_id=69756&player=1&season=2&index=5',
@@ -31,6 +35,7 @@ class RTLnowIE(InfoExtractor):
         u'params': {
             u'skip_download': True,
         },
+        u'skip': u'Only works from Germany',
     },
     {
         u'url': u'www.voxnow.de/voxtours/suedafrika-reporter-ii.php?film_id=13883&player=1&season=17',
@@ -53,6 +58,14 @@ def _real_extract(self,url):
         video_id = mobj.group(u'video_id')
 
         webpage = self._download_webpage(webpage_url, video_id)
+
+        note_m = re.search(r'''(?sx)
+            <div[ ]style="margin-left:[ ]20px;[ ]font-size:[ ]13px;">(.*?)
+            <div[ ]id="playerteaser">''', webpage)
+        if note_m:
+            msg = clean_html(note_m.group(1))
+            raise ExtractorError(msg)
+
         video_title = self._html_search_regex(r'<title>(?P<title>[^<]+)</title>',
             webpage, u'title')
         playerdata_url = self._html_search_regex(r'\'playerdata\': \'(?P<playerdata_url>[^\']+)\'',

youtube_dl/version.py

@@ -1,2 +1,2 @@
 
-__version__ = '2013.08.17'
+__version__ = '2013.08.21'