Merge 'rg3/master' into fork_master

README.md

@@ -51,7 +51,10 @@ ## Filesystem Options:
                              %(upload_date)s for the upload date (YYYYMMDD),
                              %(extractor)s for the provider (youtube, metacafe,
                              etc), %(id)s for the video id and %% for a literal
-                             percent. Use - to output to stdout.
+                             percent. Use - to output to stdout. Can also be
+                             used to download to a different directory, for
+                             example with -o '/my/downloads/%(uploader)s/%(title
+                             )s-%(id)s.%(ext)s' .
     --restrict-filenames     Restrict filenames to only ASCII characters, and
                              avoid "&" and spaces in filenames
     -a, --batch-file FILE    file containing URLs to download ('-' for stdin)
@@ -194,3 +197,5 @@ # BUGS
 * The output of `youtube-dl --version`
 * The output of `python --version`
 * The name and version of your Operating System ("Ubuntu 11.04 x64" or "Windows 7 x64" is usually enough).
+
+For discussions, join us in the irc channel #youtube-dl on freenode.

test/gentests.py

@@ -34,31 +34,19 @@
 def _file_md5(fn):
     with open(fn, 'rb') as f:
         return hashlib.md5(f.read()).hexdigest()
-
-def md5_for_file(filename, block_size=2**20):
-    with open(filename) as f:
-        md5 = hashlib.md5()
-        while True:
-            data = f.read(block_size)
-            if not data:
-                break
-            md5.update(data)
-        return md5.hexdigest()
-_file_md5 = md5_for_file
-
-
 try:
     _skip_unless = unittest.skipUnless
 except AttributeError: # Python 2.6
     def _skip_unless(cond, reason='No reason given'):
         def resfunc(f):
-            def wfunc(*args, **kwargs):
+            # Start the function name with test to appease nosetests-2.6
+            def test_wfunc(*args, **kwargs):
                 if cond:
                     return f(*args, **kwargs)
                 else:
                     print('Skipped test')
                     return
-            return wfunc
+            return test_wfunc
         return resfunc
 _skip = lambda *args, **kwargs: _skip_unless(False, *args, **kwargs)
 

test/test_download.py

@@ -20,31 +20,19 @@
 def _file_md5(fn):
     with open(fn, 'rb') as f:
         return hashlib.md5(f.read()).hexdigest()
-
-def md5_for_file(filename, block_size=2**20):
-    with open(filename) as f:
-        md5 = hashlib.md5()
-        while True:
-            data = f.read(block_size)
-            if not data:
-                break
-            md5.update(data)
-        return md5.hexdigest()
-_file_md5 = md5_for_file
-
-
 try:
     _skip_unless = unittest.skipUnless
 except AttributeError: # Python 2.6
     def _skip_unless(cond, reason='No reason given'):
         def resfunc(f):
-            def wfunc(*args, **kwargs):
+            # Start the function name with test to appease nosetests-2.6
+            def test_wfunc(*args, **kwargs):
                 if cond:
                     return f(*args, **kwargs)
                 else:
                     print('Skipped test')
                     return
-            return wfunc
+            return test_wfunc
         return resfunc
 _skip = lambda *args, **kwargs: _skip_unless(False, *args, **kwargs)
 
@@ -79,7 +67,7 @@ def test_Dailymotion(self):
 
     @_skip_unless(youtube_dl.InfoExtractors.MetacafeIE._WORKING, "IE marked as not _WORKING")
     def test_Metacafe(self):
-        filename = 'aUehQsCQtM.flv'
+        filename = '_aUehQsCQtM.flv'
         fd = FileDownloader(self.parameters)
         fd.add_info_extractor(youtube_dl.InfoExtractors.MetacafeIE())
         fd.add_info_extractor(youtube_dl.InfoExtractors.YoutubeIE())
@@ -120,7 +108,7 @@ def test_Vimeo(self):
 
     @_skip_unless(youtube_dl.InfoExtractors.SoundcloudIE._WORKING, "IE marked as not _WORKING")
     def test_Soundcloud(self):
-        filename = 'n6FLbx6ZzMiu.mp3'
+        filename = '62986583.mp3'
         fd = FileDownloader(self.parameters)
         fd.add_info_extractor(youtube_dl.InfoExtractors.SoundcloudIE())
         fd.download(['http://soundcloud.com/ethmusic/lostin-powers-she-so-heavy'])
@@ -159,26 +147,38 @@ def test_XNXX(self):
         md5_for_file = _file_md5(filename)
         self.assertEqual(md5_for_file, 'c5c67df477eb0d9b058200351448ba4c')
 
+    @_skip_unless(youtube_dl.InfoExtractors.YoukuIE._WORKING, "IE marked as not _WORKING")
+    def test_Youku(self):
+        filename = 'XNDgyMDQ2NTQw_part00.flv'
+        fd = FileDownloader(self.parameters)
+        fd.add_info_extractor(youtube_dl.InfoExtractors.YoukuIE())
+        fd.download(['http://v.youku.com/v_show/id_XNDgyMDQ2NTQw.html'])
+        self.assertTrue(os.path.exists(filename))
+        md5_for_file = _file_md5(filename)
+        self.assertEqual(md5_for_file, 'ffe3f2e435663dc2d1eea34faeff5b5b')
+
 
     def tearDown(self):
         if os.path.exists('BaW_jenozKc.mp4'):
             os.remove('BaW_jenozKc.mp4')
         if os.path.exists('x33vw9.mp4'):
             os.remove('x33vw9.mp4')
-        if os.path.exists('aUehQsCQtM.flv'):
+        if os.path.exists('_aUehQsCQtM.flv'):
-            os.remove('aUehQsCQtM.flv')
+            os.remove('_aUehQsCQtM.flv')
         if os.path.exists('5779306.m4v'):
             os.remove('5779306.m4v')
         if os.path.exists('939581.flv'):
             os.remove('939581.flv')
         # No file specified for Vimeo
-        if os.path.exists('n6FLbx6ZzMiu.mp3'):
+        if os.path.exists('62986583.mp3'):
-            os.remove('n6FLbx6ZzMiu.mp3')
+            os.remove('62986583.mp3')
         if os.path.exists('PracticalUnix_intro-environment.mp4'):
             os.remove('PracticalUnix_intro-environment.mp4')
         # No file specified for CollegeHumor
         if os.path.exists('1135332.flv'):
             os.remove('1135332.flv')
+        if os.path.exists('XNDgyMDQ2NTQw_part00.flv'):
+            os.remove('XNDgyMDQ2NTQw_part00.flv')
 
 
 

test/test_execution.py

@@ -0,0 +1,26 @@
+import unittest
+
+import sys
+import os
+import subprocess
+
+rootDir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+
+try:
+    _DEV_NULL = subprocess.DEVNULL
+except AttributeError:
+    _DEV_NULL = open(os.devnull, 'wb')
+
+class TestExecution(unittest.TestCase):
+    def test_import(self):
+        subprocess.check_call([sys.executable, '-c', 'import youtube_dl'], cwd=rootDir)
+
+    def test_module_exec(self):
+        if sys.version_info >= (2,7): # Python 2.6 doesn't support package execution
+            subprocess.check_call([sys.executable, '-m', 'youtube_dl', '--version'], cwd=rootDir, stdout=_DEV_NULL)
+
+    def test_main_exec(self):
+        subprocess.check_call([sys.executable, 'youtube_dl/__main__.py', '--version'], cwd=rootDir, stdout=_DEV_NULL)
+
+if __name__ == '__main__':
+    unittest.main()

test/test_import.py

@@ -1,13 +0,0 @@
-import unittest
-
-import sys
-import os.path
-import subprocess
-
-class TestImport(unittest.TestCase):
-    def test_import(self):
-        rootDir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
-        subprocess.check_call([sys.executable, '-c', 'import youtube_dl'], cwd=rootDir)
-
-if __name__ == '__main__':
-    unittest.main()

test/test_utils.py

@@ -1,3 +1,5 @@
+#!/usr/bin/env python
+
 # Various small unit tests
 
 import sys
@@ -79,6 +81,11 @@ def test_sanitize_filename_restricted(self):
         self.assertTrue(sanitize_filename('-', restricted=True) != '')
         self.assertTrue(sanitize_filename(':', restricted=True) != '')
 
+    def test_sanitize_ids(self):
+        self.assertEquals(sanitize_filename('_n_cd26wFpw', is_id=True), '_n_cd26wFpw')
+        self.assertEquals(sanitize_filename('_BD_eEpuzXw', is_id=True), '_BD_eEpuzXw')
+        self.assertEquals(sanitize_filename('N0Y__7-UOdI', is_id=True), 'N0Y__7-UOdI')
+
     def test_ordered_set(self):
         self.assertEqual(orderedSet([1, 1, 2, 3, 4, 4, 5, 6, 7, 3, 5]), [1, 2, 3, 4, 5, 6, 7])
         self.assertEqual(orderedSet([]), [])

test/test_youtube_playlist_ids.py

@@ -0,0 +1,22 @@
+#!/usr/bin/env python
+
+import sys
+import unittest
+
+# Allow direct execution
+import os
+sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+from youtube_dl.InfoExtractors import YoutubeIE, YoutubePlaylistIE
+
+class TestYoutubePlaylistMatching(unittest.TestCase):
+    def test_playlist_matching(self):
+        assert YoutubePlaylistIE().suitable(u'ECUl4u3cNGP61MdtwGTqZA0MreSaDybji8')
+        assert YoutubePlaylistIE().suitable(u'PL63F0C78739B09958')
+        assert not YoutubePlaylistIE().suitable(u'PLtS2H6bU1M')
+
+    def test_youtube_matching(self):
+        assert YoutubeIE().suitable(u'PLtS2H6bU1M')
+
+if __name__ == '__main__':
+    unittest.main()

test/tests.json

@@ -16,7 +16,7 @@
     "size":  5754305,
     "addIEs": ["Youtube"],
     "url":  "http://www.metacafe.com/watch/yt-_aUehQsCQtM/the_electric_company_short_i_pbs_kids_go/",
-    "file":  "aUehQsCQtM.flv"
+    "file":  "_aUehQsCQtM.flv"
   },
   {
     "name": "BlipTV",
@@ -40,7 +40,7 @@
     "name": "Soundcloud",
     "md5":  "c1b9b9ea8bfd620b96b2628664576e1c",
     "url":  "http://soundcloud.com/ethmusic/lostin-powers-she-so-heavy",
-    "file":  "n6FLbx6ZzMiu.mp3"
+    "file":  "62986583.mp3"
   },
   {
     "name": "StanfordOpenClassroom",
@@ -59,5 +59,11 @@
     "md5":  "c5c67df477eb0d9b058200351448ba4c",
     "url":  "http://video.xnxx.com/video1135332/lida_naked_funny_actress_5_",
     "file":  "1135332.flv"
+  },
+  {
+    "name": "Youku",
+    "url": "http://v.youku.com/v_show/id_XNDgyMDQ2NTQw.html",
+    "file": "XNDgyMDQ2NTQw_part00.flv",
+    "md5": "ffe3f2e435663dc2d1eea34faeff5b5b"
   }
 ]

youtube_dl/FileDownloader.py

@@ -334,8 +334,11 @@ def prepare_filename(self, info_dict):
             template_dict['epoch'] = int(time.time())
             template_dict['autonumber'] = u'%05d' % self._num_downloads
 
-            template_dict = dict((key, u'NA' if val is None else val) for key, val in template_dict.items())
+            sanitize = lambda k,v: sanitize_filename(
-            template_dict = dict((k, sanitize_filename(compat_str(v), self.params.get('restrictfilenames'))) for k,v in template_dict.items())
+                u'NA' if v is None else compat_str(v),
+                restricted=self.params.get('restrictfilenames'),
+                is_id=(k==u'id'))
+            template_dict = dict((k, sanitize(k, v)) for k,v in template_dict.items())
 
             filename = self.params['outtmpl'] % template_dict
             return filename

youtube_dl/InfoExtractors.py

@@ -1671,7 +1671,7 @@ def _download_n_results(self, query, n):
 class YoutubePlaylistIE(InfoExtractor):
     """Information Extractor for YouTube playlists."""
 
-    _VALID_URL = r'(?:(?:https?://)?(?:\w+\.)?youtube\.com/(?:(?:course|view_play_list|my_playlists|artist|playlist)\?.*?(p|a|list)=|user/.*?/user/|p/|user/.*?#[pg]/c/)(?:PL|EC)?|PL|EC)([0-9A-Za-z-_]+)(?:/.*?/([0-9A-Za-z_-]+))?.*'
+    _VALID_URL = r'(?:(?:https?://)?(?:\w+\.)?youtube\.com/(?:(?:course|view_play_list|my_playlists|artist|playlist)\?.*?(p|a|list)=|user/.*?/user/|p/|user/.*?#[pg]/c/)(?:PL|EC)?|PL|EC)([0-9A-Za-z-_]{10,})(?:/.*?/([0-9A-Za-z_-]+))?.*'
     _TEMPLATE_URL = 'http://www.youtube.com/%s?%s=%s&page=%s&gl=US&hl=en'
     _VIDEO_INDICATOR_TEMPLATE = r'/watch\?v=(.+?)&([^&"]+&)*list=.*?%s'
     _MORE_PAGES_INDICATOR = r'yt-uix-pager-next'
@@ -2803,13 +2803,13 @@ class SoundcloudIE(InfoExtractor):
     def __init__(self, downloader=None):
         InfoExtractor.__init__(self, downloader)
 
-    def report_webpage(self, video_id):
+    def report_resolve(self, video_id):
         """Report information extraction."""
-        self._downloader.to_screen(u'[%s] %s: Downloading webpage' % (self.IE_NAME, video_id))
+        self._downloader.to_screen(u'[%s] %s: Resolving id' % (self.IE_NAME, video_id))
 
     def report_extraction(self, video_id):
         """Report information extraction."""
-        self._downloader.to_screen(u'[%s] %s: Extracting information' % (self.IE_NAME, video_id))
+        self._downloader.to_screen(u'[%s] %s: Retrieving stream' % (self.IE_NAME, video_id))
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
@@ -2818,65 +2818,47 @@ def _real_extract(self, url):
             return
 
         # extract uploader (which is in the url)
-        uploader = mobj.group(1).decode('utf-8')
+        uploader = mobj.group(1)
         # extract simple title (uploader + slug of song title)
-        slug_title =  mobj.group(2).decode('utf-8')
+        slug_title =  mobj.group(2)
         simple_title = uploader + u'-' + slug_title
 
-        self.report_webpage('%s/%s' % (uploader, slug_title))
+        self.report_resolve('%s/%s' % (uploader, slug_title))
 
-        request = compat_urllib_request.Request('http://soundcloud.com/%s/%s' % (uploader, slug_title))
+        url = 'http://soundcloud.com/%s/%s' % (uploader, slug_title)
+        resolv_url = 'http://api.soundcloud.com/resolve.json?url=' + url + '&client_id=b45b1aa10f1ac2941910a7f0d10f8e28'
+        request = compat_urllib_request.Request(resolv_url)
         try:
-            webpage = compat_urllib_request.urlopen(request).read()
+            info_json_bytes = compat_urllib_request.urlopen(request).read()
+            info_json = info_json_bytes.decode('utf-8')
         except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
             self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % compat_str(err))
             return
 
+        info = json.loads(info_json)
+        video_id = info['id']
         self.report_extraction('%s/%s' % (uploader, slug_title))
 
-        # extract uid and stream token that soundcloud hands out for access
+        streams_url = 'https://api.sndcdn.com/i1/tracks/' + str(video_id) + '/streams?client_id=b45b1aa10f1ac2941910a7f0d10f8e28'
-        mobj = re.search('"uid":"([\w\d]+?)".*?stream_token=([\w\d]+)', webpage)
+        request = compat_urllib_request.Request(streams_url)
-        if mobj:
-            video_id = mobj.group(1)
-            stream_token = mobj.group(2)
-
-        # extract unsimplified title
-        mobj = re.search('"title":"(.*?)",', webpage)
-        if mobj:
-            title = mobj.group(1).decode('utf-8')
-        else:
-            title = simple_title
-
-        # construct media url (with uid/token)
-        mediaURL = "http://media.soundcloud.com/stream/%s?stream_token=%s"
-        mediaURL = mediaURL % (video_id, stream_token)
-
-        # description
-        description = u'No description available'
-        mobj = re.search('track-description-value"><p>(.*?)</p>', webpage)
-        if mobj:
-            description = mobj.group(1)
-
-        # upload date
-        upload_date = None
-        mobj = re.search("pretty-date'>on ([\w]+ [\d]+, [\d]+ \d+:\d+)</abbr></h2>", webpage)
-        if mobj:
         try:
-                upload_date = datetime.datetime.strptime(mobj.group(1), '%B %d, %Y %H:%M').strftime('%Y%m%d')
+            stream_json_bytes = compat_urllib_request.urlopen(request).read()
-            except Exception as err:
+            stream_json = stream_json_bytes.decode('utf-8')
-                self._downloader.to_stderr(compat_str(err))
+        except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
+            self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % compat_str(err))
+            return
 
-        # for soundcloud, a request to a cross domain is required for cookies
+        streams = json.loads(stream_json)
-        request = compat_urllib_request.Request('http://media.soundcloud.com/crossdomain.xml', std_headers)
+        mediaURL = streams['http_mp3_128_url']
 
         return [{
-            'id':       video_id.decode('utf-8'),
+            'id':       info['id'],
             'url':      mediaURL,
-            'uploader': uploader.decode('utf-8'),
+            'uploader': info['user']['username'],
-            'upload_date':  upload_date,
+            'upload_date':  info['created_at'],
-            'title':    title,
+            'title':    info['title'],
             'ext':      u'mp3',
-            'description': description.decode('utf-8')
+            'description': info['description'],
         }]
 
 

youtube_dl/__init__.py

@@ -305,7 +305,7 @@ def _find_term_columns():
             action='store_true', dest='autonumber',
             help='number downloaded files starting from 00000', default=False)
     filesystem.add_option('-o', '--output',
-            dest='outtmpl', metavar='TEMPLATE', help='output filename template. Use %(title)s to get the title, %(uploader)s for the uploader name, %(autonumber)s to get an automatically incremented number, %(ext)s for the filename extension, %(upload_date)s for the upload date (YYYYMMDD), %(extractor)s for the provider (youtube, metacafe, etc), %(id)s for the video id and %% for a literal percent. Use - to output to stdout.')
+            dest='outtmpl', metavar='TEMPLATE', help='output filename template. Use %(title)s to get the title, %(uploader)s for the uploader name, %(autonumber)s to get an automatically incremented number, %(ext)s for the filename extension, %(upload_date)s for the upload date (YYYYMMDD), %(extractor)s for the provider (youtube, metacafe, etc), %(id)s for the video id and %% for a literal percent. Use - to output to stdout. Can also be used to download to a different directory, for example with -o \'/my/downloads/%(uploader)s/%(title)s-%(id)s.%(ext)s\' .')
     filesystem.add_option('--restrict-filenames',
             action='store_true', dest='restrictfilenames',
             help='Restrict filenames to only ASCII characters, and avoid "&" and spaces in filenames', default=False)

youtube_dl/utils.py

@@ -317,9 +317,10 @@ def timeconvert(timestr):
         timestamp = email.utils.mktime_tz(timetuple)
     return timestamp
 
-def sanitize_filename(s, restricted=False):
+def sanitize_filename(s, restricted=False, is_id=False):
     """Sanitizes a string so it could be used as part of a filename.
     If restricted is set, use a stricter subset of allowed characters.
+    Set is_id if this is not an arbitrary string, but an ID that should be kept if possible
     """
     def replace_insane(char):
         if char == '?' or ord(char) < 32 or ord(char) == 127:
@@ -337,6 +338,7 @@ def replace_insane(char):
         return char
 
     result = u''.join(map(replace_insane, s))
+    if not is_id:
         while '__' in result:
             result = result.replace('__', '_')
         result = result.strip('_')
@@ -504,3 +506,6 @@ def http_response(self, req, resp):
             resp = self.addinfourl_wrapper(gz, old_resp.headers, old_resp.url, old_resp.code)
             resp.msg = old_resp.msg
         return resp
+
+    https_request = http_request
+    https_response = http_response