From d2272fcf6e2796583b516f4786733577459dec43 Mon Sep 17 00:00:00 2001
From: felix <m.p.isaev@yandex.com>
Date: Mon, 6 Apr 2015 09:54:19 +0200
Subject: [PATCH 1/6] crooksandliars.com extractor

---
 youtube_dl/extractor/__init__.py       |  1 +
 youtube_dl/extractor/crooksandliars.py | 71 ++++++++++++++++++++++++++
 2 files changed, 72 insertions(+)
 create mode 100644 youtube_dl/extractor/crooksandliars.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 7eb9b4fbbb..dc272af826 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -90,6 +90,7 @@
 from .condenast import CondeNastIE
 from .cracked import CrackedIE
 from .criterion import CriterionIE
+from .crooksandliars import CrooksAndLiarsIE, CrooksAndLiarsArticleIE
 from .crunchyroll import (
     CrunchyrollIE,
     CrunchyrollShowPlaylistIE
diff --git a/youtube_dl/extractor/crooksandliars.py b/youtube_dl/extractor/crooksandliars.py
new file mode 100644
index 0000000000..afccca3545
--- /dev/null
+++ b/youtube_dl/extractor/crooksandliars.py
@@ -0,0 +1,71 @@
+from __future__ import unicode_literals
+
+import json
+
+from .common import InfoExtractor
+from ..utils import (
+    mimetype2ext,
+)
+
+
+class CrooksAndLiarsIE(InfoExtractor):
+    _VALID_URL = r'(?:https?:)?//embed.crooksandliars.com/embed/(?P<id>[A-Za-z0-9]+)(?:$|[?#])'
+
+    _TESTS = [{
+        'url': 'https://embed.crooksandliars.com/embed/8RUoRhRi',
+        'info_dict': {
+            'id': 'https://embed.crooksandliars.com/embed/8RUoRhRi',
+            'title': "Fox & Friends Says Protecting Atheists From Discrimination Is Anti-Christian!",
+            'description': "Fox News, Fox & Friends Weekend, April 4, 2015. Read more... http://crooksandliars.com/2015/04/fox-friends-says-protecting-atheists",
+            'timestamp': 1428207000,
+            'thumbnail': '//crooksandliars.com/files/mediaposters/2015/04/31235.jpg?ts=1428207050',
+            'uploader': "Heather",
+        }
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+
+        manifest = json.loads(self._html_search_regex(r'var manifest = ({.*?})\n', webpage, 'manifest JSON'))
+
+        formats = []
+        for item in manifest['flavors']:
+            if not item['mime'].startswith('video/'): # XXX: or item['exclude']?
+                continue
+            formats.append({
+                'format_id': item['type'],
+                'ext': mimetype2ext(item['mime']),
+                'url': item['url'],
+            })
+
+        # XXX: manifest['url']?
+        return {
+            'url': url,
+            'id': video_id,
+            'uploader': manifest['author'],
+            'title': manifest['title'],
+            'description': manifest['description'],
+            'thumbnail': manifest['poster'],
+            'duration': manifest['duration'],
+            'timestamp': int(manifest['created']),
+            'formats': formats,
+        }
+
+class CrooksAndLiarsArticleIE(InfoExtractor):
+    _VALID_URL = r'(?:https?:)?//crooksandliars.com/\d+/\d+/(?P<id>[a-z\-]+)(?:/|$)'
+
+    _TESTS = [{
+        'url': 'http://crooksandliars.com/2015/04/fox-friends-says-protecting-atheists',
+        'only_matching': True,
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+        player_url = self._proto_relative_url(self._html_search_regex(r'<iframe src="(//embed.crooksandliars.com/.*)"', webpage, 'embedded player'))
+
+        return {
+            '_type': 'url',
+            'url': player_url
+        }

From 6e53c91608d1c43a9fe1614f13a15db74e877a91 Mon Sep 17 00:00:00 2001
From: felix <m.p.isaev@yandex.com>
Date: Mon, 6 Apr 2015 10:12:43 +0200
Subject: [PATCH 2/6] [crooksandliars] resolve protocol-relative URLs

---
 youtube_dl/extractor/crooksandliars.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/crooksandliars.py b/youtube_dl/extractor/crooksandliars.py
index afccca3545..cee0603f44 100644
--- a/youtube_dl/extractor/crooksandliars.py
+++ b/youtube_dl/extractor/crooksandliars.py
@@ -18,7 +18,7 @@ class CrooksAndLiarsIE(InfoExtractor):
             'title': "Fox & Friends Says Protecting Atheists From Discrimination Is Anti-Christian!",
             'description': "Fox News, Fox & Friends Weekend, April 4, 2015. Read more... http://crooksandliars.com/2015/04/fox-friends-says-protecting-atheists",
             'timestamp': 1428207000,
-            'thumbnail': '//crooksandliars.com/files/mediaposters/2015/04/31235.jpg?ts=1428207050',
+            'thumbnail': 'https://crooksandliars.com/files/mediaposters/2015/04/31235.jpg?ts=1428207050',
             'uploader': "Heather",
         }
     }]
@@ -46,7 +46,7 @@ def _real_extract(self, url):
             'uploader': manifest['author'],
             'title': manifest['title'],
             'description': manifest['description'],
-            'thumbnail': manifest['poster'],
+            'thumbnail': self._proto_relative_url(manifest['poster']),
             'duration': manifest['duration'],
             'timestamp': int(manifest['created']),
             'formats': formats,

From 7a91d1fc43f8873e6636c316006c908d7efe2f07 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 11 Apr 2015 20:03:12 +0600
Subject: [PATCH 3/6] [crooksandliars] Improve embed extractor and remove
 article extractor

---
 youtube_dl/extractor/__init__.py       |  2 +-
 youtube_dl/extractor/crooksandliars.py | 87 +++++++++++---------------
 2 files changed, 39 insertions(+), 50 deletions(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 894aa5b43f..bbf3be41d1 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -90,7 +90,7 @@
 from .condenast import CondeNastIE
 from .cracked import CrackedIE
 from .criterion import CriterionIE
-from .crooksandliars import CrooksAndLiarsIE, CrooksAndLiarsArticleIE
+from .crooksandliars import CrooksAndLiarsIE
 from .crunchyroll import (
     CrunchyrollIE,
     CrunchyrollShowPlaylistIE
diff --git a/youtube_dl/extractor/crooksandliars.py b/youtube_dl/extractor/crooksandliars.py
index cee0603f44..1435090042 100644
--- a/youtube_dl/extractor/crooksandliars.py
+++ b/youtube_dl/extractor/crooksandliars.py
@@ -1,71 +1,60 @@
 from __future__ import unicode_literals
 
-import json
-
 from .common import InfoExtractor
 from ..utils import (
-    mimetype2ext,
+    int_or_none,
+    qualities,
 )
 
 
 class CrooksAndLiarsIE(InfoExtractor):
-    _VALID_URL = r'(?:https?:)?//embed.crooksandliars.com/embed/(?P<id>[A-Za-z0-9]+)(?:$|[?#])'
-
+    _VALID_URL = r'https?://embed\.crooksandliars\.com/(?:embed|v)/(?P<id>[A-Za-z0-9]+)'
     _TESTS = [{
         'url': 'https://embed.crooksandliars.com/embed/8RUoRhRi',
         'info_dict': {
-            'id': 'https://embed.crooksandliars.com/embed/8RUoRhRi',
+            'id': '8RUoRhRi',
+            'ext': 'mp4',
             'title': "Fox & Friends Says Protecting Atheists From Discrimination Is Anti-Christian!",
-            'description': "Fox News, Fox & Friends Weekend, April 4, 2015. Read more... http://crooksandliars.com/2015/04/fox-friends-says-protecting-atheists",
+            'description': 'md5:e1a46ad1650e3a5ec7196d432799127f',
+            'thumbnail': 're:^https?://.*\.jpg',
             'timestamp': 1428207000,
-            'thumbnail': 'https://crooksandliars.com/files/mediaposters/2015/04/31235.jpg?ts=1428207050',
-            'uploader': "Heather",
+            'upload_date': '20150405',
+            'uploader': 'Heather',
+            'duration': 236,
         }
-    }]
-
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-        webpage = self._download_webpage(url, video_id)
-
-        manifest = json.loads(self._html_search_regex(r'var manifest = ({.*?})\n', webpage, 'manifest JSON'))
-
-        formats = []
-        for item in manifest['flavors']:
-            if not item['mime'].startswith('video/'): # XXX: or item['exclude']?
-                continue
-            formats.append({
-                'format_id': item['type'],
-                'ext': mimetype2ext(item['mime']),
-                'url': item['url'],
-            })
-
-        # XXX: manifest['url']?
-        return {
-            'url': url,
-            'id': video_id,
-            'uploader': manifest['author'],
-            'title': manifest['title'],
-            'description': manifest['description'],
-            'thumbnail': self._proto_relative_url(manifest['poster']),
-            'duration': manifest['duration'],
-            'timestamp': int(manifest['created']),
-            'formats': formats,
-        }
-
-class CrooksAndLiarsArticleIE(InfoExtractor):
-    _VALID_URL = r'(?:https?:)?//crooksandliars.com/\d+/\d+/(?P<id>[a-z\-]+)(?:/|$)'
-
-    _TESTS = [{
-        'url': 'http://crooksandliars.com/2015/04/fox-friends-says-protecting-atheists',
+    }, {
+        'url': 'http://embed.crooksandliars.com/v/MTE3MjUtMzQ2MzA',
         'only_matching': True,
     }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
-        webpage = self._download_webpage(url, video_id)
-        player_url = self._proto_relative_url(self._html_search_regex(r'<iframe src="(//embed.crooksandliars.com/.*)"', webpage, 'embedded player'))
+
+        webpage = self._download_webpage(
+            'http://embed.crooksandliars.com/embed/%s' % video_id, video_id)
+
+        manifest = self._parse_json(
+            self._search_regex(
+                r'var\s+manifest\s*=\s*({.+?})\n', webpage, 'manifest JSON'),
+            video_id)
+
+        quality = qualities(('webm_low', 'mp4_low', 'webm_high', 'mp4_high'))
+
+        formats = [{
+            'url': item['url'],
+            'format_id': item['type'],
+            'quality': quality(item['type']),
+        } for item in manifest['flavors'] if item['mime'].startswith('video/')]
+        self._sort_formats(formats)
 
         return {
-            '_type': 'url',
-            'url': player_url
+            'url': url,
+            'id': video_id,
+            'title': manifest['title'],
+            'description': manifest.get('description'),
+            'thumbnail': self._proto_relative_url(manifest.get('poster')),
+            'timestamp': int_or_none(manifest.get('created')),
+            'uploader': manifest.get('author'),
+            'duration': int_or_none(manifest.get('duration')),
+            'formats': formats,
         }

From 18153f1b3249701a82c24ec492f5d58929292caf Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 11 Apr 2015 20:20:20 +0600
Subject: [PATCH 4/6] [generic] Add support for Crooks and Liars embeds

---
 youtube_dl/extractor/generic.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 6c212efac4..0dc52ed23a 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -1275,6 +1275,12 @@ def _playlist_from_matches(matches, getter=None, ie=None):
         if mobj is not None:
             return self.url_result('5min:%s' % mobj.group('id'), 'FiveMin')
 
+        # Look for Crooks and Liars embeds
+        mobj = re.search(
+            r'<(?:iframe[^>]+src|param[^>]+value)=(["\'])(?P<url>(?:https?:)?//embed\.crooksandliars\.com/(?:embed|v)/.+?)\1', webpage)
+        if mobj is not None:
+            return self.url_result(mobj.group('url'))
+
         # Look for NBC Sports VPlayer embeds
         nbc_sports_url = NBCSportsVPlayerIE._extract_url(webpage)
         if nbc_sports_url:

From a4257017ef2ee665d35dd71905db03ca3913c92e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 11 Apr 2015 20:26:42 +0600
Subject: [PATCH 5/6] [generic] Add tests for Crooks and Liars embeds

---
 youtube_dl/extractor/generic.py | 26 ++++++++++++++++++++++++++
 1 file changed, 26 insertions(+)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 0dc52ed23a..7ad555e9f5 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -642,6 +642,32 @@ class GenericIE(InfoExtractor):
                 'title': 'pdv_maddow_netcast_m4v-02-27-2015-201624',
             }
         },
+        # Crooks and Liars embed
+        {
+            'url': 'http://crooksandliars.com/2015/04/fox-friends-says-protecting-atheists',
+            'info_dict': {
+                'id': '8RUoRhRi',
+                'ext': 'mp4',
+                'title': "Fox & Friends Says Protecting Atheists From Discrimination Is Anti-Christian!",
+                'description': 'md5:e1a46ad1650e3a5ec7196d432799127f',
+                'timestamp': 1428207000,
+                'upload_date': '20150405',
+                'uploader': 'Heather',
+            },
+        },
+        # Crooks and Liars external embed
+        {
+            'url': 'http://theothermccain.com/2010/02/02/video-proves-that-bill-kristol-has-been-watching-glenn-beck/comment-page-1/',
+            'info_dict': {
+                'id': 'MTE3MjUtMzQ2MzA',
+                'ext': 'mp4',
+                'title': 'md5:5e3662a81a4014d24c250d76d41a08d5',
+                'description': 'md5:9b8e9542d6c3c5de42d6451b7d780cec',
+                'timestamp': 1265032391,
+                'upload_date': '20100201',
+                'uploader': 'Heather',
+            },
+        },
         # NBC Sports vplayer embed
         {
             'url': 'http://www.riderfans.com/forum/showthread.php?121827-Freeman&s=e98fa1ea6dc08e886b1678d35212494a',

From ed5641e2499de9dfbfa78c6684e92e9581402f16 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 11 Apr 2015 20:27:39 +0600
Subject: [PATCH 6/6] [crooksandliars] Quotes consistency

---
 youtube_dl/extractor/crooksandliars.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/crooksandliars.py b/youtube_dl/extractor/crooksandliars.py
index 1435090042..443eb7691c 100644
--- a/youtube_dl/extractor/crooksandliars.py
+++ b/youtube_dl/extractor/crooksandliars.py
@@ -14,7 +14,7 @@ class CrooksAndLiarsIE(InfoExtractor):
         'info_dict': {
             'id': '8RUoRhRi',
             'ext': 'mp4',
-            'title': "Fox & Friends Says Protecting Atheists From Discrimination Is Anti-Christian!",
+            'title': 'Fox & Friends Says Protecting Atheists From Discrimination Is Anti-Christian!',
             'description': 'md5:e1a46ad1650e3a5ec7196d432799127f',
             'thumbnail': 're:^https?://.*\.jpg',
             'timestamp': 1428207000,