diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 00000000000..5ca706b3fa6 --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,187 @@ +name: Build + +on: workflow_dispatch + +jobs: + build_unix: + runs-on: ubuntu-latest + outputs: + version_suffix: ${{ steps.version_suffix.outputs.version_suffix }} + ytdl_version: ${{ steps.bump_version.outputs.ytdl_version }} + upload_url: ${{ steps.create_release.outputs.upload_url }} + sha256_bin: ${{ steps.sha256_bin.outputs.sha256_bin }} + sha512_bin: ${{ steps.sha512_bin.outputs.sha512_bin }} + sha256_tar: ${{ steps.sha256_tar.outputs.sha256_tar }} + sha512_tar: ${{ steps.sha512_tar.outputs.sha512_tar }} + + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 0 + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: '3.8' + - name: Install packages + run: sudo apt-get -y install zip pandoc man + - name: Set version suffix + id: version_suffix + env: + PUSH_VERSION_COMMIT: ${{ secrets.PUSH_VERSION_COMMIT }} + if: "env.PUSH_VERSION_COMMIT == ''" + run: echo ::set-output name=version_suffix::$(date -u +"%H%M%S") + - name: Bump version + id: bump_version + run: | + python devscripts/update-version.py ${{ steps.version_suffix.outputs.version_suffix }} + make issuetemplates + - name: Push to release + id: push_release + run: echo ::set-output name=head_sha::$(git rev-parse HEAD) + - name: Get Changelog + id: get_changelog + run: | + changelog=$(cat ChangeLog | grep -oPz '(?s)(?<=version ${{ steps.bump_version.outputs.ytdl_version }}\n{2}).+?(?=\n{2,3}version)') || true + echo "changelog<> $GITHUB_ENV + echo "$changelog" >> $GITHUB_ENV + echo "EOF" >> $GITHUB_ENV + + - name: Run Make + run: make all tar + - name: Get SHA2-256SUMS for youtube-dl + id: sha256_bin + run: echo "::set-output name=sha256_bin::$(sha256sum youtube-dl | awk '{print $1}')" + - name: Get SHA2-256SUMS for youtube-dl.tar.gz + id: sha256_tar + run: echo "::set-output name=sha256_tar::$(sha256sum youtube-dl.tar.gz | awk '{print $1}')" + - name: Get SHA2-512SUMS for youtube-dl + id: sha512_bin + run: echo "::set-output name=sha512_bin::$(sha512sum youtube-dl | awk '{print $1}')" + - name: Get SHA2-512SUMS for youtube-dl.tar.gz + id: sha512_tar + run: echo "::set-output name=sha512_tar::$(sha512sum youtube-dl.tar.gz | awk '{print $1}')" + + - name: Create Release + id: create_release + uses: actions/create-release@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + tag_name: ${{ steps.bump_version.outputs.ytdl_version }} + release_name: youtube-dl ${{ steps.bump_version.outputs.ytdl_version }} + commitish: ${{ steps.push_release.outputs.head_sha }} + body: ${{ env.changelog }} + draft: false + prerelease: false + - name: Upload youtube-dl Unix binary + id: upload-release-asset + uses: actions/upload-release-asset@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + upload_url: ${{ steps.create_release.outputs.upload_url }} + asset_path: ./youtube-dl + asset_name: youtube-dl + asset_content_type: application/octet-stream + - name: Upload Source tar + uses: actions/upload-release-asset@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + upload_url: ${{ steps.create_release.outputs.upload_url }} + asset_path: ./youtube-dl.tar.gz + asset_name: youtube-dl-${{ steps.bump_version.outputs.ytdl_version }}.tar.gz + asset_content_type: application/gzip + + build_windows: + runs-on: windows-2022 + needs: build_unix + outputs: + sha256_win: ${{ steps.sha256_win.outputs.sha256_win }} + sha512_win: ${{ steps.sha512_win.outputs.sha512_win }} + + steps: + - uses: actions/checkout@v2 + # reason to choose 3.4: https://media.discordapp.net/attachments/807245652072857613/942409077701619742/unknown.png + - name: Set up Python 3.4 + uses: actions/setup-python@v2 + with: + python-version: '3.4' + architecture: 'x86' + - name: Install packages + # https://setuptools.pypa.io/en/latest/history.html#v44-0-0 + # https://pypi.org/project/py2exe/0.9.2.2/ + # https://pip.pypa.io/en/stable/news/#v19-2 + # https://wheel.readthedocs.io/en/stable/news.html + run: python -m pip install --upgrade "pip<19.2" "setuptools<44" "wheel<0.34.0" py2exe==0.9.2.2 + - name: Bump version + id: bump_version + env: + version_suffix: ${{ needs.build_unix.outputs.version_suffix }} + run: python devscripts/update-version.py ${{ env.version_suffix }} + # - name: Run PyInstaller Script + # run: python -m PyInstaller --onefile --console --distpath dist/ -n youtube-dl youtube_dl\__main__.py + - name: Build EXE file + run: python setup.py py2exe + - name: Upload youtube-dl.exe Windows binary + id: upload-release-windows + uses: actions/upload-release-asset@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + upload_url: ${{ needs.build_unix.outputs.upload_url }} + asset_path: ./youtube-dl.exe + asset_name: youtube-dl.exe + asset_content_type: application/vnd.microsoft.portable-executable + - name: Get SHA2-256SUMS for youtube-dl.exe + id: sha256_win + run: echo "::set-output name=sha256_win::$((Get-FileHash youtube-dl.exe -Algorithm SHA256).Hash.ToLower())" + - name: Get SHA2-512SUMS for youtube-dl.exe + id: sha512_win + run: echo "::set-output name=sha512_win::$((Get-FileHash youtube-dl.exe -Algorithm SHA512).Hash.ToLower())" + + finish: + runs-on: ubuntu-latest + needs: [build_unix, build_windows] + env: + YTDL_VERSION: ${{ needs.build_unix.outputs.ytdl_version }} + + steps: + - name: Make SHA2-256SUMS file + env: + SHA256_BIN: ${{ needs.build_unix.outputs.sha256_bin }} + SHA256_TAR: ${{ needs.build_unix.outputs.sha256_tar }} + SHA256_WIN: ${{ needs.build_windows.outputs.sha256_win }} + run: | + echo "${{ env.SHA256_BIN }} youtube-dl" >> SHA2-256SUMS + echo "${{ env.SHA256_TAR }} youtube-dl-${YTDL_VERSION}.tar.gz" >> SHA2-256SUMS + echo "${{ env.SHA256_WIN }} youtube-dl.exe" >> SHA2-256SUMS + - name: Upload 256SUMS file + id: upload-sums + uses: actions/upload-release-asset@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + upload_url: ${{ needs.build_unix.outputs.upload_url }} + asset_path: ./SHA2-256SUMS + asset_name: SHA2-256SUMS + asset_content_type: text/plain + - name: Make SHA2-512SUMS file + env: + SHA512_BIN: ${{ needs.build_unix.outputs.sha512_bin }} + SHA512_TAR: ${{ needs.build_unix.outputs.sha512_tar }} + SHA512_WIN: ${{ needs.build_windows.outputs.sha512_win }} + run: | + echo "${{ env.SHA512_BIN }} youtube-dl" >> SHA2-512SUMS + echo "${{ env.SHA512_TAR }} youtube-dl-${YTDL_VERSION}.tar.gz" >> SHA2-512SUMS + echo "${{ env.SHA512_WIN }} youtube-dl.exe" >> SHA2-512SUMS + - name: Upload 512SUMS file + id: upload-512sums + uses: actions/upload-release-asset@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + upload_url: ${{ needs.build_unix.outputs.upload_url }} + asset_path: ./SHA2-512SUMS + asset_name: SHA2-512SUMS + asset_content_type: text/plain diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 58ab3a4b894..ff40cef784f 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -150,7 +150,7 @@ After you have ensured this site is distributing its content legally, you can fo # TODO more properties (see youtube_dl/extractor/common.py) } ``` -5. Add an import in [`youtube_dl/extractor/extractors.py`](https://github.com/ytdl-org/youtube-dl/blob/master/youtube_dl/extractor/extractors.py). +5. Add an import in [`youtube_dl/extractor/extractors.py`](https://github.com/ytdl-org/youtube-dl/blob/master/youtube_dl/extractor/extractors.py). This makes the extractor available for use, as long as the class ends with `IE`. 6. Run `python test/test_download.py TestDownload.test_YourExtractor`. This *should fail* at first, but you can continually re-run it until you're done. If you decide to add more than one test, then rename ``_TEST`` to ``_TESTS`` and make it into a list of dictionaries. The tests will then be named `TestDownload.test_YourExtractor`, `TestDownload.test_YourExtractor_1`, `TestDownload.test_YourExtractor_2`, etc. Note that tests with `only_matching` key in test's dict are not counted in. 7. Have a look at [`youtube_dl/extractor/common.py`](https://github.com/ytdl-org/youtube-dl/blob/master/youtube_dl/extractor/common.py) for possible helper methods and a [detailed description of what your extractor should and may return](https://github.com/ytdl-org/youtube-dl/blob/7f41a598b3fba1bcab2817de64a08941200aa3c8/youtube_dl/extractor/common.py#L94-L303). Add tests and code for as many as you want. 8. Make sure your code follows [youtube-dl coding conventions](#youtube-dl-coding-conventions) and check the code with [flake8](https://flake8.pycqa.org/en/latest/index.html#quickstart): diff --git a/devscripts/update-version.py b/devscripts/update-version.py new file mode 100644 index 00000000000..f317672c857 --- /dev/null +++ b/devscripts/update-version.py @@ -0,0 +1,34 @@ +#!/usr/bin/env python3 +from __future__ import unicode_literals + +from datetime import datetime +import sys + + +with open('youtube_dl/version.py', 'rt') as f: + exec(compile(f.read(), 'youtube_dl/version.py', 'exec')) +old_version = locals()['__version__'] + +old_version_list = old_version.split('.') + +old_ver = '.'.join(old_version_list[:3]) +old_rev = old_version_list[3] if len(old_version_list) > 3 else '' + +ver = datetime.utcnow().strftime("%Y.%m.%d") + +rev = (sys.argv[1:] or [''])[0] # Use first argument, if present as revision number +if not rev: + rev = str(int(old_rev or 0) + 1) if old_ver == ver else '' + +VERSION = '.'.join((ver, rev)) if rev else ver + +VERSION_FILE = '''# Autogenerated by devscripts/update-version.py + +__version__ = {!r} +'''.format(VERSION) + +with open('youtube_dl/version.py', 'wt') as f: + f.write(VERSION_FILE) + +print('::set-output name=ytdl_version::' + VERSION) +print('\nVersion = %s' % VERSION) diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py index 452caeade71..751fc38b61a 100644 --- a/youtube_dl/extractor/extractors.py +++ b/youtube_dl/extractor/extractors.py @@ -479,6 +479,7 @@ ) from .howcast import HowcastIE from .howstuffworks import HowStuffWorksIE +from .hrfernsehen import HRFernsehenIE from .hrti import ( HRTiIE, HRTiPlaylistIE, diff --git a/youtube_dl/extractor/hrfernsehen.py b/youtube_dl/extractor/hrfernsehen.py new file mode 100644 index 00000000000..11b879dbdd4 --- /dev/null +++ b/youtube_dl/extractor/hrfernsehen.py @@ -0,0 +1,101 @@ +# coding: utf-8 + +from __future__ import unicode_literals + +import json +import re + +from ..utils import ( + int_or_none, + unified_timestamp, + unescapeHTML +) +from .common import InfoExtractor + + +class HRFernsehenIE(InfoExtractor): + IE_NAME = 'hrfernsehen' + _VALID_URL = r'^https?://www\.(?:hr-fernsehen|hessenschau)\.de/.*,video-(?P[0-9]{6})\.html' + + _TESTS = [{ + 'url': 'https://www.hessenschau.de/tv-sendung/hessenschau-vom-26082020,video-130546.html', + 'md5': '5c4e0ba94677c516a2f65a84110fc536', + 'info_dict': { + 'id': '130546', + 'ext': 'mp4', + 'description': 'Sturmtief Kirsten fegt über Hessen / Die Corona-Pandemie – eine Chronologie / ' + 'Sterbehilfe: Die Lage in Hessen / Miss Hessen leitet zwei eigene Unternehmen / ' + 'Pop-Up Museum zeigt Schwarze Unterhaltung und Black Music', + 'subtitles': {'de': [{ + 'url': 'https://hr-a.akamaihd.net/video/as/hessenschau/2020_08/hrLogo_200826200407_L385592_512x288-25p-500kbit.vtt' + }]}, + 'timestamp': 1598470200, + 'upload_date': '20200826', + 'thumbnail': 'https://www.hessenschau.de/tv-sendung/hs_ganz-1554~_t-1598465545029_v-16to9__medium.jpg', + 'title': 'hessenschau vom 26.08.2020' + } + }, { + 'url': 'https://www.hr-fernsehen.de/sendungen-a-z/mex/sendungen/fair-und-gut---was-hinter-aldis-eigenem-guetesiegel-steckt,video-130544.html', + 'only_matching': True + }] + + _GEO_COUNTRIES = ['DE'] + + def extract_airdate(self, loader_data): + airdate_str = loader_data.get('mediaMetadata', {}).get('agf', {}).get('airdate') + + if airdate_str is None: + return None + + return unified_timestamp(airdate_str) + + def extract_formats(self, loader_data): + stream_formats = [] + for stream_obj in loader_data["videoResolutionLevels"]: + stream_format = { + 'format_id': str(stream_obj['verticalResolution']) + "p", + 'height': stream_obj['verticalResolution'], + 'url': stream_obj['url'], + } + + quality_information = re.search(r'([0-9]{3,4})x([0-9]{3,4})-([0-9]{2})p-([0-9]{3,4})kbit', + stream_obj['url']) + if quality_information: + stream_format['width'] = int_or_none(quality_information.group(1)) + stream_format['height'] = int_or_none(quality_information.group(2)) + stream_format['fps'] = int_or_none(quality_information.group(3)) + stream_format['tbr'] = int_or_none(quality_information.group(4)) + + stream_formats.append(stream_format) + + self._sort_formats(stream_formats) + return stream_formats + + def _real_extract(self, url): + video_id = self._match_id(url) + webpage = self._download_webpage(url, video_id) + + title = self._html_search_meta( + ['og:title', 'twitter:title', 'name'], webpage) + description = self._html_search_meta( + ['description'], webpage) + + loader_str = unescapeHTML(self._search_regex(r"data-new-hr-mediaplayer-loader='([^']*)'", webpage, "ardloader")) + loader_data = json.loads(loader_str) + + info = { + 'id': video_id, + 'title': title, + 'description': description, + 'formats': self.extract_formats(loader_data), + 'timestamp': self.extract_airdate(loader_data) + } + + if "subtitle" in loader_data: + info["subtitles"] = {"de": [{"url": loader_data["subtitle"]}]} + + thumbnails = list(set([t for t in loader_data.get("previewImageUrl", {}).values()])) + if len(thumbnails) > 0: + info["thumbnails"] = [{"url": t} for t in thumbnails] + + return info diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py index 4ff27db3d14..8aa2a43a2cb 100644 --- a/youtube_dl/utils.py +++ b/youtube_dl/utils.py @@ -2151,7 +2151,7 @@ def sanitize_url(url): for mistake, fixup in COMMON_TYPOS: if re.match(mistake, url): return re.sub(mistake, fixup, url) - return url + return escape_url(url) def sanitized_Request(url, *args, **kwargs):