Lesmiscore build yml

.github/workflows/build.yml

@@ -0,0 +1,187 @@
+name: Build
+
+on: workflow_dispatch
+
+jobs:
+  build_unix:
+    runs-on: ubuntu-latest
+    outputs:
+      version_suffix: ${{ steps.version_suffix.outputs.version_suffix }}
+      ytdl_version: ${{ steps.bump_version.outputs.ytdl_version }}
+      upload_url: ${{ steps.create_release.outputs.upload_url }}
+      sha256_bin: ${{ steps.sha256_bin.outputs.sha256_bin }}
+      sha512_bin: ${{ steps.sha512_bin.outputs.sha512_bin }}
+      sha256_tar: ${{ steps.sha256_tar.outputs.sha256_tar }}
+      sha512_tar: ${{ steps.sha512_tar.outputs.sha512_tar }}
+
+    steps:
+    - uses: actions/checkout@v2
+      with:
+        fetch-depth: 0
+    - name: Set up Python
+      uses: actions/setup-python@v2
+      with:
+          python-version: '3.8'
+    - name: Install packages
+      run: sudo apt-get -y install zip pandoc man
+    - name: Set version suffix
+      id: version_suffix
+      env:
+        PUSH_VERSION_COMMIT: ${{ secrets.PUSH_VERSION_COMMIT }}
+      if: "env.PUSH_VERSION_COMMIT == ''"
+      run: echo ::set-output name=version_suffix::$(date -u +"%H%M%S")
+    - name: Bump version
+      id: bump_version
+      run: |
+        python devscripts/update-version.py ${{ steps.version_suffix.outputs.version_suffix }}
+        make issuetemplates
+    - name: Push to release
+      id: push_release
+      run: echo ::set-output name=head_sha::$(git rev-parse HEAD)
+    - name: Get Changelog
+      id: get_changelog
+      run: |
+        changelog=$(cat ChangeLog | grep -oPz '(?s)(?<=version ${{ steps.bump_version.outputs.ytdl_version }}\n{2}).+?(?=\n{2,3}version)') || true
+        echo "changelog<<EOF" >> $GITHUB_ENV
+        echo "$changelog" >> $GITHUB_ENV
+        echo "EOF" >> $GITHUB_ENV
+
+    - name: Run Make
+      run: make all tar
+    - name: Get SHA2-256SUMS for youtube-dl
+      id: sha256_bin
+      run: echo "::set-output name=sha256_bin::$(sha256sum youtube-dl | awk '{print $1}')"
+    - name: Get SHA2-256SUMS for youtube-dl.tar.gz
+      id: sha256_tar
+      run: echo "::set-output name=sha256_tar::$(sha256sum youtube-dl.tar.gz | awk '{print $1}')"
+    - name: Get SHA2-512SUMS for youtube-dl
+      id: sha512_bin
+      run: echo "::set-output name=sha512_bin::$(sha512sum youtube-dl | awk '{print $1}')"
+    - name: Get SHA2-512SUMS for youtube-dl.tar.gz
+      id: sha512_tar
+      run: echo "::set-output name=sha512_tar::$(sha512sum youtube-dl.tar.gz | awk '{print $1}')"
+
+    - name: Create Release
+      id: create_release
+      uses: actions/create-release@v1
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      with:
+        tag_name: ${{ steps.bump_version.outputs.ytdl_version }}
+        release_name: youtube-dl ${{ steps.bump_version.outputs.ytdl_version }}
+        commitish: ${{ steps.push_release.outputs.head_sha }}
+        body: ${{ env.changelog }}
+        draft: false
+        prerelease: false
+    - name: Upload youtube-dl Unix binary
+      id: upload-release-asset
+      uses: actions/upload-release-asset@v1
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      with:
+        upload_url: ${{ steps.create_release.outputs.upload_url }}
+        asset_path: ./youtube-dl
+        asset_name: youtube-dl
+        asset_content_type: application/octet-stream
+    - name: Upload Source tar
+      uses: actions/upload-release-asset@v1
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      with:
+        upload_url: ${{ steps.create_release.outputs.upload_url }}
+        asset_path: ./youtube-dl.tar.gz
+        asset_name: youtube-dl-${{ steps.bump_version.outputs.ytdl_version }}.tar.gz
+        asset_content_type: application/gzip
+
+  build_windows:
+    runs-on: windows-2022
+    needs: build_unix
+    outputs:
+      sha256_win: ${{ steps.sha256_win.outputs.sha256_win }}
+      sha512_win: ${{ steps.sha512_win.outputs.sha512_win }}
+
+    steps:
+    - uses: actions/checkout@v2
+    # reason to choose 3.4: https://media.discordapp.net/attachments/807245652072857613/942409077701619742/unknown.png
+    - name: Set up Python 3.4
+      uses: actions/setup-python@v2
+      with:
+        python-version: '3.4'
+        architecture: 'x86'
+    - name: Install packages
+      # https://setuptools.pypa.io/en/latest/history.html#v44-0-0
+      # https://pypi.org/project/py2exe/0.9.2.2/
+      # https://pip.pypa.io/en/stable/news/#v19-2
+      # https://wheel.readthedocs.io/en/stable/news.html
+      run: python -m pip install --upgrade "pip<19.2" "setuptools<44" "wheel<0.34.0" py2exe==0.9.2.2
+    - name: Bump version
+      id: bump_version
+      env:
+        version_suffix: ${{ needs.build_unix.outputs.version_suffix }}
+      run: python devscripts/update-version.py ${{ env.version_suffix }}
+    # - name: Run PyInstaller Script
+    #   run: python -m PyInstaller --onefile --console --distpath dist/ -n youtube-dl youtube_dl\__main__.py
+    - name: Build EXE file
+      run: python setup.py py2exe
+    - name: Upload youtube-dl.exe Windows binary
+      id: upload-release-windows
+      uses: actions/upload-release-asset@v1
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      with:
+        upload_url: ${{ needs.build_unix.outputs.upload_url }}
+        asset_path: ./youtube-dl.exe
+        asset_name: youtube-dl.exe
+        asset_content_type: application/vnd.microsoft.portable-executable
+    - name: Get SHA2-256SUMS for youtube-dl.exe
+      id: sha256_win
+      run: echo "::set-output name=sha256_win::$((Get-FileHash youtube-dl.exe -Algorithm SHA256).Hash.ToLower())"
+    - name: Get SHA2-512SUMS for youtube-dl.exe
+      id: sha512_win
+      run: echo "::set-output name=sha512_win::$((Get-FileHash youtube-dl.exe -Algorithm SHA512).Hash.ToLower())"
+
+  finish:
+    runs-on: ubuntu-latest
+    needs: [build_unix, build_windows]
+    env:
+      YTDL_VERSION: ${{ needs.build_unix.outputs.ytdl_version }}
+
+    steps:
+    - name: Make SHA2-256SUMS file
+      env:
+        SHA256_BIN: ${{ needs.build_unix.outputs.sha256_bin }}
+        SHA256_TAR: ${{ needs.build_unix.outputs.sha256_tar }}
+        SHA256_WIN: ${{ needs.build_windows.outputs.sha256_win }}
+      run: |
+        echo "${{ env.SHA256_BIN }}  youtube-dl" >> SHA2-256SUMS
+        echo "${{ env.SHA256_TAR }}  youtube-dl-${YTDL_VERSION}.tar.gz" >> SHA2-256SUMS
+        echo "${{ env.SHA256_WIN }}  youtube-dl.exe" >> SHA2-256SUMS
+    - name: Upload 256SUMS file
+      id: upload-sums
+      uses: actions/upload-release-asset@v1
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      with:
+        upload_url: ${{ needs.build_unix.outputs.upload_url }}
+        asset_path: ./SHA2-256SUMS
+        asset_name: SHA2-256SUMS
+        asset_content_type: text/plain
+    - name: Make SHA2-512SUMS file
+      env:
+        SHA512_BIN: ${{ needs.build_unix.outputs.sha512_bin }}
+        SHA512_TAR: ${{ needs.build_unix.outputs.sha512_tar }}
+        SHA512_WIN: ${{ needs.build_windows.outputs.sha512_win }}
+      run: |
+        echo "${{ env.SHA512_BIN }}  youtube-dl" >> SHA2-512SUMS
+        echo "${{ env.SHA512_TAR }}  youtube-dl-${YTDL_VERSION}.tar.gz" >> SHA2-512SUMS
+        echo "${{ env.SHA512_WIN }}  youtube-dl.exe" >> SHA2-512SUMS
+    - name: Upload 512SUMS file
+      id: upload-512sums
+      uses: actions/upload-release-asset@v1
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      with:
+        upload_url: ${{ needs.build_unix.outputs.upload_url }}
+        asset_path: ./SHA2-512SUMS
+        asset_name: SHA2-512SUMS
+        asset_content_type: text/plain

CONTRIBUTING.md

@@ -150,7 +150,7 @@ After you have ensured this site is distributing its content legally, you can fo
                 # TODO more properties (see youtube_dl/extractor/common.py)
             }
     ```
-5. Add an import in [`youtube_dl/extractor/extractors.py`](https://github.com/ytdl-org/youtube-dl/blob/master/youtube_dl/extractor/extractors.py).
+5. Add an import in [`youtube_dl/extractor/extractors.py`](https://github.com/ytdl-org/youtube-dl/blob/master/youtube_dl/extractor/extractors.py). This makes the extractor available for use, as long as the class ends with `IE`.
 6. Run `python test/test_download.py TestDownload.test_YourExtractor`. This *should fail* at first, but you can continually re-run it until you're done. If you decide to add more than one test, then rename ``_TEST`` to ``_TESTS`` and make it into a list of dictionaries. The tests will then be named `TestDownload.test_YourExtractor`, `TestDownload.test_YourExtractor_1`, `TestDownload.test_YourExtractor_2`, etc. Note that tests with `only_matching` key in test's dict are not counted in.
 7. Have a look at [`youtube_dl/extractor/common.py`](https://github.com/ytdl-org/youtube-dl/blob/master/youtube_dl/extractor/common.py) for possible helper methods and a [detailed description of what your extractor should and may return](https://github.com/ytdl-org/youtube-dl/blob/7f41a598b3fba1bcab2817de64a08941200aa3c8/youtube_dl/extractor/common.py#L94-L303). Add tests and code for as many as you want.
 8. Make sure your code follows [youtube-dl coding conventions](#youtube-dl-coding-conventions) and check the code with [flake8](https://flake8.pycqa.org/en/latest/index.html#quickstart):

devscripts/update-version.py

@@ -0,0 +1,34 @@
+#!/usr/bin/env python3
+from __future__ import unicode_literals
+
+from datetime import datetime
+import sys
+
+
+with open('youtube_dl/version.py', 'rt') as f:
+    exec(compile(f.read(), 'youtube_dl/version.py', 'exec'))
+old_version = locals()['__version__']
+
+old_version_list = old_version.split('.')
+
+old_ver = '.'.join(old_version_list[:3])
+old_rev = old_version_list[3] if len(old_version_list) > 3 else ''
+
+ver = datetime.utcnow().strftime("%Y.%m.%d")
+
+rev = (sys.argv[1:] or [''])[0]  # Use first argument, if present as revision number
+if not rev:
+    rev = str(int(old_rev or 0) + 1) if old_ver == ver else ''
+
+VERSION = '.'.join((ver, rev)) if rev else ver
+
+VERSION_FILE = '''# Autogenerated by devscripts/update-version.py
+
+__version__ = {!r}
+'''.format(VERSION)
+
+with open('youtube_dl/version.py', 'wt') as f:
+    f.write(VERSION_FILE)
+
+print('::set-output name=ytdl_version::' + VERSION)
+print('\nVersion = %s' % VERSION)

youtube_dl/extractor/extractors.py

@@ -479,6 +479,7 @@
 )
 from .howcast import HowcastIE
 from .howstuffworks import HowStuffWorksIE
+from .hrfernsehen import HRFernsehenIE
 from .hrti import (
     HRTiIE,
     HRTiPlaylistIE,

youtube_dl/extractor/hrfernsehen.py

@@ -0,0 +1,101 @@
+# coding: utf-8
+
+from __future__ import unicode_literals
+
+import json
+import re
+
+from ..utils import (
+    int_or_none,
+    unified_timestamp,
+    unescapeHTML
+)
+from .common import InfoExtractor
+
+
+class HRFernsehenIE(InfoExtractor):
+    IE_NAME = 'hrfernsehen'
+    _VALID_URL = r'^https?://www\.(?:hr-fernsehen|hessenschau)\.de/.*,video-(?P<id>[0-9]{6})\.html'
+
+    _TESTS = [{
+        'url': 'https://www.hessenschau.de/tv-sendung/hessenschau-vom-26082020,video-130546.html',
+        'md5': '5c4e0ba94677c516a2f65a84110fc536',
+        'info_dict': {
+            'id': '130546',
+            'ext': 'mp4',
+            'description': 'Sturmtief Kirsten fegt über Hessen / Die Corona-Pandemie – eine Chronologie / '
+                           'Sterbehilfe: Die Lage in Hessen / Miss Hessen leitet zwei eigene Unternehmen / '
+                           'Pop-Up Museum zeigt Schwarze Unterhaltung und Black Music',
+            'subtitles': {'de': [{
+                'url': 'https://hr-a.akamaihd.net/video/as/hessenschau/2020_08/hrLogo_200826200407_L385592_512x288-25p-500kbit.vtt'
+            }]},
+            'timestamp': 1598470200,
+            'upload_date': '20200826',
+            'thumbnail': 'https://www.hessenschau.de/tv-sendung/hs_ganz-1554~_t-1598465545029_v-16to9__medium.jpg',
+            'title': 'hessenschau vom 26.08.2020'
+        }
+    }, {
+        'url': 'https://www.hr-fernsehen.de/sendungen-a-z/mex/sendungen/fair-und-gut---was-hinter-aldis-eigenem-guetesiegel-steckt,video-130544.html',
+        'only_matching': True
+    }]
+
+    _GEO_COUNTRIES = ['DE']
+
+    def extract_airdate(self, loader_data):
+        airdate_str = loader_data.get('mediaMetadata', {}).get('agf', {}).get('airdate')
+
+        if airdate_str is None:
+            return None
+
+        return unified_timestamp(airdate_str)
+
+    def extract_formats(self, loader_data):
+        stream_formats = []
+        for stream_obj in loader_data["videoResolutionLevels"]:
+            stream_format = {
+                'format_id': str(stream_obj['verticalResolution']) + "p",
+                'height': stream_obj['verticalResolution'],
+                'url': stream_obj['url'],
+            }
+
+            quality_information = re.search(r'([0-9]{3,4})x([0-9]{3,4})-([0-9]{2})p-([0-9]{3,4})kbit',
+                                            stream_obj['url'])
+            if quality_information:
+                stream_format['width'] = int_or_none(quality_information.group(1))
+                stream_format['height'] = int_or_none(quality_information.group(2))
+                stream_format['fps'] = int_or_none(quality_information.group(3))
+                stream_format['tbr'] = int_or_none(quality_information.group(4))
+
+            stream_formats.append(stream_format)
+
+        self._sort_formats(stream_formats)
+        return stream_formats
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+
+        title = self._html_search_meta(
+            ['og:title', 'twitter:title', 'name'], webpage)
+        description = self._html_search_meta(
+            ['description'], webpage)
+
+        loader_str = unescapeHTML(self._search_regex(r"data-new-hr-mediaplayer-loader='([^']*)'", webpage, "ardloader"))
+        loader_data = json.loads(loader_str)
+
+        info = {
+            'id': video_id,
+            'title': title,
+            'description': description,
+            'formats': self.extract_formats(loader_data),
+            'timestamp': self.extract_airdate(loader_data)
+        }
+
+        if "subtitle" in loader_data:
+            info["subtitles"] = {"de": [{"url": loader_data["subtitle"]}]}
+
+        thumbnails = list(set([t for t in loader_data.get("previewImageUrl", {}).values()]))
+        if len(thumbnails) > 0:
+            info["thumbnails"] = [{"url": t} for t in thumbnails]
+
+        return info

youtube_dl/utils.py

@@ -2151,7 +2151,7 @@ def sanitize_url(url):
     for mistake, fixup in COMMON_TYPOS:
         if re.match(mistake, url):
             return re.sub(mistake, fixup, url)
-    return url
+    return escape_url(url)
 
 
 def sanitized_Request(url, *args, **kwargs):