process_yle_subtitles.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "! pip install pysrt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import pysrt\n",
    "import time\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "subtitles_path = 'data/urheiluruutu/'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def to_seconds(str_time):\n",
    "    str_time = str_time.replace(',', '.')\n",
    "    if '.' not in str_time: \n",
    "        str_time = str_time + '.000'\n",
    "    assert(time.strptime(str_time,'%H:%M:%S.%f'))\n",
    "    seconds = sum(x * float(t) for x, t in zip([3600, 60, 1], str_time.split(\":\")))\n",
    "    return seconds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = []\n",
    "for filename in os.listdir(subtitles_path):\n",
    "    print(subtitles_path + filename)\n",
    "    program_id = filename.split(\".mp4_fi.sr\")[0]\n",
    "    subs =  pysrt.open(subtitles_path + filename)\n",
    "    for l in subs:\n",
    "        sub = {'program': program_id}\n",
    "        sub['content'] = str(l.text)\n",
    "        sub['start'] = str(l.start)\n",
    "        sub['start_s'] = to_seconds(str(l.start))\n",
    "        sub['end'] = str(l.end)\n",
    "        sub['end_s'] = to_seconds(str(l.end))\n",
    "        sub['duration_s'] = sub['end_s'] - sub['start_s']\n",
    "        data.append(sub)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.DataFrame(data)[['program', 'content', 'start', 'end', 'start_s', 'end_s', 'duration_s']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "! mkdir data/yle_subtitles"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.to_csv('data/yle_subtitles/urheiluruutu_subs.csv', index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Sparql Query for the segments"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "select ?part ?program_uri ?program_id ?title ?start ?duration ?end\n",
    "from <http://data.memad.eu/graph/yle>\n",
    "where\n",
    "{\n",
    "  <http://data.memad.eu/yle/urheiluruutu> ebucore:isParentOf ?program_uri. \n",
    "  ?program_uri ebucore:hasPart ?part;\n",
    "               ebucore:hasIdentifier ?program_id;\n",
    "               ebucore:isInstantiatedBy/ebucore:locator ?medialink.\n",
    "\n",
    "  ?part ebucore:description ?title;\n",
    "        ebucore:start ?start;\n",
    "        ebucore:end ?end;\n",
    "        ebucore:duration ?duration.\n",
    "\n",
    "  FILTER (lang(?title) = 'fi')\n",
    "  FILTER (strstarts(?program_id, 'PROG'))\n",
    "}\n",
    "ORDER BY ?program_id"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Sparql Query for the programs"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "select ?program_uri ?program_id ?title ?duration\n",
    "from <http://data.memad.eu/graph/yle>\n",
    "where\n",
    "{\n",
    "  <http://data.memad.eu/yle/urheiluruutu> ebucore:isParentOf ?program_uri. \n",
    "  ?program_uri ebucore:hasIdentifier ?program_id;\n",
    "               ebucore:title ?title;\n",
    "               ebucore:duration ?duration;\n",
    "               ebucore:isInstantiatedBy/ebucore:locator ?medialink.\n",
    "        \n",
    "  FILTER (lang(?title) = 'fi')\n",
    "  FILTER (strstarts(?program_id, 'PROG'))\n",
    "}\n",
    "ORDER BY ?program_id"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Process metadata files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import xml.etree.ElementTree as ET\n",
    "from tqdm.notebook import tqdm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_path = '../rdf-converter/data/yle/15-urheiluruutu'\n",
    "files = os.listdir(data_path)\n",
    "for ii, file in enumerate(tqdm(files)):\n",
    "\n",
    "    root = ET.parse(data_path+'/'+file).getroot()\n",
    "    guid = root.find(\"./MAObject[1]/GUID\").text\n",
    "\n",
    "    fi_title       = root.find(\"./MAObject[1]/Meta/[@name='FI_TITLE']\").text\n",
    "    duration_tc    = root.find(\"./MAObject[1]/Meta/[@name='SYSTEM_DURATION_TC']\").text \n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_parts.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_programs = pd.read_csv('data/urheiluruutu_programs.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_programs.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_programs['segment_id'] = df_programs['program_id']\n",
    "df_programs['program'] = df_programs['program_id']\n",
    "\n",
    "df_programs['duration_s'] = df_programs['duration']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_programs.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_parts['segment_id'] = df_parts['part'].map(lambda u: u.split('/')[-1])\n",
    "df_parts['program'] = df_parts['program_id']\n",
    "\n",
    "df_parts['start_s'] = df_parts['start'].map(to_seconds)\n",
    "df_parts['end_s'] = df_parts['end'].map(to_seconds)\n",
    "df_parts['duration_s'] = df_parts['end_s'] - df_parts['start_s']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "df_parts.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_segs = df_parts[['segment_id', 'program', 'title', 'start', 'duration', 'start_s', 'duration_s', 'end_s']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "segs = pd.concat([df_segs, df_programs])[['segment_id', 'program', 'title', 'start', 'duration', 'start_s', 'duration_s', 'end_s']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "segs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "segs.to_csv('data/yle_subtitles/urheiluruutu_segs.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "segs[segs.segment_id.str.startswith('PROG')]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}