Add files via upload

xlniu · Feb 1, 2019 · 64f5427 · 64f5427
1 parent db6387b
commit 64f5427
Show file tree

Hide file tree

Showing 4 changed files with 706 additions and 0 deletions.
diff --git a/2017_de-en.ipynb b/2017_de-en.ipynb
@@ -0,0 +1,234 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Using TensorFlow backend.\n"
+     ]
+    }
+   ],
+   "source": [
+    "import os\n",
+    "import json\n",
+    "import math\n",
+    "import pickle\n",
+    "from keras.layers import *\n",
+    "from keras.models import Model\n",
+    "from scipy.stats import pearsonr,spearmanr\n",
+    "from sklearn.metrics import mean_absolute_error,mean_squared_error"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# 参数设置\n",
+    "batch_size = 128\n",
+    "num_train = 25000\n",
+    "os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"0\""
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def padding(x):\n",
+    "    # padding至batch内的最大长度\n",
+    "    ml = max([len(i) for i in x])\n",
+    "    return [i + list(np.zeros(((ml-len(i)),768))) for i in x]\n",
+    "def process_line(line_X,line_y):\n",
+    "    feature = []\n",
+    "    line_X = line_X.strip().split('\\t')\n",
+    "    for l in line_X:\n",
+    "        l = l.split()\n",
+    "        feature.append(list(map(float,l)))\n",
+    "    score = float(line_y.strip())\n",
+    "    return feature,score"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# 读入验证集\n",
+    "path = './2017/de_en/sentence_level/task1_de-en_training-dev_corrected_version/'\n",
+    "f_X = open(path+'dev.features','r')\n",
+    "f_y = open(path+'dev.hter','r')\n",
+    "X_dev,y_dev = [],[]\n",
+    "for line_X,line_y in zip(f_X,f_y):\n",
+    "    feature, score = process_line(line_X,line_y)\n",
+    "    X_dev.append(feature)\n",
+    "    y_dev.append(score)\n",
+    "f_X.close()\n",
+    "f_y.close()\n",
+    "X_dev = np.array(padding(X_dev))\n",
+    "y_dev = np.array(y_dev)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "_________________________________________________________________\n",
+      "Layer (type)                 Output Shape              Param #   \n",
+      "=================================================================\n",
+      "input_1 (InputLayer)         (None, None, 768)         0         \n",
+      "_________________________________________________________________\n",
+      "bidirectional_1 (Bidirection (None, 256)               919552    \n",
+      "_________________________________________________________________\n",
+      "dense_1 (Dense)              (None, 1)                 257       \n",
+      "=================================================================\n",
+      "Total params: 919,809\n",
+      "Trainable params: 919,809\n",
+      "Non-trainable params: 0\n",
+      "_________________________________________________________________\n",
+      "None\n"
+     ]
+    }
+   ],
+   "source": [
+    "# 构建模型\n",
+    "input_features = Input(shape=(None,768), dtype='float32')\n",
+    "hidden = Bidirectional(CuDNNLSTM(128))(input_features)\n",
+    "score = Dense(1, activation='sigmoid')(hidden)\n",
+    "\n",
+    "model = Model(inputs=[input_features], outputs=score)\n",
+    "model.compile(loss='mean_squared_error',\n",
+    "        optimizer='adam',\n",
+    "        metrics=['mean_absolute_error','mean_squared_error'])\n",
+    "print(model.summary())"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Epoch 1/3\n",
+      "196/195 [==============================] - 347s 2s/step - loss: 0.0396 - mean_absolute_error: 0.1533 - mean_squared_error: 0.0396 - val_loss: 0.0312 - val_mean_absolute_error: 0.1405 - val_mean_squared_error: 0.0312\n",
+      "Epoch 2/3\n",
+      "196/195 [==============================] - 311s 2s/step - loss: 0.0286 - mean_absolute_error: 0.1259 - mean_squared_error: 0.0286 - val_loss: 0.0230 - val_mean_absolute_error: 0.1086 - val_mean_squared_error: 0.0230\n",
+      "Epoch 3/3\n",
+      "196/195 [==============================] - 306s 2s/step - loss: 0.0243 - mean_absolute_error: 0.1131 - mean_squared_error: 0.0243 - val_loss: 0.0240 - val_mean_absolute_error: 0.1147 - val_mean_squared_error: 0.0240\n"
+     ]
+    }
+   ],
+   "source": [
+    "# 准备训练集\n",
+    "def data_generator():\n",
+    "    # 数据生成器\n",
+    "    X,Y = [],[]\n",
+    "    while True:\n",
+    "        f_X = open(path+'train.features','r')\n",
+    "        f_y = open(path+'train.hter.shuffle','r')\n",
+    "        for line_X,line_y in zip(f_X,f_y):\n",
+    "            feature, score = process_line(line_X,line_y)\n",
+    "            X.append(feature)\n",
+    "            Y.append(score)\n",
+    "            if len(X) == batch_size:\n",
+    "                X = np.array(padding(X))\n",
+    "                Y = np.array(Y)\n",
+    "                yield (X,Y)\n",
+    "                X,Y = [],[]\n",
+    "        f_X.close()\n",
+    "        f_y.close()\n",
+    "hist = model.fit_generator(data_generator(),\n",
+    "    steps_per_epoch=num_train/batch_size,\n",
+    "    validation_data=(X_dev,y_dev),\n",
+    "    epochs=3)\n",
+    "#model.fit(X_train, y_train,validation_data=(X_test, y_test), \n",
+    "                 #epochs=50, batch_size=batch_size, shuffle=True)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# 读入测试集\n",
+    "test_path = './2017/de_en/sentence_level/task1_de-en_test/'\n",
+    "f_X = open(test_path+'test.features','r')\n",
+    "f_y = open(test_path+'de-en_task1_test.2017.hter','r')\n",
+    "X_test,y_test = [],[]\n",
+    "for line_X,line_y in zip(f_X,f_y):\n",
+    "    feature, score = process_line(line_X,line_y)\n",
+    "    X_test.append(feature)\n",
+    "    y_test.append(score)\n",
+    "f_X.close()\n",
+    "f_y.close()\n",
+    "X_test = np.array(padding(X_test))\n",
+    "y_test = np.array(y_test)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Pearson  (0.64568892196925576, 2.5586214860262072e-236)\n",
+      "SpearmanrResult(correlation=0.62524612581646222, pvalue=2.166729810488068e-217)\n",
+      "MAE  0.115002965491\n",
+      "RMSE  0.1548615904876137\n"
+     ]
+    }
+   ],
+   "source": [
+    "y_pred = model.predict(X_test)\n",
+    "y_ = []\n",
+    "for i in y_pred:\n",
+    "    y_.append(i[0])\n",
+    "print(\"Pearson \",pearsonr(y_test, y_))\n",
+    "print(spearmanr(y_test, y_))\n",
+    "print(\"MAE \",mean_absolute_error(y_test, y_))\n",
+    "print(\"RMSE \",math.sqrt(mean_squared_error(y_test, y_)))"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.6.8"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}