Building_B.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "from weighted_levenshtein import lev\n",
    "import sklearn.cluster\n",
    "from sklearn.metrics.cluster import adjusted_rand_score\n",
    "import re\n",
    "from itertools import combinations \n",
    "from nltk import ngrams\n",
    "from eppy import modeleditor\n",
    "from eppy.modeleditor import IDF\n",
    "IDF.setiddname('C:/EnergyPlusV8-8-0/Energy+.idd')\n",
    "\n",
    "import warnings\n",
    "warnings.simplefilter(action='ignore', category=DeprecationWarning)\n",
    "\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Extracting core information from BMS point tags"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>raw</th>\n",
       "      <th>tag</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>WEATHER STATION.OA_CO2</td>\n",
       "      <td>OA_CO2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>WEATHER STATION.RAINFAIL</td>\n",
       "      <td>RAINFAIL</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Weather Station.Wind speed</td>\n",
       "      <td>Wind speed</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Weather Station.Wind direction in degrees</td>\n",
       "      <td>Wind direction in degrees</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Weather Station.Temp</td>\n",
       "      <td>Temp</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                         raw                        tag\n",
       "0                     WEATHER STATION.OA_CO2                     OA_CO2\n",
       "1                   WEATHER STATION.RAINFAIL                   RAINFAIL\n",
       "2                 Weather Station.Wind speed                 Wind speed\n",
       "3  Weather Station.Wind direction in degrees  Wind direction in degrees\n",
       "4                       Weather Station.Temp                       Temp"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "raw = pd.read_csv('../.csv')\n",
    "OPCtag = raw.copy()\n",
    "OPCtag['tag'] = OPCtag['raw']\n",
    "# the core information of this dataset is separated with the delimiter and therefore was easily extracted\n",
    "OPCtag['tag'] = OPCtag['tag'].apply(lambda x: x.split('.')[-1])\n",
    "OPCtag['tag'][188:332] = OPCtag['tag'][188:332].apply(lambda x: '_'.join(x.split('_')[3:]))\n",
    "\n",
    "OPCtag.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Tokenization of BMS tags using X-gram"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def razors(string):\n",
    "    m = len(string)-1\n",
    "    result = []\n",
    "    for k in range(min(m+1,7)):\n",
    "        for bits in combinations(range(m), k):\n",
    "            s = [1] * m\n",
    "            for bit in bits:\n",
    "                s[bit] = 0\n",
    "            result.append(s)\n",
    "    return result\n",
    "\n",
    "def Xgrams(string):\n",
    "    string = ''.join(re.split('[ _,:]',string))\n",
    "    ngrams = []\n",
    "    for item in razors(string):\n",
    "        ngram = []\n",
    "        flag = 0\n",
    "        for i in range(len(item)):\n",
    "            if item[i]:\n",
    "                ngram.append(string[flag:i+1])\n",
    "                flag = i+1\n",
    "        ngram.append(string[flag:])\n",
    "        ngram.sort(key = len,reverse=True)\n",
    "        ngrams.append(ngram)\n",
    "    return ngrams"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>tag</th>\n",
       "      <th>Xgrams</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>180</th>\n",
       "      <td>MSB1 KWH NET BALANCE</td>\n",
       "      <td>MSB1 KWH NET BALANCE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>181</th>\n",
       "      <td>MSB2 KWH NET DELIVERED</td>\n",
       "      <td>MSB2 KWH NET DELIVERED</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>182</th>\n",
       "      <td>MSB2 KWH NET RECEIVED</td>\n",
       "      <td>MSB2 KWH NET RECEIVED</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>183</th>\n",
       "      <td>MSB2 KWH NET BALANCE</td>\n",
       "      <td>MSB2 KWH NET BALANCE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>184</th>\n",
       "      <td>SDE4 NET ELEC ENERGY BALANCE</td>\n",
       "      <td>SDE4 NET ELEC ENERGY BALANCE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>185</th>\n",
       "      <td>MAIN BTU ENERGY IN KWH NET</td>\n",
       "      <td>MAIN BTU ENERGY IN KWH NET</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>186</th>\n",
       "      <td>BTU COOLING ELEC ENERGY</td>\n",
       "      <td>BTU COOLING ELEC ENERGY</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>187</th>\n",
       "      <td>SDE4 NET ENERGY BALANCE</td>\n",
       "      <td>SDE4 NET ENERGY BALANCE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>188</th>\n",
       "      <td>CMD</td>\n",
       "      <td>[[C, M, D], [CM, D], [MD, C], [CMD]]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>189</th>\n",
       "      <td>PFILT</td>\n",
       "      <td>[[P, F, I, L, T], [PF, I, L, T], [FI, P, L, T]...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                              tag  \\\n",
       "180          MSB1 KWH NET BALANCE   \n",
       "181        MSB2 KWH NET DELIVERED   \n",
       "182         MSB2 KWH NET RECEIVED   \n",
       "183          MSB2 KWH NET BALANCE   \n",
       "184  SDE4 NET ELEC ENERGY BALANCE   \n",
       "185    MAIN BTU ENERGY IN KWH NET   \n",
       "186       BTU COOLING ELEC ENERGY   \n",
       "187       SDE4 NET ENERGY BALANCE   \n",
       "188                           CMD   \n",
       "189                         PFILT   \n",
       "\n",
       "                                                Xgrams  \n",
       "180                               MSB1 KWH NET BALANCE  \n",
       "181                             MSB2 KWH NET DELIVERED  \n",
       "182                              MSB2 KWH NET RECEIVED  \n",
       "183                               MSB2 KWH NET BALANCE  \n",
       "184                       SDE4 NET ELEC ENERGY BALANCE  \n",
       "185                         MAIN BTU ENERGY IN KWH NET  \n",
       "186                            BTU COOLING ELEC ENERGY  \n",
       "187                            SDE4 NET ENERGY BALANCE  \n",
       "188               [[C, M, D], [CM, D], [MD, C], [CMD]]  \n",
       "189  [[P, F, I, L, T], [PF, I, L, T], [FI, P, L, T]...  "
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "OPCtag['tag'] = OPCtag['tag'].apply(lambda x: str.upper(x))\n",
    "OPCtag['Xgrams'] = OPCtag['tag']\n",
    "OPCtag['Xgrams'][188:332] = OPCtag['tag'][188:332].apply(lambda x: Xgrams(x))\n",
    "# OPCtag = OPCtag[OPCtag['tag'].str.len()<7]\n",
    "OPCtag.iloc[:,1:][180:190]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Defining the dictionary used to tokenize the EnergyPlus variables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def updateABB(abb,full):\n",
    "    global ABBdict\n",
    "    for i in full:\n",
    "        if i in list(ABBdict['name']):\n",
    "            ABBdict['abb'].loc[ABBdict['name']==i].item().add(abb[full.index(i)])\n",
    "        else:\n",
    "            ABBdict = ABBdict.append({'abb': set([abb[full.index(i)]]),'name':i}, ignore_index=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "dict1 = pd.read_csv('../HVACacronym.csv')\n",
    "dict2 = pd.read_csv('../HVACacronym1.csv')\n",
    "\n",
    "dict1['full']=dict1['full'].apply(lambda x: str.lower(x))\n",
    "dict2['name']=dict2['name'].apply(lambda x: str.lower(x))\n",
    "\n",
    "ABBdict = pd.concat([dict1.rename({'full':'name'}, axis='columns'),dict2])\n",
    "ABBdict['abb']= ABBdict[['abb']].values.tolist()\n",
    "ABBdict = pd.DataFrame(ABBdict.groupby('name')['abb'].sum())\n",
    "ABBdict['name']=ABBdict.index\n",
    "ABBdict.rename(index=str, columns={0: \"abb\"},inplace=True)\n",
    "ABBdict['abb']= ABBdict['abb'].apply(lambda x: set(x))\n",
    "\n",
    "ABBdict.reset_index(drop=True,inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "abb = ['OC','TMP','TEMP','SA','STATIC','RA','RH','RM','BM','FLWR','RET','KWATT','KWHR','PM','ENGRY','FWD','PWR','AHU','FAN','SPEED','BYP','CLG','WATER','VLV','TP','SMK','FA','PM','BM','CURR','VOLT','PW']\n",
    "full=['off coil','temperature','temperature','supply air','pressure','return air','relative humidity','room','btu meter','flow rate','return','power','energy','power meter','energy','supply','power','ahu','fan','speed','bypass','cooling','water','valve','time program','smoke','fresh air','power meter','btu meter','current','voltage','power']\n",
    "updateABB(abb,full)\n",
    "\n",
    "EPname = ['zone','air','energy','heating','cooling','air terminal','pump','boiler','condenser','setpoint','heat exchanger','humidifier','heater','water','people','window','infiltration','equipment','outdoor','drybulb','dewpoint','speed','angle','radiation','pressure','precipitation','occupant','lights','humidity ratio','facility','mass flow rate','interior','gas','office','room','level','extract','fan','supply','air loop','site','power','radiant','fan coil unit','indoor','heat pump','vav','vrf','return','inlet','outlet','relief air','rate']\n",
    "EPabb = ['RM','A','KWHR','HT','CL','VAV','PMP','B','COND','SP','HX','HUMID','HTR','WTR','PPL','WD','INFIL','EQUIP','O','DB','DP','SPD','AGL','R','PR','P','OCC','LT','HR','FAC','FLWR','INTR','G','OFC','RM','LV','E','F','S','AHU','LOC','KWATT','R','FCU','I','CU','VAV','VRF','R','S','R','EA','KW']\n",
    "updateABB(EPabb,EPname)\n",
    "\n",
    "NODEname = ['supply side inlet','supply side outlet','coil air outlet','mixed inlet','splitter outlet','coil outlet','inlet','outlet','supply inlet']\n",
    "NODEabb = ['R','S','OC','R','S','S','FWD','RET','R']\n",
    "updateABB(NODEabb,NODEname)\n",
    "\n",
    "testName = ['electricity','energy','power','demand','rate','coil air outlet','coil outlet']\n",
    "testABB = ['KWHR','ENGRY','PWR','ENGRY','PWR','S','OC']\n",
    "updateABB(testABB,testName)\n",
    "\n",
    "iwName = ['dewpoint','relative humidity','humidity ratio','wind','direction','global','solar','zone','electricity','water','mass flow rate']\n",
    "iwABB = ['DWP','HM','HM','W','DIREC','GLO','S','R','TOTKW','W','FL']\n",
    "updateABB(iwABB,iwName)\n",
    "\n",
    "iwName = ['humidity ratio','relative humidity','mass flow rate']\n",
    "iwABB = ['HUM','HUM','FLO']\n",
    "updateABB(iwABB,iwName)\n",
    "\n",
    "sdeName = ['temperature','dewpoint','relative humidity','humidity ratio','pressure','barometric','wind','direction','solar','radiation','precipitation','electricity','energy','purchased','net','produced']\n",
    "sdeABB = ['TEMPERATURE','DEW','HUMIDITY','HUMIDITY','PRESSURE','BAROMETRIC','WIND','DIRECTION','SOLAR','RADIATION','PRECIPITATION','POWER','ENERGY','RECEIVED','BALANCE','DELIVERED']\n",
    "updateABB(sdeABB,sdeName)\n",
    "\n",
    "sdeABB = ['KWH','SP']\n",
    "sdeName = ['electricity','pressure']\n",
    "updateABB(sdeABB,sdeName)\n",
    "\n",
    "CSLabb = ['DEWPOINT','H','H','OCCUPANCY','USAGE','SUPPLY','CFM','STPT','ZN']\n",
    "CSLname = ['dewpoint','humidity ratio','relative humidity','occupant','energy','inlet','flow rate','setpoint','zone']\n",
    "updateABB(CSLabb,CSLname)\n",
    "\n",
    "CSLabb = ['KCFM','SETPOINT','RETURN','RETURN','ZONE','SUPPLY','RETURN']\n",
    "CSLname = ['flow rate','setpoint','supply inlet','outlet','zone','supply side outlet','return']\n",
    "updateABB(CSLabb,CSLname)\n",
    "\n",
    "CSL3abb = ['FLOW','RETURN']\n",
    "CSL3name = ['flow rate','supply side inlet']\n",
    "updateABB(CSL3abb,CSL3name)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### read the EnergyPlus variables as well as the groudtruth"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "truth = pd.read_csv('../truthSDE.csv')\n",
    "truth.drop(columns=['alter'],inplace = True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## fuzzy string matching"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "def match(s,OPCtag,th):\n",
    "    \n",
    "    ngram = []\n",
    "    for i in range(3,0,-1):\n",
    "        grams = ngrams(s.split(' '),i)\n",
    "        for gram in grams:\n",
    "            ngram.append(' '.join(list(gram)))\n",
    "    \n",
    "#     ngram.sort(key = lambda s: len(s),reverse = True)\n",
    "    ABBs = []\n",
    "    while len(ngram)>0:\n",
    "        i = ngram[0]\n",
    "        ngram.remove(i)\n",
    "#         print(ngram)\n",
    "        try:\n",
    "            ABBs.append(ABBdict.loc[ABBdict['name'] == i,'abb'].values[0])\n",
    "            ngram = [s for s in ngram if (s not in i)]\n",
    "        except:\n",
    "            pass\n",
    "\n",
    "    details = []\n",
    "    for i in OPCtag.index:\n",
    "        r = 0\n",
    "        if i >187 and i <332:\n",
    "            idx = 0\n",
    "            m = 0\n",
    "            for j in range(len(OPCtag['Xgrams'][i])):\n",
    "                used = [0]*len(ABBs)    # so that a set of ABBs is only matched once\n",
    "                count = 0\n",
    "                for gram in OPCtag.loc[i,'Xgrams'][j]:\n",
    "                    for c in range(len(ABBs)):\n",
    "                        if gram in ABBs[c] and used[c]==0:\n",
    "                            used[c] = 1\n",
    "                            count += len(gram)\n",
    "                            break\n",
    "                if count > m:\n",
    "                    m = count # to avoid division by zero error\n",
    "                    # harmonic average of ratio on both side\n",
    "                    r = 2/(len(ABBs)/sum(used) + len(OPCtag['tag'][i])/m)\n",
    "                    idx = j\n",
    "            if r >= .01:\n",
    "                details.append([round(r,4),OPCtag['tag'][i],tuple(OPCtag['Xgrams'][i][idx])])\n",
    "        else:\n",
    "            used = [0]*len(ABBs)\n",
    "            m=0\n",
    "            for s in OPCtag['Xgrams'][i].split(' '):\n",
    "                for c in range(len(ABBs)):\n",
    "                    if s in ABBs[c] and used[c]==0:\n",
    "                        used[c] = 1\n",
    "                        m += 1\n",
    "                        break\n",
    "            if m > 0:\n",
    "                r = 2/(len(ABBs)/ sum(used) + len(OPCtag['Xgrams'][i].split(' '))/m)\n",
    "                details.append([round(r,4),OPCtag['tag'][i],tuple(OPCtag['Xgrams'][i].split(' '))])\n",
    "\n",
    "    details = list(set(tuple(x) for x in details))\n",
    "    details.sort()\n",
    "    matches = [i[1] for i in details][-th:]\n",
    "\n",
    "    return [matches,details,ABBs]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "def check(result,th):\n",
    "    for i in result.index:\n",
    "        if len(result.loc[i,'matches']) == 0:\n",
    "            if result.loc[i,'truth'] in [np.nan]:\n",
    "                result.loc[i,'test'] = 'TN'\n",
    "            else:\n",
    "                result.loc[i,'test'] = 'FN'\n",
    "        elif result.loc[i,'truth'] in [np.nan]:\n",
    "            result.loc[i,'test'] = 'FP'\n",
    "        elif result.loc[i,'truth'] in result.loc[i,'matches'][-th:]:\n",
    "            result.loc[i,'test'] = 'TP'\n",
    "        else:\n",
    "            result.loc[i,'test'] = 'FP1'\n",
    "    return result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "top5 = pd.DataFrame(columns = ['input','abbs','matches'])\n",
    "ls = truth['input'].unique()\n",
    "for i in range(len(ls)):\n",
    "    test = match(ls[i],OPCtag,5)\n",
    "    top5.loc[i,'input'] = ls[i]\n",
    "    top5.loc[i,'abbs'] = test[2]\n",
    "    top5.loc[i,'matches'] = test[0]\n",
    "top5 = top5.join(truth.set_index('input'), on='input')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "FP     427\n",
       "TP      26\n",
       "TN      10\n",
       "FP1      3\n",
       "Name: test, dtype: int64"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "t5 = check(top5,5)\n",
    "t5['test'].value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### the proposed framework made mistake on BMS points tagged with full words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>input</th>\n",
       "      <th>abbs</th>\n",
       "      <th>matches</th>\n",
       "      <th>truth</th>\n",
       "      <th>test</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>site outdoor air drybulb temperature</td>\n",
       "      <td>[{LOC}, {O}, {A}, {DB}, {TEMP, T, TMP, TEMPERA...</td>\n",
       "      <td>[RA_CO2, SA-TSP, SA_CO2, OC-T, OC_T, RA_T, SA_T]</td>\n",
       "      <td>TEMP</td>\n",
       "      <td>FP1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>zone mean air temperature</td>\n",
       "      <td>[{RM, R}, {A}, {TEMP, T, TMP, TEMPERATURE}]</td>\n",
       "      <td>[RA_CO2, SA-TSP, SA_RH, TEMP, SA_T, TRP, RA_T]</td>\n",
       "      <td>STATUS TEMP LEVEL</td>\n",
       "      <td>FP1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>283</th>\n",
       "      <td>zone air node temperature</td>\n",
       "      <td>[{RM, R}, {A}, {TEMP, T, TMP, TEMPERATURE}]</td>\n",
       "      <td>[RA_CO2, SA-TSP, SA_RH, TEMP, SA_T, TRP, RA_T]</td>\n",
       "      <td>STATUS TEMP LEVEL</td>\n",
       "      <td>FP1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                    input  \\\n",
       "0    site outdoor air drybulb temperature   \n",
       "25              zone mean air temperature   \n",
       "283             zone air node temperature   \n",
       "\n",
       "                                                  abbs  \\\n",
       "0    [{LOC}, {O}, {A}, {DB}, {TEMP, T, TMP, TEMPERA...   \n",
       "25         [{RM, R}, {A}, {TEMP, T, TMP, TEMPERATURE}]   \n",
       "283        [{RM, R}, {A}, {TEMP, T, TMP, TEMPERATURE}]   \n",
       "\n",
       "                                              matches              truth test  \n",
       "0    [RA_CO2, SA-TSP, SA_CO2, OC-T, OC_T, RA_T, SA_T]               TEMP  FP1  \n",
       "25     [RA_CO2, SA-TSP, SA_RH, TEMP, SA_T, TRP, RA_T]  STATUS TEMP LEVEL  FP1  \n",
       "283    [RA_CO2, SA-TSP, SA_RH, TEMP, SA_T, TRP, RA_T]  STATUS TEMP LEVEL  FP1  "
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "top5[top5['test']=='FP1']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "FP     427\n",
       "FP1     15\n",
       "TP      14\n",
       "TN      10\n",
       "Name: test, dtype: int64"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "t1 = check(top5,1)\n",
    "t1['test'].value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## baseline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "def onNgram(s,OPCtag,th):\n",
    "    \n",
    "    ngram = []\n",
    "    for i in range(3,0,-1):\n",
    "        grams = ngrams(s.split(' '),i)\n",
    "        for gram in grams:\n",
    "            ngram.append(' '.join(list(gram)))\n",
    "    \n",
    "    ABBs = []\n",
    "    while len(ngram)>0:\n",
    "        i = ngram[0]\n",
    "        ngram.remove(i)\n",
    "        try:\n",
    "            ABBs.append(ABBdict.loc[ABBdict['name'] == i,'abb'].values[0])\n",
    "            ngram = [s for s in ngram if (s not in i)]\n",
    "        except:\n",
    "            pass\n",
    "    \n",
    "    detail = []\n",
    "    for i in OPCtag.index:\n",
    "        r = 0\n",
    "        used = [0]*len(ABBs)    # so that a set of ABBs is only matched once\n",
    "        if i >187 and i <332:\n",
    "            count = 0\n",
    "            for gram in OPCtag['ngram'][i]:\n",
    "                for c in range(len(ABBs)):\n",
    "                    if gram in ABBs[c] and used[c]==0:\n",
    "                        used[c] = 1\n",
    "                        count += 1\n",
    "                        break\n",
    "            if count >= 1:\n",
    "                r = count/(len(ABBs)+len(OPCtag['ngram'][i])-count)\n",
    "                detail.append([round(r,4),OPCtag['tag'][i]])\n",
    "        else:\n",
    "            m=0\n",
    "            for s in OPCtag['ngram'][i].split(' '):\n",
    "                for c in range(len(ABBs)):\n",
    "                    if s in ABBs[c] and used[c]==0:\n",
    "                        used[c] = 1\n",
    "                        m += 1\n",
    "                        break\n",
    "            if m > 0:\n",
    "                r = m/(len(ABBs)+len(OPCtag['ngram'][i].split(' '))-m)\n",
    "                detail.append([round(r,4),OPCtag['tag'][i],tuple(OPCtag['Xgrams'][i].split(' '))])\n",
    "\n",
    "            \n",
    "    detail = list(set(tuple(x) for x in detail))\n",
    "    detail.sort()\n",
    "    matches = [i[1] for i in detail][-th:]\n",
    "            \n",
    "    return [matches,detail,ABBs]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "# tokenization\n",
    "OPCtag['ngram'] = OPCtag['tag']\n",
    "for i in range(188,332):\n",
    "    ngram = []\n",
    "    for n in range(1,min(7,len(OPCtag['tag'][i])+1)):\n",
    "        grams = ngrams(OPCtag.loc[i,('tag')],n)\n",
    "        for gram in grams:\n",
    "            ngram.append(''.join(list(gram)))\n",
    "    OPCtag.at[i,'ngram'] = ngram"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "top5 = pd.DataFrame(columns = ['input','abbs','matches'])\n",
    "ls = truth['input'].unique()\n",
    "for i in range(len(ls)):\n",
    "    test = onNgram(ls[i],OPCtag,5)\n",
    "    top5.loc[i,'input'] = ls[i]\n",
    "    top5.loc[i,'abbs'] = test[2]\n",
    "    top5.loc[i,'matches'] = test[0]\n",
    "top5 = top5.join(truth.set_index('input'), on='input')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "FP     427\n",
       "TP      26\n",
       "TN      10\n",
       "FP1      3\n",
       "Name: test, dtype: int64"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "t5 = check(top5,5)\n",
    "t5['test'].value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### the proposed framework made mistake on BMS points tagged with abbreviations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>input</th>\n",
       "      <th>abbs</th>\n",
       "      <th>matches</th>\n",
       "      <th>truth</th>\n",
       "      <th>test</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>94</th>\n",
       "      <td>demand side inlet 1 setpoint temperature</td>\n",
       "      <td>[{ENGRY}, {FWD, S}, {SP}, {TEMP, T, TMP, TEMPE...</td>\n",
       "      <td>[DEW POINT TEMPERATURE, SA_T, STATUS TEMP LEVE...</td>\n",
       "      <td>SA-TSP</td>\n",
       "      <td>FP1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>148</th>\n",
       "      <td>supply side outlet setpoint temperature</td>\n",
       "      <td>[{S}, {SP}, {TEMP, T, TMP, TEMPERATURE}]</td>\n",
       "      <td>[SA_T, DEW POINT TEMPERATURE, STATUS TEMP LEVE...</td>\n",
       "      <td>SA-TSP</td>\n",
       "      <td>FP1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>457</th>\n",
       "      <td>cooling coil outlet temperature</td>\n",
       "      <td>[{OC, S}, {CL, CLG}, {TEMP, T, TMP, TEMPERATURE}]</td>\n",
       "      <td>[SA_T, DEW POINT TEMPERATURE, STATUS TEMP LEVE...</td>\n",
       "      <td>OC_T</td>\n",
       "      <td>FP1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                        input  \\\n",
       "94   demand side inlet 1 setpoint temperature   \n",
       "148   supply side outlet setpoint temperature   \n",
       "457           cooling coil outlet temperature   \n",
       "\n",
       "                                                  abbs  \\\n",
       "94   [{ENGRY}, {FWD, S}, {SP}, {TEMP, T, TMP, TEMPE...   \n",
       "148           [{S}, {SP}, {TEMP, T, TMP, TEMPERATURE}]   \n",
       "457  [{OC, S}, {CL, CLG}, {TEMP, T, TMP, TEMPERATURE}]   \n",
       "\n",
       "                                               matches   truth test  \n",
       "94   [DEW POINT TEMPERATURE, SA_T, STATUS TEMP LEVE...  SA-TSP  FP1  \n",
       "148  [SA_T, DEW POINT TEMPERATURE, STATUS TEMP LEVE...  SA-TSP  FP1  \n",
       "457  [SA_T, DEW POINT TEMPERATURE, STATUS TEMP LEVE...    OC_T  FP1  "
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "top5[top5['test']=='FP1']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "FP     427\n",
       "FP1     18\n",
       "TP      11\n",
       "TN      10\n",
       "Name: test, dtype: int64"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "t1 = check(top5,1)\n",
    "t1['test'].value_counts()"
   ]
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}