work/1_panel_design/20200305_fix_primer_info.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "As part of work for integration of Ag1000g and sequencing data, problems with reference panel files were detected:\n",
    "- insert coordinates are important, as the output data corresponds to the insert only\n",
    "- at least one primer has gap char '-', we should delete it"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "PANEL = '../../data/panel_info.csv'\n",
    "PANEL_EXT = '../../data/panel_extended_info.csv'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "panel = pd.read_csv(PANEL, index_col='amplicon')\n",
    "# remove gaps\n",
    "panel['F'] = panel.F.str.replace('-','')\n",
    "panel['R'] = panel.R.str.replace('-','')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "({'A', 'B', 'C', 'D', 'G', 'M', 'R', 'S', 'T', 'Y', 'a', 'c', 'g', 't'},\n",
       " {'A', 'C', 'G', 'R', 'S', 'T', 'V', 'W', 'Y', 'a', 'c', 'g', 't'})"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# check remaining chars\n",
    "set(panel.F.str.cat()), set(panel.R.str.cat())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>chrom</th>\n",
       "      <th>start</th>\n",
       "      <th>end</th>\n",
       "      <th>F</th>\n",
       "      <th>R</th>\n",
       "      <th>conc_multiplier</th>\n",
       "      <th>aligned_ampl_length</th>\n",
       "      <th>agam_ampl_length</th>\n",
       "      <th>aligned_insert_length</th>\n",
       "      <th>agam_insert_length</th>\n",
       "      <th>exon</th>\n",
       "      <th>gene</th>\n",
       "      <th>intron</th>\n",
       "      <th>mRNA</th>\n",
       "      <th>repeat</th>\n",
       "      <th>utr</th>\n",
       "      <th>aligned_in_outgroup</th>\n",
       "      <th>start_insert</th>\n",
       "      <th>end_insert</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>amplicon</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2L</td>\n",
       "      <td>10356417</td>\n",
       "      <td>10356608</td>\n",
       "      <td>TGTSTACGGTCTGAAGAACATc</td>\n",
       "      <td>TTATCCGGCTCCAAGTTAAGG</td>\n",
       "      <td>10.0</td>\n",
       "      <td>259</td>\n",
       "      <td>191.0</td>\n",
       "      <td>216.0</td>\n",
       "      <td>148.0</td>\n",
       "      <td>E016098A;E016099A</td>\n",
       "      <td>AGAP005134</td>\n",
       "      <td>Yes</td>\n",
       "      <td>AGAP005134-RA</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>No</td>\n",
       "      <td>10356439</td>\n",
       "      <td>10356587</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         chrom     start       end                       F  \\\n",
       "amplicon                                                     \n",
       "0           2L  10356417  10356608  TGTSTACGGTCTGAAGAACATc   \n",
       "\n",
       "                              R  conc_multiplier  aligned_ampl_length  \\\n",
       "amplicon                                                                \n",
       "0         TTATCCGGCTCCAAGTTAAGG             10.0                  259   \n",
       "\n",
       "          agam_ampl_length  aligned_insert_length  agam_insert_length  \\\n",
       "amplicon                                                                \n",
       "0                    191.0                  216.0               148.0   \n",
       "\n",
       "                       exon        gene intron           mRNA repeat   utr  \\\n",
       "amplicon                                                                     \n",
       "0         E016098A;E016099A  AGAP005134    Yes  AGAP005134-RA   None  None   \n",
       "\n",
       "         aligned_in_outgroup  start_insert  end_insert  \n",
       "amplicon                                                \n",
       "0                         No      10356439    10356587  "
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# calculate insert coordinates\n",
    "panel['start_insert'] = panel.start + panel.F.str.len()\n",
    "panel['end_insert'] = panel.end - panel.R.str.len()\n",
    "panel.head(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "amplicon,chrom,start,end,F,R,conc_multiplier,aligned_ampl_length,agam_ampl_length,aligned_insert_length,agam_insert_length,exon,gene,intron,mRNA,repeat,utr,aligned_in_outgroup,start_insert,end_insert\r\n",
      "0,2L,10356417,10356608,TGTSTACGGTCTGAAGAACATc,TTATCCGGCTCCAAGTTAAGG,10.0,259,191.0,216.0,148.0,E016098A;E016099A,AGAP005134,Yes,AGAP005134-RA,None,None,No,10356439,10356587\r\n",
      "1,2L,10357458,10357648,GAGCGtGCGGCcAAGATG,ACAgACCGACGTTAATGGC,10.0,190,190.0,153.0,153.0,E016100A,AGAP005134,None,AGAP005134-RA,None,None,Yes,10357476,10357629\r\n",
      "2,2L,12084495,12084690,CAGTCAAATTTCCAGACAATCT,CGGAAGTGCATTTGAAGGAAaA,5.8,234,195.0,190.0,152.0,None,None,None,None,None,None,,12084517,12084668\r\n",
      "3,2L,13862864,13863055,GaTATAAATTGTCGATCACACAAACT,TGCATTTATCGTAGTACAATCTCA,2.6,255,191.0,205.0,141.0,None,None,None,None,None,None,,13862890,13863031\r\n",
      "4,2L,18693481,18693714,ATGcTBGTCATgATGATGATCT,CCGATCCACGATAAGGAGTAC,10.0,270,233.0,227.0,190.0,E018545A,AGAP005681,None,AGAP005681-RA,None,None,Yes,18693503,18693693\r\n",
      "5,2L,19065171,19065363,GCTGGCGCATAATTATCaCAAA,tTTCCACTTCATCGCTCGC,10.0,269,192.0,228.0,151.0,None,None,None,None,None,None,,19065193,19065344\r\n",
      "6,2L,20840388,20840620,GCAAAATTTCCGTCCCATTA,TGTAATTAGCTGTGTCTTGTG,4.0,263,232.0,222.0,191.0,None,None,None,None,None,None,,20840408,20840599\r\n",
      "7,2L,26760700,26760890,GTcTCgGAGCACATYGTG,TCGTACTTCATTATTCTTTGGACTG,4.55,328,190.0,285.0,147.0,E020405C;E020406C;E020405B;E020406B;E020405A;E020406A,AGAP006107,Yes,AGAP006107-RC;AGAP006107-RB;AGAP006107-RA,None,None,No,26760718,26760865\r\n",
      "8,2L,27674646,27674836,AGTGRCTCCAGACGGTgTT,CCAAGGATTTGCTACTACCAcT,1.8,307,190.0,266.0,149.0,E020688A;E020689A;E020688B;E020689B,AGAP006176,Yes,AGAP006176-RA;AGAP006176-RB,None,None,,27674665,27674814\r\n"
     ]
    }
   ],
   "source": [
    "panel.to_csv(PANEL)\n",
    "! head {PANEL}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "({'A', 'B', 'C', 'D', 'G', 'M', 'R', 'S', 'T', 'Y', 'a', 'c', 'g', 't'},\n",
       " {'A', 'C', 'G', 'R', 'S', 'T', 'V', 'W', 'Y', 'a', 'c', 'g', 't'})"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# repeat for extended panel\n",
    "panel = pd.read_csv(PANEL_EXT, index_col='Primary_ID')\n",
    "panel['F'] = panel.F.str.replace('-','')\n",
    "panel['R'] = panel.R.str.replace('-','')\n",
    "set(panel.F.str.cat()), set(panel.R.str.cat())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Primary_ID,ID,SeekDeep_ID,SNP,F,R,chr,start,end,F_deg,R_deg,identical_seq,idenified_lineages,aligned_spp,unaligned_spp,aligned_ampl_length,agam_ampl_length,aligned_insert_length,agam_insert_length,total_snvs,total_indels,insert_snvs,insert_indels,prop_id_lineages,exon,gene,intron,mRNA,repeat,utr,AaegL5_identity,AaegL5_q_span,AaegL5_s_span,AgamP3_identity,AgamP3_q_span,AgamP3_s_span,BDGP6_identity,BDGP6_q_span,BDGP6_s_span,CpipJ2_identity,CpipJ2_q_span,CpipJ2_s_span,Comments,Outgroup ampl,conc_multiplier,annotation,start_insert,end_insert\r\n",
      "0,31_var,V0,high freq,TGTSTACGGTCTGAAGAACATc,TTATCCGGCTCCAAGTTAAGG,2L,10356417,10356608,TGT[G/C]TACGGTCTGAAGAACATc,TT[A/G]TCCGGCTCCAAGTTAAGG,\"[{'AsteI2', 'AsteS1'}]\",20.0,21.0,,259,191.0,216.0,148.0,33.0,139.0,30.0,139.0,0.9523809520000001,E016098A;E016099A,AGAP005134,Yes,AGAP005134-RA,None,None,0.96969697,1-66,2:204342881-204342946,1.0,1-191,2L:10356418-10356608,0.951612903,1-62,2R:22800075-22800136,0.968253968,1-63,supercont3.30:1160215-1160153,F primer perfect match in Cpip&Aaeg,No,10.0,intron,10356439,10356587\r\n",
      "1,33_other,O0,low freq,GAGCGtGCGGCcAAGATG,ACAgACCGACGTTAATGGC,2L,10357458,10357648,GAGCGtGCGGCcAAGATG,ACAgACCGACGTTAATGGC,\"[{'AquaS1', 'AgamP3', 'AmelC1', 'AaraD1', 'AgamM1', 'AmerM1', 'AgamS1'}, {'AsteI2', 'AsteS1'}, {'AfarF1', 'AdirW1'}]\",13.0,21.0,,190,190.0,153.0,153.0,36.0,0.0,33.0,0.0,0.6190476189999999,E016100A,AGAP005134,None,AGAP005134-RA,None,None,0.894736842,1-190,2:204349481-204349670,1.0,1-190,2L:10357459-10357648,,,,0.926315789,1-190,supercont3.30:1159280-1159091,\"Cpip - perfect, Aaeg - 5` subst\",Yes,10.0,exon,10357476,10357629\r\n",
      "2,44_other,O1,low freq,CAGTCAAATTTCCAGACAATCT,CGGAAGTGCATTTGAAGGAAaA,2L,12084495,12084690,CAGTCAAATTTCCAGACAATCT,CGGAAGTGCATTTGAAGG-AAaA,\"[{'AgamP3', 'AgamS1', 'AgamM1'}]\",14.0,16.0,AchrA1;AsinS1;AatrE1;AdarC2;AalbS1,234,195.0,190.0,152.0,42.0,81.0,41.0,80.0,0.875,None,None,None,None,None,None,,,,1.0,1-195,2L:12084496-12084690,,,,,,,,,5.800000000000001,intergenic,12084517,12084668\r\n",
      "3,47_other,O2,low freq,GaTATAAATTGTCGATCACACAAACT,TGCATTTATCGTAGTACAATCTCA,2L,13862864,13863055,GaTATAAATTGTCGATCACACAAACT,TGCATTTATCGTAGTACAATCTCA,\"[{'AquaS1', 'AgamP3', 'AmelC1', 'AaraD1', 'AgamM1', 'AmerM1', 'AgamS1'}, {'AminM1', 'AfunF1'}, {'AsteI2', 'AsteS1'}]\",13.0,21.0,,255,191.0,205.0,141.0,12.0,107.0,11.0,107.0,0.6190476189999999,None,None,None,None,None,None,,,,1.0,1-191,2L:13862865-13863055,,,,,,,,,2.6,intergenic,13862890,13863031\r\n",
      "4,58_cons,C0,high freq,ATGcTBGTCATgATGATGATCT,CCGATCCACGATAAGGAGTAC,2L,18693481,18693714,ATGcT[C/T/G]GTCATgATGATGATCT,CCGATCCACGATAAGGAGTAC,\"[{'AquaS1', 'AgamP3', 'AaraD1', 'AgamM1', 'AmerM1', 'AgamS1'}, {'AsteI2', 'AsteS1'}]\",15.0,21.0,,270,233.0,227.0,190.0,32.0,74.0,29.0,74.0,0.7142857140000001,E018545A,AGAP005681,None,AGAP005681-RA,None,None,0.846153846,1-233,2:74623848-74623616,1.0,1-233,2L:18693482-18693714,,,,0.834042553,1-233,supercont3.137:804142-804374,5`subst in Aaeg&Cpip,Yes,10.0,exon,18693503,18693693\r\n",
      "5,59_other,O3,low freq,GCTGGCGCATAATTATCaCAAA,tTTCCACTTCATCGCTCGC,2L,19065171,19065363,GCTGGCGCATAATTATCaCAAA,tTTCCACTTCATCGCTCGC,\"[{'AsteI2', 'AsteS1'}]\",15.0,16.0,AfarF1;AsinS1;AatrE1;AdarC2;AalbS1,269,192.0,228.0,151.0,43.0,135.0,41.0,135.0,0.9375,None,None,None,None,None,None,,,,1.0,1-192,2L:19065172-19065363,,,,,,,,,10.0,intergenic,19065193,19065344\r\n",
      "6,65_other,O4,None,GCAAAATTTCCGTCCCATTA,TGTAATTAGCTGTGTCTTGTG,2L,20840388,20840620,GCAAAATTTCCGTCCCATTA,TGTAATTAGCTGTGTCTTGTG,[],15.0,15.0,AfarF1;AdirW1;AsinS1;AatrE1;AdarC2;AalbS1,263,232.0,222.0,191.0,30.0,124.0,30.0,124.0,1.0,None,None,None,None,None,None,,,,1.0,1-232,2L:20840389-20840620,,,,,,,,,4.0,intergenic,20840408,20840599\r\n",
      "7,77_var,V1,high freq,GTcTCgGAGCACATYGTG,TCGTACTTCATTATTCTTTGGACTG,2L,26760700,26760890,GTcTCgGAGCACAT[C/T]GTG,TCGTACTTCATTATTCTTTGGACTG,\"[{'AgamP3', 'AgamM1'}]\",20.0,21.0,,328,190.0,285.0,147.0,46.0,157.0,43.0,157.0,0.9523809520000001,E020405C;E020406C;E020405B;E020406B;E020405A;E020406A,AGAP006107,Yes,AGAP006107-RC;AGAP006107-RB;AGAP006107-RA,None,None,0.951807229,1-83,2:205452944-205453026,1.0,1-190,2L:26760701-26760890,,,,,,,No R ,No,4.55,intron,26760718,26760865\r\n",
      "8,83_var,V2,high freq,AGTGRCTCCAGACGGTgTT,CCAAGGATTTGCTACTACCAcT,2L,27674646,27674836,AGTG[A/G]CTCCAGACGGTgTT,CC[A/T/G]AGGATTTGCTACTACCAcT,\"[{'AsteI2', 'AsteS1'}]\",20.0,21.0,,307,190.0,266.0,149.0,31.0,178.0,27.0,178.0,0.9523809520000001,E020688A;E020689A;E020688B;E020689B,AGAP006176,Yes,AGAP006176-RA;AGAP006176-RB,None,None,,,,1.0,1-190,2L:27674647-27674836,,,,,,,,,1.8,intron,27674665,27674814\r\n"
     ]
    }
   ],
   "source": [
    "panel['start_insert'] = panel.start + panel.F.str.len()\n",
    "panel['end_insert'] = panel.end - panel.R.str.len()\n",
    "panel.to_csv(PANEL_EXT)\n",
    "! head {PANEL_EXT}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}