Sigma_beta optimization and rerun

banskt · Feb 9, 2020 · e0a3552 · e0a3552
1 parent 6118451
commit e0a3552
Show file tree

Hide file tree

Showing 69 changed files with 1,966 additions and 544 deletions.
diff --git a/.gitignore b/.gitignore
@@ -2,25 +2,21 @@
 jobsubs
 .ipynb_checkpoints
 __pycache__
-analysis/jupyter/cardiogenics_mono_macro
-analysis/jupyter/*.pkl
+main/archive
+analysis/jupyter/archive
+preprocess/gtex/archive
+*.pkl
 analysis/external/DHS_Index_and_Vocabulary_hg38_WM20190703.txt*
-analysis/external/multi-tissue.master.ntypes.simple.hg19*
-analysis/external/multi-tissue.master.ntypes.simple.hg38_hglft_genome_620db_fada10*
+analysis/external/multi-tissue.master.ntypes.simple.*
+analysis/external/human_protein_coding_genes_gencode_v26.txt
 analysis/external/tissue_specific_TFs
-main/tissue_table.txt
-main/01a_resubmit_tejaas_rand.sh
-main/00b_copy_expression.sh
-main/tmp_postprocess
+analysis/external/segway_encyclopedia
+analysis/external/roadmap_chromhmm
 analysis/plots
-preprocess/gtex/slurmfiles/add_DSentry_*
+analysis/cis-eGene_biotype_enrichment
+analysis/cre_enrichment
 analysis/dhs_enrichment
 analysis/distance_from_pcgenes
 analysis/distance_from_transcription_factor
-analysis/cre_enrichment
-analysis/external/human_protein_coding_genes_gencode_v26.txt
-analysis/external/segway_encyclopedia
-analysis/external/roadmap_chromhmm
-analysis/roadmap*
-analysis/segway_with_lines_01.txt
+analysis/gencode_annotation_enrichment
 analysis/tfdist_enrichment_wrt_genes.py
diff --git a/analysis/cre_enrichment_tissue_specific_chromhmm_withpval.py b/analysis/cre_enrichment_tissue_specific_chromhmm_withpval.py
@@ -62,6 +62,13 @@ def parse_args():
                         metavar='DIR',
                         help='name of the result directory')
 
+
+    parser.add_argument('--resfile',
+                        type=str,
+                        dest='resfile',
+                        metavar='FILE',
+                        help='name of the result file (without directory path)')
+
     parser.add_argument('--match',
                         type=str,
                         dest='matchfile',
@@ -209,6 +216,7 @@ def empr_overlap(rand_dicts, dhsdir, eidlist, labeldict, outdir):
 if __name__ == '__main__':
     opts = parse_args()
     resdir = opts.resdir
+    resfile = opts.resfile
     dhsdir = opts.dhsdir
     matchfile = opts.matchfile
     eidfile = opts.eidfile
@@ -222,7 +230,7 @@ def empr_overlap(rand_dicts, dhsdir, eidlist, labeldict, outdir):
 
     tissue = opts.tissuename
 
-    resfilename = os.path.join(resdir, tissue, 'trans_eqtls.txt')
+    resfilename = os.path.join(resdir, tissue, resfile)
     print("Reading trans-eQTL results.")
     transeqtls = read_tejaas_results.transeqtls(resfilename)
     nteqtl = len(transeqtls)

diff --git a/analysis/dhs_enrichment.py b/analysis/dhs_enrichment.py
@@ -130,12 +130,23 @@ def generate_empirical_dhs_enrichment(dirname, dhs_file, dhs_frac_rand):
 print(f'Fraction of DHS overlap for randomly selected SNPs: {dhs_frac_rand :7.4f}')
 enrichment_rand = generate_empirical_dhs_enrichment(random_snp_dir, dhsfile, dhs_frac_rand)
 
+#import pickle
+##filehandler = open('dhs_frac_rand.pkl', 'wb')
+##pickle.dump(dhs_frac_rand, filehandler)
+##print(dhs_frac_rand)
+##filehandler = open('enrichment_rand.pkl', 'wb')
+##pickle.dump(enrichment_rand, filehandler)
+#dhs_frac_rand = pickle.load( open( "dhs_frac_rand.pkl", "rb" ) )
+#enrichment_rand = pickle.load( open( "enrichment_rand.pkl", "rb" ) )
+#print(f'Fraction of DHS overlap for randomly selected SNPs: {dhs_frac_rand :7.4f}')
+
 fout = open(outfile, 'w')
 fout.write(f'TISSUE\tN_TRANSEQTLS\tDHS_FRAC\tENRICHMENT\tP_VALUE\n')
 
 for tissue in tissuelist:
 
     resfilename = os.path.join(resdir, tissue, 'trans_eqtls.txt')
+    #resfilename = os.path.join(resdir, tissue, 'trans_eqtls_mod_ld1.txt')
     transeqtls = read_tejaas_results.transeqtls(resfilename)
     print(f'{tissue}: {len(transeqtls)} trans-eQTLs')
     nteqtl = len(transeqtls)

diff --git a/analysis/jupyter/archive/MWE_corrections_in_gene_expression_breaks_null_model.ipynb b/analysis/jupyter/archive/MWE_corrections_in_gene_expression_breaks_null_model.ipynb
@@ -185,7 +185,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 209,
+   "execution_count": 2,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -1138,7 +1138,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.6.8"
+   "version": "3.7.5"
   }
  },
  "nbformat": 4,

diff --git a/analysis/jupyter/archive/MWE_knn.ipynb b/analysis/jupyter/archive/MWE_knn.ipynb
@@ -18,7 +18,7 @@
     "#plt.switch_backend('agg')\n",
     "import matplotlib\n",
     "import sys\n",
-    "sys.path.append('../')\n",
+    "sys.path.append('../../')\n",
     "from utils import mpl_stylesheet\n",
     "mpl_stylesheet.banskt_presentation(fontfamily = 'latex-clearsans', fontsize = 18, colors = 'banskt', dpi = 72)"
    ]

diff --git a/analysis/jupyter/archive/check_Qmean_Qstd.ipynb b/analysis/jupyter/archive/check_Qmean_Qstd.ipynb
@@ -446,7 +446,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.6.8"
+   "version": "3.7.5"
   }
  },
  "nbformat": 4,

diff --git a/analysis/jupyter/archive/check_linear_model_correction.ipynb b/analysis/jupyter/archive/check_linear_model_correction.ipynb
@@ -343,7 +343,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.6.8"
+   "version": "3.7.5"
   }
  },
  "nbformat": 4,

diff --git a/analysis/jupyter/archive/compare_shuffled_covariate_effect.ipynb b/analysis/jupyter/archive/compare_shuffled_covariate_effect.ipynb
@@ -373,7 +373,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.6.8"
+   "version": "3.7.5"
   }
  },
  "nbformat": 4,

diff --git a/analysis/jupyter/archive/norm_is_the_problem.ipynb b/analysis/jupyter/archive/norm_is_the_problem.ipynb
@@ -221,7 +221,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.6.8"
+   "version": "3.7.5"
   }
  },
  "nbformat": 4,

diff --git a/analysis/jupyter/archive/what_is_happening_in_real_data.ipynb b/analysis/jupyter/archive/what_is_happening_in_real_data.ipynb
@@ -453,7 +453,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.6.8"
+   "version": "3.7.5"
   }
  },
  "nbformat": 4,

diff --git a/analysis/jupyter/distance_from_human_TFs.ipynb b/analysis/jupyter/distance_from_human_TFs.ipynb
@@ -586,7 +586,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.6.8"
+   "version": "3.7.5"
   }
  },
  "nbformat": 4,

diff --git a/analysis/jupyter/distance_from_human_TFs_with_random_sampling_from_genotypes.ipynb b/analysis/jupyter/distance_from_human_TFs_with_random_sampling_from_genotypes.ipynb
diff --git a/analysis/jupyter/manhattan_plot.ipynb b/analysis/jupyter/manhattan_plot.ipynb
diff --git a/analysis/jupyter/plot_Qscore_vs_pvalue.ipynb b/analysis/jupyter/plot_Qscore_vs_pvalue.ipynb
diff --git a/analysis/jupyter/plot_cistf_enrichment.ipynb b/analysis/jupyter/plot_cistf_enrichment.ipynb
@@ -174,7 +174,7 @@
     "    ax1.text(xpos, ypos, f'{significance[i]}', ha='left', va='center')\n",
     "\n",
     "fig.canvas.draw()\n",
-    "xticklabels = [f'{x.get_position()[0] + 1 :3.1f}' for x in ax1.get_xticklabels()]\n",
+    "xticklabels = [f'{x.get_position()[0]:3.1f}' for x in ax1.get_xticklabels()]\n",
     "ax1.set_xticklabels(xticklabels)\n",
     "\n",
     "#plt.savefig(outfile, bbox_inches='tight')\n",
@@ -205,7 +205,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.6.8"
+   "version": "3.7.5"
   }
  },
  "nbformat": 4,

diff --git a/analysis/jupyter/plot_dhs_enrichment.ipynb b/analysis/jupyter/plot_dhs_enrichment.ipynb
@@ -199,7 +199,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.6.8"
+   "version": "3.7.5"
   }
  },
  "nbformat": 4,

diff --git a/analysis/jupyter/plot_gtex_tissue_specific_trans-eqtls_overview.ipynb b/analysis/jupyter/plot_gtex_tissue_specific_trans-eqtls_overview.ipynb
@@ -24,7 +24,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.6.8"
+   "version": "3.7.5"
   }
  },
  "nbformat": 4,

diff --git a/analysis/jupyter/plot_numbers_only.ipynb b/analysis/jupyter/plot_numbers_only.ipynb
@@ -275,7 +275,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.6.8"
+   "version": "3.7.5"
   }
  },
  "nbformat": 4,

diff --git a/analysis/jupyter/plot_tissue_specific_cre.ipynb b/analysis/jupyter/plot_tissue_specific_cre.ipynb
@@ -739,7 +739,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.6.8"
+   "version": "3.7.5"
   }
  },
  "nbformat": 4,

diff --git a/analysis/jupyter/plot_transeqtls_overlap.ipynb b/analysis/jupyter/plot_transeqtls_overlap.ipynb
@@ -120,7 +120,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.6.8"
+   "version": "3.7.5"
   }
  },
  "nbformat": 4,