data_extractor.py

#!/usr/bin/python !/usr/bin/env python
# -*- coding: utf-8 -*


# Functions to extract knowledge from medical text. Everything related to
# extraction needed for the knowledge base. Also, some wrappers for SemRep,
# MetaMap and Reverb. Contains some enrichment routines for utilizing UTS
# services.


import json
import subprocess
import urllib2
import pymongo
import numpy as np
from nltk.tokenize import sent_tokenize
from config import settings
from pymetamap import MetaMap
from utilities import time_log, get_concept_from_cui, get_concept_from_source
from itertools import product
from multiprocessing import cpu_count, Pool
from unidecode import unidecode

def metamap_wrapper(text):
    """
    Function-wrapper for metamap binary. Extracts concepts
    found in text.

    !!!! REMEMBER TO START THE METAMAP TAGGER AND
        WordSense DISAMBIGUATION SERVER !!!!

    Input:
        - text: str,
        a piece of text or sentence
    Output:
       - a dictionary with key sents and values
       a list of the concepts found
    """

    # Tokenize into sentences
    sents = sent_tokenize(text)
    # Load Metamap Instance
    mm = MetaMap.get_instance(settings['load']['path']['metamap'])
    concepts, errors = mm.extract_concepts(sents, range(len(sents)))
    # Keep the sentence ids
    ids = np.array([int(concept[0]) for concept in concepts])
    sentences = []
    for i in xrange(len(sents)):
        tmp = {'sent_id': i+1, 'entities': [], 'relations': []}
        # Wanted concepts according to sentence
        wanted = np.where(ids == i)[0].tolist()
        for w_ind in wanted:
            w_conc = concepts[w_ind]
            if hasattr(w_conc, 'cui'):
                tmp_conc = {'label': w_conc.preferred_name, 'cui': w_conc.cui,
                            'sem_types': w_conc.semtypes, 'score': w_conc.score}
                tmp['entities'].append(tmp_conc)
        sentences.append(tmp)
    if errors:
        time_log('Errors with extracting concepts!')
        time_log(errors)
    return {'sents': sentences, 'sent_text':text}


def runProcess(exe, working_dir):
    """
    Function that opens a command line and runs a command.
    Captures the output and returns.
    Input:
        - exe: str,
        string of the command to be run. ! REMEMBER TO ESCAPE CHARS!
        - working_dir: str,
        directory where the cmd should be executed
    Output:
        - lines: list,
        list of strings generated from the command
    """

    p = subprocess.Popen(exe, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, cwd=working_dir, shell=True)
    lines = p.stdout.readlines()
    return lines


def stopw_removal(inp, stop):
    """
    Stopwords removal in line of text.
    Input:
        - inp: str,
        string of the text input
        - stop: list,
        list of stop-words to be removed
    """

    # Final string to be returned
    final = ''
    for w in inp.lower().split():
        if w not in stop:
            final += w + ' '
    # Remove last whitespace that was added ' '
    final = final[:-1]
    return final


def create_text_batches(text, N=5000, buffer_ = 100):
    """
    Function that takes a long string and split it into
    batches of approximately length N. The actual length
    of each batch differs, as each batch end in the next
    dot found in the string after the N chars.
    Input:
        - text: str,
        piece of text to clean
        - N: int,
        split into strings of 5000 characters each
    Output:
        - chunks: list,
        list containing the string parts
    """
    M = len(text)
    chunks_num = M // N
    if M % N != 0:
        chunks_num += 1
    chunks = []
    end_ind = 0
    start_ind = 0
    i = 0
    while i < chunks_num:
        start_ind = end_ind
        prob_text = text[start_ind + N: start_ind + N + buffer_]
        if '.' in prob_text:
            end_ind = start_ind + N + prob_text.index('.')+1
        else:
            end_ind = start_ind + N
        chunks.append(text[start_ind:end_ind])
        i += 1
    chunks = [ch for ch in chunks if ch]
    return chunks



def reverb_wrapper(text, stop=None):
    """
    Function-wrapper for ReVerb binary. Extracts relations
    found in text.
    Input:
        - text: str,
        a piece of text or sentence
        - stop: list,
        list of stopwords to remove from the relations
    Output:
        - total: list,
        list of lists. Each inner list contains one relation in the form
        [subject, predicate, object]
    """
    total = []
    for sent in sent_tokenize(text):
        cmd = 'echo "' + sent + '"' "| ./reverb -q | tr '\t' '\n' | cat -n"
        reverb_dir = settings['load']['path']['reverb']
        result = runProcess(cmd, reverb_dir)
        # Extract relations from reverb output
        result = result[-3:]
        result = [row.split('\t')[1].strip('\n') for row in result]
        # Remove common stopwords from relations
        if stop:
            result = [stopw_removal(res, stop) for res in result]
        total.append(result)
    # Remove empty relations
    total = [t for t in total if t]
    return total




def cui_to_uri(api_key, cui):
    """
    Function to map from cui to uri if possible. Uses biontology portal
    Input:
        - api_key: str,
        api usage key change it in setting.yaml
        - cui: str,
        cui of the entity we wish to map the uri
    Output:
        - the uri found in string format or None
    """

    REST_URL = "http://data.bioontology.org"
    annotations = get_json_with_api(api_key, REST_URL + "/search?include_properties=true&q=" + urllib2.quote(cui))
    try:
        return annotations['collection'][0]['@id']
    except Exception, e:
        time_log(Exception)
        time_log(e)
        return None

def get_json_with_api(api_key, url):
    """
    Helper funtion to retrieve a json from a url through urlib2
    Input:
        - api_key: str,
        api usage key change it in setting.yaml
        - url: str,
        url to curl
    Output:
        - json-style dictionary with the curl results
    """

    opener = urllib2.build_opener()
    opener.addheaders = [('Authorization', 'apikey token=' + api_key)]
    return json.loads(opener.open(url).read())


def threshold_concepts(concepts, hard_num=3, score=None):
    """
    Thresholding concepts from metamap to keep only the most probable ones.
    Currently supporting thresholding on the first-N (hard_num) or based on
    the concept score.
    Input:
        - concepts: list,
        list of Metamap Class concepts
        - hard_num: int,
        the first-N concepts to keep, if this thresholidng is selected
        - score: float,
        lowest accepted concept score, if this thresholidng is selected
    """

    if hard_num:
        if hard_num >= len(concepts):
            return concepts
        elif hard_num < len(concepts):
            return concepts[:hard_num]
    elif score:
            return [c for c in concepts if c.score > score]
    else:
        return concepts




def get_name_concept(concept):
    """
    Get name from the metamap concept. Tries different variations and
    returns the name found.
    Input:
        - concept: Metamap class concept, as generated from mmap_extract
        for example
    Output:
        - name: str,
        the name found for this concept
    """

    name = ''
    if hasattr(concept, 'preferred_name'):
        name = concept.preferred_name
    elif hasattr(concept, 'long_form') and hasattr(concept, 'short_form'):
        name = concept.long_form + '|' + concept.short_form
    elif hasattr(concept, 'long_form'):
        name = concept.long_form
    elif hasattr(concept, 'short_form'):
        name =  concept.short_form
    else:
        name = 'NO NAME IN CONCEPT'
    return name



def metamap_ents(x):
    """
    Function to get entities in usable form.
    Exctracts metamap concepts first, thresholds them and
    tries to extract names and uris for the concepts to be
    more usable.
    Input:
        - x: str,
        sentence to extract entities
    Output:
        - ents: list,
        list of entities found. Each entity is a dictionary with
        fields id (no. found in sentence), name if retrieved, cui if
        available and uri if found
    """

    # API KEY to biontology mapping from cui to uri
    API_KEY = settings['apis']['biont']
    concepts = mmap_extract(x)
    concepts = threshold_concepts(concepts)
    ents = []
    for i, concept in enumerate(concepts):
        ent = {}
        ent['ent_id'] = i
        ent['name'] = get_name_concept(concept)
        if hasattr(concept, 'cui'):
            ent['cui'] = concept.cui
            ent['uri'] = cui_to_uri(API_KEY, ent['cui'])
        else:
            ent['cui'] = None
            ent['uri'] = None
        ents.append(ent)
    return ents


def extract_entities(text, json_={}):
    """
    Extract entities from a given text using metamap and
    generate a json, preserving infro regarding the sentence
    of each entity that was found. For the time being, we preserve
    both concepts and the entities related to them
    Input:
         - text: str,
        a piece of text or sentence
        - json_: dic,
        sometimes the json to be returned is given to us to be enriched
        Defaults to an empty json_
    Output:
        - json_: dic,
        json with fields text, sents, concepts and entities
        containg the final results
    """
    json_['text'] = text
    # Tokenize the text
    sents = sent_tokenize(text)
    json_['sents'] = [{'sent_id': i, 'sent_text': sent} for i, sent in enumerate(sents)]
    json_['concepts'], _ = mmap_extract(text)
    json_['entities'] = {}
    for i, sent in enumerate(json_['sents']):
        ents = metamap_ents(sent)
        json_['entities'][sent['sent_id']] = ents
    return json_

def extract_metamap(json_, key):
    """
    Task function to parse and extract concepts from json_ style dic, using
    the MetaMap binary.
    Input:
        - json_ : dic,
        json-style dictionary generated from the Parse object related
        to the specific type of input
        - key : str,
        string denoting the type of medical text to read from. Used to
        find the correct paragraph in the settings.yaml file.
    Output:
        - json_ : dic,
        the previous json-style dictionary enriched with medical concepts
    """
    # outerfield for the documents in json
    docfield = settings['out']['json']['itemfield']
    # textfield to read text from
    textfield = settings['out']['json']['json_text_field']
    N = len(json_[docfield])
    for i, doc in enumerate(json_[docfield]):
        text = clean_text(doc[textfield])
        if len(text) > 5000:
            chunks = create_text_batches(text)
            results = {'text': text, 'sents': []}
            sent_id = 0
            for chunk in chunks:
                tmp = metamap_wrapper(chunk)
                for sent in tmp['sents']:
                    sent['sent_id'] = sent_id
                    sent_id += 1
                    results['sents'].append(sent)
        else:
            results = metamap_wrapper(text)
        json_[docfield][i].update(results)
        proc = int(i/float(N)*100)
        if proc % 10 == 0 and proc > 0:
            time_log('We are at %d/%d documents -- %0.2f %%' % (i, N, proc))
    return json_


def enrich_with_triples(results, subject, pred='MENTIONED_IN'):
    """
    Enrich with rdf triples a json dictionary in the form of:
    entity-URI -- MENTIONED_IN -- 'Text 'Title'. Only entities with
    uri's are considered.
    Input:
        - results: dic,
        json-style dictionary genereated from the extract_entities function
        - subject: str,
        the name of the text document in which the entities are mentioned
        - pred: str,
        the predicate to be used as a link between the uri and the title
    Output:
        - results: dic,
        the same dictionary with one more
    """
    triples = []
    for sent_key, ents in results['entities'].iteritems():
        for ent in ents:
            if ent['uri']:
               triples.append({'subj': ent['uri'], 'pred': pred, 'obj': subject})
    results['triples'] = triples
    return results

def force_to_unicode(text):
    "If text is unicode, it is returned as is. If it's str, convert it to Unicode using UTF-8 encoding"
    return text if isinstance(text, unicode) else text.decode('utf8', 'ignore')


def toAscii_wrapper(text):
    """
    Function wrapper for Lexical Tool toAscii:
    https://lexsrv3.nlm.nih.gov/LexSysGroup/Projects/lvg/current/docs/userDoc/tools/toAscii.html
    Converts input to ascii ready for SemRep
    Input:
        - text: str,
        a piece of text or sentence'
    Output:
        - text: str,
        the same text with changes
    """
    text = clean_text(text)
    #text = repr(text)
    cmd = 'echo "' + text + '" | ./toAscii'
    toAscii_dir = settings['load']['path']['toAscii']
    lines = runProcess(cmd, toAscii_dir)
    return lines[0]

def semrep_wrapper(text):
    """
    Function wrapper for SemRep binary. It is called with flags
    -F only and changing this will cause this parsing to fail, cause
    the resulting lines won't have the same structure.
    Input:
        - text: str,
        a piece of text or sentence
    Output:
        - results: dic,
        jston-style dictionary with fields text and sents. Each
        sentence has entities and relations found in it. Each entity and
        each relation has attributes denoted in the corresponding
        mappings dictionary.
    """
    # Exec the binary
    # THIS SHOULD FIX ENCODING PROBLEMS???
    text = clean_text(text)
    utf8 = force_to_unicode(text)
    text = unidecode(utf8)
    # text = toAscii_wrapper(text)
    # THIS IS NEEDED FOR ANY ARTIFACTS!
    text = repr(text)
    cmd = "echo " + text + " | ./semrep.v1.7 -L 2015 -Z 2015AA -F"
    #print cmd
    semrep_dir = settings['load']['path']['semrep']
    lines = runProcess(cmd, semrep_dir)
    #print(lines)
    # mapping of line elements to fields
    mappings = {
        "text": {
            "sent_id": 4,
            "sent_text": 6
        },
        "entity": {
            'cuid': 6,
            'label': 7,
            'sem_types': 8,
            'score': 15
        },
        "relation": {
            'subject__cui': 8,
            'subject__label': 9,
            'subject__sem_types': 10,
            'subject__sem_type': 11,
            'subject__score': 18,
            'predicate__type': 21,
            'predicate': 22,
            'negation': 23,
            'object__cui': 28,
            'object__label': 29,
            'object__sem_types': 30,
            'object__sem_type': 31,
            'object__score': 38,
        }
    }
    results = {'sents': [], 'text': text}
    for line in lines:
        # If Sentence
        if line.startswith('SE'):
            ##### DEPRECATED AS IN CLEAN TEXT WE REMOVE TABS FROM TEXT #######
            # Temporary workaround to remove read |-delimited semrep output
            # Without mixing up tabs contained in the text
            # line = line.replace('\|', '!@#$')
            # elements = line.split('|')
            # elements = [el.replace('!@#$', '\|') for el in elements]
            #########################  DEPRECATED ###########################
            elements = line.split('|')
            # New sentence that was processed
            if elements[5] == 'text':
                tmp = {"entities": [], "relations": []}
                for key, ind in mappings['text'].iteritems():
                    tmp[key] = elements[ind]
                results['sents'].append(tmp)
            # A line containing entity info
            if elements[5] == 'entity':
                tmp = {}
                for key, ind in mappings['entity'].iteritems():
                    if key == 'sem_types':
                        tmp[key] = elements[ind].split(',')
                    tmp[key] = elements[ind]
                results['sents'][-1]['entities'].append(tmp)
            # A line containing relation info
            if elements[5] == 'relation':
                tmp = {}
                for key, ind in mappings['relation'].iteritems():
                    if 'sem_types' in key:
                        tmp[key] = elements[ind].split(',')
                    else:
                        tmp[key] = elements[ind]
                results['sents'][-1]['relations'].append(tmp)
    return results


def clean_text(text):
    """
    Escape specific characters for command line call of SemRep. This
    could be updated in the future to more sophisticated transformations.
    Input:
        - text: str,
        piece of text to clean
    Output:
        - text: str,
        the same text with cmd escaped parenthesis and removing '
    """
    replace_chars = [('(', ' '), (')', ' '), ("'",  ' '), ('\n', " "), ('\t', ' '), (';', " "),
                     ("}", " "), ("{", " "), ("|", " "), ("&", " "), ("/", ' ')]
    for unw_pair in replace_chars:
        text = text.replace(unw_pair[0], unw_pair[1])
    text = ' '.join(text.split())
    return text


def extract_semrep(json_, key):
    """
    Task function to parse and extract concepts from json_ style dic, using
    the SemRep binary.
    Input:
        - json_ : dic,
        json-style dictionary generated from the Parse object related
        to the specific type of input
        - key : str,
        string denoting the type of medical text to read from. Used to
        find the correct paragraph in the settings.yaml file.
    Output:
        - json_ : dic,
        the previous json-style dictionary enriched with medical concepts
    """
    # outerfield for the documents in json
    if key == 'mongo':
        key = 'json'
    docfield = settings['out']['json']['itemfield']
    # textfield to read text from
    textfield = settings['out']['json']['json_text_field']
    N = len(json_[docfield])
    for i, doc in enumerate(json_[docfield]):
        print doc['id']
        text = doc[textfield]
        if len(text) > 5000:
            chunks = create_text_batches(text)
            results = {'text': text, 'sents': []}
            sent_id = 0
            c = 0
            for chunk in chunks:
                c += 1
                tmp = semrep_wrapper(chunk)
                for sent in tmp['sents']:
                    sent['sent_id'] = sent_id
                    sent_id += 1
                    results['sents'].append(sent)
        else:
            results = semrep_wrapper(text)
        json_[docfield][i].update(results)
        proc = int(i/float(N)*100)
        if proc % 10 == 0 and proc > 0:
            time_log('We are at %d/%d documents -- %0.2f %%' % (i, N, proc))
    return json_




def extract_semrep_parallel(json_, key):
    """
    Task function to parse and extract concepts from json_ style dic, using
    the SemRep binary. It uses multiprocessing for efficiency.
    Input:
        - json_ : dic,
        json-style dictionary generated from the Parse object related
        to the specific type of input
        - key : str,
        string denoting the type of medical text to read from. Used to
        find the correct paragraph in the settings.yaml file.
    Output:
        - json_ : dic,
        the previous json-style dictionary enriched with medical concepts
    """
    # outerfield for the documents in json
    docfield = settings['out']['json']['itemfield']
    N = len(json_[docfield])
    try:
        N_THREADS = int(settings['num_cores'])
    except:
        N_THREADS = cpu_count()
    batches = chunk_document_collection(json_[docfield], N_THREADS)
    len_col = " | ".join([str(len(b)) for b in batches])
    time_log('Will break the collection into batches of: %s documents!' % len_col)
    batches = [{docfield: batch} for batch in batches]
    data = zip(batches, [key for batch in batches])
    pool = Pool(N_THREADS, maxtasksperchild=1)
    res = pool.map(semrep_parallel_worker, data)
    pool.close()
    pool.join()
    del pool
    tmp = {docfield: []}
    for batch_res in res:
        tmp[docfield].extend(batch_res[docfield])
    for i, sub_doc in enumerate(json_[docfield]):
        for sub_doc_new in tmp[docfield]:
            if sub_doc_new['id'] == sub_doc['id']:
                json_[docfield][i].update(sub_doc_new)
                break
    time_log('Completed multiprocessing extraction!')
    return json_


def chunk_document_collection(seq, num):
    """
    Helper function to break a collection of N = len(seq) documents
    to num batches.
    Input:
        - seq: list,
        a list of documents
        - num: int,
        number of batches to be broken into. This will usually be
        equal to the number of cores available
    Output:
        - out: list,
        a list of lists. Each sublist contains the batch-collection
        of documents to be used.
    """
    avg = len(seq) / float(num)
    out = []
    last = 0.0

    while last < len(seq):
        out.append(seq[int(last):int(last + avg)])
        last += avg

    return out


def semrep_parallel_worker((json_, key)):
    """
    Just a worker interface for the different SemRep
    executions.
    Input:
        - json_ : dic,
        json-style dictionary generated from the Parse object related
        to the specific type of input
        - key : str,
        string denoting the type of medical text to read from. Used to
        find the correct paragraph in the settings.yaml file.
    Output:
        - res : dic,
        the previous json-style dictionary enriched with medical concepts

    """
    res = extract_semrep(json_, key)
    return res



def get_concepts_from_edges_parallel(json_, key):
    """
    Same work as the get_concepts_from_edges_paralle. It uses multiprocessing
    for efficiency.
    Input:
        - json: dict,
        json-style dictionary with a field containing
        relations
        - key : str,
        string denoting the type of medical text to read from. Used to
        find the correct paragraph in the settings.yaml file.
    Output:
        - json: dict,
        the updated json-style dictionary where the relations
        in the list have been updated and each subject-object has been
        mapped to the according

    """
    outfield = settings['load'][key]['itemfield']
    N = len(json_[outfield])
    try:
        N_THREADS = int(settings['num_cores'])
    except:
        N_THREADS = cpu_count()
    batches = chunk_document_collection(json_[outfield], N_THREADS)
    len_col = " | ".join([str(len(b)) for b in batches])
    time_log('Will break the edges into batches of: %s documents!' % len_col)
    batches = [{outfield: batch} for batch in batches]
    data = zip(batches, [key for batch in batches])
    pool = Pool(N_THREADS, maxtasksperchild=1)
    res = pool.map(edges_parallel_worker, data)
    pool.close()
    pool.join()
    del pool
    json_ = {outfield: []}
    for batch_res in res:
        json_[outfield].extend(batch_res[outfield])
    time_log('Completed multiprocessing extraction!')
    return json_




def edges_parallel_worker((json_, key)):
    """
    Just a worker interface for the parallel enrichment
    executions.
    Input:
        - json_ : dic,
        json-style dictionary generated from the Parse object related
        to the specific type of input
        - key : str,
        string denoting the type of medical text to read from. Used to
        find the correct paragraph in the settings.yaml file.
    Output:
        - res : dic,
        expected outcome of get_concepts_from_edges

    """
    res = get_concepts_from_edges(json_, key)
    return res


def get_concepts_from_edges(json_, key):
    """
    Get concept-specific info related to an entity from a list
    containing relations. Each subject-object in the relations
    list is expressed in a another data source(MESH, DRUGBANK etc)
    and their unique identifier is provided. Also, articles and new
    kinde of sub-obj are handled.
    Input:
        - json: dict,
        json-style dictionary with a field containing
        relations
        - key : str,
        string denoting the type of medical text to read from. Used to
        find the correct paragraph in the settings.yaml file.
    Output:
        - json: dict,
        the updated json-style dictionary where the relations
        in the list have been updated and each subject-object has been
        mapped to the according

    """

    # docfield containing list of elements containing the relations
    outfield = settings['load'][key]['itemfield']
    # field containing the type of the node for the subject
    sub_type = settings['load'][key]['sub_type']
    # field containing the source of the node for the subject
    sub_source = settings['load'][key]['sub_source']
    # field containing the type of the node for the object
    obj_type = settings['load'][key]['obj_type']
    # field containing the source of the node for the object
    obj_source = settings['load'][key]['obj_source']
    new_relations = []
    uri = settings['load']['mongo']['uri']
    db_name = settings['load']['mongo']['db']
    collection_name = settings['load']['mongo']['cache_collection']
    client = pymongo.MongoClient(uri)
    db = client[db_name]
    collection = db[collection_name]
    cur = collection.find({})
    cache = {}
    for item in cur:
        cache[item['key']] = item['value']
    N = len(json_[outfield])
    for ii, triple in enumerate(json_[outfield]):
        print triple
        try:
            if sub_source == 'UMLS':
                if not(triple['s'] in cache):
                    ent = get_concept_from_cui(triple['s'])
                    cache[triple['s']] = ent
                    collection.insert_one({'key':triple['s'],'value':ent})
                    print 'INSERTED in UMLS %s' % triple['s']
                else:
                    ent = cache[triple['s']]
                if (type(ent['sem_types']) == list and len(ent['sem_types']) > 1):
                    sem_types = ';'.join(ent['sem_types'])
                elif (',' in ent['sem_types']):
                    sem_types = ';'.join(ent['sem_types'].split(','))
                else:
                    sem_types = ent['sem_types']

                triple_subj = [{'id:ID': ent['cuid'],
                                'label': ent['label'],
                                'sem_types:string[]': sem_types}]
            elif (sub_source == 'PMC') or (sub_source == 'TEXT') or (sub_source == 'None'):
                triple_subj = [{'id:ID': triple['s']}]
            else:
                if not(triple['s'] in cache):
                    ents = get_concept_from_source(triple['s'], sub_source)
                    cache[triple['s']] = ents
                    collection.insert_one({'key':triple['s'],'value':ents})
                    print 'INSERTED in other %s' % triple['s']
                else:
                    ents = cache[triple['s']]
                triple_subj = []
                for ent in ents:
                    if (type(ent['sem_types']) == list and len(ent['sem_types']) > 1):
                        sem_types = ';'.join(ent['sem_types'])
                    elif (',' in ent['sem_types']):
                        sem_types = ';'.join(ent['sem_types'].split(','))
                    else:
                        sem_types = ent['sem_types']

                    triple_subj.append({'id:ID': ent['cuid'],
                                    'label': ent['label'],
                                    'sem_types:string[]': sem_types})
            if obj_source == 'UMLS':
                if not(triple['o'] in cache):
                    ent = get_concept_from_cui(triple['o'])
                    cache[triple['o']] = ent
                    collection.insert_one({'key':triple['o'],'value':ent})
                    print 'INSERTED in UMLS %s' % triple['o']
                else:
                    ent = cache[triple['o']]
                if (type(ent['sem_types']) == list and len(ent['sem_types']) > 1):
                    sem_types = ';'.join(ent['sem_types'])
                elif (',' in ent['sem_types']):
                    sem_types = ';'.join(ent['sem_types'].split(','))
                else:
                    sem_types = ent['sem_types']
                triple_obj = [{'id:ID': ent['cuid'],
                                'label': ent['label'],
                                'sem_types:string[]': sem_types}]
            elif (obj_source == 'PMC') or (obj_source == 'TEXT') or (obj_source == 'None'):
                triple_obj = [{'id:ID': triple['o']}]
            else:
                if not(triple['o'] in cache):
                    ents = get_concept_from_source(triple['o'], obj_source)
                    cache[triple['o']] = ents
                    collection.insert_one({'key':triple['o'],'value':ents})
                    print 'INSERTED in other %s' % triple['o']
                else:
                    ents = cache[triple['o']]
                triple_obj = []
                for ent in ents:
                    if (type(ent['sem_types']) == list and len(ent['sem_types']) > 1):
                        sem_types = ';'.join(ent['sem_types'])
                    elif (',' in ent['sem_types']):
                        sem_types = ';'.join(ent['sem_types'].split(','))
                    else:
                        sem_types = ent['sem_types']

                    triple_obj.append({'id:ID': ent['cuid'],
                                    'label': ent['label'],
                                    'sem_types:string[]': sem_types})
            combs = product(triple_subj, triple_obj)
            for comb in combs:
                new_relations.append({'s':comb[0], 'p':triple['p'], 'o':comb[1]})
        except Exception, e:
            time_log(e)
            time_log('S: %s | P: %s | O: %s' % (triple['s'],triple['p'],triple['o']))
            time_log('Skipped the above edge! Probably due to concept-fetching errors!')
        proc = int(ii/float(N)*100)
        if proc % 10 == 0 and proc > 0:
            time_log('We are at %d/%d edges transformed -- %0.2f %%' % (ii, N, proc))
        # if ii % 100 == 0 and ii > 9:
        #     time_log("Edges Transformation Process: %d -- %0.2f %%" % (ii, 100*ii/float(len(json_[outfield]))))
    json_[outfield] = new_relations
    return json_