llod_quality.py

# -*- coding: utf-8 -*-
"""LLOD_quality.ipynb

Automatically generated by Colab.

Original file is located at
    https://colab.research.google.com/drive/1_M8ixdOAgkws4CpLFWStGXMFz1iQWVXF
"""

import pandas as pd

import json

import matplotlib.pyplot as plt

import numpy as np

table = {}

f = open("sample_data/LLOD.json")
data=json.load(f)
df = pd.DataFrame(data)

simplified = df[['title','keywords','sparql','full_download','other_download','triples', 'license']]
simplified.head()

results = pd.DataFrame(data={})
results['title'] = simplified[['title']]
results.head()

results['amount_of_data'] = simplified[['triples']]
results['amount_of_data'] = results['amount_of_data'].str.replace(',', '')
results['amount_of_data'] = results['amount_of_data'].fillna(0).astype(float).astype(int)
results

sparql_list = []
for sparql in simplified['sparql']:
  status = 0;
  if len(sparql)==0:
    status = -1 #missing
  else:
    for entry in sparql:
      if "OK" in entry['status']:
        status = 1
  sparql_list.append(status)

results['sparql'] = sparql_list
results.head()

download_list = []
for download in simplified['full_download']:
  status = 0;
  if len(download)==0:
    status = -1 #missing
  else:
    for entry in download:
      if "OK" in entry['status']:
        status = 1
  download_list.append(status)

results['full_download'] = download_list
results.head()

download_list = []
for download in simplified['other_download']:
  status = 0;
  if len(download)==0:
    status = -1 #missing
  else:
    for entry in download:
      if "OK" in entry['status']:
        status = 1
  download_list.append(status)

results['other_download'] = download_list
results.head()

openlicenses = [
    'http://www.opendefinition.org/licenses/cc-by',
    'http://creativecommons.org/licenses/by-nc/2.0/', #deprecated
    'https://wordnet.princeton.edu/license-and-commercial-use',
    'https://opendefinition.org/licenses/cc-zero/',
    'https://opendefinition.org/licenses/cc-by-sa/',
    'https://opendefinition.org/licenses/odc-by/',
    'https://opensource.org/license/MIT',
    'https://creativecommons.org/licenses/by-sa/3.0/', #deprecated
    'https://opendefinition.org/licenses/odc-odbl/',
    'https://www.apache.org/licenses/LICENSE-2.0',
    'https://creativecommons.org/licenses/by/3.0/', #deprecated
    'https://opendefinition.org/licenses/gfdl/',
    'https://opendatacommons.org/licenses/by/1-0/',
    'http://www.cecill.info/licences/Licence_CeCILL-C_V1-en.html',
    'https://creativecommons.org/publicdomain/zero/1.0/' #deprecated
]

licenses = []
for license in simplified['license']:
  if pd.isnull(license):
    licenses.append(-1)
  elif license in openlicenses:
    licenses.append(1)
  else:
    licenses.append(0)

results['openlicense'] = licenses
results.head()

access = []
for index, row in results.iterrows():
  access.append(max(int(row['sparql']),int(row['full_download']),int(row['other_download'])))
results['access'] = access
results.head()

category = []
for keywords in simplified['keywords']:

  if 'corpus' in keywords:
    category.append("Corpora")
  elif 'thesaurus' in keywords:
      category.append("Terminologies, Thesauri and Knowledge Bases")
  elif 'lexicon' in keywords:
    category.append("Lexicons and Dictionaries")
  elif 'metadata' in keywords:
    category.append("Linguistic Resource Metadata")
  elif 'typological-database' in keywords and 'typology'in keywords:
    category.append("Typological Databases")
  elif 'linguistic' in keywords:
    category.append("Linguistic Data Categories")
  else:
    category.append("Other")

results['category'] = category
print(results.groupby("category").count())

results.head()

results.describe().transpose()

results.boxplot(['sparql', 'access', 'openlicense', 'amount_of_data'], by = 'category', layout=(1,4), figsize = (20,5), showmeans = True, sharex=False,  vert=False)

results.boxplot(['amount_of_data'], by = 'category', layout=(2,3), figsize = (15,10), showmeans = True, sharex=False,  vert=False, showfliers = False)

syntesis = pd.DataFrame(data={})
for name, group in results.groupby("category"):
  print(name)
  print(group.describe().transpose())

print(len(results['access']))
print("Open license", (results['openlicense']==1).sum())

print("Access", (results['access']==1).sum())
print("Data via Access", (results['amount_of_data'][results['access']==1].sum()))

print("SPARQL", (results['sparql']==1).sum())
print("Data via SPARQL", (results['amount_of_data'][results['sparql']==1].sum()))

print("Open license AND Access", ((results['openlicense']==1)&(results['access']==1)).sum())
print("Data via Open license AND Access", (results['amount_of_data'][(results['openlicense']==1)&(results['access']==1)].sum()))

print("Open license AND SPARQL", ((results['openlicense']==1)&(results['sparql']==1)).sum())
print("Data via Open license AND SPARQL", (results['amount_of_data'][(results['openlicense']==1)&(results['sparql']==1)].sum()))

print(results[(results['openlicense']==1)&(results['sparql']==1)])

for name, group in results.groupby("category"):
  print(name)
  print(len(group))
  print("Open license", (group['openlicense']==1).sum())

  print("Access", (group['access']==1).sum())
  print("Data via Access", (group['amount_of_data'][group['access']==1].sum()))

  print("SPARQL", (group['sparql']==1).sum())
  print("Data via SPARQL", (group['amount_of_data'][group['sparql']==1].sum()))

  print("Open license AND Access ", ((group['openlicense']==1)&(group['access']==1)).sum())
  print("Data via Open license AND Access", (group['amount_of_data'][(group['openlicense']==1)&(group['access']==1)].sum()))

  print("Open license AND SPARQL", ((group['openlicense']==1)&(group['sparql']==1)).sum())
  print("Data via Open license AND SPARQL", (group['amount_of_data'][(group['openlicense']==1)&(group['sparql']==1)].sum()))

  print(group[(group['openlicense']==1)&(group['sparql']==1)])
  print("---")