forked from OBOFoundry/OBOFoundry.github.io
-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathMakefile
153 lines (117 loc) · 5.83 KB
/
Makefile
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
# Makefile for OBOFoundry.github.io
#
# This file contains commands for validating and generating
# configuration files for the OBO Foundry website.
# The `util/` directory contains additional scripts
# called from this Makefile.
# It is intended to be run on a Unix system (Linux, Mac OS X)
# **before** the site is built using
# [Jekyll](http://jekyllrb.com).
#
# Software requirements:
#
# - [GNU Make](http://www.gnu.org/software/make/)
# - [Apache Jena](https://jena.apache.org/download/)
# - [Python 3](https://www.python.org/downloads/)
# - [PyYAML](http://pyyaml.org/wiki/PyYAML)
# - [SPARQLWrapper](https://pypi.python.org/pypi/SPARQLWrapper)
#
# Run `make` in this directory to update all generated files
# (i.e. the default `all` task).
# Make does its best to detect changes and run only the required tasks,
# but sometimes it helps to delete the target files first by running `make clean`
#
# WARNING: Makefiles contain significant tab characters!
# Before editing this file, ensure that your editor is not set up to convert tabs
# to spaces, and then use tabs to indent recipe lines.
### Configuration
# All ontology .md files
ONTS := $(wildcard ontology/*.md)
# All principles .md files
PRINCIPLES := $(wildcard principles/*.md)
### Main Tasks
.PHONY: all pull_and_build test pull clean
all: _config.yml registry/ontologies.ttl registry/publications.md registry/obo_context.jsonld registry/obo_prefixes.ttl
pull:
git pull
pull_and_build: pull all
test: reports/metadata-grid.html _config.yml
integration-test: test valid-purl-report.txt
# Remove and/or revert all targets to their repository versions:
clean:
rm -Rf registry/ontologies.nt registry/ontologies.ttl registry/ontologies.yml registry/publications.md sparql-consistency-report.txt jenkins-output.txt valid-purl-report.txt valid-purl-report.txt.tmp _site/ tmp/ reports/
git checkout _config.yml registry/ontologies.jsonld registry/ontologies.ttl registry/ontologies.yml registry/publications.md
### Directories:
tmp:
mkdir -p $@
reports:
mkdir -p $@
### Build Configuration Files
# Create the site-wide config file by combining all metadata on ontologies + principles
# and combining with site-wide metadata.
#
# Note that anything in _config.yml is accessible to any liquid template via the
# `sites` object - think of it like the global database
#
# (this is somewhat hacky, but concatenating these yamls is safe)
_config.yml: _config_header.yml registry/ontologies.yml principles/all.yml
cat $^ > [email protected] && mv [email protected] $@
# Sort ontologies based on the validation (metadata-grid)
registry/ontologies.yml: reports/metadata-grid.csv
./util/sort-ontologies.py tmp/unsorted-ontologies.yml $< $@ && rm -rf tmp
# Extract the metadata from each principle in the principles/ directory, and concatenate
# into a single yaml file in that directory
principles/all.yml: $(PRINCIPLES)
./util/extract-metadata.py concat-principles -o [email protected] $^ && mv [email protected] $@
# Use a generic yaml->json conversion, but adding a @content
registry/ontologies.jsonld: registry/ontologies.yml
./util/yaml2json.py $< > [email protected] && mv [email protected] $@
registry/obo_context.jsonld: registry/ontologies.yml
./util/processor.py -i $< extract-context > [email protected] && mv [email protected] $@
# generate triples mapping prefixes to their corresponding PURLs.
# we use the SHACL vocabulary for this
registry/obo_prefixes.ttl: registry/ontologies.yml
./util/make-shacl-prefixes.py $< > [email protected] && mv [email protected] $@
# Use Apache-Jena RIOT to convert jsonld to n-triples
# NOTE: UGLY HACK. If there is a problem then Jena will write WARN message (to stdout!!!), there appears to
# be no way to get it to flag this even with strict and check options, so we do a check with grep, ugh.
# see: http://stackoverflow.com/questions/20860222/why-do-i-have-these-warnings-with-jena-2-11-0
registry/ontologies.nt: registry/ontologies.jsonld
riot --base=http://purl.obolibrary.org/obo/ --strict --check -q registry/context.jsonld $< > [email protected] && mv [email protected] $@ && egrep '(WARN|ERROR)' $@ && exit 1 || echo ok
registry/ontologies.ttl: registry/ontologies.nt
riot --base=http://purl.obolibrary.org/obo/ --out=ttl $< > [email protected] && mv [email protected] $@
# Generate a list of primary publications
registry/publications.md: registry/ontologies.yml
util/extract-publications.py $< $@
### Validate Configuration Files
# generate both a report of the violations and a grid of all results
# the grid is later used to sort the ontologies on the home page
RESULTS = reports/metadata-violations.tsv reports/metadata-grid.csv
reports/metadata-grid.csv: tmp/unsorted-ontologies.yml | extract-metadata reports
./util/validate-metadata.py $< $(RESULTS)
# generate an HTML output of the metadata grid
# TODO: determine where this output belongs
reports/metadata-grid.html: reports/metadata-grid.csv
./util/create-html-grid.py $< $@
# Extract metadata from each ontology .md file and combine into single yaml
tmp/unsorted-ontologies.yml: $(ONTS) | tmp
./util/extract-metadata.py concat -o [email protected] $^ && mv [email protected] $@
extract-metadata: $(ONTS)
./util/extract-metadata.py validate $^
# Note this should *not* be run as part of general travis jobs, it is expensive
# and may be prone to false positives as it is inherently network-based
#
# TODO: Other non-travis CI job. Nightly?
# TODO: Integrate this with some kind of OCLC query check
#
# See: https://github.com/OBOFoundry/OBOFoundry.github.io/issues/18
valid-purl-report.txt: registry/ontologies.yml
./util/processor.py -i $< check-urls > [email protected] && mv [email protected] $@
sparql-consistency-report.txt: registry/ontologies.yml
./util/processor.py -i $< sparql-compare > [email protected] && mv [email protected] $@
# output of central OBO build
# See FAQ for more details, and also README.md
jenkins-output.txt:
wget http://build.berkeleybop.org/job/simple-build-obo-all/lastBuild/consoleFull -O $@
reports/%.csv: registry/ontologies.ttl sparql/%.sparql
arq --data $< --query sparql/$*.sparql --results csv > [email protected] && mv [email protected] $@