This project demonstrates how to provide object detection as a microservice based on a trained deep learning model using the detectron2 framework. Furthermore the microservice
An example implementation is made for the use case of plant organ detection, using the trained model for this purpose from @2younis and his repo 2younis/plant-organ-detection. His work is described in the following publication:
Younis et al. (2020): Detection and Annotation of Plant Organs from Digitized Herbarium Scans using Deep Learning https://arxiv.org/abs/2007.13106
The idea is that an image can be sent to the microservice via http, which then applies the trained model on the image and returns the detected plant organ instances in JSON format. Also an example is given of this service could interact with a Cordra instance.
Requirement: Python3.8 (!),(higher Python versions would require a change of the current dependency config) with the venv module installed
# copy this repository
git clone https://github.com/jgrieb/object-detection-service
# create a virtual environment
python3 -m venv object-detection-service
cd object-detection-service && source bin/activate
# install dependencies
pip install -r requirements.txt
# if you run into an error during installation of the pycocotools
# try running the following (depending on the Python version on the machine):
# sudo apt-get install python3.8-dev python3-opencv
# Then try again: pip install -r requirements.txtAfterwards download a trained .pth model for the object detection and save it in the data folder as model_final.pth. For this example we use the model from @2younis mentioned above for plant organ detection.
wget https://github.com/2younis/plant-organ-detection/releases/download/v1.0/model_final.pth -P data/
Finally customize the config/custom_model_config.yaml according to the trained model and set the CLASS_LIST parameter and the configuration loading in main.py accordingly. This is already preconfigured for the plant organ detection example.
For interaction with Cordra you will also need to cp config/cordraConfig.json.template config/cordraConfig.json and configure, see below Interaction with Cordra for further information.
Run the service with bin/uvicorn main:app (you might want to add --host 0.0.0.0 --port 8000 or another configuration)
You can then either access it by passing a url to an image:
curl -XPOST localhost:8000/object-detection -H "Content-Type: application/json" -d '{"url":"http://oxalis.br.fgov.be/images/BR0/000/013/701/604/BR0000013701604.jpg"}'
Or by uploading the image directly:
curl -XPOST localhost:8000/object-detection/image-upload -F 'payload=@images/BR0000013701604.jpg'
By default JSON is returned, but you can change this so that the service returns the image with the detected bounding boxes drawn on top (in this case you should specify the output file with the -o flag)
curl -XPOST localhost:8000/object-detection -H "Content-Type: application/json" -d '{"url":"http://oxalis.br.fgov.be/images/BR0/000/013/701/604/BR0000013701604.jpg", "returnType":"image"}' -o output.jpg
curl -XPOST localhost:8000/object-detection/image-upload -F 'payload=@images/BR0000013701604.jpg' -F 'returnType=image' -o output.jpg
Returned JSON data has the following format:
{
"success":true,
"instances":[
{
"class":"leaf",
"score":0.9999428987503052,
"boundingBox":[2470,3957,3229,4348]
}
]
}where the boundingBox are pixel coordinates of [xmin, ymin, xmax, ymax].
Thanks to the fastapi framework an OpenApi-conformant documentation of the REST Api is autogenerated when running the service and can be accessed under localhost:8000/docs. See here for further explanation of the parameters.
This microservice is desigend for interaction with Digital Objects managed in Cordra. The lifecycle hooks of the Digital Object can be used to automatically trigger the object detection when a new image is uploaded as a payload to a Digital Object. Currently the setup is in that way that this microservice uploads new Digital Objects of type AnnotationList when objects are detected and links them to the object of origin. For this a service account with Cordra access is required: cp config/cordraConfig.json.template config/cordraConfig.json to copy the template and then put in the credentials there. An example lifecycle configuration for the Digital Object type is given in the following
// Nashorn JDK cannot use XmlHttpRequest, therefore we must use Java Http requests
// see for reference: https://gist.github.com/billybong/a462152889b6616deb02
var cordra = require("cordra");
var cordraBaseUri = cordra.get('design').content.handleMintingConfig.baseUri;
// ensure that ends with a slash
if (cordraBaseUri[cordraBaseUri.length - 1] !== '/'){
cordraBaseUri += '/'
}
var imageServiceUri = 'http://localhost:8000/object-detection';
exports.afterCreateOrUpdate = afterCreateOrUpdate;
exports.methods = {};
exports.methods.getAnnotations = getAnnotations;
exports.methods.getAnnotations.allowGet = true;
function getAnnotations(object, context){
if (Array.isArray(object.payloads) && object.payloads.length > 0){
var query = 'type:AnnotationList AND /ods\\:linkedDigitalObject:' + object.id + ' AND (';
for(var i=0; i<object.payloads.length; i++){
var payloadName = object.payloads[i].name;
var payloadUrl = cordraBaseUri + 'objects/' + object.id + '?payload=' + payloadName;
var payloadUrlEscaped = payloadUrl.replace(/([\!\*\+\&\|\(\)\[\]\{\}\^\~\?\:\"])/g, "\\$1");
if(i>0){
query += ' OR ';
}
query += '/ods\\:imageURI:"' + payloadUrlEscaped + '"';
}
query += ')';
var queryResult = cordra.search(query);
return queryResult.results;
}
return [];
}
function afterCreateOrUpdate(object, context) {
// todo: need validation of uploaded payload type and appropriate handling
if (Array.isArray(object.payloads) && object.payloads.length > 0){
for(var i=0; i<object.payloads.length; i++){
var payloadName = object.payloads[i].name;
var payloadUrl = cordraBaseUri + 'objects/' + object.id + '?payload=' + payloadName;
var payloadUrlEscaped = payloadUrl.replace(/([\!\*\+\&\|\(\)\[\]\{\}\^\~\?\:\"])/g, "\\$1");
var query = 'type:AnnotationList AND /ods\\:linkedDigitalObject:' + object.id;
query += ' AND /ods\\:imageURI:"' + payloadUrlEscaped + '"';
// search if AnnotationLists for the image exist already in Cordra
var queryResult = cordra.search(query);
// if not send to the microservice for object detection
if (queryResult.results.length === 0){
var data = JSON.stringify({
"linkedDigitalObjectId": object.id,
"url": payloadUrl,
"returnType": "cordra",
"runAsync": true
});
var con = new java.net.URL(imageServiceUri).openConnection();
con.setConnectTimeout(10000); // 10 seconds
con.setReadTimeout(10000); // 10 seconds
var response = {};
try {
write(con, data);
response = JSON.parse(read(con.inputStream));
} catch (e){
console.log("HTTP Error: ", e);
}
}
}
}
}
function read(inputStream){
var inReader = new java.io.BufferedReader(new java.io.InputStreamReader(inputStream));
var inputLine;
var response = new java.lang.StringBuffer();
while ((inputLine = inReader.readLine()) !== null) {
response.append(inputLine);
}
inReader.close();
return response.toString();
}
function write(con, data){
con.requestMethod = "POST";
con.setRequestProperty( "Content-Type", "application/json; charset=utf-8");
con.doOutput=true;
var wr = new java.io.DataOutputStream(con.outputStream);
wr.writeBytes(data);
wr.flush();
wr.close();
}The generated AnnotationList currently does not make use of a controlled RDF vocabulary, this should be changed in the future and when decided uploaded in json-ld format.