diff --git a/.github/my_actions/check_membership/package-lock.json b/.github/my_actions/check_membership/package-lock.json deleted file mode 100644 index 6dac377..0000000 --- a/.github/my_actions/check_membership/package-lock.json +++ /dev/null @@ -1,321 +0,0 @@ -{ - "name": "my_actions", - "version": "1.0.0", - "lockfileVersion": 3, - "requires": true, - "packages": { - "": { - "name": "my_actions", - "version": "1.0.0", - "license": "MIT", - "dependencies": { - "@actions/core": "^1.10.0", - "@actions/github": "^5.1.1", - "node-fetch": "^3.3.2" - } - }, - "node_modules/@actions/core": { - "version": "1.10.0", - "resolved": "https://registry.npmjs.org/@actions/core/-/core-1.10.0.tgz", - "integrity": "sha512-2aZDDa3zrrZbP5ZYg159sNoLRb61nQ7awl5pSvIq5Qpj81vwDzdMRKzkWJGJuwVvWpvZKx7vspJALyvaaIQyug==", - "dependencies": { - "@actions/http-client": "^2.0.1", - "uuid": "^8.3.2" - } - }, - "node_modules/@actions/github": { - "version": "5.1.1", - "resolved": "https://registry.npmjs.org/@actions/github/-/github-5.1.1.tgz", - "integrity": "sha512-Nk59rMDoJaV+mHCOJPXuvB1zIbomlKS0dmSIqPGxd0enAXBnOfn4VWF+CGtRCwXZG9Epa54tZA7VIRlJDS8A6g==", - "dependencies": { - "@actions/http-client": "^2.0.1", - "@octokit/core": "^3.6.0", - "@octokit/plugin-paginate-rest": "^2.17.0", - "@octokit/plugin-rest-endpoint-methods": "^5.13.0" - } - }, - "node_modules/@actions/http-client": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/@actions/http-client/-/http-client-2.1.1.tgz", - "integrity": "sha512-qhrkRMB40bbbLo7gF+0vu+X+UawOvQQqNAA/5Unx774RS8poaOhThDOG6BGmxvAnxhQnDp2BG/ZUm65xZILTpw==", - "dependencies": { - "tunnel": "^0.0.6" - } - }, - "node_modules/@octokit/auth-token": { - "version": "2.5.0", - "resolved": "https://registry.npmjs.org/@octokit/auth-token/-/auth-token-2.5.0.tgz", - "integrity": "sha512-r5FVUJCOLl19AxiuZD2VRZ/ORjp/4IN98Of6YJoJOkY75CIBuYfmiNHGrDwXr+aLGG55igl9QrxX3hbiXlLb+g==", - "dependencies": { - "@octokit/types": "^6.0.3" - } - }, - "node_modules/@octokit/core": { - "version": "3.6.0", - "resolved": "https://registry.npmjs.org/@octokit/core/-/core-3.6.0.tgz", - "integrity": "sha512-7RKRKuA4xTjMhY+eG3jthb3hlZCsOwg3rztWh75Xc+ShDWOfDDATWbeZpAHBNRpm4Tv9WgBMOy1zEJYXG6NJ7Q==", - "dependencies": { - "@octokit/auth-token": "^2.4.4", - "@octokit/graphql": "^4.5.8", - "@octokit/request": "^5.6.3", - "@octokit/request-error": "^2.0.5", - "@octokit/types": "^6.0.3", - "before-after-hook": "^2.2.0", - "universal-user-agent": "^6.0.0" - } - }, - "node_modules/@octokit/endpoint": { - "version": "6.0.12", - "resolved": "https://registry.npmjs.org/@octokit/endpoint/-/endpoint-6.0.12.tgz", - "integrity": "sha512-lF3puPwkQWGfkMClXb4k/eUT/nZKQfxinRWJrdZaJO85Dqwo/G0yOC434Jr2ojwafWJMYqFGFa5ms4jJUgujdA==", - "dependencies": { - "@octokit/types": "^6.0.3", - "is-plain-object": "^5.0.0", - "universal-user-agent": "^6.0.0" - } - }, - "node_modules/@octokit/graphql": { - "version": "4.8.0", - "resolved": "https://registry.npmjs.org/@octokit/graphql/-/graphql-4.8.0.tgz", - "integrity": "sha512-0gv+qLSBLKF0z8TKaSKTsS39scVKF9dbMxJpj3U0vC7wjNWFuIpL/z76Qe2fiuCbDRcJSavkXsVtMS6/dtQQsg==", - "dependencies": { - "@octokit/request": "^5.6.0", - "@octokit/types": "^6.0.3", - "universal-user-agent": "^6.0.0" - } - }, - "node_modules/@octokit/openapi-types": { - "version": "12.11.0", - "resolved": "https://registry.npmjs.org/@octokit/openapi-types/-/openapi-types-12.11.0.tgz", - "integrity": "sha512-VsXyi8peyRq9PqIz/tpqiL2w3w80OgVMwBHltTml3LmVvXiphgeqmY9mvBw9Wu7e0QWk/fqD37ux8yP5uVekyQ==" - }, - "node_modules/@octokit/plugin-paginate-rest": { - "version": "2.21.3", - "resolved": "https://registry.npmjs.org/@octokit/plugin-paginate-rest/-/plugin-paginate-rest-2.21.3.tgz", - "integrity": "sha512-aCZTEf0y2h3OLbrgKkrfFdjRL6eSOo8komneVQJnYecAxIej7Bafor2xhuDJOIFau4pk0i/P28/XgtbyPF0ZHw==", - "dependencies": { - "@octokit/types": "^6.40.0" - }, - "peerDependencies": { - "@octokit/core": ">=2" - } - }, - "node_modules/@octokit/plugin-rest-endpoint-methods": { - "version": "5.16.2", - "resolved": "https://registry.npmjs.org/@octokit/plugin-rest-endpoint-methods/-/plugin-rest-endpoint-methods-5.16.2.tgz", - "integrity": "sha512-8QFz29Fg5jDuTPXVtey05BLm7OB+M8fnvE64RNegzX7U+5NUXcOcnpTIK0YfSHBg8gYd0oxIq3IZTe9SfPZiRw==", - "dependencies": { - "@octokit/types": "^6.39.0", - "deprecation": "^2.3.1" - }, - "peerDependencies": { - "@octokit/core": ">=3" - } - }, - "node_modules/@octokit/request": { - "version": "5.6.3", - "resolved": "https://registry.npmjs.org/@octokit/request/-/request-5.6.3.tgz", - "integrity": "sha512-bFJl0I1KVc9jYTe9tdGGpAMPy32dLBXXo1dS/YwSCTL/2nd9XeHsY616RE3HPXDVk+a+dBuzyz5YdlXwcDTr2A==", - "dependencies": { - "@octokit/endpoint": "^6.0.1", - "@octokit/request-error": "^2.1.0", - "@octokit/types": "^6.16.1", - "is-plain-object": "^5.0.0", - "node-fetch": "^2.6.7", - "universal-user-agent": "^6.0.0" - } - }, - "node_modules/@octokit/request-error": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/@octokit/request-error/-/request-error-2.1.0.tgz", - "integrity": "sha512-1VIvgXxs9WHSjicsRwq8PlR2LR2x6DwsJAaFgzdi0JfJoGSO8mYI/cHJQ+9FbN21aa+DrgNLnwObmyeSC8Rmpg==", - "dependencies": { - "@octokit/types": "^6.0.3", - "deprecation": "^2.0.0", - "once": "^1.4.0" - } - }, - "node_modules/@octokit/request/node_modules/node-fetch": { - "version": "2.7.0", - "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz", - "integrity": "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==", - "dependencies": { - "whatwg-url": "^5.0.0" - }, - "engines": { - "node": "4.x || >=6.0.0" - }, - "peerDependencies": { - "encoding": "^0.1.0" - }, - "peerDependenciesMeta": { - "encoding": { - "optional": true - } - } - }, - "node_modules/@octokit/types": { - "version": "6.41.0", - "resolved": "https://registry.npmjs.org/@octokit/types/-/types-6.41.0.tgz", - "integrity": "sha512-eJ2jbzjdijiL3B4PrSQaSjuF2sPEQPVCPzBvTHJD9Nz+9dw2SGH4K4xeQJ77YfTq5bRQ+bD8wT11JbeDPmxmGg==", - "dependencies": { - "@octokit/openapi-types": "^12.11.0" - } - }, - "node_modules/before-after-hook": { - "version": "2.2.3", - "resolved": "https://registry.npmjs.org/before-after-hook/-/before-after-hook-2.2.3.tgz", - "integrity": "sha512-NzUnlZexiaH/46WDhANlyR2bXRopNg4F/zuSA3OpZnllCUgRaOF2znDioDWrmbNVsuZk6l9pMquQB38cfBZwkQ==" - }, - "node_modules/data-uri-to-buffer": { - "version": "4.0.1", - "resolved": "https://registry.npmjs.org/data-uri-to-buffer/-/data-uri-to-buffer-4.0.1.tgz", - "integrity": "sha512-0R9ikRb668HB7QDxT1vkpuUBtqc53YyAwMwGeUFKRojY/NWKvdZ+9UYtRfGmhqNbRkTSVpMbmyhXipFFv2cb/A==", - "engines": { - "node": ">= 12" - } - }, - "node_modules/deprecation": { - "version": "2.3.1", - "resolved": "https://registry.npmjs.org/deprecation/-/deprecation-2.3.1.tgz", - "integrity": "sha512-xmHIy4F3scKVwMsQ4WnVaS8bHOx0DmVwRywosKhaILI0ywMDWPtBSku2HNxRvF7jtwDRsoEwYQSfbxj8b7RlJQ==" - }, - "node_modules/fetch-blob": { - "version": "3.2.0", - "resolved": "https://registry.npmjs.org/fetch-blob/-/fetch-blob-3.2.0.tgz", - "integrity": "sha512-7yAQpD2UMJzLi1Dqv7qFYnPbaPx7ZfFK6PiIxQ4PfkGPyNyl2Ugx+a/umUonmKqjhM4DnfbMvdX6otXq83soQQ==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/jimmywarting" - }, - { - "type": "paypal", - "url": "https://paypal.me/jimmywarting" - } - ], - "dependencies": { - "node-domexception": "^1.0.0", - "web-streams-polyfill": "^3.0.3" - }, - "engines": { - "node": "^12.20 || >= 14.13" - } - }, - "node_modules/formdata-polyfill": { - "version": "4.0.10", - "resolved": "https://registry.npmjs.org/formdata-polyfill/-/formdata-polyfill-4.0.10.tgz", - "integrity": "sha512-buewHzMvYL29jdeQTVILecSaZKnt/RJWjoZCF5OW60Z67/GmSLBkOFM7qh1PI3zFNtJbaZL5eQu1vLfazOwj4g==", - "dependencies": { - "fetch-blob": "^3.1.2" - }, - "engines": { - "node": ">=12.20.0" - } - }, - "node_modules/is-plain-object": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/is-plain-object/-/is-plain-object-5.0.0.tgz", - "integrity": "sha512-VRSzKkbMm5jMDoKLbltAkFQ5Qr7VDiTFGXxYFXXowVj387GeGNOCsOH6Msy00SGZ3Fp84b1Naa1psqgcCIEP5Q==", - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/node-domexception": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/node-domexception/-/node-domexception-1.0.0.tgz", - "integrity": "sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/jimmywarting" - }, - { - "type": "github", - "url": "https://paypal.me/jimmywarting" - } - ], - "engines": { - "node": ">=10.5.0" - } - }, - "node_modules/node-fetch": { - "version": "3.3.2", - "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-3.3.2.tgz", - "integrity": "sha512-dRB78srN/l6gqWulah9SrxeYnxeddIG30+GOqK/9OlLVyLg3HPnr6SqOWTWOXKRwC2eGYCkZ59NNuSgvSrpgOA==", - "dependencies": { - "data-uri-to-buffer": "^4.0.0", - "fetch-blob": "^3.1.4", - "formdata-polyfill": "^4.0.10" - }, - "engines": { - "node": "^12.20.0 || ^14.13.1 || >=16.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/node-fetch" - } - }, - "node_modules/once": { - "version": "1.4.0", - "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", - "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", - "dependencies": { - "wrappy": "1" - } - }, - "node_modules/tr46": { - "version": "0.0.3", - "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", - "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==" - }, - "node_modules/tunnel": { - "version": "0.0.6", - "resolved": "https://registry.npmjs.org/tunnel/-/tunnel-0.0.6.tgz", - "integrity": "sha512-1h/Lnq9yajKY2PEbBadPXj3VxsDDu844OnaAo52UVmIzIvwwtBPIuNvkjuzBlTWpfJyUbG3ez0KSBibQkj4ojg==", - "engines": { - "node": ">=0.6.11 <=0.7.0 || >=0.7.3" - } - }, - "node_modules/universal-user-agent": { - "version": "6.0.0", - "resolved": "https://registry.npmjs.org/universal-user-agent/-/universal-user-agent-6.0.0.tgz", - "integrity": "sha512-isyNax3wXoKaulPDZWHQqbmIx1k2tb9fb3GGDBRxCscfYV2Ch7WxPArBsFEG8s/safwXTT7H4QGhaIkTp9447w==" - }, - "node_modules/uuid": { - "version": "8.3.2", - "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz", - "integrity": "sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==", - "bin": { - "uuid": "dist/bin/uuid" - } - }, - "node_modules/web-streams-polyfill": { - "version": "3.2.1", - "resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-3.2.1.tgz", - "integrity": "sha512-e0MO3wdXWKrLbL0DgGnUV7WHVuw9OUvL4hjgnPkIeEvESk74gAITi5G606JtZPp39cd8HA9VQzCIvA49LpPN5Q==", - "engines": { - "node": ">= 8" - } - }, - "node_modules/webidl-conversions": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", - "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==" - }, - "node_modules/whatwg-url": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz", - "integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==", - "dependencies": { - "tr46": "~0.0.3", - "webidl-conversions": "^3.0.0" - } - }, - "node_modules/wrappy": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", - "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==" - } - } -} diff --git a/.github/my_actions/check_membership/package.json b/.github/my_actions/check_membership/package.json deleted file mode 100644 index 7c7a14c..0000000 --- a/.github/my_actions/check_membership/package.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "name": "my_actions", - "version": "1.0.0", - "main": "index.js", - "scripts": { - "test": "echo \"Error: no test specified\" && exit 1" - }, - "keywords": [], - "author": "", - "license": "MIT", - "dependencies": { - "@actions/core": "^1.10.0", - "@actions/github": "^5.1.1", - "node-fetch": "^3.3.2" - }, - "description": "" -} diff --git a/.github/my_actions/repository_dispatch/package-lock.json b/.github/my_actions/repository_dispatch/package-lock.json deleted file mode 100644 index 6dac377..0000000 --- a/.github/my_actions/repository_dispatch/package-lock.json +++ /dev/null @@ -1,321 +0,0 @@ -{ - "name": "my_actions", - "version": "1.0.0", - "lockfileVersion": 3, - "requires": true, - "packages": { - "": { - "name": "my_actions", - "version": "1.0.0", - "license": "MIT", - "dependencies": { - "@actions/core": "^1.10.0", - "@actions/github": "^5.1.1", - "node-fetch": "^3.3.2" - } - }, - "node_modules/@actions/core": { - "version": "1.10.0", - "resolved": "https://registry.npmjs.org/@actions/core/-/core-1.10.0.tgz", - "integrity": "sha512-2aZDDa3zrrZbP5ZYg159sNoLRb61nQ7awl5pSvIq5Qpj81vwDzdMRKzkWJGJuwVvWpvZKx7vspJALyvaaIQyug==", - "dependencies": { - "@actions/http-client": "^2.0.1", - "uuid": "^8.3.2" - } - }, - "node_modules/@actions/github": { - "version": "5.1.1", - "resolved": "https://registry.npmjs.org/@actions/github/-/github-5.1.1.tgz", - "integrity": "sha512-Nk59rMDoJaV+mHCOJPXuvB1zIbomlKS0dmSIqPGxd0enAXBnOfn4VWF+CGtRCwXZG9Epa54tZA7VIRlJDS8A6g==", - "dependencies": { - "@actions/http-client": "^2.0.1", - "@octokit/core": "^3.6.0", - "@octokit/plugin-paginate-rest": "^2.17.0", - "@octokit/plugin-rest-endpoint-methods": "^5.13.0" - } - }, - "node_modules/@actions/http-client": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/@actions/http-client/-/http-client-2.1.1.tgz", - "integrity": "sha512-qhrkRMB40bbbLo7gF+0vu+X+UawOvQQqNAA/5Unx774RS8poaOhThDOG6BGmxvAnxhQnDp2BG/ZUm65xZILTpw==", - "dependencies": { - "tunnel": "^0.0.6" - } - }, - "node_modules/@octokit/auth-token": { - "version": "2.5.0", - "resolved": "https://registry.npmjs.org/@octokit/auth-token/-/auth-token-2.5.0.tgz", - "integrity": "sha512-r5FVUJCOLl19AxiuZD2VRZ/ORjp/4IN98Of6YJoJOkY75CIBuYfmiNHGrDwXr+aLGG55igl9QrxX3hbiXlLb+g==", - "dependencies": { - "@octokit/types": "^6.0.3" - } - }, - "node_modules/@octokit/core": { - "version": "3.6.0", - "resolved": "https://registry.npmjs.org/@octokit/core/-/core-3.6.0.tgz", - "integrity": "sha512-7RKRKuA4xTjMhY+eG3jthb3hlZCsOwg3rztWh75Xc+ShDWOfDDATWbeZpAHBNRpm4Tv9WgBMOy1zEJYXG6NJ7Q==", - "dependencies": { - "@octokit/auth-token": "^2.4.4", - "@octokit/graphql": "^4.5.8", - "@octokit/request": "^5.6.3", - "@octokit/request-error": "^2.0.5", - "@octokit/types": "^6.0.3", - "before-after-hook": "^2.2.0", - "universal-user-agent": "^6.0.0" - } - }, - "node_modules/@octokit/endpoint": { - "version": "6.0.12", - "resolved": "https://registry.npmjs.org/@octokit/endpoint/-/endpoint-6.0.12.tgz", - "integrity": "sha512-lF3puPwkQWGfkMClXb4k/eUT/nZKQfxinRWJrdZaJO85Dqwo/G0yOC434Jr2ojwafWJMYqFGFa5ms4jJUgujdA==", - "dependencies": { - "@octokit/types": "^6.0.3", - "is-plain-object": "^5.0.0", - "universal-user-agent": "^6.0.0" - } - }, - "node_modules/@octokit/graphql": { - "version": "4.8.0", - "resolved": "https://registry.npmjs.org/@octokit/graphql/-/graphql-4.8.0.tgz", - "integrity": "sha512-0gv+qLSBLKF0z8TKaSKTsS39scVKF9dbMxJpj3U0vC7wjNWFuIpL/z76Qe2fiuCbDRcJSavkXsVtMS6/dtQQsg==", - "dependencies": { - "@octokit/request": "^5.6.0", - "@octokit/types": "^6.0.3", - "universal-user-agent": "^6.0.0" - } - }, - "node_modules/@octokit/openapi-types": { - "version": "12.11.0", - "resolved": "https://registry.npmjs.org/@octokit/openapi-types/-/openapi-types-12.11.0.tgz", - "integrity": "sha512-VsXyi8peyRq9PqIz/tpqiL2w3w80OgVMwBHltTml3LmVvXiphgeqmY9mvBw9Wu7e0QWk/fqD37ux8yP5uVekyQ==" - }, - "node_modules/@octokit/plugin-paginate-rest": { - "version": "2.21.3", - "resolved": "https://registry.npmjs.org/@octokit/plugin-paginate-rest/-/plugin-paginate-rest-2.21.3.tgz", - "integrity": "sha512-aCZTEf0y2h3OLbrgKkrfFdjRL6eSOo8komneVQJnYecAxIej7Bafor2xhuDJOIFau4pk0i/P28/XgtbyPF0ZHw==", - "dependencies": { - "@octokit/types": "^6.40.0" - }, - "peerDependencies": { - "@octokit/core": ">=2" - } - }, - "node_modules/@octokit/plugin-rest-endpoint-methods": { - "version": "5.16.2", - "resolved": "https://registry.npmjs.org/@octokit/plugin-rest-endpoint-methods/-/plugin-rest-endpoint-methods-5.16.2.tgz", - "integrity": "sha512-8QFz29Fg5jDuTPXVtey05BLm7OB+M8fnvE64RNegzX7U+5NUXcOcnpTIK0YfSHBg8gYd0oxIq3IZTe9SfPZiRw==", - "dependencies": { - "@octokit/types": "^6.39.0", - "deprecation": "^2.3.1" - }, - "peerDependencies": { - "@octokit/core": ">=3" - } - }, - "node_modules/@octokit/request": { - "version": "5.6.3", - "resolved": "https://registry.npmjs.org/@octokit/request/-/request-5.6.3.tgz", - "integrity": "sha512-bFJl0I1KVc9jYTe9tdGGpAMPy32dLBXXo1dS/YwSCTL/2nd9XeHsY616RE3HPXDVk+a+dBuzyz5YdlXwcDTr2A==", - "dependencies": { - "@octokit/endpoint": "^6.0.1", - "@octokit/request-error": "^2.1.0", - "@octokit/types": "^6.16.1", - "is-plain-object": "^5.0.0", - "node-fetch": "^2.6.7", - "universal-user-agent": "^6.0.0" - } - }, - "node_modules/@octokit/request-error": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/@octokit/request-error/-/request-error-2.1.0.tgz", - "integrity": "sha512-1VIvgXxs9WHSjicsRwq8PlR2LR2x6DwsJAaFgzdi0JfJoGSO8mYI/cHJQ+9FbN21aa+DrgNLnwObmyeSC8Rmpg==", - "dependencies": { - "@octokit/types": "^6.0.3", - "deprecation": "^2.0.0", - "once": "^1.4.0" - } - }, - "node_modules/@octokit/request/node_modules/node-fetch": { - "version": "2.7.0", - "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz", - "integrity": "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==", - "dependencies": { - "whatwg-url": "^5.0.0" - }, - "engines": { - "node": "4.x || >=6.0.0" - }, - "peerDependencies": { - "encoding": "^0.1.0" - }, - "peerDependenciesMeta": { - "encoding": { - "optional": true - } - } - }, - "node_modules/@octokit/types": { - "version": "6.41.0", - "resolved": "https://registry.npmjs.org/@octokit/types/-/types-6.41.0.tgz", - "integrity": "sha512-eJ2jbzjdijiL3B4PrSQaSjuF2sPEQPVCPzBvTHJD9Nz+9dw2SGH4K4xeQJ77YfTq5bRQ+bD8wT11JbeDPmxmGg==", - "dependencies": { - "@octokit/openapi-types": "^12.11.0" - } - }, - "node_modules/before-after-hook": { - "version": "2.2.3", - "resolved": "https://registry.npmjs.org/before-after-hook/-/before-after-hook-2.2.3.tgz", - "integrity": "sha512-NzUnlZexiaH/46WDhANlyR2bXRopNg4F/zuSA3OpZnllCUgRaOF2znDioDWrmbNVsuZk6l9pMquQB38cfBZwkQ==" - }, - "node_modules/data-uri-to-buffer": { - "version": "4.0.1", - "resolved": "https://registry.npmjs.org/data-uri-to-buffer/-/data-uri-to-buffer-4.0.1.tgz", - "integrity": "sha512-0R9ikRb668HB7QDxT1vkpuUBtqc53YyAwMwGeUFKRojY/NWKvdZ+9UYtRfGmhqNbRkTSVpMbmyhXipFFv2cb/A==", - "engines": { - "node": ">= 12" - } - }, - "node_modules/deprecation": { - "version": "2.3.1", - "resolved": "https://registry.npmjs.org/deprecation/-/deprecation-2.3.1.tgz", - "integrity": "sha512-xmHIy4F3scKVwMsQ4WnVaS8bHOx0DmVwRywosKhaILI0ywMDWPtBSku2HNxRvF7jtwDRsoEwYQSfbxj8b7RlJQ==" - }, - "node_modules/fetch-blob": { - "version": "3.2.0", - "resolved": "https://registry.npmjs.org/fetch-blob/-/fetch-blob-3.2.0.tgz", - "integrity": "sha512-7yAQpD2UMJzLi1Dqv7qFYnPbaPx7ZfFK6PiIxQ4PfkGPyNyl2Ugx+a/umUonmKqjhM4DnfbMvdX6otXq83soQQ==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/jimmywarting" - }, - { - "type": "paypal", - "url": "https://paypal.me/jimmywarting" - } - ], - "dependencies": { - "node-domexception": "^1.0.0", - "web-streams-polyfill": "^3.0.3" - }, - "engines": { - "node": "^12.20 || >= 14.13" - } - }, - "node_modules/formdata-polyfill": { - "version": "4.0.10", - "resolved": "https://registry.npmjs.org/formdata-polyfill/-/formdata-polyfill-4.0.10.tgz", - "integrity": "sha512-buewHzMvYL29jdeQTVILecSaZKnt/RJWjoZCF5OW60Z67/GmSLBkOFM7qh1PI3zFNtJbaZL5eQu1vLfazOwj4g==", - "dependencies": { - "fetch-blob": "^3.1.2" - }, - "engines": { - "node": ">=12.20.0" - } - }, - "node_modules/is-plain-object": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/is-plain-object/-/is-plain-object-5.0.0.tgz", - "integrity": "sha512-VRSzKkbMm5jMDoKLbltAkFQ5Qr7VDiTFGXxYFXXowVj387GeGNOCsOH6Msy00SGZ3Fp84b1Naa1psqgcCIEP5Q==", - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/node-domexception": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/node-domexception/-/node-domexception-1.0.0.tgz", - "integrity": "sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/jimmywarting" - }, - { - "type": "github", - "url": "https://paypal.me/jimmywarting" - } - ], - "engines": { - "node": ">=10.5.0" - } - }, - "node_modules/node-fetch": { - "version": "3.3.2", - "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-3.3.2.tgz", - "integrity": "sha512-dRB78srN/l6gqWulah9SrxeYnxeddIG30+GOqK/9OlLVyLg3HPnr6SqOWTWOXKRwC2eGYCkZ59NNuSgvSrpgOA==", - "dependencies": { - "data-uri-to-buffer": "^4.0.0", - "fetch-blob": "^3.1.4", - "formdata-polyfill": "^4.0.10" - }, - "engines": { - "node": "^12.20.0 || ^14.13.1 || >=16.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/node-fetch" - } - }, - "node_modules/once": { - "version": "1.4.0", - "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", - "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", - "dependencies": { - "wrappy": "1" - } - }, - "node_modules/tr46": { - "version": "0.0.3", - "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", - "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==" - }, - "node_modules/tunnel": { - "version": "0.0.6", - "resolved": "https://registry.npmjs.org/tunnel/-/tunnel-0.0.6.tgz", - "integrity": "sha512-1h/Lnq9yajKY2PEbBadPXj3VxsDDu844OnaAo52UVmIzIvwwtBPIuNvkjuzBlTWpfJyUbG3ez0KSBibQkj4ojg==", - "engines": { - "node": ">=0.6.11 <=0.7.0 || >=0.7.3" - } - }, - "node_modules/universal-user-agent": { - "version": "6.0.0", - "resolved": "https://registry.npmjs.org/universal-user-agent/-/universal-user-agent-6.0.0.tgz", - "integrity": "sha512-isyNax3wXoKaulPDZWHQqbmIx1k2tb9fb3GGDBRxCscfYV2Ch7WxPArBsFEG8s/safwXTT7H4QGhaIkTp9447w==" - }, - "node_modules/uuid": { - "version": "8.3.2", - "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz", - "integrity": "sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==", - "bin": { - "uuid": "dist/bin/uuid" - } - }, - "node_modules/web-streams-polyfill": { - "version": "3.2.1", - "resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-3.2.1.tgz", - "integrity": "sha512-e0MO3wdXWKrLbL0DgGnUV7WHVuw9OUvL4hjgnPkIeEvESk74gAITi5G606JtZPp39cd8HA9VQzCIvA49LpPN5Q==", - "engines": { - "node": ">= 8" - } - }, - "node_modules/webidl-conversions": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", - "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==" - }, - "node_modules/whatwg-url": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz", - "integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==", - "dependencies": { - "tr46": "~0.0.3", - "webidl-conversions": "^3.0.0" - } - }, - "node_modules/wrappy": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", - "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==" - } - } -} diff --git a/.github/my_actions/repository_dispatch/package.json b/.github/my_actions/repository_dispatch/package.json deleted file mode 100644 index 7c7a14c..0000000 --- a/.github/my_actions/repository_dispatch/package.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "name": "my_actions", - "version": "1.0.0", - "main": "index.js", - "scripts": { - "test": "echo \"Error: no test specified\" && exit 1" - }, - "keywords": [], - "author": "", - "license": "MIT", - "dependencies": { - "@actions/core": "^1.10.0", - "@actions/github": "^5.1.1", - "node-fetch": "^3.3.2" - }, - "description": "" -} diff --git a/.vscode/settings.json b/.vscode/settings.json deleted file mode 100644 index b76ca5e..0000000 --- a/.vscode/settings.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "files.associations": {"*.wic": "yaml"}, - "yaml.schemas": { - "autogenerated/schemas/wic.json": "*.wic" - }, - "yaml.customTags": [ - "!&", - "!*", - "!ii", - "!ii mapping", - "!ii sequence" - ], - "[yaml]": { - "editor.suggest.showWords": false - }, -} \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..592f585 --- /dev/null +++ b/README.md @@ -0,0 +1,135 @@ +# Sophios Workflows for Imaging Datasets + +CWL feature extraction workflow for imaging dataset + +## Workflow Stepup: + +Create a [Conda](https://conda.io/projects/conda/en/latest/user-guide/tasks/manage-environments.html#activating-an-environment) environment with Python between versions 3.9 and 3.11 + +``` +conda create -n image-workflow-env python=">=3.9,<3.12" +conda activate image-workflow-env +``` + +#### 1. Install image-workflows. + +- clone the image-workflows repository +```bash +git clone https://github.com/PolusAI/image-workflows.git +``` +- Navigate to the cloned directory `image-workflows` +```bash +cd image-workflows +``` +- Switch to the sophios_workflow branch: +```bash +git checkout -b sophios_workflow remotes/origin/sophios_workflow +``` +- `pip install .` + +#### 3. Generate and Configure the JSON Configuration File. +Execute the following command to generate the configuration file in your home directory `/home/user/wic/` + +```bash +sophios --generate_config +``` + +Directory and file structure after running the command: +``` +├── cwl_adapters +├── examples +└── global_config.json + +``` +Edit the `global_config.json` file to update the `search_paths_cwl`: + +``` + "search_paths_cwl": { + "global": [ + "/home/user/image-workflows/cwl_adapters" + ], + "gpu": [] + }, +``` + +#### Note: +Ensure that the [docker-desktop](https://www.docker.com/products/docker-desktop/) is running in the background. To verify that it's operational, you can use the following command: +`docker run -d -p 80:80 docker/getting-started` +This command will launch the `docker/getting-started container` in detached mode (-d flag), exposing port 80 on your local machine (-p 80:80). It's a simple way to test if Docker Desktop is functioning correctly. + +## Details +This workflow integrates eleven distinct plugins, starting from data retrieval from [Broad Bioimage Benchmark Collection](https://bbbc.broadinstitute.org/), renaming files, correcting uneven illumination, segmenting nuclear objects, and culminating in the extraction of features from identified objects + +Below are the specifics of the plugins employed in the workflow +1. [bbbc-download-plugin](https://github.com/saketprem/polus-plugins/tree/bbbc_download/utils/bbbc-download-plugin) +2. [file-renaming-tool](https://github.com/PolusAI/image-tools/tree/master/formats/file-renaming-tool) +3. [ome-converter-tool](https://github.com/PolusAI/image-tools/tree/master/formats/ome-converter-tool) +4. [basic-flatfield-estimation-tool](https://github.com/PolusAI/image-tools/tree/master/regression/basic-flatfield-estimation-tool) +5. [apply-flatfield-tool](https://github.com/PolusAI/image-tools/tree/master/transforms/images/apply-flatfield-tool) +6. [kaggle-nuclei-segmentation](https://github.com/hamshkhawar/image-tools/tree/kaggle-nuclei_seg/segmentation/kaggle-nuclei-segmentation) +7. [polus-ftl-label-plugin](https://github.com/hamshkhawar/image-tools/tree/kaggle-nuclei_seg/transforms/images/polus-ftl-label-plugin) +8. [nyxus-tool](https://github.com/PolusAI/image-tools/tree/master/features/nyxus-tool) +9. [montage-tool](https://github.com/PolusAI/image-tools/tree/master/transforms/images/montage-tool) +10. [image-assembler-tool](https://github.com/PolusAI/image-tools/tree/master/transforms/images/image-assembler-tool) +11. [precompute-slide-tool](https://github.com/PolusAI/image-tools/tree/master/visualization/precompute-slide-tool) + + +## Execute CWL workflows +Three different CWL workflows can be executed for specific datasets +1. segmentation +2. analysis +3. visualization + +During the execution of the segmentation workflow, `1 to 7` plugins will be utilized. However, for executing the analysis workflow, `1 to 8` plugins will be employed. +If a user wishes to execute a workflow for a new dataset, they can utilize a sample YAML file to input parameter values. This YAML file can be saved in the desired subdirectory of the `configuration` folder with the name `dataset.yml` + +If a user opts to run a workflow without background correction, they can set `background_correction` to false. In this case, the workflow will skip steps `4 and 5` + +`python -m image.workflows --name="BBBC001" --workflow=analysis --outDir=path/to/outputs` + +All outputs are stored within the `outDir` directory. +``` +outputs +├── experiment +│ └── cwl_adapters +| experiment.cwl +| experiment.yml +| +└── outdir + └── experiment + ├── step 1 BbbcDownload + │ └── outDir + │ └── bbbc.outDir + │ └── BBBC + │ └── BBBC039 + │ └── raw + │ ├── Ground_Truth + │ │ ├── masks + │ │ └── metadata + │ └── Images + │ └── images + ├── step 2 FileRenaming + │ └── outDir + │ └── rename.outDir + ├── step 3 OmeConverter + │ └── outDir + │ └── ome_converter.outDir + ├── step 4 BasicFlatfieldEstimation + │ └── outDir + │ └── estimate_flatfield.outDir + ├── step 5 ApplyFlatfield + │ └── outDir + │ └── apply_flatfield.outDir + ├── step 6 KaggleNucleiSegmentation + │ └── outDir + │ └── kaggle_nuclei_segmentation.outDir + ├── step 7 FtlLabel + │ └── outDir + │ └── ftl_plugin.outDir + └── step 8 NyxusPlugin + └── outDir + └── nyxus_plugin.outDir + +``` +#### Note: +Step 7 and step 8 are executed only in the case of the `analysis` workflow. `9-11` plugins will be used in `visualization` workflow \ No newline at end of file diff --git a/configuration/__init__.py b/configuration/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/configuration/analysis/BBBC001.yml b/configuration/analysis/BBBC001.yml new file mode 100644 index 0000000..7efe214 --- /dev/null +++ b/configuration/analysis/BBBC001.yml @@ -0,0 +1,14 @@ +--- +name : BBBC001 +file_pattern : /.*/.*/.*/Images/.*/.*_{row:c}{col:dd}f{f:dd}d{channel:d}.tif +out_file_pattern : x{row:dd}_y{col:dd}_p{f:dd}_c{channel:d}.tif +image_pattern: x{x:dd}_y{y:dd}_p{p:dd}_c{c:d}.ome.tif +seg_pattern: x{x:dd}_y{y:dd}_p{p:dd}_c0.ome.tif +ff_pattern: "x00_y03_p0\\(0-5\\)_c{c:d}_flatfield.ome.tif" +df_pattern: "x00_y03_p0\\(0-5\\)_c{c:d}_darkfield.ome.tif" +group_by: c +map_directory: false +features: ALL +file_extension: pandas +background_correction: false + diff --git a/configuration/analysis/BBBC039.yml b/configuration/analysis/BBBC039.yml new file mode 100644 index 0000000..308a274 --- /dev/null +++ b/configuration/analysis/BBBC039.yml @@ -0,0 +1,13 @@ +--- +name : BBBC039 +file_pattern : /.*/.*/.*/Images/.*/.*_{row:c}{col:dd}_s{s:d}_w{channel:d}.*.tif +out_file_pattern : x{row:dd}_y{col:dd}_p{s:dd}_c{channel:d}.tif +image_pattern: x{x:dd}_y{y:dd}_p{p:dd}_c{c:d}.ome.tif +seg_pattern: x{x:dd}_y{y:dd}_p{p:dd}_c1.ome.tif +ff_pattern: "x\\(00-15\\)_y\\(01-24\\)_p0\\(1-9\\)_c{c:d}_flatfield.ome.tif" +df_pattern: "x\\(00-15\\)_y\\(01-24\\)_p0\\(1-9\\)_c{c:d}_darkfield.ome.tif" +group_by: c +map_directory: false +features: "ALL_INTENSITY" +file_extension: pandas +background_correction: false \ No newline at end of file diff --git a/configuration/analysis/__init__.py b/configuration/analysis/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/configuration/analysis/sample.yml b/configuration/analysis/sample.yml new file mode 100644 index 0000000..47ffb02 --- /dev/null +++ b/configuration/analysis/sample.yml @@ -0,0 +1,13 @@ +--- +name : +file_pattern : +out_file_pattern : +image_pattern: +seg_pattern: +ff_pattern: +df_pattern: +group_by: +map_directory: +features: +file_extension: +background_correction: \ No newline at end of file diff --git a/configuration/segmentation/BBBC001.yml b/configuration/segmentation/BBBC001.yml new file mode 100644 index 0000000..a4cbe36 --- /dev/null +++ b/configuration/segmentation/BBBC001.yml @@ -0,0 +1,11 @@ +--- +name: BBBC001 +file_pattern: /.*/.*/.*/Images/.*/.*_{row:c}{col:dd}f{f:dd}d{channel:d}.tif +out_file_pattern: x{row:dd}_y{col:dd}_p{f:dd}_c{channel:d}.tif +image_pattern: x{x:dd}_y{y:dd}_p{p:dd}_c{c:d}.ome.tif +seg_pattern: x{x:dd}_y{y:dd}_p{p:dd}_c0.ome.tif +ff_pattern: "x00_y03_p0\\(0-5\\)_c{c:d}_flatfield.ome.tif" +df_pattern: "x00_y03_p0\\(0-5\\)_c{c:d}_darkfield.ome.tif" +group_by: c +map_directory: false +background_correction: false diff --git a/configuration/segmentation/BBBC039.yml b/configuration/segmentation/BBBC039.yml new file mode 100644 index 0000000..1884878 --- /dev/null +++ b/configuration/segmentation/BBBC039.yml @@ -0,0 +1,11 @@ +--- +name : BBBC039 +file_pattern : /.*/.*/.*/Images/.*/.*_{row:c}{col:dd}_s{s:d}_w{channel:d}.*.tif +out_file_pattern : x{row:dd}_y{col:dd}_p{s:dd}_c{channel:d}.tif +image_pattern: x{x:dd}_y{y:dd}_p{p:dd}_c{c:d}.ome.tif +seg_pattern: x{x:dd}_y{y:dd}_p{p:dd}_c1.ome.tif +ff_pattern: "x\\(00-15\\)_y\\(01-24\\)_p0\\(1-9\\)_c{c:d}_flatfield.ome.tif" +df_pattern: "x\\(00-15\\)_y\\(01-24\\)_p0\\(1-9\\)_c{c:d}_darkfield.ome.tif" +group_by: c +map_directory: false +background_correction: false \ No newline at end of file diff --git a/configuration/segmentation/__init__.py b/configuration/segmentation/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/configuration/segmentation/sample.yml b/configuration/segmentation/sample.yml new file mode 100644 index 0000000..ecc82e1 --- /dev/null +++ b/configuration/segmentation/sample.yml @@ -0,0 +1,12 @@ +--- +name : +file_pattern : +out_file_pattern : +image_pattern: +seg_pattern: +ff_pattern: +df_pattern: +group_by: +map_directory: +features: +file_extension: \ No newline at end of file diff --git a/configuration/visualization/BBBC001.yml b/configuration/visualization/BBBC001.yml new file mode 100644 index 0000000..7f5cb25 --- /dev/null +++ b/configuration/visualization/BBBC001.yml @@ -0,0 +1,14 @@ +--- +name: BBBC001 +file_pattern: /.*/.*/.*/Images/.*/.*_{row:c}{col:dd}f{f:dd}d{channel:d}.tif +out_file_pattern: x{row:dd}_y{col:dd}_p{f:dd}_c{channel:d}.tif +image_pattern: x{x:dd}_y{y:dd}_p{p:dd}_c{c:d}.ome.tif +seg_pattern: x{x:dd}_y{y:dd}_p{p:dd}_c0.ome.tif +ff_pattern: "x00_y03_p0\\(0-5\\)_c{c:d}_flatfield.ome.tif" +df_pattern: "x00_y03_p0\\(0-5\\)_c{c:d}_darkfield.ome.tif" +group_by: c +map_directory: false +background_correction: false +layout: p +pyramid_type: Zarr +image_type: Intensity diff --git a/configuration/visualization/BBBC039.yml b/configuration/visualization/BBBC039.yml new file mode 100644 index 0000000..1884878 --- /dev/null +++ b/configuration/visualization/BBBC039.yml @@ -0,0 +1,11 @@ +--- +name : BBBC039 +file_pattern : /.*/.*/.*/Images/.*/.*_{row:c}{col:dd}_s{s:d}_w{channel:d}.*.tif +out_file_pattern : x{row:dd}_y{col:dd}_p{s:dd}_c{channel:d}.tif +image_pattern: x{x:dd}_y{y:dd}_p{p:dd}_c{c:d}.ome.tif +seg_pattern: x{x:dd}_y{y:dd}_p{p:dd}_c1.ome.tif +ff_pattern: "x\\(00-15\\)_y\\(01-24\\)_p0\\(1-9\\)_c{c:d}_flatfield.ome.tif" +df_pattern: "x\\(00-15\\)_y\\(01-24\\)_p0\\(1-9\\)_c{c:d}_darkfield.ome.tif" +group_by: c +map_directory: false +background_correction: false \ No newline at end of file diff --git a/configuration/visualization/__init__.py b/configuration/visualization/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/configuration/visualization/sample.yml b/configuration/visualization/sample.yml new file mode 100644 index 0000000..ecc82e1 --- /dev/null +++ b/configuration/visualization/sample.yml @@ -0,0 +1,12 @@ +--- +name : +file_pattern : +out_file_pattern : +image_pattern: +seg_pattern: +ff_pattern: +df_pattern: +group_by: +map_directory: +features: +file_extension: \ No newline at end of file diff --git a/cwl_adapters/ApplyFlatfield.cwl b/cwl_adapters/ApplyFlatfield.cwl new file mode 100644 index 0000000..b27cef3 --- /dev/null +++ b/cwl_adapters/ApplyFlatfield.cwl @@ -0,0 +1,49 @@ +class: CommandLineTool +cwlVersion: v1.2 +inputs: + dfPattern: + inputBinding: + prefix: --dfPattern + type: string? + ffDir: + inputBinding: + prefix: --ffDir + type: Directory + ffPattern: + inputBinding: + prefix: --ffPattern + type: string + imgDir: + inputBinding: + prefix: --imgDir + type: Directory + imgPattern: + inputBinding: + prefix: --imgPattern + type: string + outDir: + inputBinding: + prefix: --outDir + type: Directory + preview: + inputBinding: + prefix: --preview + type: boolean? +outputs: + outDir: + outputBinding: + glob: $(inputs.outDir.basename) + type: Directory +requirements: + DockerRequirement: + dockerPull: polusai/apply-flatfield-tool:2.0.1 + EnvVarRequirement: + envDef: + HOME: /home/polusai + InitialWorkDirRequirement: + listing: + - entry: $(inputs.outDir) + writable: true + InlineJavascriptRequirement: {} + NetworkAccess: + networkAccess: true diff --git a/cwl_adapters/BasicFlatfieldEstimation.cwl b/cwl_adapters/BasicFlatfieldEstimation.cwl new file mode 100644 index 0000000..c695d3a --- /dev/null +++ b/cwl_adapters/BasicFlatfieldEstimation.cwl @@ -0,0 +1,41 @@ +class: CommandLineTool +cwlVersion: v1.2 +inputs: + filePattern: + inputBinding: + prefix: --filePattern + type: string + getDarkfield: + inputBinding: + prefix: --getDarkfield + type: boolean + groupBy: + inputBinding: + prefix: --groupBy + type: string? + inpDir: + inputBinding: + prefix: --inpDir + type: Directory + outDir: + inputBinding: + prefix: --outDir + type: Directory +outputs: + outDir: + outputBinding: + glob: $(inputs.outDir.basename) + type: Directory +requirements: + DockerRequirement: + dockerPull: polusai/basic-flatfield-estimation-tool:2.1.2 + EnvVarRequirement: + envDef: + HOME: /home/polusai + InitialWorkDirRequirement: + listing: + - entry: $(inputs.outDir) + writable: true + InlineJavascriptRequirement: {} + NetworkAccess: + networkAccess: true diff --git a/cwl_adapters/BbbcDownload.cwl b/cwl_adapters/BbbcDownload.cwl new file mode 100644 index 0000000..9eeba93 --- /dev/null +++ b/cwl_adapters/BbbcDownload.cwl @@ -0,0 +1,29 @@ +class: CommandLineTool +cwlVersion: v1.2 +inputs: + name: + inputBinding: + prefix: --name + type: string + outDir: + inputBinding: + prefix: --outDir + type: Directory +outputs: + outDir: + outputBinding: + glob: $(inputs.outDir.basename) + type: Directory +requirements: + DockerRequirement: + dockerPull: polusai/bbbc-download-plugin:0.1.0-dev1 + EnvVarRequirement: + envDef: + HOME: /home/polusai + InitialWorkDirRequirement: + listing: + - entry: $(inputs.outDir) + writable: true + InlineJavascriptRequirement: {} + NetworkAccess: + networkAccess: true diff --git a/cwl_adapters/FileRenaming.cwl b/cwl_adapters/FileRenaming.cwl new file mode 100644 index 0000000..1e154ed --- /dev/null +++ b/cwl_adapters/FileRenaming.cwl @@ -0,0 +1,41 @@ +class: CommandLineTool +cwlVersion: v1.2 +inputs: + filePattern: + inputBinding: + prefix: --filePattern + type: string + inpDir: + inputBinding: + prefix: --inpDir + type: Directory + mapDirectory: + inputBinding: + prefix: --mapDirectory + type: boolean? + outDir: + inputBinding: + prefix: --outDir + type: Directory + outFilePattern: + inputBinding: + prefix: --outFilePattern + type: string +outputs: + outDir: + outputBinding: + glob: $(inputs.outDir.basename) + type: Directory +requirements: + DockerRequirement: + dockerPull: polusai/file-renaming-tool:0.2.4-dev1 + EnvVarRequirement: + envDef: + HOME: /home/polusai + InitialWorkDirRequirement: + listing: + - entry: $(inputs.outDir) + writable: true + InlineJavascriptRequirement: {} + NetworkAccess: + networkAccess: true diff --git a/cwl_adapters/FtlLabel.cwl b/cwl_adapters/FtlLabel.cwl new file mode 100644 index 0000000..227be2a --- /dev/null +++ b/cwl_adapters/FtlLabel.cwl @@ -0,0 +1,37 @@ +class: CommandLineTool +cwlVersion: v1.2 +inputs: + binarizationThreshold: + inputBinding: + prefix: --binarizationThreshold + type: double + connectivity: + inputBinding: + prefix: --connectivity + type: string + inpDir: + inputBinding: + prefix: --inpDir + type: Directory + outDir: + inputBinding: + prefix: --outDir + type: Directory +outputs: + outDir: + outputBinding: + glob: $(inputs.outDir.basename) + type: Directory +requirements: + DockerRequirement: + dockerPull: polusai/ftl-label-plugin:0.3.12-dev5 + EnvVarRequirement: + envDef: + HOME: /home/polusai + InitialWorkDirRequirement: + listing: + - entry: $(inputs.outDir) + writable: true + InlineJavascriptRequirement: {} + NetworkAccess: + networkAccess: true diff --git a/cwl_adapters/ImageAssembler.cwl b/cwl_adapters/ImageAssembler.cwl new file mode 100644 index 0000000..8b06233 --- /dev/null +++ b/cwl_adapters/ImageAssembler.cwl @@ -0,0 +1,41 @@ +class: CommandLineTool +cwlVersion: v1.2 +inputs: + imgPath: + inputBinding: + prefix: --imgPath + type: Directory + outDir: + inputBinding: + prefix: --outDir + type: Directory + preview: + inputBinding: + prefix: --preview + type: boolean? + stitchPath: + inputBinding: + prefix: --stitchPath + type: Directory + timesliceNaming: + inputBinding: + prefix: --timesliceNaming + type: boolean? +outputs: + outDir: + outputBinding: + glob: $(inputs.outDir.basename) + type: Directory +requirements: + DockerRequirement: + dockerPull: polusai/image-assembler-tool:1.4.2 + EnvVarRequirement: + envDef: + HOME: /home/polusai + InitialWorkDirRequirement: + listing: + - entry: $(inputs.outDir) + writable: true + InlineJavascriptRequirement: {} + NetworkAccess: + networkAccess: true diff --git a/cwl_adapters/KaggleNucleiSegmentation.cwl b/cwl_adapters/KaggleNucleiSegmentation.cwl new file mode 100644 index 0000000..f27f358 --- /dev/null +++ b/cwl_adapters/KaggleNucleiSegmentation.cwl @@ -0,0 +1,37 @@ +class: CommandLineTool +cwlVersion: v1.2 +inputs: + filePattern: + inputBinding: + prefix: --filePattern + type: string? + inpDir: + inputBinding: + prefix: --inpDir + type: Directory + outDir: + inputBinding: + prefix: --outDir + type: Directory + preview: + inputBinding: + prefix: --preview + type: boolean? +outputs: + outDir: + outputBinding: + glob: $(inputs.outDir.basename) + type: Directory +requirements: + DockerRequirement: + dockerPull: polusai/kaggle-nuclei-segmentation-tool:0.1.5-dev2 + EnvVarRequirement: + envDef: + HOME: /home/polusai + InitialWorkDirRequirement: + listing: + - entry: $(inputs.outDir) + writable: true + InlineJavascriptRequirement: {} + NetworkAccess: + networkAccess: true diff --git a/cwl_adapters/Montage.cwl b/cwl_adapters/Montage.cwl new file mode 100644 index 0000000..e3ee6ea --- /dev/null +++ b/cwl_adapters/Montage.cwl @@ -0,0 +1,49 @@ +class: CommandLineTool +cwlVersion: v1.2 +inputs: + filePattern: + inputBinding: + prefix: --filePattern + type: string + flipAxis: + inputBinding: + prefix: --flipAxis + type: string? + gridSpacing: + inputBinding: + prefix: --gridSpacing + type: string? + imageSpacing: + inputBinding: + prefix: --imageSpacing + type: string? + inpDir: + inputBinding: + prefix: --inpDir + type: Directory + layout: + inputBinding: + prefix: --layout + type: string? + outDir: + inputBinding: + prefix: --outDir + type: Directory +outputs: + outDir: + outputBinding: + glob: $(inputs.outDir.basename) + type: Directory +requirements: + DockerRequirement: + dockerPull: polusai/montage-tool:0.5.1 + EnvVarRequirement: + envDef: + HOME: /home/polusai + InitialWorkDirRequirement: + listing: + - entry: $(inputs.outDir) + writable: true + InlineJavascriptRequirement: {} + NetworkAccess: + networkAccess: true diff --git a/cwl_adapters/NyxusPlugin.cwl b/cwl_adapters/NyxusPlugin.cwl new file mode 100644 index 0000000..bfbe6bb --- /dev/null +++ b/cwl_adapters/NyxusPlugin.cwl @@ -0,0 +1,61 @@ +class: CommandLineTool +cwlVersion: v1.2 +inputs: + features: + inputBinding: + prefix: --features + type: string? + fileExtension: + inputBinding: + prefix: --fileExtension + type: string + inpDir: + inputBinding: + prefix: --inpDir + type: Directory + intPattern: + inputBinding: + prefix: --intPattern + type: string + neighborDist: + inputBinding: + prefix: --neighborDist + type: double? + outDir: + inputBinding: + prefix: --outDir + type: Directory + pixelPerMicron: + inputBinding: + prefix: --pixelPerMicron + type: double? + segDir: + inputBinding: + prefix: --segDir + type: Directory + segPattern: + inputBinding: + prefix: --segPattern + type: string + singleRoi: + inputBinding: + prefix: --singleRoi + type: boolean? +outputs: + outDir: + outputBinding: + glob: $(inputs.outDir.basename) + type: Directory +requirements: + DockerRequirement: + dockerPull: polusai/nyxus-tool:0.1.8-dev0 + EnvVarRequirement: + envDef: + HOME: /home/polusai + InitialWorkDirRequirement: + listing: + - entry: $(inputs.outDir) + writable: true + InlineJavascriptRequirement: {} + NetworkAccess: + networkAccess: true diff --git a/cwl_adapters/OmeConverter.cwl b/cwl_adapters/OmeConverter.cwl new file mode 100644 index 0000000..c66d2cf --- /dev/null +++ b/cwl_adapters/OmeConverter.cwl @@ -0,0 +1,33 @@ +class: CommandLineTool +cwlVersion: v1.2 +inputs: + filePattern: + inputBinding: + prefix: --filePattern + type: string + inpDir: + inputBinding: + prefix: --inpDir + type: Directory + outDir: + inputBinding: + prefix: --outDir + type: Directory +outputs: + outDir: + outputBinding: + glob: $(inputs.outDir.basename) + type: Directory +requirements: + DockerRequirement: + dockerPull: polusai/ome-converter-tool:0.3.3-dev1 + EnvVarRequirement: + envDef: + HOME: /home/polusai + InitialWorkDirRequirement: + listing: + - entry: $(inputs.outDir) + writable: true + InlineJavascriptRequirement: {} + NetworkAccess: + networkAccess: true diff --git a/cwl_adapters/PrecomputeSlide.cwl b/cwl_adapters/PrecomputeSlide.cwl new file mode 100644 index 0000000..4dc1234 --- /dev/null +++ b/cwl_adapters/PrecomputeSlide.cwl @@ -0,0 +1,41 @@ +class: CommandLineTool +cwlVersion: v1.2 +inputs: + filePattern: + inputBinding: + prefix: --filePattern + type: string? + imageType: + inputBinding: + prefix: --imageType + type: string? + inpDir: + inputBinding: + prefix: --inpDir + type: Directory + outDir: + inputBinding: + prefix: --outDir + type: Directory + pyramidType: + inputBinding: + prefix: --pyramidType + type: string +outputs: + outDir: + outputBinding: + glob: $(inputs.outDir.basename) + type: Directory +requirements: + DockerRequirement: + dockerPull: polusai/precompute-slide-tool:1.7.2 + EnvVarRequirement: + envDef: + HOME: /home/polusai + InitialWorkDirRequirement: + listing: + - entry: $(inputs.outDir) + writable: true + InlineJavascriptRequirement: {} + NetworkAccess: + networkAccess: true diff --git a/cwl_adapters/basic-flatfield-estimation.cwl b/cwl_adapters/basic-flatfield-estimation.cwl deleted file mode 100644 index 85c18ea..0000000 --- a/cwl_adapters/basic-flatfield-estimation.cwl +++ /dev/null @@ -1,111 +0,0 @@ -#!/usr/bin/env cwl-runner -class: CommandLineTool -cwlVersion: v1.0 - -label: BaSiC Flatfield Estimation - -doc: |- - This WIPP plugin will take a collection of images and use the BaSiC flatfield correction algorithm to generate a flatfield image, a darkfield image, and a photobleach offset. - https://github.com/PolusAI/polus-plugins/tree/master/regression/basic-flatfield-estimation-plugin - -requirements: - DockerRequirement: - dockerPull: polusai/basic-flatfield-estimation-plugin:2.1.1 - # See https://www.commonwl.org/v1.0/CommandLineTool.html#InitialWorkDirRequirement - InitialWorkDirRequirement: - listing: - - entry: $(inputs.outDir) - writable: true # Output directories must be writable - InlineJavascriptRequirement: {} - -# "jax._src.xla_bridge - WARNING - An NVIDIA GPU may be present on this machine, but a CUDA-enabled jaxlib is not installed. Falling back to cpu." -hints: - cwltool:CUDARequirement: - cudaVersionMin: "11.4" - cudaComputeCapabilityMin: "3.0" - cudaDeviceCountMin: 1 - cudaDeviceCountMax: 1 - -inputs: - inpDir: - label: Path to input images - doc: |- - Path to input images - type: Directory - inputBinding: - prefix: --inpDir - - getDarkfield: - label: If 'true', will calculate darkfield image - doc: |- - If 'true', will calculate darkfield image - type: boolean? - inputBinding: - prefix: --getDarkfield - - # photobleach: - # label: If 'true', will calculate photobleach scalar - # doc: |- - # If 'true', will calculate photobleach scalar - # type: boolean? - # inputBinding: - # prefix: --photobleach - - filePattern: - label: File pattern to subset data - doc: |- - File pattern to subset data - type: string? - inputBinding: - prefix: --filePattern - - groupBy: - label: Variables to group together - doc: |- - Variables to group together - type: string? - inputBinding: - prefix: --groupBy - - preview: - label: Generate a JSON file describing what the outputs should be - doc: |- - Generate a JSON file describing what the outputs should be - type: boolean? - inputBinding: - prefix: --preview - - outDir: - label: Output image collection - doc: |- - Output image collection - type: Directory - inputBinding: - prefix: --outDir - -outputs: - outDir: - label: Output image collection - doc: |- - Output image collection - type: Directory - outputBinding: - glob: $(inputs.outDir.basename) - - preview_json: - label: JSON file describing what the outputs should be - doc: |- - JSON file describing what the outputs should be - type: File? # if --preview - format: edam:format_3464 - outputBinding: - glob: preview.json - -$namespaces: - edam: https://edamontology.org/ - cwltool: http://commonwl.org/cwltool# - -$schemas: -- https://raw.githubusercontent.com/edamontology/edamontology/master/EDAM_dev.owl - -# manifest: https://raw.githubusercontent.com/PolusAI/polus-plugins/master/regression/basic-flatfield-estimation-plugin/plugin.json \ No newline at end of file diff --git a/cwl_adapters/bbbcdownload.cwl b/cwl_adapters/bbbcdownload.cwl deleted file mode 100644 index 9fba292..0000000 --- a/cwl_adapters/bbbcdownload.cwl +++ /dev/null @@ -1,57 +0,0 @@ -class: CommandLineTool -cwlVersion: v1.1 - -label: BBBC Download - -doc: |- - Downloads the datasets on the Broad Bioimage Benchmark Collection website - https://github.com/saketprem/polus-plugins/tree/bbbc_download/utils/bbbc-download-plugin - -requirements: - DockerRequirement: - dockerPull: polusai/bbbc-download-plugin:0.1.0-dev1 - # See https://www.commonwl.org/v1.0/CommandLineTool.html#InitialWorkDirRequirement - InitialWorkDirRequirement: - listing: - - entry: $(inputs.outDir) - writable: true # Output directories must be writable - InlineJavascriptRequirement: {} - # NOTE: By default, "tools must not assume network access, except for localhost" - # See https://www.commonwl.org/v1.1/CommandLineTool.html#NetworkAccess - NetworkAccess: - networkAccess: true - -inputs: - name: - label: The name of the dataset(s) to be downloaded (separate the datasets with a comma. eg BBBC001,BBBC002,BBBC003) - doc: |- - The name of the dataset(s) to be downloaded (separate the datasets with a comma. eg BBBC001,BBBC002,BBBC003) - inputBinding: - prefix: --name - type: string - # default: BBBC001 - - outDir: - label: Output collection - doc: |- - Output collection - inputBinding: - prefix: --outDir - type: Directory - -outputs: - outDir: - label: Output collection - doc: |- - Output collection - type: Directory - outputBinding: - glob: $(inputs.outDir.basename) - -$namespaces: - edam: https://edamontology.org/ - -$schemas: -- https://raw.githubusercontent.com/edamontology/edamontology/master/EDAM_dev.owl - -# manifest: "https://raw.githubusercontent.com/saketprem/polus-plugins/bbbc_download/utils/bbbc-download-plugin/plugin.json" \ No newline at end of file diff --git a/cwl_adapters/file-renaming.cwl b/cwl_adapters/file-renaming.cwl deleted file mode 100644 index 29628cd..0000000 --- a/cwl_adapters/file-renaming.cwl +++ /dev/null @@ -1,81 +0,0 @@ -class: CommandLineTool -cwlVersion: v1.0 - -label: File Renaming - -doc: |- - Rename and store image collection files in a new image collection - https://github.com/PolusAI/polus-plugins/tree/master/formats/file-renaming-plugin - -requirements: - DockerRequirement: - dockerPull: polusai/file-renaming-plugin:0.2.1-dev0 # NOTE: 0.2.3 not pushed yet - # See https://www.commonwl.org/v1.0/CommandLineTool.html#InitialWorkDirRequirement - InitialWorkDirRequirement: - listing: - - entry: $(inputs.outDir) - writable: true # Output directories must be writable - InlineJavascriptRequirement: {} - -inputs: - inpDir: - inputBinding: - prefix: --inpDir - type: Directory - - filePattern: - inputBinding: - prefix: --filePattern - type: string - - mapDirectory: - inputBinding: - prefix: --mapDirectory - type: string? # enum: raw, map, default - - preview: - label: Generate a JSON file describing what the outputs should be - doc: |- - Generate a JSON file describing what the outputs should be - inputBinding: - prefix: --preview - type: boolean? - - outFilePattern: - inputBinding: - prefix: --outFilePattern - type: string - - outDir: - label: Output collection - doc: |- - Output collection - inputBinding: - prefix: --outDir - type: Directory - -outputs: - outDir: - label: Output collection - doc: |- - Output collection - type: Directory - outputBinding: - glob: $(inputs.outDir.basename) - - preview_json: - label: JSON file describing what the outputs should be - doc: |- - JSON file describing what the outputs should be - type: File? # if --preview - format: edam:format_3464 - outputBinding: - glob: preview.json - -$namespaces: - edam: https://edamontology.org/ - -$schemas: -- https://raw.githubusercontent.com/edamontology/edamontology/master/EDAM_dev.owl - -# manifest: https://raw.githubusercontent.com/PolusAI/polus-plugins/master/formats/file-renaming-plugin/plugin.json \ No newline at end of file diff --git a/cwl_adapters/image_assembler.cwl b/cwl_adapters/image_assembler.cwl deleted file mode 100644 index f6d179f..0000000 --- a/cwl_adapters/image_assembler.cwl +++ /dev/null @@ -1,97 +0,0 @@ -#!/usr/bin/env cwl-runner -class: CommandLineTool -cwlVersion: v1.0 - -label: Image Assembler - -doc: |- - This plugin assembles images into a stitched image using an image stitching vector. - https://github.com/PolusAI/polus-plugins/tree/master/transforms/images/image-assembler-plugin - -requirements: - DockerRequirement: - dockerPull: polusai/image-assembler-plugin:1.4.0-dev0 - # See https://www.commonwl.org/v1.0/CommandLineTool.html#InitialWorkDirRequirement - InitialWorkDirRequirement: - listing: - - $(inputs.stitchPath) # Must stage inputs for tools which do not accept full paths. - - entry: $(inputs.outDir) - writable: true # Output directories must be writable - InlineJavascriptRequirement: {} - -inputs: - stitchPath: - label: Path to directory containing "stitching vector" file img-global-positions-0.txt - doc: |- - Path to directory containing "stitching vector" file img-global-positions-0.txt - type: Directory - inputBinding: - prefix: --stitchPath - - imgPath: - label: Path to input image collection - doc: |- - Path to input image collection - type: Directory - inputBinding: - prefix: --imgPath - - timesliceNaming: - label: Label images by timeslice rather than analyzing input image names - doc: |- - Label images by timeslice rather than analyzing input image names - inputBinding: - prefix: --timesliceNaming - type: boolean? - - preview: - label: Generate a JSON file describing what the outputs should be - doc: |- - Generate a JSON file describing what the outputs should be - type: boolean? - inputBinding: - prefix: --preview - - outDir: - label: Output collection - doc: |- - Output collection - type: Directory - inputBinding: - prefix: --outDir - -outputs: - outDir: - label: Output collection - doc: |- - Output collection - type: Directory - outputBinding: - glob: $(inputs.outDir.basename) - - assembled_image: - label: The assembled montage image - doc: |- - JSON file with outputs - type: File? # if not --preview - # See https://bioportal.bioontology.org/ontologies/EDAM?p=classes&conceptid=format_3727 - format: edam:format_3727 - outputBinding: - glob: "*.ome.tif" - - preview_json: - label: JSON file with outputs - doc: |- - JSON file with outputs - type: File? # if --preview - format: edam:format_3464 - outputBinding: - glob: preview.json - -$namespaces: - edam: https://edamontology.org/ - -$schemas: -- https://raw.githubusercontent.com/edamontology/edamontology/master/EDAM_dev.owl - -# manifest: https://raw.githubusercontent.com/PolusAI/polus-plugins/master/transforms/images/image-assembler-plugin/plugin.json \ No newline at end of file diff --git a/cwl_adapters/montage.cwl b/cwl_adapters/montage.cwl deleted file mode 100644 index bc90bb0..0000000 --- a/cwl_adapters/montage.cwl +++ /dev/null @@ -1,119 +0,0 @@ -#!/usr/bin/env cwl-runner -class: CommandLineTool -cwlVersion: v1.0 - -label: Montage - -doc: |- - This plugin generates a stitching vector that will montage images together. - https://github.com/PolusAI/polus-plugins/tree/master/transforms/images/montage-plugin - -requirements: - DockerRequirement: - dockerPull: polusai/montage-plugin:0.5.0 - # See https://www.commonwl.org/v1.0/CommandLineTool.html#InitialWorkDirRequirement - InitialWorkDirRequirement: - listing: - - entry: $(inputs.outDir) - writable: true # Output directories must be writable - InlineJavascriptRequirement: {} - -inputs: - inpDir: - label: Input image collection to be processed by this plugin - doc: |- - Input image collection to be processed by this plugin - type: Directory - inputBinding: - prefix: --inpDir - - filePattern: - label: Filename pattern used to parse data - doc: |- - Filename pattern used to parse data - type: string - inputBinding: - prefix: --filePattern - - layout: - label: Specify montage organization - doc: |- - Specify montage organization - type: string? - # optional array of strings? - inputBinding: - prefix: --layout - - gridSpacing: - label: Specify spacing between images in the lowest grid - doc: |- - Specify spacing between images in the lowest grid - inputBinding: - prefix: --gridSpacing - type: int? - - imageSpacing: - label: Specify spacing multiplier between grids - doc: |- - Specify spacing multiplier between grids - inputBinding: - prefix: --imageSpacing - type: int? - - flipAxis: - label: Axes to flip when laying out images - doc: |- - Axes to flip when laying out images - inputBinding: - prefix: --flipAxis - type: string? - - preview: - label: Generate a JSON file describing what the outputs should be - doc: |- - Generate a JSON file describing what the outputs should be - type: boolean? - inputBinding: - prefix: --preview - - outDir: - label: Output collection - doc: |- - Output collection - type: Directory - inputBinding: - prefix: --outDir - -outputs: - outDir: - label: Output collection - doc: |- - Output collection - type: Directory - outputBinding: - glob: $(inputs.outDir.basename) - - global_positions: - label: The "stitching vector", i.e. the positions of the individual images in the montage - doc: |- - The "stitching vector", i.e. the positions of the individual images in the montage - type: File? # if not --preview - outputBinding: - glob: $(inputs.outDir.basename)/img-global-positions-0.txt - - preview_json: - label: JSON file describing what the outputs should be - doc: |- - JSON file describing what the outputs should be - type: File? # if --preview - format: edam:format_3464 - outputBinding: - glob: preview.json - -$namespaces: - edam: https://edamontology.org/ - -$schemas: -- https://raw.githubusercontent.com/edamontology/edamontology/master/EDAM_dev.owl - -# manifest: https://raw.githubusercontent.com/PolusAI/polus-plugins/master/transforms/images/montage-plugin/plugin.json \ No newline at end of file diff --git a/cwl_adapters/ome-converter.cwl b/cwl_adapters/ome-converter.cwl deleted file mode 100644 index 7dd5607..0000000 --- a/cwl_adapters/ome-converter.cwl +++ /dev/null @@ -1,81 +0,0 @@ -#!/usr/bin/env cwl-runner -class: CommandLineTool -cwlVersion: v1.0 - -label: OME Zarr Converter - -doc: |- - This WIPP plugin converts BioFormats supported data types to the OME Zarr file format. - https://github.com/PolusAI/polus-plugins/tree/master/formats/ome-converter-plugin - -requirements: - DockerRequirement: - dockerPull: polusai/ome-converter-plugin:0.3.2-dev2 - # See https://www.commonwl.org/v1.0/CommandLineTool.html#InitialWorkDirRequirement - InitialWorkDirRequirement: - listing: - - entry: $(inputs.outDir) - writable: true # Output directories must be writable - InlineJavascriptRequirement: {} -# NOTE: polusai/ome-converter-plugin:0.3.1 uses the base image -# polusai/bfio:2.3.2 which now un-bundles the java maven package -# ome:formats-gpl:7.1.0 due to licensing reasons. -# To avoid requiring network access at runtime, in the bfio Dockerfile -# it is pre-installed and saved in ~/.m2/ However, by default -# CWL hides all environment variables (including HOME), so we need to -# set HOME here so that at runtime we get a cache hit on the maven install. - EnvVarRequirement: -# See https://www.commonwl.org/user_guide/topics/environment-variables.html - envDef: - HOME: /home/polusai - -inputs: - inpDir: - label: Input generic data collection to be processed by this plugin - doc: |- - Input generic data collection to be processed by this plugin - type: Directory - inputBinding: - prefix: --inpDir - - filePattern: - label: A filepattern, used to select data for conversion - doc: |- - A filepattern, used to select data for conversion - type: string - inputBinding: - prefix: --filePattern - - fileExtension: - label: The file extension - doc: |- - The file extension - type: string - inputBinding: - prefix: --fileExtension - default: "default" # enum: .ome.tiff, .ome.zarr, default - - outDir: - label: Output collection - doc: |- - Output collection - type: Directory - inputBinding: - prefix: --outDir - -outputs: - outDir: - label: Output collection - doc: |- - Output collection - type: Directory - outputBinding: - glob: $(inputs.outDir.basename) - -$namespaces: - edam: https://edamontology.org/ - -$schemas: -- https://raw.githubusercontent.com/edamontology/edamontology/master/EDAM_dev.owl - -# manifest: https://raw.githubusercontent.com/PolusAI/polus-plugins/master/formats/ome-converter-plugin/plugin.json \ No newline at end of file diff --git a/cwl_adapters/precompute_slide.cwl b/cwl_adapters/precompute_slide.cwl deleted file mode 100644 index 2419170..0000000 --- a/cwl_adapters/precompute_slide.cwl +++ /dev/null @@ -1,77 +0,0 @@ -#!/usr/bin/env cwl-runner -class: CommandLineTool -cwlVersion: v1.0 - -label: Precompute Slide - -doc: |- - This plugin generates image pyramids in multiple viewing formats. - https://github.com/PolusAI/polus-plugins/tree/master/visualization/polus-precompute-slide-plugin - -requirements: - DockerRequirement: - dockerPull: polusai/precompute-slide-plugin:1.7.0-dev0 - # See https://www.commonwl.org/v1.0/CommandLineTool.html#InitialWorkDirRequirement - InitialWorkDirRequirement: - listing: - - entry: $(inputs.outDir) - writable: true # Output directories must be writable - InlineJavascriptRequirement: {} - -inputs: - inpDir: - label: Input generic data collection to be processed by this plugin - doc: |- - Input generic data collection to be processed by this plugin - type: Directory - inputBinding: - prefix: --inpDir - - pyramidType: - label: Build a DeepZoom, Neuroglancer, Zarr pyramid - doc: |- - Build a DeepZoom, Neuroglancer, Zarr pyramid - type: string # enum: DeepZoom, Neuroglancer, Zarr - inputBinding: - prefix: --pyramidType - - imageType: - label: Image is either Segmentation or Image - doc: |- - Image is either Segmentation or Image - inputBinding: - prefix: --imageType - type: string - - filePattern: - label: Filename pattern used to parse data - doc: |- - Filename pattern used to parse data - type: string? - inputBinding: - prefix: --filePattern - - outDir: - label: Output collection - doc: |- - Output collection - type: Directory - inputBinding: - prefix: --outDir - -outputs: - outDir: - label: Output collection - doc: |- - Output collection - type: Directory - outputBinding: - glob: $(inputs.outDir.basename) - -$namespaces: - edam: https://edamontology.org/ - -$schemas: -- https://raw.githubusercontent.com/edamontology/edamontology/master/EDAM_dev.owl - -# \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..ba436a8 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,42 @@ +[tool.poetry] +name = "image-workflows" +version = "0.1.2-dev0" +description = "Build and execute pipelines of polus plugins on Compute." +authors = ["Hamdah Shafqat Abbasi "] +readme = "README.md" +packages = [{include = "image", from = "src"}] + +[tool.poetry.dependencies] +python = ">=3.9,<3.12" +typer = "^0.9.0" +pyyaml = "^6.0.1" +pydantic = "^2.6.1" +graphviz = "^0.20.3" +toil = "7.0.0" +sophios = "^0.1.4" +cwlref-runner = "1.0" +cwltool = "3.1.20240508115724" +polus-tools = { git = "https://github.com/PolusAI/tools.git" } + + + +[tool.poetry.group.dev.dependencies] +jupyter = "^1.0.0" +nbconvert = "^7.11.0" +pytest = "^7.4.4" +bump2version = "^1.0.1" +pre-commit = "^3.3.3" +black = "^23.3.0" +ruff = "^0.0.274" +mypy = "^1.4.0" +pytest-xdist = "^3.3.1" +pytest-sugar = "^0.9.7" + +[build-system] +requires = ["poetry-core>=1.0.0"] +build-backend = "poetry.core.masonry.api" + +[tool.pytest.ini_options] +addopts = [ + "--import-mode=importlib", +] \ No newline at end of file diff --git a/src/image/workflows/__init__.py b/src/image/workflows/__init__.py new file mode 100644 index 0000000..d50ea40 --- /dev/null +++ b/src/image/workflows/__init__.py @@ -0,0 +1,3 @@ +from .cwl_analysis import CWLAnalysisWorkflow +from .cwl_nuclear_segmentation import CWLSegmentationWorkflow +from .cwl_visualization import CWLVisualizationWorkflow \ No newline at end of file diff --git a/src/image/workflows/__main__.py b/src/image/workflows/__main__.py new file mode 100644 index 0000000..99013b2 --- /dev/null +++ b/src/image/workflows/__main__.py @@ -0,0 +1,122 @@ +"""CWL Workflow.""" +import logging +import typer +from pathlib import Path +from typing import Optional +from image.workflows.utils import LoadYaml +from image.workflows.cwl_analysis import CWLAnalysisWorkflow +from image.workflows.cwl_nuclear_segmentation import CWLSegmentationWorkflow +from image.workflows.cwl_visualization import CWLVisualizationWorkflow + + + + +app = typer.Typer() + +# Initialize the logger +logging.basicConfig( + format="%(asctime)s - %(name)-8s - %(levelname)-8s - %(message)s", + datefmt="%d-%b-%y %H:%M:%S", +) +logger = logging.getLogger("WIC Python API") +logger.setLevel(logging.INFO) + + +# Mapping of workflow names to their corresponding classes +WORKFLOW_CLASSES = { + "analysis": CWLAnalysisWorkflow, + "segmentation": CWLSegmentationWorkflow, + "visualization": CWLVisualizationWorkflow +} + + +@app.command() +def main( + name: str = typer.Option( + ..., + "--name", + "-n", + help="Name of imaging dataset of Broad Bioimage Benchmark Collection (https://bbbc.broadinstitute.org/image_sets)" + ), + workflow: str = typer.Option( + ..., + "--workflow", + "-w", + help="Name of cwl workflow" + ), + out_dir: Optional[Path] = typer.Option( + None, + "--outDir", + "-o", + help="Name of cwl workflow" + ) +) -> None: + + """ + Execute the specified CWL Workflow. + + Attributes: + name (str): The name of the imaging dataset. + workflow (str): The name of the CWL workflow to execute. + out_dir (Path): The output directory for workflow results. + """ + + logger.info(f"name = {name}") + logger.info(f"workflow = {workflow}") + logger.info(f"outDir = {out_dir}") + + config_path = Path.cwd().joinpath(f"configuration/{workflow}/{name}.yml") + work_dir = Path.cwd() + + try: + # Load the YAML configuration + model = LoadYaml(workflow=workflow, config_path=config_path) + params = model.parse_yaml() + + # Set output directory + out_dir = out_dir or Path.cwd() + params["out_dir"] = out_dir + params["work_dir"] = work_dir + + # Get the workflow class + workflow_class = WORKFLOW_CLASSES.get(workflow) + if not workflow_class: + logger.error(f"Workflow '{workflow}' is not recognized. Available workflows: {list(WORKFLOW_CLASSES.keys())}") + raise ValueError(f"Unknown workflow: {workflow}") + + logger.info(f"Executing {workflow} workflow.") + # Initialize and execute the workflow class + workflow_instance = workflow_class(**params) + workflow_instance.workflow() + + except FileNotFoundError as e: + logger.error(f"Configuration file not found: {e}") + except Exception as e: + logger.error(f"An error occurred while executing the workflow: {e}") + raise + + # model = LoadYaml(workflow=workflow, config_path=config_path) + # params = model.parse_yaml() + + # out_dir = out_dir or Path.cwd() + + # params["out_dir"] = out_dir + # params["work_dir"] = work_dir + + # # Validate workflow and execute corresponding class + # workflow_class = WORKFLOW_CLASSES.get(workflow) + # print(workflow_class) + # if workflow_class: + # logger.info(f"Executing {workflow} workflow.") + # model = workflow_class(**params) + # model.workflow() + # else: + # logger.error(f"Invalid workflow: {workflow}. Available options: {', '.join(WORKFLOW_CLASSES.keys())}") + # raise ValueError(f"Workflow '{workflow}' is not recognized.") + + # logger.info(f"Completed {workflow} workflow execution!") + + + +if __name__ == "__main__": + app() \ No newline at end of file diff --git a/workflows/bbbc.py b/src/image/workflows/bbbc.py similarity index 100% rename from workflows/bbbc.py rename to src/image/workflows/bbbc.py diff --git a/workflows/bbbc.wic b/src/image/workflows/bbbc.wic similarity index 100% rename from workflows/bbbc.wic rename to src/image/workflows/bbbc.wic diff --git a/src/image/workflows/bbbc.yml b/src/image/workflows/bbbc.yml new file mode 100644 index 0000000..76541d9 --- /dev/null +++ b/src/image/workflows/bbbc.yml @@ -0,0 +1,39 @@ +steps: +- bbbcdownload: + in: + name: BBBC001 + outDir: '&bbbcdownload.outDir' +- subdirectory: + in: + directory: '*bbbcdownload.outDir' + glob_pattern: bbbcdownload.outDir/BBBC/BBBC001/raw/Images/human_ht29_colon_cancer_1_images/ + subdirectory: '&subdirectory.subdirectory' +- file-renaming: + in: + filePattern: .*_{row:c}{col:dd}f{f:dd}d{channel:d}.tif + inpDir: '*subdirectory.subdirectory' + outDir: '&file-renaming.outDir' + outFilePattern: x{row:dd}_y{col:dd}_p{f:dd}_c{channel:d}.tif +- ome-converter: + in: + fileExtension: .ome.tif + filePattern: .*.tif + inpDir: '*file-renaming.outDir' + outDir: '&omeconverter.outDir' +- montage: + in: + filePattern: x00_y03_p{p:dd}_c0.ome.tif + inpDir: '*omeconverter.outDir' + layout: p + outDir: '&montage.outDir' +- image_assembler: + in: + imgPath: '*omeconverter.outDir' + outDir: '&image_assembler.outDir' + stitchPath: '*montage.outDir' +- precompute_slide: + in: + imageType: image + inpDir: '*image_assembler.outDir' + outDir: precompute_slide.outDir + pyramidType: Zarr diff --git a/workflows/bbbc_sub.py b/src/image/workflows/bbbc_sub.py similarity index 100% rename from workflows/bbbc_sub.py rename to src/image/workflows/bbbc_sub.py diff --git a/src/image/workflows/cwl_analysis.py b/src/image/workflows/cwl_analysis.py new file mode 100644 index 0000000..a6bc0b0 --- /dev/null +++ b/src/image/workflows/cwl_analysis.py @@ -0,0 +1,216 @@ +import logging +import re +import shutil +from pathlib import Path +import typing +import yaml + +from sophios.api.pythonapi import Step, Workflow +import polus.tools.plugins as pp +from image.workflows.utils import OUT_PATH, MANIFEST_URLS + + +# Initialize the logger +logger = logging.getLogger(__name__) +logger.setLevel(logging.INFO) + + +class CWLAnalysisWorkflow: + """ + A CWL feature extraction or Analysis pipeline. + + Attributes: + name: Name of the imaging dataset. + file_pattern: Pattern for parsing raw filenames. + out_file_pattern: Desired format for output filenames. + image_pattern: Pattern for parsing intensity image filenames. + seg_pattern: Pattern to parse segmentation image filenames. + map_directory: Enable mapping of folder names. + ff_pattern: Filename pattern for selecting flatfield components. + df_pattern: Filename pattern for selecting darkfield components. + group_by: Variables used for grouping the file pattern. + features:Features from Nyxus (https://github.com/PolusAI/nyxus/) that need extraction + file_extension: Output file format + background_correction: Flag to enable background correction. + out_dir: Directory for saving outputs. + """ + def __init__( + self, + work_dir: Path, + name: str, + file_pattern: str, + out_file_pattern: str, + image_pattern: str, + seg_pattern: str, + ff_pattern: typing.Optional[str] = '', + df_pattern: typing.Optional[str] = '', + group_by: typing.Optional[str] = '', + map_directory: typing.Optional[bool] = False, + features: typing.Optional[str]="ALL", + file_extension: typing.Optional[str]="arrowipc", + background_correction: typing.Optional[bool] = False, + out_dir: typing.Optional[Path] = OUT_PATH + ): + self.name = name + self.file_pattern = file_pattern + self.out_file_pattern = out_file_pattern + self.image_pattern = image_pattern + self.seg_pattern = seg_pattern + self.ff_pattern = ff_pattern + self.df_pattern = df_pattern + self.group_by = group_by + self.map_directory = map_directory + self.features = features + self.file_extension=file_extension + self.background_correction = background_correction + self.out_dir = out_dir + self.work_dir = work_dir + self.adapters_path = self.work_dir.joinpath("cwl_adapters") + if not self.adapters_path.exists(): + self.adapters_path.mkdir(exist_ok=True, parents=True) + + def _move_outputs(self) -> None: + """Move output files and directories to the specified output directory.""" + logger.info("Moving directories and JSON files to output directory.") + dir_names = {"autogenerated", "cachedir", "provenance", "outdir"} + + # Move specified directories + for directory in self.work_dir.iterdir(): + if directory.name in dir_names: + shutil.move(directory, self.out_dir) + + # Move JSON files + for json_file in self.work_dir.rglob("*.json"): + shutil.move(json_file, self.out_dir) + + @staticmethod + def _to_camel_case(name: str) -> str: + """Convert a plugin name to camel case.""" + return re.sub(r"(_|-)+", " ", name).title().replace(" ", "") + + @staticmethod + def _extract_file_extension(pattern: str) -> str: + """Extract and return the string after a period in the pattern.""" + match = re.search(r"\.(.*)", pattern) + return f".*.{match.group(1)}" if match else "" + + @staticmethod + def _escape_parentheses(pattern: str) -> str: + """Escape parentheses and generate patterns for flatfield and darkfield correction.""" + pattern = re.sub(r"(\()|(\))", r"\\\1\2", pattern) + return pattern.split("_c")[0] + "_c{c:d}" + re.sub(r"\d", "", pattern.split("_c")[1]) + + def create_step(self, plugin_url: str) -> Step: + """Create a step for the workflow based on the plugin manifest.""" + manifest = dict(pp.submit_plugin(plugin_url)) + plugin_version = str(manifest['version']) + cwl_tool = pp.get_plugin(self._to_camel_case(manifest['name']), plugin_version).save_cwl( + self.adapters_path.joinpath(f"{self._to_camel_case(manifest['name'])}.cwl") + ) + self._modify_cwl() + return Step(cwl_tool) + + def _get_manifest_url(self, plugin_name: str) -> str: + """Retrieve the URL for the plugin manifest from GitHub.""" + return MANIFEST_URLS.get(plugin_name, "") + + def _modify_cwl(self) -> None: + """Modify CWL files to include environmental variables and permissions.""" + for cwl_file in self.adapters_path.rglob("*.cwl"): + if "cwl" in cwl_file.name: + try: + with cwl_file.open("r") as file: + config = yaml.safe_load(file) + config.setdefault("requirements", {}) + config["requirements"]["NetworkAccess"] = {"networkAccess": True} + config["requirements"]["EnvVarRequirement"] = {"envDef": {"HOME": "/home/polusai"}} + with cwl_file.open("w") as out_file: + yaml.dump(config, out_file) + except FileNotFoundError: + logger.error(f"Error processing file: {cwl_file}") + + def workflow(self) -> None: + """Execute the CWL nuclear segmentation pipeline.""" + logger.info("Starting CWL nuclear segmentation workflow.") + + # Step: BBBC Download + bbbc = self.create_step(self._get_manifest_url("bbbc_download")) + bbbc.name = self.name + bbbc.outDir = Path("bbbc.outDir") + + # Step: File Renaming + rename = self.create_step(self._get_manifest_url("file_renaming")) + rename.filePattern = self.file_pattern + rename.outFilePattern = self.out_file_pattern + rename.mapDirectory = self.map_directory + rename.inpDir = bbbc.outDir + rename.outDir = Path("rename.outDir") + + # Step: OME Converter + ome_converter = self.create_step(self._get_manifest_url("ome_converter")) + ome_converter.filePattern = self._extract_file_extension(self.out_file_pattern) + ome_converter.fileExtension = ".ome.tif" + ome_converter.inpDir = rename.outDir + ome_converter.outDir = Path("ome_converter.outDir") + + # Optional: Background correction + if self.background_correction: + estimate_flatfield = self.create_step(self._get_manifest_url("estimate_flatfield")) + estimate_flatfield.inpDir = ome_converter.outDir + estimate_flatfield.filePattern = self.image_pattern + estimate_flatfield.groupBy = self.group_by + estimate_flatfield.getDarkfield = True + estimate_flatfield.outDir = Path("estimate_flatfield.outDir") + + apply_flatfield = self.create_step(self._get_manifest_url("apply_flatfield")) + apply_flatfield.imgDir = ome_converter.outDir + apply_flatfield.imgPattern = self.image_pattern + apply_flatfield.ffDir = estimate_flatfield.outDir + apply_flatfield.ffPattern = self.ff_pattern + apply_flatfield.dfPattern = self.df_pattern + apply_flatfield.outDir = Path("apply_flatfield.outDir") + + # Step: Kaggle Nuclei Segmentation + kaggle_segmentation = self.create_step(self._get_manifest_url("kaggle_nuclei_segmentation")) + kaggle_segmentation.inpDir = apply_flatfield.outDir if self.background_correction else ome_converter.outDir + kaggle_segmentation.filePattern = self.image_pattern + kaggle_segmentation.outDir = Path("kaggle_nuclei_segmentation.outDir") + + # Step: FTL Label Plugin + ftl_plugin = self.create_step(self._get_manifest_url("ftl_plugin")) + ftl_plugin.inpDir = kaggle_segmentation.outDir + ftl_plugin.connectivity = "1" + ftl_plugin.binarizationThreshold = 0.5 + ftl_plugin.outDir = Path("ftl_plugin.outDir") + + + # # ## Nyxus Plugin + nyxus_plugin = self.create_step(self._get_manifest_url("nyxus_plugin")) + nyxus_plugin.inpDir = apply_flatfield.outDir if self.background_correction else ome_converter.outDir + nyxus_plugin.segDir = ftl_plugin.outDir + nyxus_plugin.intPattern = self.image_pattern + nyxus_plugin.segPattern = self.seg_pattern + nyxus_plugin.features = self.features + nyxus_plugin.fileExtension = self.file_extension + nyxus_plugin.neighborDist = 5 + nyxus_plugin.pixelPerMicron = 1.0 + nyxus_plugin.outDir = Path("nyxus_plugin.outDir") + + # Run the workflow + + steps = [ + bbbc, rename, ome_converter, + estimate_flatfield if self.background_correction else None, + apply_flatfield if self.background_correction else None, + kaggle_segmentation, + ftl_plugin, + nyxus_plugin + ] + + workflow = Workflow(steps, f"{self.name}_workflow") + # Compile and run using WIC python API + workflow.compile() + workflow.run() + self._move_outputs() + logger.info("Completed CWL analysis workflow.") + return diff --git a/src/image/workflows/cwl_nuclear_segmentation.py b/src/image/workflows/cwl_nuclear_segmentation.py new file mode 100644 index 0000000..3ba1087 --- /dev/null +++ b/src/image/workflows/cwl_nuclear_segmentation.py @@ -0,0 +1,198 @@ +import logging +import re +import shutil +from pathlib import Path +import typing +import yaml + +from sophios.api.pythonapi import Step, Workflow +import polus.tools.plugins as pp +from image.workflows.utils import OUT_PATH,MANIFEST_URLS + + +# Initialize the logger +logger = logging.getLogger(__name__) +logger.setLevel(logging.INFO) + + +class CWLSegmentationWorkflow: + """ + A CWL Nuclear Segmentation pipeline to process imaging datasets. + + Attributes: + work_dir: Path to working directory. + name: Name of the imaging dataset. + file_pattern: Pattern for parsing raw filenames. + out_file_pattern: Desired format for output filenames. + image_pattern: Pattern for parsing intensity image filenames. + seg_pattern: Pattern to parse segmentation image filenames. + ff_pattern: Filename pattern for selecting flatfield components. + df_pattern: Filename pattern for selecting darkfield components. + group_by: Variables used for grouping the file pattern. + background_correction: Flag to enable background correction. + map_directory: Enable mapping of folder names. + out_dir: Directory for saving outputs. + """ + def __init__( + self, + work_dir: Path, + name: str, + file_pattern: str, + out_file_pattern: str, + image_pattern: str, + seg_pattern: str, + ff_pattern: typing.Optional[str] = '', + df_pattern: typing.Optional[str] = '', + group_by: typing.Optional[str] = '', + map_directory: typing.Optional[bool] = False, + background_correction: typing.Optional[bool] = False, + out_dir: typing.Optional[Path] = OUT_PATH + ): + self.name = name + self.file_pattern = file_pattern + self.out_file_pattern = out_file_pattern + self.image_pattern = image_pattern + self.seg_pattern = seg_pattern + self.ff_pattern = ff_pattern + self.df_pattern = df_pattern + self.group_by = group_by + self.map_directory = map_directory + self.background_correction = background_correction + self.out_dir = out_dir + self.work_dir = work_dir + self.adapters_path = self.work_dir.joinpath("cwl_adapters") + if not self.adapters_path.exists(): + self.adapters_path.mkdir(exist_ok=True, parents=True) + + def _move_outputs(self) -> None: + """Move output files and directories to the specified output directory.""" + logger.info("Moving directories and JSON files to output directory.") + dir_names = {"autogenerated", "cachedir", "provenance", "outdir"} + + # Move specified directories + for directory in self.work_dir.iterdir(): + if directory.name in dir_names: + shutil.move(directory, self.out_dir) + + # Move JSON files + for json_file in self.work_dir.rglob("*.json"): + shutil.move(json_file, self.out_dir) + + @staticmethod + def _to_camel_case(name: str) -> str: + """Convert a plugin name to camel case.""" + return re.sub(r"(_|-)+", " ", name).title().replace(" ", "") + + @staticmethod + def _extract_file_extension(pattern: str) -> str: + """Extract and return the string after a period in the pattern.""" + match = re.search(r"\.(.*)", pattern) + return f".*.{match.group(1)}" if match else "" + + @staticmethod + def _escape_parentheses(pattern: str) -> str: + """Escape parentheses and generate patterns for flatfield and darkfield correction.""" + pattern = re.sub(r"(\()|(\))", r"\\\1\2", pattern) + return pattern.split("_c")[0] + "_c{c:d}" + re.sub(r"\d", "", pattern.split("_c")[1]) + + def create_step(self, plugin_url: str) -> Step: + """Create a step for the workflow based on the plugin manifest.""" + manifest = dict(pp.submit_plugin(plugin_url)) + plugin_version = str(manifest['version']) + cwl_tool = pp.get_plugin(self._to_camel_case(manifest['name']), plugin_version).save_cwl( + self.adapters_path.joinpath(f"{self._to_camel_case(manifest['name'])}.cwl") + ) + self._modify_cwl() + return Step(cwl_tool) + + def _get_manifest_url(self, plugin_name: str) -> str: + """Retrieve the URL for the plugin manifest from GitHub.""" + return MANIFEST_URLS.get(plugin_name, "") + + def _modify_cwl(self) -> None: + """Modify CWL files to include environmental variables and permissions.""" + for cwl_file in self.adapters_path.rglob("*.cwl"): + if "cwl" in cwl_file.name: + try: + with cwl_file.open("r") as file: + config = yaml.safe_load(file) + config.setdefault("requirements", {}) + config["requirements"]["NetworkAccess"] = {"networkAccess": True} + config["requirements"]["EnvVarRequirement"] = {"envDef": {"HOME": "/home/polusai"}} + with cwl_file.open("w") as out_file: + yaml.dump(config, out_file) + except FileNotFoundError: + logger.error(f"Error processing file: {cwl_file}") + + def workflow(self) -> None: + """Execute the CWL nuclear segmentation pipeline.""" + logger.info("Starting CWL nuclear segmentation workflow.") + + # Step: BBBC Download + bbbc = self.create_step(self._get_manifest_url("bbbc_download")) + bbbc.name = self.name + bbbc.outDir = Path("bbbc.outDir") + + # Step: File Renaming + rename = self.create_step(self._get_manifest_url("file_renaming")) + rename.filePattern = self.file_pattern + rename.outFilePattern = self.out_file_pattern + rename.mapDirectory = self.map_directory + rename.inpDir = bbbc.outDir + rename.outDir = Path("rename.outDir") + + # Step: OME Converter + ome_converter = self.create_step(self._get_manifest_url("ome_converter")) + ome_converter.filePattern = self._extract_file_extension(self.out_file_pattern) + ome_converter.fileExtension = ".ome.tif" + ome_converter.inpDir = rename.outDir + ome_converter.outDir = Path("ome_converter.outDir") + + # Optional: Background correction + if self.background_correction: + estimate_flatfield = self.create_step(self._get_manifest_url("estimate_flatfield")) + estimate_flatfield.inpDir = ome_converter.outDir + estimate_flatfield.filePattern = self.image_pattern + estimate_flatfield.groupBy = self.group_by + estimate_flatfield.getDarkfield = True + estimate_flatfield.outDir = Path("estimate_flatfield.outDir") + + apply_flatfield = self.create_step(self._get_manifest_url("apply_flatfield")) + apply_flatfield.imgDir = ome_converter.outDir + apply_flatfield.imgPattern = self.image_pattern + apply_flatfield.ffDir = estimate_flatfield.outDir + apply_flatfield.ffPattern = self.ff_pattern + apply_flatfield.dfPattern = self.df_pattern + apply_flatfield.outDir = Path("apply_flatfield.outDir") + + # Step: Kaggle Nuclei Segmentation + kaggle_segmentation = self.create_step(self._get_manifest_url("kaggle_nuclei_segmentation")) + kaggle_segmentation.inpDir = apply_flatfield.outDir if self.background_correction else ome_converter.outDir + kaggle_segmentation.filePattern = self.image_pattern + kaggle_segmentation.outDir = Path("kaggle_nuclei_segmentation.outDir") + + # Step: FTL Label Plugin + ftl_plugin = self.create_step(self._get_manifest_url("ftl_plugin")) + ftl_plugin.inpDir = kaggle_segmentation.outDir + ftl_plugin.connectivity = "1" + ftl_plugin.binarizationThreshold = 0.5 + ftl_plugin.outDir = Path("ftl_plugin.outDir") + + # Run the workflow + steps = [ + bbbc, + rename, ome_converter, + estimate_flatfield if self.background_correction else None, + apply_flatfield if self.background_correction else None, + kaggle_segmentation, + ftl_plugin + + ] + + workflow = Workflow(steps, f"{self.name}_workflow") + # Compile and run using WIC python API + workflow.compile() + workflow.run() + self._move_outputs() + logger.info("Completed CWL nuclear segmentation workflow.") + return diff --git a/src/image/workflows/cwl_visualization.py b/src/image/workflows/cwl_visualization.py new file mode 100644 index 0000000..cc401a2 --- /dev/null +++ b/src/image/workflows/cwl_visualization.py @@ -0,0 +1,223 @@ +import logging +import re +import shutil +from pathlib import Path +import typing +import yaml + +from sophios.api.pythonapi import Step, Workflow +import polus.tools.plugins as pp +from image.workflows.utils import OUT_PATH,MANIFEST_URLS + + +# Initialize the logger +logger = logging.getLogger(__name__) +logger.setLevel(logging.INFO) + + +class CWLVisualizationWorkflow: + """ + A CWL visualization pipeline to process imaging datasets. + + Attributes: + work_dir: Path to working directory. + name: Name of the imaging dataset. + file_pattern: Pattern for parsing raw filenames. + out_file_pattern: Desired format for output filenames. + image_pattern: Pattern for parsing intensity image filenames. + seg_pattern: Pattern to parse segmentation image filenames. + layout : A list indicating the grid layout. + pyramid_type : A list indicating the grid layout. + ff_pattern: Filename pattern for selecting flatfield components. + df_pattern: Filename pattern for selecting darkfield components. + group_by: Variables used for grouping the file pattern. + background_correction: Flag to enable background correction. + map_directory: Enable mapping of folder names. + out_dir: Directory for saving outputs. + """ + def __init__( + self, + work_dir: Path, + name: str, + file_pattern: str, + out_file_pattern: str, + image_pattern: str, + seg_pattern: str, + layout: str, + pyramid_type: str, + image_type: str, + ff_pattern: typing.Optional[str] = '', + df_pattern: typing.Optional[str] = '', + group_by: typing.Optional[str] = '', + map_directory: typing.Optional[bool] = False, + background_correction: typing.Optional[bool] = False, + out_dir: typing.Optional[Path] = OUT_PATH + ): + self.name = name + self.file_pattern = file_pattern + self.out_file_pattern = out_file_pattern + self.image_pattern = image_pattern + self.seg_pattern = seg_pattern + self.layout = layout + self.pyramid_type = pyramid_type + self.image_type = image_type + self.ff_pattern = ff_pattern + self.df_pattern = df_pattern + self.group_by = group_by + self.map_directory = map_directory + self.background_correction = background_correction + self.out_dir = out_dir + self.work_dir = work_dir + self.adapters_path = self.work_dir.joinpath("cwl_adapters") + if not self.adapters_path.exists(): + self.adapters_path.mkdir(exist_ok=True, parents=True) + + def _move_outputs(self) -> None: + """Move output files and directories to the specified output directory.""" + logger.info("Moving directories and JSON files to output directory.") + dir_names = {"autogenerated", "cachedir", "provenance", "outdir"} + + # Move specified directories + for directory in self.work_dir.iterdir(): + if directory.name in dir_names: + shutil.move(directory, self.out_dir) + + # Move JSON files + for json_file in self.work_dir.rglob("*.json"): + shutil.move(json_file, self.out_dir) + + @staticmethod + def _to_camel_case(name: str) -> str: + """Convert a plugin name to camel case.""" + return re.sub(r"(_|-)+", " ", name).title().replace(" ", "") + + @staticmethod + def _extract_file_extension(pattern: str) -> str: + """Extract and return the string after a period in the pattern.""" + match = re.search(r"\.(.*)", pattern) + return f".*.{match.group(1)}" if match else "" + + @staticmethod + def _escape_parentheses(pattern: str) -> str: + """Escape parentheses and generate patterns for flatfield and darkfield correction.""" + pattern = re.sub(r"(\()|(\))", r"\\\1\2", pattern) + return pattern.split("_c")[0] + "_c{c:d}" + re.sub(r"\d", "", pattern.split("_c")[1]) + + def create_step(self, plugin_url: str) -> Step: + """Create a step for the workflow based on the plugin manifest.""" + manifest = dict(pp.submit_plugin(plugin_url)) + plugin_version = str(manifest['version']) + cwl_tool = pp.get_plugin(self._to_camel_case(manifest['name']), plugin_version).save_cwl( + self.adapters_path.joinpath(f"{self._to_camel_case(manifest['name'])}.cwl") + ) + self._modify_cwl() + return Step(cwl_tool) + + def _get_manifest_url(self, plugin_name: str) -> str: + """Retrieve the URL for the plugin manifest from GitHub.""" + + return MANIFEST_URLS.get(plugin_name, "") + + def _modify_cwl(self) -> None: + """Modify CWL files to include environmental variables and permissions.""" + for cwl_file in self.adapters_path.rglob("*.cwl"): + if "cwl" in cwl_file.name: + try: + with cwl_file.open("r") as file: + config = yaml.safe_load(file) + config.setdefault("requirements", {}) + config["requirements"]["NetworkAccess"] = {"networkAccess": True} + config["requirements"]["EnvVarRequirement"] = {"envDef": {"HOME": "/home/polusai"}} + with cwl_file.open("w") as out_file: + yaml.dump(config, out_file) + except FileNotFoundError: + logger.error(f"Error processing file: {cwl_file}") + + def workflow(self) -> None: + """ + A CWL visualization pipeline. + """ + # Step: BBBC Download + bbbc = self.create_step(self._get_manifest_url("bbbc_download")) + bbbc.name = self.name + bbbc.outDir = Path("bbbc.outDir") + + # Step: File Renaming + rename = self.create_step(self._get_manifest_url("file_renaming")) + rename.filePattern = self.file_pattern + rename.outFilePattern = self.out_file_pattern + rename.mapDirectory = self.map_directory + rename.inpDir = bbbc.outDir + rename.outDir = Path("rename.outDir") + + # Step: OME Converter + ome_converter = self.create_step(self._get_manifest_url("ome_converter")) + ome_converter.filePattern = self._extract_file_extension(self.out_file_pattern) + ome_converter.fileExtension = ".ome.tif" + ome_converter.inpDir = rename.outDir + ome_converter.outDir = Path("ome_converter.outDir") + + # Optional: Background correction + if self.background_correction: + estimate_flatfield = self.create_step(self._get_manifest_url("estimate_flatfield")) + estimate_flatfield.inpDir = ome_converter.outDir + estimate_flatfield.filePattern = self.image_pattern + estimate_flatfield.groupBy = self.group_by + estimate_flatfield.getDarkfield = True + estimate_flatfield.outDir = Path("estimate_flatfield.outDir") + + # Apply Flatfield + apply_flatfield = self.create_step(self._get_manifest_url("apply_flatfield")) + apply_flatfield.imgDir = ome_converter.outDir + apply_flatfield.imgPattern = self.image_pattern + apply_flatfield.ffDir = estimate_flatfield.outDir + apply_flatfield.ffPattern = self.ff_pattern + apply_flatfield.dfPattern = self.df_pattern + apply_flatfield.outDir = Path("apply_flatfield.outDir") + + + # Montage + montage = self.create_step(self._get_manifest_url("montage_url")) + montage.inpDir = apply_flatfield.outDir if self.background_correction else ome_converter.outDir + montage.filePattern = self.image_pattern + montage.layout = self.layout + montage.outDir = Path("montage.outDir") + + # # Image Assembler + image_assembler = self.create_step( + self._get_manifest_url("image_assembler_url") + ) + image_assembler.imgPath = apply_flatfield.outDir if self.background_correction else ome_converter.outDir + image_assembler.stitchPath = montage.outDir + image_assembler.outDir = Path("image_assembler.outDir") + + + # Precompute Slide + precompute_slide = self.create_step( + self._get_manifest_url("precompute_slide_url") + ) + precompute_slide.pyramidType = self.pyramid_type + precompute_slide.imageType = self.image_type + precompute_slide.inpDir = image_assembler.outDir + precompute_slide.outDir = Path("precompute_slide.outDir") + + logger.info("Initiating CWL Visualization Workflow!!!") + steps = [ + bbbc, + rename, + ome_converter, + estimate_flatfield if self.background_correction else None, + apply_flatfield if self.background_correction else None, + montage, + image_assembler, + precompute_slide # At the moment precompute slide is not working. PR will be open for the fix + ] + workflow = Workflow(steps, f"{self.name}_workflow") + # Compile and run using WIC python API + + workflow.compile() + workflow.run() + self._move_outputs() + logger.info("Completed CWL visualization workflow.") + + return \ No newline at end of file diff --git a/src/image/workflows/utils.py b/src/image/workflows/utils.py new file mode 100644 index 0000000..aae6100 --- /dev/null +++ b/src/image/workflows/utils.py @@ -0,0 +1,100 @@ +import pydantic +from pathlib import Path +from typing import Dict +from typing import Union +import yaml + + +GITHUB_TAG = "https://raw.githubusercontent.com" + +OUT_PATH = Path.cwd() + + + +MANIFEST_URLS = { + "bbbc_download": f"{GITHUB_TAG}/saketprem/polus-plugins/bbbc_download/utils/bbbc-download-plugin/plugin.json", + "file_renaming": f"{GITHUB_TAG}/hamshkhawar/image-tools/filepattern_filerenaming/formats/file-renaming-tool/plugin.json", + "ome_converter": f"{GITHUB_TAG}/hamshkhawar/image-tools/refs/heads/fix_endian_bug_omeconverter/formats/ome-converter-tool/plugin.json", + "estimate_flatfield": f"{GITHUB_TAG}/PolusAI/image-tools/refs/heads/master/regression/basic-flatfield-estimation-tool/plugin.json", + "apply_flatfield": f"{GITHUB_TAG}/PolusAI/image-tools/refs/heads/master/transforms/images/apply-flatfield-tool/plugin.json", + "kaggle_nuclei_segmentation": f"{GITHUB_TAG}/hamshkhawar/image-tools/refs/heads/kaggle_update_dependencies/segmentation/kaggle-nuclei-segmentation-tool/plugin.json", + "ftl_plugin": f"{GITHUB_TAG}/nishaq503/image-tools/fix/ftl-label/transforms/images/polus-ftl-label-plugin/plugin.json", + "nyxus_plugin": f"{GITHUB_TAG}/hamshkhawar/image-tools/refs/heads/nyxus_fix_entrypoint/features/nyxus-tool/plugin.json", + "montage_url" :f"{GITHUB_TAG}/PolusAI/image-tools/refs/heads/master/transforms/images/montage-tool/plugin.json", + "image_assembler_url": f"{GITHUB_TAG}/PolusAI/image-tools/refs/heads/master/transforms/images/image-assembler-tool/plugin.json", + "precompute_slide_url": f"{GITHUB_TAG}/PolusAI/image-tools/refs/heads/master/visualization/precompute-slide-tool/plugin.json" + } + + +# Define keys as frozensets for immutability +ANALYSIS_KEYS = frozenset([ + "name", "file_pattern", "out_file_pattern", "image_pattern", "seg_pattern", + "ff_pattern", "df_pattern", "group_by", "map_directory", "features", + "file_extension", "background_correction" +]) + +SEG_KEYS = frozenset([ + "name", "file_pattern", "out_file_pattern", "image_pattern", "seg_pattern", + "ff_pattern", "df_pattern", "group_by", "map_directory", "background_correction" +]) + +VIZ_KEYS = frozenset([ + "name", "file_pattern", "out_file_pattern", "image_pattern", "seg_pattern", + "layout", "pyramid_type", "image_type", "ff_pattern", "df_pattern", "group_by", + "map_directory", "background_correction" +]) + +# Mapping workflows to their respective keys +WORKFLOW_KEYS = { + "analysis": ANALYSIS_KEYS, + "segmentation": SEG_KEYS, + "visualization": VIZ_KEYS, +} + + +class DataModel(pydantic.BaseModel): + data: Dict[str, Dict[str, Union[str, bool]]] + + +class LoadYaml(pydantic.BaseModel): + """Validation of Dataset YAML.""" + workflow: str + config_path: Union[str, Path] + + @pydantic.validator("config_path", pre=True) + @classmethod + def validate_path(cls, value: Union[str, Path]) -> Path: + """Validate the configuration file path.""" + path = Path(value) + if not path.exists(): + raise ValueError(f"{value} does not exist! Please check the path again.") + return path + + @pydantic.validator("workflow", pre=True) + @classmethod + def validate_workflow_name(cls, value: str) -> str: + """Validate workflow name.""" + valid_workflows = WORKFLOW_KEYS.keys() + if value not in valid_workflows: + raise ValueError(f"Invalid workflow: {value}. Please choose one of {', '.join(valid_workflows)}.") + return value + + def parse_yaml(self) -> Dict[str, Union[str, bool]]: + """Parse the YAML configuration file for each dataset.""" + with open(self.config_path, 'r') as f: + data = yaml.safe_load(f) + + # Check missing values in the YAML + if any(v is None for v in data.values()): + raise ValueError("All parameters are not defined! Please check the YAML file.") + + # Validate keys against the workflow's expected keys + self._validate_workflow_keys(data) + + return data + + def _validate_workflow_keys(self, data: Dict[str, Union[str, bool]]) -> None: + """Validate that the keys in the YAML match the expected keys for the selected workflow.""" + expected_keys = WORKFLOW_KEYS[self.workflow] + if data.get("background_correction", False) and set(data.keys()) != expected_keys: + raise ValueError(f"Invalid parameters for {self.workflow} workflow. Expected keys: {expected_keys}") \ No newline at end of file