diff --git a/docker/celeryconfig.py b/docker/celeryconfig.py index 6c8b64e3..97a62d43 100644 --- a/docker/celeryconfig.py +++ b/docker/celeryconfig.py @@ -1,4 +1,4 @@ -broker_url = "amqp://guest@rabbit" +broker_url = "amqp://guest@rabbit:5672" track_started = True send_events = True imports = ("wooey.tasks",) diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 271d5971..33ad5bad 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -23,7 +23,7 @@ services: depends_on: - rabbit - db - command: watchmedo auto-restart --directory=$BUILD_DIR/wooey --recursive --ignore-patterns="*.pyc" -- celery worker -A $WOOEY_PROJECT -c 4 -B -l debug -s schedule + command: watchmedo auto-restart --directory=$BUILD_DIR/wooey --recursive --ignore-patterns="*.pyc" -- celery -A $WOOEY_PROJECT worker -c 4 -B -l debug -s schedule rabbit: image: rabbitmq:3.9.29-management-alpine diff --git a/docs/api.rst b/docs/api.rst new file mode 100644 index 00000000..e4973a48 --- /dev/null +++ b/docs/api.rst @@ -0,0 +1,128 @@ +Wooey API +========= + +Wooey's API allows for programmatic access to manage scripts as well as submit and query jobs. +All use of the API requires a user to be authenticated via :ref:`API keys ` and +the API to be enabled by setting `WOOEY_ENABLE_API_KEYS` in your user_settings.py file. + +Script Management API +~~~~~~~~~~~~~~~~~~~~~ + +Adding and updating a script use the same endpoint, **api/scripts/v1/add-or-update/**. + +.. code-block:: python + + import requests + + response = requests.post( + 'https://wooey.fly.dev/api/scripts/v1/add-or-update/', + data={ + "group": "The script group", # optional + "script-name": open("path_to_script.py", "rb") + }, + headers={'Authorization': 'Bearer your_token_here'}, + ) + +For updating an existing script, the same code can be used with the new version. By default, +updating a script will make that version the default version to run for submissions. To disable +this, add `default: False` to the payload. Multiple scripts can be uploaded at once by simply +providing multiple files. The name of the script is the key used for the file. Thus, for this example +our script name would be `script-name`. + + +Job API +~~~~~~~ + +Creating a new Job +################## + +A script can be ran via the **api/scripts/v1//submit/** endpoint. A `script_slug` is the +script name, with any invalid url characters removed. This will normally be the lowercase version of the +script's name, but can be found by looking at the url of a given script. + +.. image:: img/script_slug_example.png + +.. code-block:: python + + import requests + + response = requests.post( + 'https://wooey.fly.dev/api/scripts/v1/cat-fetcher/submit/', + data={ + "job_name": "test job", + "command": "--count 5 --breed bengal" + }, + headers={'Authorization': 'Bearer your_token_here'}, + ) + + # A valid response will contain + data = response.json() + # {"job_id": 123, "valid": True} + +For jobs that require files, the uploaded file can be provided and referenced in the `command` parameter. For example: + +.. code-block:: python + + import requests + + response = requests.post( + 'https://wooey.fly.dev/api/scripts/v1/protein-translation/submit/', + data={ + "job_name": "test job", + "command": "--fasta protein_sequences" + }, + files={ + "protein_sequences": open('./proteins.fasta') + }, + headers={'Authorization': 'Bearer your_token_here'}, + ) + +Currently, this is only supported if the parameter is marked as a filetype (such as `here `_). + +Querying Jobs +############# + +A job can be queried by its id. While the UI allows sharing and management of jobs via a shareable UUID, that +currently does not exist for Wooey's API as there is no public access permitted. **Importantly, these requests +are GET requests** + +There are 2 endpoints for querying: **api/jobs/v1//status/** and **api/jobs/v1//details/**. + +**api/jobs/v1//status/** will provide information if the job is complete and should be used for polling. +Once the job is complete, **api/jobs/v1//details/** will provide rich details about the job, including +all assets generated by it and URLs to programatically download assets. + +.. code-block:: python + + import requests + + requests.get( + 'https://wooey.fly.dev/api/jobs/v1/123/status/', + headers={'Authorization': 'Bearer your_token_here'}, + ) + # {"status": "running", "is_complete": False} + ... + requests.get( + 'https://wooey.fly.dev/api/jobs/v1/123/status/', + headers={'Authorization': 'Bearer your_token_here'}, + ) + # {"status": "completed", "is_complete": True} + requests.get( + 'https://wooey.fly.dev/api/jobs/v1/123/details/', + headers={'Authorization': 'Bearer your_token_here'}, + ) + # { + # "status": "completed", + # "is_complete": True, + # "job_name": "test job", + # "job_description": "", + # "assets": [{"name": "assert 1", "url": "https://...", ...}], + # "stdout": "This job's output, errors and other information would appear here", + # "stderr": "This job's error output, errors and other information would appear here", + # "uuid": "The sharable UUID, this can be used to provide someone a permalink to the UI view of the Job" + # } + +.. toctree:: + :maxdepth: 1 + + api_keys diff --git a/docs/img/script_slug_example.png b/docs/img/script_slug_example.png new file mode 100644 index 00000000..7b7dfba4 Binary files /dev/null and b/docs/img/script_slug_example.png differ diff --git a/docs/index.rst b/docs/index.rst index 075aa255..0130574b 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -37,6 +37,8 @@ Getting Started running_wooey scripts wooey_ui + api + api_keys customizations remote upgrade_help diff --git a/docs/wooey_ui.rst b/docs/wooey_ui.rst index 8749e836..e2335d2f 100644 --- a/docs/wooey_ui.rst +++ b/docs/wooey_ui.rst @@ -43,8 +43,3 @@ to parse this information), updates to the script version will result in a new version being created. If a command line library doesn't support versioning or the version has not been updated in a script, the Script Iteration counter will be incremented. - -.. toctree:: - :maxdepth: 2 - - api_keys diff --git a/wooey/api/__init__.py b/wooey/api/__init__.py new file mode 100644 index 00000000..7970845c --- /dev/null +++ b/wooey/api/__init__.py @@ -0,0 +1,8 @@ +from .jobs import ( # noqa: F401 + job_details, + job_status, +) +from .scripts import ( # noqa: F401 + add_or_update_script, + submit_script, +) diff --git a/wooey/api/forms.py b/wooey/api/forms.py new file mode 100644 index 00000000..ad3bc3cd --- /dev/null +++ b/wooey/api/forms.py @@ -0,0 +1,19 @@ +from django import forms + + +class SubmitForm(forms.Form): + job_name = forms.CharField() + job_description = forms.CharField(required=False) + version = forms.CharField(required=False) + iteration = forms.IntegerField(required=False) + command = forms.CharField(required=False) + + +class AddScriptForm(forms.Form): + group = forms.CharField(required=False) + default = forms.NullBooleanField(required=False) + + def clean_default(self): + if self.cleaned_data["default"] is None: + return True + return self.cleaned_data["default"] diff --git a/wooey/api/jobs.py b/wooey/api/jobs.py new file mode 100644 index 00000000..1c38144c --- /dev/null +++ b/wooey/api/jobs.py @@ -0,0 +1,78 @@ +from django.http import JsonResponse +from django.utils.encoding import force_str +from django.utils.translation import gettext_lazy as _ +from django.views.decorators.csrf import csrf_exempt +from django.views.decorators.http import require_http_methods + +from .. import models +from ..utils import requires_login + + +@csrf_exempt +@require_http_methods(["GET"]) +@requires_login +def job_status(request, job_id): + job = models.WooeyJob.objects.get(id=job_id) + if job.can_user_view(request.user): + return JsonResponse( + { + "status": job.status, + "is_complete": job.status in models.WooeyJob.TERMINAL_STATES, + } + ) + else: + return JsonResponse( + { + "valid": False, + "errors": { + "__all__": [ + force_str(_("You are not permitted to access this job.")) + ] + }, + }, + status=403, + ) + + +@csrf_exempt +@require_http_methods(["GET"]) +@requires_login +def job_details(request, job_id): + job = models.WooeyJob.objects.get(id=job_id) + if job.can_user_view(request.user): + assets = [] + is_terminal = job.status in models.WooeyJob.TERMINAL_STATES + if is_terminal: + for asset in job.userfile_set.all(): + assets.append( + { + "name": asset.filename, + "url": request.build_absolute_uri( + asset.system_file.filepath.url + ), + } + ) + return JsonResponse( + { + "status": job.status, + "is_complete": is_terminal, + "uuid": job.uuid, + "job_name": job.job_name, + "job_description": job.job_description, + "stdout": job.stdout, + "stderr": job.stderr, + "assets": assets, + } + ) + else: + return JsonResponse( + { + "valid": False, + "errors": { + "__all__": [ + force_str(_("You are not permitted to access this job.")) + ] + }, + }, + status=403, + ) diff --git a/wooey/api/scripts.py b/wooey/api/scripts.py new file mode 100644 index 00000000..20909a53 --- /dev/null +++ b/wooey/api/scripts.py @@ -0,0 +1,233 @@ +import argparse +import json +import os +import shlex +from itertools import groupby + +from django.http import JsonResponse +from django.utils.encoding import force_str +from django.utils.translation import gettext_lazy as _ +from django.views.decorators.csrf import csrf_exempt +from django.views.decorators.http import require_http_methods + +from .. import models +from ..backend import utils +from .forms import AddScriptForm, SubmitForm +from ..utils import requires_login +from .. import settings as wooey_settings + + +def create_argparser(script_version): + """From a script version, return an argparse cli + + This is meant to assist in processing a command sent to the Wooey API for + running a script. This is an attempt at a shortcut instead of building + our own tokenizer and parser. One downside at the moment is we do not + store alternative parameters that a script can have (e.g. if `--foo` can also + be specified as `--old-foo`). So if a user enters the one not saved, it will + be an error. + """ + parser = argparse.ArgumentParser(prog="wooey-temp") + subparsers = None + parameters = list(script_version.get_parameters()) + grouped_parameters = { + subparser: list(arguments) + for subparser, arguments in groupby( + sorted(parameters, key=lambda x: [x.parser.name, x.param_order]), + key=lambda x: x.parser.name, + ) + } + + def log_error(parser): + error_func = parser.error + + def inner(message): + parser._wooey_error = message + return error_func(message) + + return inner + + parser.error = log_error(parser) + + for subparser_name, arguments in grouped_parameters.items(): + if subparser_name: + if not subparsers: + subparsers = parser.add_subparsers(dest="wooey_subparser") + active_parser = subparsers.add_parser(subparser_name) + else: + active_parser = parser + for argument in arguments: + argument_kwargs = {"dest": argument.form_slug, "default": argument.default} + if argument.multiple_choice: + argument_kwargs["nargs"] = "*" + if argument.short_param: + argument_kwargs["required"] = argument.required + active_parser.add_argument(argument.short_param, **argument_kwargs) + else: + active_parser.add_argument(**argument_kwargs) + return parser + + +@csrf_exempt +@require_http_methods(["POST"]) +@requires_login +def submit_script(request, slug=None): + if "application/json" in request.headers.get("Content-Type", "").lower(): + submitted_data = json.loads(request.body) + else: + submitted_data = request.POST + files = request.FILES + + form = SubmitForm(submitted_data) + if not form.is_valid(): + return JsonResponse({"valid": False, "errors": form.errors}) + data = form.cleaned_data + + version = data["version"] + iteration = data["iteration"] + command = data["command"] + qs = models.ScriptVersion.objects.filter(script__slug=slug) + if not version and not iteration: + qs = qs.filter(default_version=True) + else: + if version: + qs = qs.filter(script_version=version) + if iteration: + qs = qs.filter(script_iteration=iteration) + try: + script_version = qs.get() + except models.ScriptVersion.DoesNotExist: + return JsonResponse( + {"valid": False, "errors": {"script": _("Unable to find script.")}} + ) + + valid = utils.valid_user(script_version.script, request.user).get("valid") + if valid: + group_valid = utils.valid_user( + script_version.script.script_group, request.user + )["valid"] + + parser = create_argparser(script_version) + try: + parsed_command = parser.parse_args(shlex.split(command)) + except SystemExit: + return JsonResponse( + {"valid": False, "errors": {"command": parser._wooey_error}}, status=400 + ) + if valid and group_valid: + job_data = vars(parsed_command) + job_data["job_name"] = data["job_name"] + if data["job_description"]: + job_data["job_description"] = data["job_description"] + subparser_id = script_version.scriptparser_set.get( + name=job_data.pop("wooey_subparser", "") + ).id + form = utils.get_master_form( + script_version=script_version, parser=subparser_id + ) + wooey_form_data = job_data.copy() + wooey_form_data["wooey_type"] = script_version.pk + + # We need to remap uploaded files to the correct slug + form_slugs = list(wooey_form_data) + for form_slug in form_slugs: + form_value = wooey_form_data[form_slug] + if isinstance(form_value, list): + to_append = [] + for index, value in enumerate(form_value): + if value in files: + to_append.append(index) + if to_append: + existing_files = files.get(form_slug, []) + files.setlist( + form_slug, + utils.flatten( + existing_files + + [files.pop(form_value[i]) for i in to_append] + ), + ) + for index in reversed(to_append): + form_value.pop(index) + else: + if form_value in files: + files.setlist(form_slug, files.pop(form_value)) + wooey_form_data[form_slug] = [""] + + utils.validate_form(form=form, data=wooey_form_data, files=files) + + if not form.errors: + job = utils.create_wooey_job( + script_parser_pk=subparser_id, + script_version_pk=script_version.id, + user=request.user, + data=form.cleaned_data, + ) + job.submit_to_celery() + return JsonResponse({"valid": True, "job_id": job.id}) + else: + return JsonResponse({"valid": False, "errors": form.errors}, status=400) + else: + return JsonResponse( + { + "valid": False, + "errors": { + "script": [ + force_str(_("You are not permitted to access this script.")) + ] + }, + }, + status=403, + ) + + +@csrf_exempt +@require_http_methods(["POST"]) +@requires_login +def add_or_update_script(request): + submitted_data = request.POST.dict() + files = request.FILES + + form = AddScriptForm(submitted_data) + if not form.is_valid(): + return JsonResponse({"valid": False, "errors": form.errors}) + + data = form.cleaned_data + group = data["group"] or wooey_settings.WOOEY_DEFAULT_SCRIPT_GROUP + + response = [] + + for script_name, script_file in files.items(): + script_path = utils.default_storage.save( + os.path.join(wooey_settings.WOOEY_SCRIPT_DIR, script_file.name), + script_file, + ) + if wooey_settings.WOOEY_EPHEMERAL_FILES: + # save it locally as well (the default_storage will default to the remote store) + script_file.seek(0) + local_storage = utils.get_storage(local=True) + local_storage.save( + os.path.join( + wooey_settings.WOOEY_SCRIPT_DIR, + script_file.name, + ), + script_file, + ) + add_kwargs = { + "script_path": script_path, + "group": group, + "script_name": script_name, + "set_default_version": data["default"], + } + results = utils.add_wooey_script(**add_kwargs) + output = { + "script": script_name, + "success": results["valid"], + "errors": results["errors"], + } + if results["valid"]: + output["version"] = results["script"].script_version + output["iteration"] = results["script"].script_iteration + output["is_default"] = results["script"].default_version + response.append(output) + + return JsonResponse(response, safe=False) diff --git a/wooey/backend/utils.py b/wooey/backend/utils.py index 7f720a99..90b6c3b2 100644 --- a/wooey/backend/utils.py +++ b/wooey/backend/utils.py @@ -22,6 +22,9 @@ from django.db.utils import OperationalError from django.core.files.storage import default_storage from django.core.files import File +from django.forms import FileField +from django.http import QueryDict +from django.utils.datastructures import MultiValueDict from django.utils.translation import gettext_lazy as _ from django.db.models import Q @@ -37,6 +40,22 @@ def sanitize_string(value): return value.replace('"', '\\"') +def ensure_list(value): + if value is None: + return [] + return value if isinstance(value, list) else [value] + + +def flatten(value): + new_list = [] + for element in value: + if isinstance(element, list): + new_list.extend(flatten(element)) + else: + new_list.append(element) + return new_list + + def get_storage(local=True): if wooey_settings.WOOEY_EPHEMERAL_FILES: storage = default_storage.local_storage if local else default_storage @@ -206,6 +225,50 @@ def validate_form(form=None, data=None, files=None): form.is_bound = True form.full_clean() + # for cloned jobs, because we do not open a file selection window again in the browser, the pointer to files will just be a list + # like ['', filename]. We need to remap these to previously submitted files and merge with any new files provided. + to_delete = [] + for field in data: + if isinstance(form.fields.get(field), FileField): + # if we have a value set, reassert this + new_values = ( + list(filter(lambda x: x, data.getlist(field))) + if isinstance(data, (MultiValueDict, QueryDict)) + else ensure_list(data.get(field)) + ) + cleaned_values = [] + for new_value in new_values: + if field not in files and ( + field not in form.cleaned_data + or ( + new_value + and ( + form.cleaned_data[field] is None + or not [j for j in form.cleaned_data[field] if j] + ) + ) + ): + # this is a previously set field, so a cloned job + if new_value is not None: + cleaned_values.append(get_storage(local=False).open(new_value)) + to_delete.append(field) + if cleaned_values: + form.cleaned_data[field] = cleaned_values + for field in to_delete: + if field in form.errors: + del form.errors[field] + + # Now append any new files into our cleaned form data + for field in files or {}: + v = ( + files.getlist(field) + if isinstance(files, (MultiValueDict, QueryDict)) + else files[field] + ) + if field in form.cleaned_data: + cleaned = ensure_list(form.cleaned_data[field]) + form.cleaned_data[field] = list(set(cleaned).union(set(v))) + def get_current_scripts(): from ..models import ScriptVersion @@ -262,7 +325,11 @@ def get_storage_object(path, local=False, close=True): def add_wooey_script( - script_version=None, script_path=None, group=None, script_name=None + script_version=None, + script_path=None, + group=None, + script_name=None, + set_default_version=True, ): # There is a class called 'Script' which contains the general information about a script. However, that is not where the file details # of the script lie. That is the ScriptVersion model. This allows the end user to tag a script as a favorite/etc. and set @@ -312,9 +379,7 @@ def add_wooey_script( ): return { "valid": False, - "errors": errors.DuplicateScriptError( - ScriptVersion.error_messages["duplicate_script"] - ), + "errors": ScriptVersion.error_messages["duplicate_script"], "script": existing_version, } @@ -409,7 +474,7 @@ def add_wooey_script( version_kwargs = { "script_version": version_string, "script_path": local_file, - "default_version": True, + "default_version": set_default_version, "checksum": checksum, } # does this script already exist in the database? @@ -420,7 +485,7 @@ def add_wooey_script( wooey_script = Script(**script_kwargs) wooey_script._script_cl_creation = True wooey_script.save() - version_kwargs.update({"script_iteration": 1}) + version_kwargs.update({"script_iteration": 1, "default_version": True}) else: # we're updating it wooey_script = Script.objects.get(**script_kwargs) @@ -439,9 +504,10 @@ def add_wooey_script( sorted([i.script_iteration for i in current_versions])[-1] + 1 ) # disable older versions - ScriptVersion.objects.filter(script=wooey_script).update( - default_version=False - ) + if set_default_version: + ScriptVersion.objects.filter(script=wooey_script).update( + default_version=False + ) version_kwargs.update({"script_iteration": next_iteration}) version_kwargs.update({"script": wooey_script}) script_version = ScriptVersion(**version_kwargs) @@ -460,9 +526,13 @@ def add_wooey_script( ).exclude(pk=script_version.pk) if len(past_versions) == 0: script_version.script_version = version_string + script_version.default_version = True script_version.script_iteration = past_versions.count() + 1 # Make all old versions non-default - ScriptVersion.objects.filter(script=wooey_script).update(default_version=False) + if set_default_version: + ScriptVersion.objects.filter(script=wooey_script).update( + default_version=False + ) script_version.default_version = True script_version.checksum = checksum wooey_script.save() diff --git a/wooey/conf/project_template/wooey_celery_app.py b/wooey/conf/project_template/wooey_celery_app.py index ad1cd0e8..7e3e6351 100644 --- a/wooey/conf/project_template/wooey_celery_app.py +++ b/wooey/conf/project_template/wooey_celery_app.py @@ -10,7 +10,10 @@ # Using a string here means the worker will not have to # pickle the object when using Windows. -app.config_from_object("django.conf:settings") +if "CELERY_CONFIG_MODULE" in os.environ: + app.config_from_envvar("CELERY_CONFIG_MODULE") +else: + app.config_from_object("django.conf:settings") app.autodiscover_tasks() diff --git a/wooey/errors.py b/wooey/errors.py index 9b3944cc..a9002344 100644 --- a/wooey/errors.py +++ b/wooey/errors.py @@ -1,6 +1,2 @@ class ParserError(Exception): pass - - -class DuplicateScriptError(Exception): - pass diff --git a/wooey/forms/factory.py b/wooey/forms/factory.py index 6518bbc3..9d4caa99 100644 --- a/wooey/forms/factory.py +++ b/wooey/forms/factory.py @@ -11,6 +11,7 @@ from django import forms from django.forms.utils import flatatt from django.http.request import QueryDict +from django.utils.datastructures import MultiValueDict from django.utils.html import format_html from django.utils.module_loading import import_string from django.utils.safestring import mark_safe @@ -70,7 +71,11 @@ def value_from_datadict(data, files, name): files, name, ) - for i in data.getlist(name) + for i in ( + data.getlist(name) + if isinstance(data, (MultiValueDict, QueryDict)) + else utils.ensure_list(data[name]) + ) ] return value_from_datadict diff --git a/wooey/management/commands/addscript.py b/wooey/management/commands/addscript.py index 181e3628..7e5d707b 100644 --- a/wooey/management/commands/addscript.py +++ b/wooey/management/commands/addscript.py @@ -47,7 +47,7 @@ def handle(self, *args, **options): ) if not os.path.exists(script): raise CommandError("{0} does not exist.".format(script)) - group = options.get("group", "Wooey Scripts") + group = options.get("group", wooey_settings.WOOEY_DEFAULT_SCRIPT_GROUP) scripts = ( [os.path.join(script, i) for i in os.listdir(script)] if os.path.isdir(script) diff --git a/wooey/models/core.py b/wooey/models/core.py index d1751692..b3b67e1b 100644 --- a/wooey/models/core.py +++ b/wooey/models/core.py @@ -148,6 +148,7 @@ class Meta: app_label = "wooey" verbose_name = _("script version") verbose_name_plural = _("script versions") + get_latest_by = "-created_date" def __str__(self): return "{}({}: {})".format( @@ -351,6 +352,9 @@ def get_stderr(self): return rt return self.stderr + def can_user_view(self, user): + return self.user is None or (user.is_authenticated and self.user == user) + class ScriptParameterGroup(models.Model): group_name = models.TextField() diff --git a/wooey/settings.py b/wooey/settings.py index 4b4dbaf8..1ab5c47c 100644 --- a/wooey/settings.py +++ b/wooey/settings.py @@ -22,7 +22,7 @@ def get(key, default): WOOEY_CELERY = get("WOOEY_CELERY", True) WOOEY_CELERY_TASKS = get("WOOEY_CELERY_TASKS", "wooey.tasks") WOOEY_CELERY_STOPPABLE_JOBS = "amqp" in str( - celery_app.conf.get("CELERY_BROKER_URL", celery_app.conf.get("broker_url", "")) + celery_app.conf.get("CELERY_BROKER_URL", celery_app.conf.get("broker_url") or "") ) # Site setup settings diff --git a/wooey/tests/mixins.py b/wooey/tests/mixins.py index c47018cb..541ea565 100644 --- a/wooey/tests/mixins.py +++ b/wooey/tests/mixins.py @@ -5,6 +5,9 @@ from ..backend import utils from .. import settings as wooey_settings from . import factories, config +from .utils import ( + get_subparser_form_slug, +) # TODO: Track down where file handles are not being closed. This is not a problem on Linux/Mac, but is on Windows @@ -56,34 +59,53 @@ def tearDown(self): class ScriptFactoryMixin(ScriptTearDown, object): def setUp(self): - self.translate_script = factories.generate_script( - os.path.join(config.WOOEY_TEST_SCRIPTS, "translate.py") - ) - self.choice_script = factories.generate_script( - os.path.join(config.WOOEY_TEST_SCRIPTS, "choices.py") + self.translate_script_path = os.path.join( + config.WOOEY_TEST_SCRIPTS, "translate.py" ) + self.translate_script = factories.generate_script(self.translate_script_path) + self.choice_script_path = os.path.join(config.WOOEY_TEST_SCRIPTS, "choices.py") + self.choice_script = factories.generate_script(self.choice_script_path) self.without_args = factories.generate_script( os.path.join(config.WOOEY_TEST_SCRIPTS, "without_args.py") ) self.subparser_script = factories.generate_script( os.path.join(config.WOOEY_TEST_SCRIPTS, "subparser_script.py") ) + self.version1_script_path = os.path.join( + config.WOOEY_TEST_SCRIPTS, "versioned_script", "v1.py" + ) self.version1_script = factories.generate_script( - os.path.join(config.WOOEY_TEST_SCRIPTS, "versioned_script", "v1.py"), + self.version1_script_path, script_name="version_test", ) + self.version2_script_path = os.path.join( + config.WOOEY_TEST_SCRIPTS, "versioned_script", "v2.py" + ) self.version2_script = factories.generate_script( - os.path.join(config.WOOEY_TEST_SCRIPTS, "versioned_script", "v2.py"), + self.version2_script_path, script_name="version_test", ) - super(ScriptFactoryMixin, self).setUp() + return super(ScriptFactoryMixin, self).setUp() + + def create_job_with_output_files(self): + script = self.translate_script + from ..backend import utils + + sequence_slug = get_subparser_form_slug(script, "sequence") + out_slug = get_subparser_form_slug(script, "out") + job = utils.create_wooey_job( + script_version_pk=script.pk, + data={"job_name": "abc", sequence_slug: "aaa", out_slug: "abc"}, + ) + job = job.submit_to_celery() + return job class FileMixin(object): def setUp(self): self.storage = utils.get_storage(local=not wooey_settings.WOOEY_EPHEMERAL_FILES) self.filename_func = lambda x: os.path.join(wooey_settings.WOOEY_SCRIPT_DIR, x) - super(FileMixin, self).setUp() + return super(FileMixin, self).setUp() def get_any_file(self): script = os.path.join(config.WOOEY_TEST_SCRIPTS, "command_order.py") diff --git a/wooey/tests/scripts/mandlebrot.py b/wooey/tests/scripts/mandlebrot.py index ad36cf90..e6c0b2b2 100644 --- a/wooey/tests/scripts/mandlebrot.py +++ b/wooey/tests/scripts/mandlebrot.py @@ -32,14 +32,14 @@ def mandel(n, m, itermax, xmin, xmax, ymin, ymax): iy.shape = n * m c.shape = n * m z = copy(c) - for i in xrange(itermax): + for i in range(itermax): if not len(z): break multiply(z, z, z) add(z, c, z) rem = abs(z) > 2.0 img[ix[rem], iy[rem]] = i + 1 - rem = -rem + rem = ~rem z = z[rem] ix, iy = ix[rem], iy[rem] c = c[rem] diff --git a/wooey/tests/scripts/nested_heatmap.py b/wooey/tests/scripts/nested_heatmap.py index 4b4f0d61..a9a13c00 100644 --- a/wooey/tests/scripts/nested_heatmap.py +++ b/wooey/tests/scripts/nested_heatmap.py @@ -70,7 +70,7 @@ def main(): multi.groupby(level=major_index).var().mean(axis=1).order(ascending=False) ) # and group by 20s - for i in xrange(11): + for i in range(11): dat = multi[ multi.index.get_level_values(major_index).isin( most_variable.index[10 * i : 10 * (i + 1)] diff --git a/wooey/tests/scripts/translate.py b/wooey/tests/scripts/translate.py index d4eec0da..4b932459 100755 --- a/wooey/tests/scripts/translate.py +++ b/wooey/tests/scripts/translate.py @@ -99,7 +99,7 @@ group = parser.add_mutually_exclusive_group(required=True) group.add_argument("--sequence", help="The sequence to translate.", type=str) group.add_argument( - "--fasta", help="The fasta file to translate.", type=argparse.FileType("rb") + "--fasta", help="The fasta file to translate.", type=argparse.FileType("r") ) simple_group = parser.add_argument_group("Parameter Group") simple_group.add_argument( @@ -110,7 +110,7 @@ default="+1", ) simple_group.add_argument( - "--out", help="The file to save translations to.", type=argparse.FileType("wb") + "--out", help="The file to save translations to.", type=argparse.FileType("w") ) @@ -126,7 +126,7 @@ def translate(seq=None, frame=None): return "".join( [ CODON_TABLE.get(seq[i : i + 3], "X") - for i in xrange(frame, len(seq), 3) + for i in range(frame, len(seq), 3) if i + 3 <= len(seq) ] ) diff --git a/wooey/tests/scripts/translate2.py b/wooey/tests/scripts/translate2.py index f2e59118..fb663d53 100755 --- a/wooey/tests/scripts/translate2.py +++ b/wooey/tests/scripts/translate2.py @@ -128,7 +128,7 @@ def translate(seq=None, frame=None): return "".join( [ CODON_TABLE.get(seq[i : i + 3], "X") - for i in xrange(frame, len(seq), 3) + for i in range(frame, len(seq), 3) if i + 3 <= len(seq) ] ) diff --git a/wooey/tests/test_api.py b/wooey/tests/test_api.py new file mode 100644 index 00000000..00bc45ae --- /dev/null +++ b/wooey/tests/test_api.py @@ -0,0 +1,283 @@ +import os +from io import BytesIO + +from django.test import Client, TransactionTestCase +from django.urls import reverse + +from ..models import WooeyJob + +from . import mixins, factories + + +class ApiTestMixin(object): + def setUp(self): + api_key = factories.APIKeyFactory() + self.client = Client(HTTP_AUTHORIZATION="Bearer {}".format(api_key._api_key)) + return super().setUp() + + +class TestJobStatus(mixins.ScriptFactoryMixin, ApiTestMixin, TransactionTestCase): + def test_reports_when_job_is_complete(self): + job = factories.generate_job(self.translate_script) + response = self.client.get( + reverse("wooey:api_job_status", kwargs={"job_id": job.id}) + ) + self.assertFalse(response.json()["is_complete"]) + job.status = WooeyJob.COMPLETED + job.save() + response = self.client.get( + reverse("wooey:api_job_status", kwargs={"job_id": job.id}) + ) + self.assertTrue(response.json()["is_complete"]) + + def test_error_when_invalid_user(self): + another_user = factories.UserFactory(username="bob") + job = factories.generate_job(self.translate_script) + job.user = another_user + job.save() + response = self.client.get( + reverse("wooey:api_job_status", kwargs={"job_id": job.id}) + ) + self.assertFalse(response.json()["valid"]) + self.assertEqual(response.status_code, 403) + + +class TestJobDetails( + mixins.ScriptFactoryMixin, + mixins.FileCleanupMixin, + ApiTestMixin, + TransactionTestCase, +): + def test_job_details(self): + job = self.create_job_with_output_files() + response = self.client.get( + reverse("wooey:api_job_details", kwargs={"job_id": job.id}) + ) + data = response.json() + self.assertIn("url", data["assets"][0]) + self.assertTrue(data["is_complete"]) + self.assertEqual(job.job_name, data["job_name"]) + + +class TestScriptAddition(mixins.ScriptFactoryMixin, ApiTestMixin, TransactionTestCase): + def test_can_add_new_scripts(self): + payload = { + "group": "test group", + "translate-script": open(self.translate_script_path, "rb"), + "choice-script": open(self.choice_script_path, "rb"), + } + response = self.client.post( + reverse("wooey:api_add_or_update_script"), + data=payload, + ) + data = response.json() + self.assertEqual(data[0]["script"], "translate-script") + self.assertEqual(data[1]["script"], "choice-script") + + def test_can_update_existing_script(self): + payload = { + "group": "test group", + "update-script": open(self.version1_script_path, "rb"), + } + response = self.client.post( + reverse("wooey:api_add_or_update_script"), + data=payload, + ) + data = response.json() + self.assertEqual(data[0]["script"], "update-script") + self.assertEqual(data[0]["iteration"], 1) + self.assertEqual(data[0]["version"], "1") + payload = { + "group": "test group", + "update-script": open(self.version2_script_path, "rb"), + } + response = self.client.post( + reverse("wooey:api_add_or_update_script"), + data=payload, + ) + data = response.json() + self.assertEqual(data[0]["script"], "update-script") + self.assertEqual(data[0]["version"], "1") + self.assertEqual(data[0]["iteration"], 2) + self.assertEqual(data[0]["is_default"], True) + + def test_can_disable_default_update(self): + payload = { + "group": "test group 2", + "update-script": open(self.version1_script_path, "rb"), + } + response = self.client.post( + reverse("wooey:api_add_or_update_script"), + data=payload, + ) + data = response.json() + self.assertEqual(data[0]["script"], "update-script") + self.assertEqual(data[0]["iteration"], 1) + self.assertEqual(data[0]["version"], "1") + payload = { + "group": "test group 2", + "default": False, + "update-script": open(self.version2_script_path, "rb"), + } + response = self.client.post( + reverse("wooey:api_add_or_update_script"), + data=payload, + ) + data = response.json() + self.assertEqual(data[0]["script"], "update-script") + self.assertEqual(data[0]["version"], "1") + self.assertEqual(data[0]["iteration"], 2) + self.assertEqual(data[0]["is_default"], False) + + +class TestScriptSubmission( + mixins.ScriptFactoryMixin, ApiTestMixin, TransactionTestCase +): + def test_defaults_to_latest_script(self): + payload = { + "group": "test group 2", + "update-script": open(self.version1_script_path, "rb"), + } + response = self.client.post( + reverse("wooey:api_add_or_update_script"), + data=payload, + ) + payload = { + "group": "test group 2", + "update-script": open(self.version2_script_path, "rb"), + } + response = self.client.post( + reverse("wooey:api_add_or_update_script"), + data=payload, + ) + payload = { + "job_name": "test", + "command": "--one 1 --two 2", + } + response = self.client.post( + reverse("wooey:api_submit_script", kwargs={"slug": "update-script"}), + data=payload, + content_type="application/json", + ) + data = response.json() + self.assertTrue(data["valid"]) + job = WooeyJob.objects.get(id=data["job_id"]) + self.assertEqual(job.script_version.script_iteration, 2) + + def test_can_specify_version(self): + payload = { + "group": "test group 2", + "update-script": open(self.version1_script_path, "rb"), + } + response = self.client.post( + reverse("wooey:api_add_or_update_script"), + data=payload, + ) + payload = { + "group": "test group 2", + "update-script": open(self.version2_script_path, "rb"), + } + response = self.client.post( + reverse("wooey:api_add_or_update_script"), + data=payload, + ) + payload = { + "job_name": "test", + "iteration": 1, + "command": "--one 1", + } + response = self.client.post( + reverse("wooey:api_submit_script", kwargs={"slug": "update-script"}), + data=payload, + content_type="application/json", + ) + data = response.json() + self.assertTrue(data["valid"]) + job = WooeyJob.objects.get(id=data["job_id"]) + self.assertEqual(job.script_version.script_iteration, 1) + + def test_can_submit_script_with_json(self): + script_version = self.translate_script + payload = { + "job_name": "test", + "job_description": "a test job", + "command": "--sequence aaa", + } + response = self.client.post( + reverse( + "wooey:api_submit_script", kwargs={"slug": script_version.script.slug} + ), + data=payload, + content_type="application/json", + ) + data = response.json() + self.assertTrue(data["valid"]) + job = WooeyJob.objects.get(id=data["job_id"]) + self.assertEqual(job.job_description, "a test job") + self.assertEqual( + job.scriptparameters_set.get(parameter__slug="sequence").value, "aaa" + ) + + def test_submit_script_with_mixed_form_data(self): + script_version = self.translate_script + contents = b">test sequence\naaaggg\n" + fasta_file = BytesIO(contents) + payload = { + "job_name": "test", + "command": "--fasta fasta", + "fasta": fasta_file, + } + response = self.client.post( + reverse( + "wooey:api_submit_script", kwargs={"slug": script_version.script.slug} + ), + data=payload, + ) + data = response.json() + self.assertTrue(data["valid"]) + job = WooeyJob.objects.get(id=data["job_id"]) + self.assertEqual( + job.scriptparameters_set.get(parameter__slug="fasta").value.read(), contents + ) + + def test_submit_multiple_files_and_inputs(self): + script_version = self.choice_script + contents1 = b"foo" + contents2 = b"bar" + file1 = BytesIO(contents1) + file2 = BytesIO(contents2) + payload = { + "job_name": "test", + "command": "--need-at-least-one-numbers 1 2 3 --multiple-file-choices file1 file2", + "file1": file1, + "file2": file2, + } + response = self.client.post( + reverse( + "wooey:api_submit_script", kwargs={"slug": script_version.script.slug} + ), + data=payload, + ) + data = response.json() + self.assertTrue(data["valid"]) + job = WooeyJob.objects.get(id=data["job_id"]) + uploaded_files = { + os.path.basename(i.value.name): i.value + for i in job.scriptparameters_set.filter( + parameter__slug="multiple_file_choices" + ) + } + self.assertEqual(len(uploaded_files), 2) + self.assertEqual(uploaded_files["file1"].read(), contents1) + self.assertEqual(uploaded_files["file2"].read(), contents2) + self.assertEqual( + sorted( + [ + i.value + for i in job.scriptparameters_set.filter( + parameter__slug="need_at_least_one_numbers" + ).all() + ] + ), + [1, 2, 3], + ) diff --git a/wooey/tests/test_forms.py b/wooey/tests/test_forms.py index 785c3722..8e04843e 100644 --- a/wooey/tests/test_forms.py +++ b/wooey/tests/test_forms.py @@ -1,8 +1,8 @@ import os from django.test import TransactionTestCase -from django.http.request import MultiValueDict from django.core.files.uploadedfile import SimpleUploadedFile +from django.utils.datastructures import MultiValueDict from ..backend import utils from ..forms import ( diff --git a/wooey/tests/test_models.py b/wooey/tests/test_models.py index dc4c73ba..4519137a 100644 --- a/wooey/tests/test_models.py +++ b/wooey/tests/test_models.py @@ -1,6 +1,7 @@ import os from urllib.parse import quote +from django.contrib.auth.models import AnonymousUser from django.test import Client, TestCase, TransactionTestCase from wooey import models @@ -196,6 +197,20 @@ def test_multiplechoices(self): self.assertEqual(choices, choice_params) job = job.submit_to_celery() + def test_anyone_can_view_anonymous_jobs(self): + job = factories.WooeyJob(user=None) + new_user = factories.UserFactory() + self.assertTrue(job.can_user_view(AnonymousUser)) + self.assertTrue(job.can_user_view(new_user)) + + def test_jobs_with_user_only_viewable_by_user(self): + job_user = factories.UserFactory(username="someone new") + other_user = factories.UserFactory(username="a different user") + job = factories.WooeyJob(user=job_user) + self.assertFalse(job.can_user_view(AnonymousUser)) + self.assertFalse(job.can_user_view(other_user)) + self.assertTrue(job.can_user_view(job_user)) + class TestCustomWidgets(TestCase): def test_widget_attributes(self): diff --git a/wooey/tests/utils.py b/wooey/tests/utils.py index c372515f..8888d3ab 100644 --- a/wooey/tests/utils.py +++ b/wooey/tests/utils.py @@ -5,8 +5,8 @@ from .. import settings as wooey_settings -def get_subparser_form_slug(script_version, slug): - return script_version.scriptparameter_set.get(script_param=slug).form_slug +def get_subparser_form_slug(script_version, param): + return script_version.scriptparameter_set.get(script_param=param).form_slug def save_script_path(script_path): diff --git a/wooey/urls.py b/wooey/urls.py index 9efeaca2..d6d8d932 100644 --- a/wooey/urls.py +++ b/wooey/urls.py @@ -2,6 +2,7 @@ from django.urls import include, re_path +from . import api from . import views from . import settings as wooey_settings @@ -48,6 +49,26 @@ views.JobJSON.as_view(), name="celery_results_json_uuid", ), + re_path( + r"^api/scripts/v1/(?P[a-zA-Z0-9\-\_]+)/submit/$", + api.submit_script, + name="api_submit_script", + ), + re_path( + r"^api/scripts/v1/add-or-update/$", + api.add_or_update_script, + name="api_add_or_update_script", + ), + re_path( + r"^api/jobs/v1/(?P[a-zA-Z0-9\-\_]+)/status/$", + api.job_status, + name="api_job_status", + ), + re_path( + r"^api/jobs/v1/(?P[a-zA-Z0-9\-\_]+)/details/$", + api.job_details, + name="api_job_details", + ), re_path( r"^scripts/(?P[a-zA-Z0-9\-\_]+)/$", views.WooeyScriptView.as_view(), diff --git a/wooey/utils/__init__.py b/wooey/utils/__init__.py index 10faaf6a..b22bb4fe 100644 --- a/wooey/utils/__init__.py +++ b/wooey/utils/__init__.py @@ -1,5 +1,6 @@ import hashlib +from django.http import HttpResponse from django.utils.crypto import get_random_string @@ -12,3 +13,13 @@ def generate_hash(value): hasher = hashlib.sha256() hasher.update(value.encode("utf-8")) return hasher.hexdigest() + + +def requires_login(func): + def inner(request, *args, **kwargs): + user = request.user + if not user.is_authenticated: + return HttpResponse("Must be authenticated to use this method.", status=403) + return func(request, *args, **kwargs) + + return inner diff --git a/wooey/views/profile.py b/wooey/views/profile.py index 5a6209c5..fee20fb1 100644 --- a/wooey/views/profile.py +++ b/wooey/views/profile.py @@ -3,16 +3,7 @@ from ..forms import APIKeyForm from ..models import APIKey, WooeyProfile - - -def requires_login(func): - def inner(request, *args, **kwargs): - user = request.user - if not user.is_authenticated: - return HttpResponse("Must be authenticated to use this method.", status=403) - return func(request, *args, **kwargs) - - return inner +from ..utils import requires_login @require_http_methods(["POST"]) diff --git a/wooey/views/views.py b/wooey/views/views.py index 65863411..f763072b 100644 --- a/wooey/views/views.py +++ b/wooey/views/views.py @@ -3,7 +3,6 @@ from django.contrib.auth import get_user_model from django.contrib.contenttypes.models import ContentType -from django.forms import FileField from django.http import JsonResponse from django.template.loader import render_to_string from django.urls import reverse @@ -44,9 +43,7 @@ def get_context_data(self, **kwargs): if job_id: job = WooeyJob.objects.get(pk=job_id) - if job.user is None or ( - self.request.user.is_authenticated and job.user == self.request.user - ): + if job.can_user_view(self.request.user): context["job_info"] = {"job_id": job_id} parser_used = None @@ -107,46 +104,7 @@ def post(self, request, *args, **kwargs): form = utils.get_master_form( pk=int(post["wooey_type"]), parser=int(post.get("wooey_parser", 0)) ) - # TODO: Check with people who know more if there's a smarter way to do this utils.validate_form(form=form, data=post, files=request.FILES) - # for cloned jobs, we don't have the files in input fields, they'll be in a list like ['', filename] - # This will cause issues. - to_delete = [] - for i in post: - if isinstance(form.fields.get(i), FileField): - # if we have a value set, reassert this - new_values = list(filter(lambda x: x, post.getlist(i))) - cleaned_values = [] - for new_value in new_values: - if i not in request.FILES and ( - i not in form.cleaned_data - or ( - new_value - and ( - form.cleaned_data[i] is None - or not [j for j in form.cleaned_data[i] if j] - ) - ) - ): - # this is a previously set field, so a cloned job - if new_value is not None: - cleaned_values.append( - utils.get_storage(local=False).open(new_value) - ) - to_delete.append(i) - if cleaned_values: - form.cleaned_data[i] = cleaned_values - for i in to_delete: - if i in form.errors: - del form.errors[i] - - # because we can have multiple files for a field, we need to update our form.cleaned_data to be a list of files - for i in request.FILES: - v = request.FILES.getlist(i) - if i in form.cleaned_data: - cleaned = form.cleaned_data[i] - cleaned = cleaned if isinstance(cleaned, list) else [cleaned] - form.cleaned_data[i] = list(set(cleaned).union(set(v))) if not form.errors: version_pk = form.cleaned_data.get("wooey_type")