From 23c0e05dc21833e45f6f911b43996e8e54a50da5 Mon Sep 17 00:00:00 2001 From: Jake Howard Date: Fri, 13 Jun 2025 16:37:25 +0100 Subject: [PATCH 1/4] Rename task status to make it clear the task may not be immediately new --- README.md | 2 +- ...t_django_task_new_ordering_idx_and_more.py | 58 +++++++++++++++++++ django_tasks/backends/database/models.py | 6 +- django_tasks/backends/dummy.py | 2 +- django_tasks/backends/immediate.py | 2 +- django_tasks/backends/rq.py | 10 ++-- django_tasks/task.py | 9 ++- tests/tests/test_database_backend.py | 28 ++++----- tests/tests/test_dummy_backend.py | 14 ++--- tests/tests/test_immediate_backend.py | 6 +- tests/tests/test_rq_backend.py | 14 ++--- tests/tests/test_tasks.py | 4 +- 12 files changed, 110 insertions(+), 45 deletions(-) create mode 100644 django_tasks/backends/database/migrations/0017_remove_dbtaskresult_django_task_new_ordering_idx_and_more.py diff --git a/README.md b/README.md index 66548bd..e33c94c 100644 --- a/README.md +++ b/README.md @@ -197,7 +197,7 @@ assert result.return_value == 42 If a result has been updated in the background, you can call `refresh` on it to update its values. Results obtained using `get_result` will always be up-to-date. ```python -assert result.status == ResultStatus.NEW +assert result.status == ResultStatus.READY result.refresh() assert result.status == ResultStatus.SUCCEEDED ``` diff --git a/django_tasks/backends/database/migrations/0017_remove_dbtaskresult_django_task_new_ordering_idx_and_more.py b/django_tasks/backends/database/migrations/0017_remove_dbtaskresult_django_task_new_ordering_idx_and_more.py new file mode 100644 index 0000000..0218678 --- /dev/null +++ b/django_tasks/backends/database/migrations/0017_remove_dbtaskresult_django_task_new_ordering_idx_and_more.py @@ -0,0 +1,58 @@ +# Generated by Django 5.2.2 on 2025-06-13 14:56 + +from django.db import migrations, models +from django.db.backends.base.schema import BaseDatabaseSchemaEditor +from django.db.migrations.state import StateApps + + +def update_status(apps: StateApps, schema_editor: BaseDatabaseSchemaEditor) -> None: + DBTaskResult = apps.get_model("django_tasks_database", "DBTaskResult") + + DBTaskResult.objects.filter(status="NEW").update(status="READY") + + +def revert_status_rename( + apps: StateApps, schema_editor: BaseDatabaseSchemaEditor +) -> None: + DBTaskResult = apps.get_model("django_tasks_database", "DBTaskResult") + + DBTaskResult.objects.filter(status="READY").update(status="NEW") + + +class Migration(migrations.Migration): + dependencies = [ + ("django_tasks_database", "0016_alter_dbtaskresult_options_and_more"), + ] + + operations = [ + migrations.RemoveIndex( + model_name="dbtaskresult", + name="django_task_new_ordering_idx", + ), + migrations.AlterField( + model_name="dbtaskresult", + name="status", + field=models.CharField( + choices=[ + ("READY", "Ready"), + ("RUNNING", "Running"), + ("FAILED", "Failed"), + ("SUCCEEDED", "Succeeded"), + ], + default="READY", + max_length=9, + verbose_name="status", + ), + ), + migrations.RunPython(update_status, revert_status_rename), + migrations.AddIndex( + model_name="dbtaskresult", + index=models.Index( + models.F("status"), + models.OrderBy(models.F("priority"), descending=True), + models.OrderBy(models.F("run_after")), + condition=models.Q(("status", "READY")), + name="django_task_new_ordering_idx", + ), + ), + ] diff --git a/django_tasks/backends/database/models.py b/django_tasks/backends/database/models.py index 59de8eb..0810bd2 100644 --- a/django_tasks/backends/database/models.py +++ b/django_tasks/backends/database/models.py @@ -56,7 +56,7 @@ def ready(self) -> "DBTaskResultQuerySet": Return tasks which are ready to be processed. """ return self.filter( - status=ResultStatus.NEW, + status=ResultStatus.READY, ).filter(models.Q(run_after=DATE_MAX) | models.Q(run_after__lte=timezone.now())) def succeeded(self) -> "DBTaskResultQuerySet": @@ -85,7 +85,7 @@ class DBTaskResult(GenericBase[P, T], models.Model): status = models.CharField( _("status"), choices=ResultStatus.choices, - default=ResultStatus.NEW, + default=ResultStatus.READY, max_length=max(len(value) for value in ResultStatus.values), ) @@ -122,7 +122,7 @@ class Meta: "status", *ordering, name="django_task_new_ordering_idx", - condition=Q(status=ResultStatus.NEW), + condition=Q(status=ResultStatus.READY), ), models.Index(fields=["queue_name"]), models.Index(fields=["backend_name"]), diff --git a/django_tasks/backends/dummy.py b/django_tasks/backends/dummy.py index e9027da..0d337f3 100644 --- a/django_tasks/backends/dummy.py +++ b/django_tasks/backends/dummy.py @@ -43,7 +43,7 @@ def enqueue( result = TaskResult[T]( task=task, id=get_random_id(), - status=ResultStatus.NEW, + status=ResultStatus.READY, enqueued_at=None, started_at=None, finished_at=None, diff --git a/django_tasks/backends/immediate.py b/django_tasks/backends/immediate.py index cfae387..51ab0a7 100644 --- a/django_tasks/backends/immediate.py +++ b/django_tasks/backends/immediate.py @@ -79,7 +79,7 @@ def enqueue( task_result = TaskResult[T]( task=task, id=get_random_id(), - status=ResultStatus.NEW, + status=ResultStatus.READY, enqueued_at=None, started_at=None, finished_at=None, diff --git a/django_tasks/backends/rq.py b/django_tasks/backends/rq.py index 4ecba0c..8ad5885 100644 --- a/django_tasks/backends/rq.py +++ b/django_tasks/backends/rq.py @@ -26,15 +26,15 @@ P = ParamSpec("P") RQ_STATUS_TO_RESULT_STATUS = { - JobStatus.QUEUED: ResultStatus.NEW, + JobStatus.QUEUED: ResultStatus.READY, JobStatus.FINISHED: ResultStatus.SUCCEEDED, JobStatus.FAILED: ResultStatus.FAILED, JobStatus.STARTED: ResultStatus.RUNNING, - JobStatus.DEFERRED: ResultStatus.NEW, - JobStatus.SCHEDULED: ResultStatus.NEW, + JobStatus.DEFERRED: ResultStatus.READY, + JobStatus.SCHEDULED: ResultStatus.READY, JobStatus.STOPPED: ResultStatus.FAILED, JobStatus.CANCELED: ResultStatus.FAILED, - None: ResultStatus.NEW, + None: ResultStatus.READY, } @@ -162,7 +162,7 @@ def enqueue( task_result = TaskResult[T]( task=task, id=get_random_id(), - status=ResultStatus.NEW, + status=ResultStatus.READY, enqueued_at=None, started_at=None, finished_at=None, diff --git a/django_tasks/task.py b/django_tasks/task.py index 745a201..ffdcac3 100644 --- a/django_tasks/task.py +++ b/django_tasks/task.py @@ -45,10 +45,17 @@ class ResultStatus(TextChoices): - NEW = ("NEW", _("New")) + READY = ("READY", _("Ready")) + """The task has just been enqueued, or is ready to be executed again (eg for a retry).""" + RUNNING = ("RUNNING", _("Running")) + """The task is currently running.""" + FAILED = ("FAILED", _("Failed")) + """The task has finished running, but resulted in an error.""" + SUCCEEDED = ("SUCCEEDED", _("Succeeded")) + """The task has finished running successfully.""" T = TypeVar("T") diff --git a/tests/tests/test_database_backend.py b/tests/tests/test_database_backend.py index 8460d1b..d27a004 100644 --- a/tests/tests/test_database_backend.py +++ b/tests/tests/test_database_backend.py @@ -79,7 +79,7 @@ def test_enqueue_task(self) -> None: with self.subTest(task), self.assertNumQueries(1): result = cast(Task, task).enqueue(1, two=3) - self.assertEqual(result.status, ResultStatus.NEW) + self.assertEqual(result.status, ResultStatus.READY) self.assertFalse(result.is_finished) self.assertIsNone(result.started_at) self.assertIsNone(result.finished_at) @@ -94,7 +94,7 @@ async def test_enqueue_task_async(self) -> None: with self.subTest(task): result = await cast(Task, task).aenqueue() - self.assertEqual(result.status, ResultStatus.NEW) + self.assertEqual(result.status, ResultStatus.READY) self.assertFalse(result.is_finished) self.assertIsNone(result.started_at) self.assertIsNone(result.finished_at) @@ -132,7 +132,7 @@ def test_refresh_result(self) -> None: return_value=42, ) - self.assertEqual(result.status, ResultStatus.NEW) + self.assertEqual(result.status, ResultStatus.READY) self.assertFalse(result.is_finished) self.assertIsNone(result.started_at) self.assertIsNone(result.finished_at) @@ -156,7 +156,7 @@ async def test_refresh_result_async(self) -> None: return_value=42, ) - self.assertEqual(result.status, ResultStatus.NEW) + self.assertEqual(result.status, ResultStatus.READY) self.assertFalse(result.is_finished) self.assertIsNone(result.started_at) self.assertIsNone(result.finished_at) @@ -195,10 +195,10 @@ def test_meaning_of_life_view(self) -> None: data = json.loads(response.content) self.assertEqual(data["result"], None) - self.assertEqual(data["status"], ResultStatus.NEW) + self.assertEqual(data["status"], ResultStatus.READY) result = default_task_backend.get_result(data["result_id"]) - self.assertEqual(result.status, ResultStatus.NEW) + self.assertEqual(result.status, ResultStatus.READY) def test_get_result_from_different_request(self) -> None: response = self.client.get(reverse("meaning-of-life")) @@ -212,7 +212,7 @@ def test_get_result_from_different_request(self) -> None: self.assertEqual( json.loads(response.content), - {"result_id": result_id, "result": None, "status": ResultStatus.NEW}, + {"result_id": result_id, "result": None, "status": ResultStatus.READY}, ) def test_invalid_task_path(self) -> None: @@ -455,12 +455,12 @@ def test_run_enqueued_task(self) -> None: result = cast(Task, task).enqueue() self.assertEqual(DBTaskResult.objects.ready().count(), 1) - self.assertEqual(result.status, ResultStatus.NEW) + self.assertEqual(result.status, ResultStatus.READY) with self.assertNumQueries(9 if connection.vendor == "mysql" else 8): self.run_worker() - self.assertEqual(result.status, ResultStatus.NEW) + self.assertEqual(result.status, ResultStatus.READY) result.refresh() self.assertIsNotNone(result.started_at) self.assertIsNotNone(result.finished_at) @@ -531,7 +531,7 @@ def test_failing_task(self) -> None: with self.assertNumQueries(9 if connection.vendor == "mysql" else 8): self.run_worker() - self.assertEqual(result.status, ResultStatus.NEW) + self.assertEqual(result.status, ResultStatus.READY) result.refresh() self.assertIsNotNone(result.started_at) self.assertIsNotNone(result.finished_at) @@ -565,7 +565,7 @@ def test_complex_exception(self) -> None: with self.assertNumQueries(9 if connection.vendor == "mysql" else 8): self.run_worker() - self.assertEqual(result.status, ResultStatus.NEW) + self.assertEqual(result.status, ResultStatus.READY) result.refresh() self.assertIsNotNone(result.started_at) self.assertIsNotNone(result.finished_at) @@ -806,7 +806,7 @@ def test_worker_with_locked_rows(self) -> None: result_1.refresh() result_2.refresh() - self.assertEqual(result_1.status, ResultStatus.NEW) + self.assertEqual(result_1.status, ResultStatus.READY) self.assertEqual(result_2.status, ResultStatus.SUCCEEDED) def test_max_tasks(self) -> None: @@ -823,7 +823,7 @@ def test_max_tasks(self) -> None: statuses = Counter(result.status for result in results) self.assertEqual(statuses[ResultStatus.SUCCEEDED], 2) - self.assertEqual(statuses[ResultStatus.NEW], 3) + self.assertEqual(statuses[ResultStatus.READY], 3) @override_settings( @@ -1358,7 +1358,7 @@ def test_run_subprocess(self) -> None: process.wait() self.assertEqual(process.returncode, 0) - self.assertEqual(result.status, ResultStatus.NEW) + self.assertEqual(result.status, ResultStatus.READY) result.refresh() diff --git a/tests/tests/test_dummy_backend.py b/tests/tests/test_dummy_backend.py index d448f50..8a61916 100644 --- a/tests/tests/test_dummy_backend.py +++ b/tests/tests/test_dummy_backend.py @@ -38,7 +38,7 @@ def test_enqueue_task(self) -> None: with self.subTest(task): result = cast(Task, task).enqueue(1, two=3) - self.assertEqual(result.status, ResultStatus.NEW) + self.assertEqual(result.status, ResultStatus.READY) self.assertFalse(result.is_finished) self.assertIsNone(result.started_at) self.assertIsNone(result.finished_at) @@ -55,7 +55,7 @@ async def test_enqueue_task_async(self) -> None: with self.subTest(task): result = await cast(Task, task).aenqueue() - self.assertEqual(result.status, ResultStatus.NEW) + self.assertEqual(result.status, ResultStatus.READY) self.assertFalse(result.is_finished) self.assertIsNone(result.started_at) self.assertIsNone(result.finished_at) @@ -89,7 +89,7 @@ def test_refresh_result(self) -> None: enqueued_result = default_task_backend.results[0] # type:ignore[attr-defined] object.__setattr__(enqueued_result, "status", ResultStatus.SUCCEEDED) - self.assertEqual(result.status, ResultStatus.NEW) + self.assertEqual(result.status, ResultStatus.READY) result.refresh() self.assertEqual(result.status, ResultStatus.SUCCEEDED) @@ -101,7 +101,7 @@ async def test_refresh_result_async(self) -> None: enqueued_result = default_task_backend.results[0] # type:ignore[attr-defined] object.__setattr__(enqueued_result, "status", ResultStatus.SUCCEEDED) - self.assertEqual(result.status, ResultStatus.NEW) + self.assertEqual(result.status, ResultStatus.READY) await result.arefresh() self.assertEqual(result.status, ResultStatus.SUCCEEDED) @@ -124,10 +124,10 @@ def test_meaning_of_life_view(self) -> None: data = json.loads(response.content) self.assertEqual(data["result"], None) - self.assertEqual(data["status"], ResultStatus.NEW) + self.assertEqual(data["status"], ResultStatus.READY) result = default_task_backend.get_result(data["result_id"]) - self.assertEqual(result.status, ResultStatus.NEW) + self.assertEqual(result.status, ResultStatus.READY) def test_get_result_from_different_request(self) -> None: response = self.client.get(reverse("meaning-of-life")) @@ -141,7 +141,7 @@ def test_get_result_from_different_request(self) -> None: self.assertEqual( json.loads(response.content), - {"result_id": result_id, "result": None, "status": ResultStatus.NEW}, + {"result_id": result_id, "result": None, "status": ResultStatus.READY}, ) def test_enqueue_on_commit(self) -> None: diff --git a/tests/tests/test_immediate_backend.py b/tests/tests/test_immediate_backend.py index 1ee6032..7f412b1 100644 --- a/tests/tests/test_immediate_backend.py +++ b/tests/tests/test_immediate_backend.py @@ -267,7 +267,7 @@ def test_wait_until_transaction_commit(self) -> None: result = test_tasks.noop_task.enqueue() self.assertIsNone(result.enqueued_at) - self.assertEqual(result.status, ResultStatus.NEW) + self.assertEqual(result.status, ResultStatus.READY) self.assertEqual(result.status, ResultStatus.SUCCEEDED) self.assertIsNotNone(result.enqueued_at) @@ -312,7 +312,7 @@ def test_wait_until_transaction_by_default(self) -> None: result = test_tasks.noop_task.enqueue() self.assertIsNone(result.enqueued_at) - self.assertEqual(result.status, ResultStatus.NEW) + self.assertEqual(result.status, ResultStatus.READY) self.assertEqual(result.status, ResultStatus.SUCCEEDED) @@ -337,6 +337,6 @@ def test_task_specific_enqueue_on_commit(self) -> None: result = test_tasks.enqueue_on_commit_task.enqueue() self.assertIsNone(result.enqueued_at) - self.assertEqual(result.status, ResultStatus.NEW) + self.assertEqual(result.status, ResultStatus.READY) self.assertEqual(result.status, ResultStatus.SUCCEEDED) diff --git a/tests/tests/test_rq_backend.py b/tests/tests/test_rq_backend.py index 8a11898..1c39157 100644 --- a/tests/tests/test_rq_backend.py +++ b/tests/tests/test_rq_backend.py @@ -98,7 +98,7 @@ def test_enqueue_task(self) -> None: with self.subTest(task): result = cast(Task, task).enqueue(1, two=3) - self.assertEqual(result.status, ResultStatus.NEW) + self.assertEqual(result.status, ResultStatus.READY) self.assertFalse(result.is_finished) self.assertIsNone(result.started_at) self.assertIsNone(result.finished_at) @@ -113,7 +113,7 @@ async def test_enqueue_task_async(self) -> None: with self.subTest(task): result = await cast(Task, task).aenqueue() - self.assertEqual(result.status, ResultStatus.NEW) + self.assertEqual(result.status, ResultStatus.READY) self.assertFalse(result.is_finished) self.assertIsNone(result.started_at) self.assertIsNone(result.finished_at) @@ -208,7 +208,7 @@ def test_refresh_result(self) -> None: self.run_worker() - self.assertEqual(result.status, ResultStatus.NEW) + self.assertEqual(result.status, ResultStatus.READY) self.assertFalse(result.is_finished) self.assertIsNone(result.started_at) self.assertIsNone(result.finished_at) @@ -228,7 +228,7 @@ def test_refresh_result_async(self) -> None: self.run_worker() - self.assertEqual(result.status, ResultStatus.NEW) + self.assertEqual(result.status, ResultStatus.READY) self.assertFalse(result.is_finished) self.assertIsNone(result.started_at) self.assertIsNone(result.finished_at) @@ -269,10 +269,10 @@ def test_meaning_of_life_view(self) -> None: data = json.loads(response.content) self.assertEqual(data["result"], None) - self.assertEqual(data["status"], ResultStatus.NEW) + self.assertEqual(data["status"], ResultStatus.READY) result = default_task_backend.get_result(data["result_id"]) - self.assertEqual(result.status, ResultStatus.NEW) + self.assertEqual(result.status, ResultStatus.READY) def test_get_result_from_different_request(self) -> None: response = self.client.get(reverse("meaning-of-life")) @@ -286,7 +286,7 @@ def test_get_result_from_different_request(self) -> None: self.assertEqual( json.loads(response.content), - {"result_id": result_id, "result": None, "status": ResultStatus.NEW}, + {"result_id": result_id, "result": None, "status": ResultStatus.READY}, ) def test_invalid_task_path(self) -> None: diff --git a/tests/tests/test_tasks.py b/tests/tests/test_tasks.py index d05a12c..c594e41 100644 --- a/tests/tests/test_tasks.py +++ b/tests/tests/test_tasks.py @@ -51,7 +51,7 @@ def test_task_decorator(self) -> None: def test_enqueue_task(self) -> None: result = test_tasks.noop_task.enqueue() - self.assertEqual(result.status, ResultStatus.NEW) + self.assertEqual(result.status, ResultStatus.READY) self.assertEqual(result.task, test_tasks.noop_task) self.assertEqual(result.args, []) self.assertEqual(result.kwargs, {}) @@ -61,7 +61,7 @@ def test_enqueue_task(self) -> None: async def test_enqueue_task_async(self) -> None: result = await test_tasks.noop_task.aenqueue() - self.assertEqual(result.status, ResultStatus.NEW) + self.assertEqual(result.status, ResultStatus.READY) self.assertEqual(result.task, test_tasks.noop_task) self.assertEqual(result.args, []) self.assertEqual(result.kwargs, {}) From 4779427c1ba746e84b06096268e689ecdc57557a Mon Sep 17 00:00:00 2001 From: Jake Howard Date: Fri, 13 Jun 2025 16:55:20 +0100 Subject: [PATCH 2/4] Support storing multiple errors --- README.md | 10 +++-- django_tasks/backends/database/models.py | 17 +++++---- django_tasks/backends/dummy.py | 1 + django_tasks/backends/immediate.py | 18 +++++++-- django_tasks/backends/rq.py | 38 +++++++++++++------ django_tasks/task.py | 48 +++++++++++++----------- tests/tests/test_database_backend.py | 34 ++++++----------- tests/tests/test_dummy_backend.py | 8 +--- tests/tests/test_immediate_backend.py | 14 ++++--- tests/tests/test_rq_backend.py | 14 ++++--- tests/tests/test_tasks.py | 37 +++++++++++++++++- tests/views.py | 2 +- 12 files changed, 152 insertions(+), 89 deletions(-) diff --git a/README.md b/README.md index e33c94c..2acc9e4 100644 --- a/README.md +++ b/README.md @@ -202,20 +202,22 @@ result.refresh() assert result.status == ResultStatus.SUCCEEDED ``` -#### Exceptions +#### Errors -If a task raised an exception, its `.exception_class` will be the exception class raised: +If a task raised an exception, its `.errors` contains information about the error: ```python -assert result.exception_class == ValueError +assert result.errors[0].exception_class == ValueError ``` Note that this is just the type of exception, and contains no other values. The traceback information is reduced to a string that you can print to help debugging: ```python -assert isinstance(result.traceback, str) +assert isinstance(result.errors[0].traceback, str) ``` +Note that currently, whilst `.errors` is a list, it will only ever contain a single element. + ### Backend introspecting Because `django-tasks` enables support for multiple different backends, those backends may not support all features, and it can be useful to determine this at runtime to ensure the chosen task queue meets the requirements, or to gracefully degrade functionality if it doesn't. diff --git a/django_tasks/backends/database/models.py b/django_tasks/backends/database/models.py index 0810bd2..6279971 100644 --- a/django_tasks/backends/database/models.py +++ b/django_tasks/backends/database/models.py @@ -20,6 +20,7 @@ MIN_PRIORITY, ResultStatus, Task, + TaskError, ) from django_tasks.utils import get_exception_traceback, get_module_path, retry @@ -163,11 +164,6 @@ def task(self) -> Task[P, T]: def task_result(self) -> "TaskResult[T]": from .backend import TaskResult - try: - exception_class = import_string(self.exception_class_path) - except ImportError: - exception_class = None - task_result = TaskResult[T]( db_result=self, task=self.task, @@ -179,10 +175,17 @@ def task_result(self) -> "TaskResult[T]": args=self.args_kwargs["args"], kwargs=self.args_kwargs["kwargs"], backend=self.backend_name, + errors=[], ) - object.__setattr__(task_result, "_exception_class", exception_class) - object.__setattr__(task_result, "_traceback", self.traceback or None) + if self.status == ResultStatus.FAILED: + task_result.errors.append( + TaskError( + exception_class_path=self.exception_class_path, + traceback=self.traceback, + ) + ) + object.__setattr__(task_result, "_return_value", self.return_value) return task_result diff --git a/django_tasks/backends/dummy.py b/django_tasks/backends/dummy.py index 0d337f3..c181fa7 100644 --- a/django_tasks/backends/dummy.py +++ b/django_tasks/backends/dummy.py @@ -50,6 +50,7 @@ def enqueue( args=args, kwargs=kwargs, backend=self.alias, + errors=[], ) if self._get_enqueue_on_commit_for_task(task) is not False: diff --git a/django_tasks/backends/immediate.py b/django_tasks/backends/immediate.py index 51ab0a7..5b01cdf 100644 --- a/django_tasks/backends/immediate.py +++ b/django_tasks/backends/immediate.py @@ -9,8 +9,13 @@ from typing_extensions import ParamSpec from django_tasks.signals import task_enqueued, task_finished, task_started -from django_tasks.task import ResultStatus, Task, TaskResult -from django_tasks.utils import get_exception_traceback, get_random_id, json_normalize +from django_tasks.task import ResultStatus, Task, TaskError, TaskResult +from django_tasks.utils import ( + get_exception_traceback, + get_module_path, + get_random_id, + json_normalize, +) from .base import BaseTaskBackend @@ -56,8 +61,12 @@ def _execute_task(self, task_result: TaskResult) -> None: object.__setattr__(task_result, "finished_at", timezone.now()) - object.__setattr__(task_result, "_traceback", get_exception_traceback(e)) - object.__setattr__(task_result, "_exception_class", type(e)) + task_result.errors.append( + TaskError( + exception_class_path=get_module_path(type(e)), + traceback=get_exception_traceback(e), + ) + ) object.__setattr__(task_result, "status", ResultStatus.FAILED) @@ -86,6 +95,7 @@ def enqueue( args=args, kwargs=kwargs, backend=self.alias, + errors=[], ) if self._get_enqueue_on_commit_for_task(task) is not False: diff --git a/django_tasks/backends/rq.py b/django_tasks/backends/rq.py index 8ad5885..eae2f56 100644 --- a/django_tasks/backends/rq.py +++ b/django_tasks/backends/rq.py @@ -8,17 +8,23 @@ from django.core.checks import messages from django.core.exceptions import SuspiciousOperation from django.db import transaction -from django.utils.module_loading import import_string from redis.client import Redis from rq.job import Callback, JobStatus from rq.job import Job as BaseJob from rq.registry import ScheduledJobRegistry +from rq.results import Result from typing_extensions import ParamSpec from django_tasks.backends.base import BaseTaskBackend from django_tasks.exceptions import ResultDoesNotExist from django_tasks.signals import task_enqueued, task_finished, task_started -from django_tasks.task import DEFAULT_PRIORITY, MAX_PRIORITY, ResultStatus, Task +from django_tasks.task import ( + DEFAULT_PRIORITY, + MAX_PRIORITY, + ResultStatus, + Task, + TaskError, +) from django_tasks.task import TaskResult as BaseTaskResult from django_tasks.utils import get_module_path, get_random_id @@ -95,19 +101,24 @@ def into_task_result(self) -> TaskResult: args=list(self.args), kwargs=self.kwargs, backend=self.meta["backend_name"], + errors=[], ) - latest_result = self.latest_result() + exception_classes = self.meta.get("_django_tasks_exceptions", []).copy() - if latest_result is not None: - if "exception_class" in self.meta: - object.__setattr__( - task_result, - "_exception_class", - import_string(self.meta["exception_class"]), + rq_results = self.results() + + for rq_result in rq_results: + if rq_result.type == Result.Type.FAILED: + task_result.errors.append( + TaskError( + exception_class_path=exception_classes.pop(), + traceback=rq_result.exc_string, # type: ignore[arg-type] + ) ) - object.__setattr__(task_result, "_traceback", latest_result.exc_string) - object.__setattr__(task_result, "_return_value", latest_result.return_value) + + if rq_results: + object.__setattr__(task_result, "_return_value", rq_results[0].return_value) return task_result @@ -120,7 +131,9 @@ def failed_callback( traceback: TracebackType, ) -> None: # Smuggle the exception class through meta - job.meta["exception_class"] = get_module_path(exception_class) + job.meta.setdefault("_django_tasks_exceptions", []).append( + get_module_path(exception_class) + ) job.save_meta() # type: ignore[no-untyped-call] task_result = job.into_task_result() @@ -169,6 +182,7 @@ def enqueue( args=args, kwargs=kwargs, backend=self.alias, + errors=[], ) job = queue.create_job( diff --git a/django_tasks/task.py b/django_tasks/task.py index ffdcac3..56a4a44 100644 --- a/django_tasks/task.py +++ b/django_tasks/task.py @@ -1,6 +1,6 @@ from dataclasses import dataclass, field, replace from datetime import datetime -from inspect import iscoroutinefunction +from inspect import isclass, iscoroutinefunction from typing import ( TYPE_CHECKING, Any, @@ -15,6 +15,7 @@ from asgiref.sync import async_to_sync, sync_to_async from django.db.models.enums import TextChoices +from django.utils.module_loading import import_string from django.utils.translation import gettext_lazy as _ from typing_extensions import ParamSpec, Self @@ -34,8 +35,7 @@ DEFAULT_PRIORITY = 0 TASK_REFRESH_ATTRS = { - "_exception_class", - "_traceback", + "errors", "_return_value", "finished_at", "started_at", @@ -227,6 +227,26 @@ def wrapper(f: Callable[P, T]) -> Task[P, T]: return wrapper +@dataclass(frozen=True) +class TaskError: + exception_class_path: str + traceback: str + + @property + def exception_class(self) -> type[BaseException]: + # Lazy resolve the exception class + exception_class = import_string(self.exception_class_path) + + if not isclass(exception_class) or not issubclass( + exception_class, BaseException + ): + raise ValueError( + f"{self.exception_class_path!r} does not reference a valid exception." + ) + + return exception_class + + @dataclass(frozen=True) class TaskResult(Generic[T]): task: Task @@ -256,15 +276,15 @@ class TaskResult(Generic[T]): backend: str """The name of the backend the task will run on""" - _exception_class: Optional[type[BaseException]] = field(init=False, default=None) - _traceback: Optional[str] = field(init=False, default=None) + errors: list[TaskError] + """The errors raised when running the task""" _return_value: Optional[T] = field(init=False, default=None) @property def return_value(self) -> Optional[T]: """ - Get the return value of the task. + The return value of the task. If the task didn't succeed, an exception is raised. This is to distinguish against the task returning None. @@ -276,22 +296,6 @@ def return_value(self) -> Optional[T]: else: raise ValueError("Task has not finished yet") - @property - def exception_class(self) -> Optional[type[BaseException]]: - """The exception raised by the task function""" - if not self.is_finished: - raise ValueError("Task has not finished yet") - - return self._exception_class - - @property - def traceback(self) -> Optional[str]: - """The traceback of the exception if the task failed""" - if not self.is_finished: - raise ValueError("Task has not finished yet") - - return self._traceback - @property def is_finished(self) -> bool: """Has the task finished?""" diff --git a/tests/tests/test_database_backend.py b/tests/tests/test_database_backend.py index d27a004..a588a16 100644 --- a/tests/tests/test_database_backend.py +++ b/tests/tests/test_database_backend.py @@ -522,12 +522,6 @@ def test_failing_task(self) -> None: result = test_tasks.failing_task_value_error.enqueue() self.assertEqual(DBTaskResult.objects.ready().count(), 1) - with self.assertRaisesMessage(ValueError, "Task has not finished yet"): - result.exception_class # noqa: B018 - - with self.assertRaisesMessage(ValueError, "Task has not finished yet"): - result.traceback # noqa: B018 - with self.assertNumQueries(9 if connection.vendor == "mysql" else 8): self.run_worker() @@ -542,12 +536,12 @@ def test_failing_task(self) -> None: with self.assertRaisesMessage(ValueError, "Task failed"): result.return_value # noqa: B018 - self.assertEqual(result.exception_class, ValueError) - assert result.traceback # So that mypy knows the next line is allowed + self.assertEqual(result.errors[0].exception_class, ValueError) + traceback = result.errors[0].traceback self.assertTrue( - result.traceback.endswith( - "ValueError: This task failed due to ValueError\n" - ) + traceback + and traceback.endswith("ValueError: This task failed due to ValueError\n"), + traceback, ) self.assertEqual(DBTaskResult.objects.ready().count(), 0) @@ -556,12 +550,6 @@ def test_complex_exception(self) -> None: result = test_tasks.complex_exception.enqueue() self.assertEqual(DBTaskResult.objects.ready().count(), 1) - with self.assertRaisesMessage(ValueError, "Task has not finished"): - result.exception_class # noqa: B018 - - with self.assertRaisesMessage(ValueError, "Task has not finished"): - result.traceback # noqa: B018 - with self.assertNumQueries(9 if connection.vendor == "mysql" else 8): self.run_worker() @@ -576,8 +564,10 @@ def test_complex_exception(self) -> None: with self.assertRaisesMessage(ValueError, "Task failed"): result.return_value # noqa: B018 - self.assertEqual(result.exception_class, ValueError) - self.assertIn('ValueError(ValueError("This task failed"))', result.traceback) # type: ignore[arg-type] + self.assertEqual(result.errors[0].exception_class, ValueError) + self.assertIn( + 'ValueError(ValueError("This task failed"))', result.errors[0].traceback + ) self.assertEqual(DBTaskResult.objects.ready().count(), 0) @@ -1432,7 +1422,7 @@ def test_repeat_ctrl_c(self) -> None: result.refresh() self.assertEqual(result.status, ResultStatus.FAILED) - self.assertEqual(result.exception_class, SystemExit) + self.assertEqual(result.errors[0].exception_class, SystemExit) @skipIf(sys.platform == "win32", "Windows doesn't support SIGKILL") def test_kill(self) -> None: @@ -1470,7 +1460,7 @@ def test_system_exit_task(self) -> None: result.refresh() self.assertEqual(result.status, ResultStatus.FAILED) - self.assertEqual(result.exception_class, SystemExit) + self.assertEqual(result.errors[0].exception_class, SystemExit) def test_keyboard_interrupt_task(self) -> None: result = test_tasks.failing_task_keyboard_interrupt.enqueue() @@ -1482,7 +1472,7 @@ def test_keyboard_interrupt_task(self) -> None: result.refresh() self.assertEqual(result.status, ResultStatus.FAILED) - self.assertEqual(result.exception_class, KeyboardInterrupt) + self.assertEqual(result.errors[0].exception_class, KeyboardInterrupt) def test_multiple_workers(self) -> None: results = [test_tasks.sleep_for.enqueue(0.1) for _ in range(10)] diff --git a/tests/tests/test_dummy_backend.py b/tests/tests/test_dummy_backend.py index 8a61916..a808193 100644 --- a/tests/tests/test_dummy_backend.py +++ b/tests/tests/test_dummy_backend.py @@ -159,14 +159,10 @@ def test_enqueue_logs(self) -> None: self.assertIn("enqueued", captured_logs.output[0]) self.assertIn(result.id, captured_logs.output[0]) - def test_exceptions(self) -> None: + def test_errors(self) -> None: result = test_tasks.noop_task.enqueue() - with self.assertRaisesMessage(ValueError, "Task has not finished yet"): - result.exception_class # noqa: B018 - - with self.assertRaisesMessage(ValueError, "Task has not finished yet"): - result.traceback # noqa: B018 + self.assertEqual(result.errors, []) def test_validate_disallowed_async_task(self) -> None: with mock.patch.multiple(default_task_backend, supports_async_task=False): diff --git a/tests/tests/test_immediate_backend.py b/tests/tests/test_immediate_backend.py index 7f412b1..804b685 100644 --- a/tests/tests/test_immediate_backend.py +++ b/tests/tests/test_immediate_backend.py @@ -90,10 +90,12 @@ def test_catches_exception(self) -> None: self.assertIsNotNone(result.finished_at) self.assertGreaterEqual(result.started_at, result.enqueued_at) # type:ignore[arg-type, misc] self.assertGreaterEqual(result.finished_at, result.started_at) # type:ignore[arg-type, misc] - self.assertEqual(result.exception_class, exception) + self.assertEqual(result.errors[0].exception_class, exception) + traceback = result.errors[0].traceback self.assertTrue( - result.traceback - and result.traceback.endswith(f"{exception.__name__}: {message}\n") + traceback + and traceback.endswith(f"{exception.__name__}: {message}\n"), + traceback, ) self.assertEqual(result.task, task) self.assertEqual(result.args, []) @@ -120,8 +122,10 @@ def test_complex_exception(self) -> None: self.assertGreaterEqual(result.finished_at, result.started_at) # type:ignore[arg-type,misc] self.assertIsNone(result._return_value) - self.assertEqual(result.exception_class, ValueError) - self.assertIn('ValueError(ValueError("This task failed"))', result.traceback) # type: ignore[arg-type] + self.assertEqual(result.errors[0].exception_class, ValueError) + self.assertIn( + 'ValueError(ValueError("This task failed"))', result.errors[0].traceback + ) self.assertEqual(result.task, test_tasks.complex_exception) self.assertEqual(result.args, []) diff --git a/tests/tests/test_rq_backend.py b/tests/tests/test_rq_backend.py index 1c39157..ec11351 100644 --- a/tests/tests/test_rq_backend.py +++ b/tests/tests/test_rq_backend.py @@ -156,10 +156,12 @@ def test_catches_exception(self) -> None: self.assertIsNotNone(result.finished_at) self.assertGreaterEqual(result.started_at, result.enqueued_at) # type:ignore[arg-type, misc] self.assertGreaterEqual(result.finished_at, result.started_at) # type:ignore[arg-type, misc] - self.assertEqual(result.exception_class, exception) + self.assertEqual(result.errors[0].exception_class, exception) + traceback = result.errors[0].traceback self.assertTrue( - result.traceback - and result.traceback.endswith(f"{exception.__name__}: {message}\n") + traceback + and traceback.endswith(f"{exception.__name__}: {message}\n"), + traceback, ) self.assertEqual(result.task, task) self.assertEqual(result.args, []) @@ -180,8 +182,10 @@ def test_complex_exception(self) -> None: self.assertGreaterEqual(result.finished_at, result.started_at) # type:ignore[arg-type,misc] self.assertIsNone(result._return_value) - self.assertEqual(result.exception_class, ValueError) - self.assertIn('ValueError(ValueError("This task failed"))', result.traceback) # type: ignore[arg-type] + self.assertEqual(result.errors[0].exception_class, ValueError) + self.assertIn( + 'ValueError(ValueError("This task failed"))', result.errors[0].traceback + ) self.assertEqual(result.task, test_tasks.complex_exception) self.assertEqual(result.args, []) diff --git a/tests/tests/test_tasks.py b/tests/tests/test_tasks.py index c594e41..498b7f7 100644 --- a/tests/tests/test_tasks.py +++ b/tests/tests/test_tasks.py @@ -31,7 +31,10 @@ "QUEUES": ["default", "queue_1"], "ENQUEUE_ON_COMMIT": False, }, - "immediate": {"BACKEND": "django_tasks.backends.immediate.ImmediateBackend"}, + "immediate": { + "BACKEND": "django_tasks.backends.immediate.ImmediateBackend", + "ENQUEUE_ON_COMMIT": False, + }, "missing": {"BACKEND": "does.not.exist"}, } ) @@ -264,3 +267,35 @@ def test_module_path(self) -> None: def test_no_backends(self) -> None: with self.assertRaises(InvalidTaskBackendError): test_tasks.noop_task.enqueue() + + def test_task_error_invalid_exception(self) -> None: + with self.assertLogs("django_tasks"): + immediate_task = test_tasks.failing_task_value_error.using( + backend="immediate" + ).enqueue() + + self.assertEqual(len(immediate_task.errors), 1) + + object.__setattr__( + immediate_task.errors[0], "exception_class_path", "subprocess.run" + ) + + with self.assertRaisesMessage( + ValueError, "'subprocess.run' does not reference a valid exception." + ): + immediate_task.errors[0].exception_class # noqa: B018 + + def test_task_error_unknown_module(self) -> None: + with self.assertLogs("django_tasks"): + immediate_task = test_tasks.failing_task_value_error.using( + backend="immediate" + ).enqueue() + + self.assertEqual(len(immediate_task.errors), 1) + + object.__setattr__( + immediate_task.errors[0], "exception_class_path", "does.not.exist" + ) + + with self.assertRaises(ImportError): + immediate_task.errors[0].exception_class # noqa: B018 diff --git a/tests/views.py b/tests/views.py index a71163d..b1af831 100644 --- a/tests/views.py +++ b/tests/views.py @@ -13,7 +13,7 @@ def get_result_value(result: TaskResult) -> Any: if result.status == ResultStatus.SUCCEEDED: return result.return_value elif result.status == ResultStatus.FAILED: - return result.traceback + return result.errors[0].traceback return None From cde6d3f5412e4b4a3f31d4f5a4691ef494f1b3cc Mon Sep 17 00:00:00 2001 From: Jake Howard Date: Fri, 13 Jun 2025 16:57:57 +0100 Subject: [PATCH 3/4] Track number of attempts --- README.md | 4 ++++ django_tasks/task.py | 9 +++++++++ tests/tests/test_database_backend.py | 12 ++++++++++++ tests/tests/test_dummy_backend.py | 2 ++ tests/tests/test_immediate_backend.py | 4 ++++ tests/tests/test_rq_backend.py | 8 ++++++++ 6 files changed, 39 insertions(+) diff --git a/README.md b/README.md index 2acc9e4..0f17346 100644 --- a/README.md +++ b/README.md @@ -218,6 +218,10 @@ assert isinstance(result.errors[0].traceback, str) Note that currently, whilst `.errors` is a list, it will only ever contain a single element. +#### Attempts + +The number of times a task has been run is stored as the `.attempts` attribute. This will currently only ever be 0 or 1. + ### Backend introspecting Because `django-tasks` enables support for multiple different backends, those backends may not support all features, and it can be useful to determine this at runtime to ensure the chosen task queue meets the requirements, or to gracefully degrade functionality if it doesn't. diff --git a/django_tasks/task.py b/django_tasks/task.py index 56a4a44..79d25bf 100644 --- a/django_tasks/task.py +++ b/django_tasks/task.py @@ -301,6 +301,15 @@ def is_finished(self) -> bool: """Has the task finished?""" return self.status in {ResultStatus.FAILED, ResultStatus.SUCCEEDED} + @property + def attempts(self) -> int: + attempts = len(self.errors) + + if self.status == ResultStatus.SUCCEEDED: + attempts += 1 + + return attempts + def refresh(self) -> None: """ Reload the cached task data from the task store diff --git a/tests/tests/test_database_backend.py b/tests/tests/test_database_backend.py index a588a16..376bdfe 100644 --- a/tests/tests/test_database_backend.py +++ b/tests/tests/test_database_backend.py @@ -88,6 +88,7 @@ def test_enqueue_task(self) -> None: self.assertEqual(result.task, task) self.assertEqual(result.args, [1]) self.assertEqual(result.kwargs, {"two": 3}) + self.assertEqual(result.attempts, 0) async def test_enqueue_task_async(self) -> None: for task in [test_tasks.noop_task, test_tasks.noop_task_async]: @@ -103,6 +104,7 @@ async def test_enqueue_task_async(self) -> None: self.assertEqual(result.task, task) self.assertEqual(result.args, []) self.assertEqual(result.kwargs, {}) + self.assertEqual(result.attempts, 0) def test_get_result(self) -> None: with self.assertNumQueries(1): @@ -136,13 +138,17 @@ def test_refresh_result(self) -> None: self.assertFalse(result.is_finished) self.assertIsNone(result.started_at) self.assertIsNone(result.finished_at) + self.assertEqual(result.attempts, 0) + with self.assertNumQueries(1): result.refresh() + self.assertIsNotNone(result.started_at) self.assertIsNotNone(result.finished_at) self.assertEqual(result.status, ResultStatus.SUCCEEDED) self.assertTrue(result.is_finished) self.assertEqual(result.return_value, 42) + self.assertEqual(result.attempts, 1) async def test_refresh_result_async(self) -> None: result = await default_task_backend.aenqueue( @@ -160,12 +166,16 @@ async def test_refresh_result_async(self) -> None: self.assertFalse(result.is_finished) self.assertIsNone(result.started_at) self.assertIsNone(result.finished_at) + self.assertEqual(result.attempts, 0) + await result.arefresh() + self.assertIsNotNone(result.started_at) self.assertIsNotNone(result.finished_at) self.assertEqual(result.status, ResultStatus.SUCCEEDED) self.assertTrue(result.is_finished) self.assertEqual(result.return_value, 42) + self.assertEqual(result.attempts, 1) def test_get_missing_result(self) -> None: with self.assertRaises(ResultDoesNotExist): @@ -461,12 +471,14 @@ def test_run_enqueued_task(self) -> None: self.run_worker() self.assertEqual(result.status, ResultStatus.READY) + self.assertEqual(result.attempts, 0) result.refresh() self.assertIsNotNone(result.started_at) self.assertIsNotNone(result.finished_at) self.assertGreaterEqual(result.started_at, result.enqueued_at) # type:ignore[arg-type,misc] self.assertGreaterEqual(result.finished_at, result.started_at) # type:ignore[arg-type,misc] self.assertEqual(result.status, ResultStatus.SUCCEEDED) + self.assertEqual(result.attempts, 1) self.assertEqual(DBTaskResult.objects.ready().count(), 0) diff --git a/tests/tests/test_dummy_backend.py b/tests/tests/test_dummy_backend.py index a808193..99618f9 100644 --- a/tests/tests/test_dummy_backend.py +++ b/tests/tests/test_dummy_backend.py @@ -47,6 +47,7 @@ def test_enqueue_task(self) -> None: self.assertEqual(result.task, task) self.assertEqual(result.args, [1]) self.assertEqual(result.kwargs, {"two": 3}) + self.assertEqual(result.attempts, 0) self.assertIn(result, default_task_backend.results) # type:ignore[attr-defined] @@ -64,6 +65,7 @@ async def test_enqueue_task_async(self) -> None: self.assertEqual(result.task, task) self.assertEqual(result.args, []) self.assertEqual(result.kwargs, {}) + self.assertEqual(result.attempts, 0) self.assertIn(result, default_task_backend.results) # type:ignore[attr-defined] diff --git a/tests/tests/test_immediate_backend.py b/tests/tests/test_immediate_backend.py index 804b685..b72069c 100644 --- a/tests/tests/test_immediate_backend.py +++ b/tests/tests/test_immediate_backend.py @@ -40,6 +40,7 @@ def test_enqueue_task(self) -> None: self.assertEqual(result.task, task) self.assertEqual(result.args, [1]) self.assertEqual(result.kwargs, {"two": 3}) + self.assertEqual(result.attempts, 1) async def test_enqueue_task_async(self) -> None: for task in [test_tasks.noop_task, test_tasks.noop_task_async]: @@ -56,6 +57,7 @@ async def test_enqueue_task_async(self) -> None: self.assertEqual(result.task, task) self.assertEqual(result.args, []) self.assertEqual(result.kwargs, {}) + self.assertEqual(result.attempts, 1) def test_catches_exception(self) -> None: test_data = [ @@ -271,10 +273,12 @@ def test_wait_until_transaction_commit(self) -> None: result = test_tasks.noop_task.enqueue() self.assertIsNone(result.enqueued_at) + self.assertEqual(result.attempts, 0) self.assertEqual(result.status, ResultStatus.READY) self.assertEqual(result.status, ResultStatus.SUCCEEDED) self.assertIsNotNone(result.enqueued_at) + self.assertEqual(result.attempts, 1) @override_settings( TASKS={ diff --git a/tests/tests/test_rq_backend.py b/tests/tests/test_rq_backend.py index ec11351..568b6b7 100644 --- a/tests/tests/test_rq_backend.py +++ b/tests/tests/test_rq_backend.py @@ -107,6 +107,7 @@ def test_enqueue_task(self) -> None: self.assertEqual(result.task, task) self.assertEqual(result.args, [1]) self.assertEqual(result.kwargs, {"two": 3}) + self.assertEqual(result.attempts, 0) async def test_enqueue_task_async(self) -> None: for task in [test_tasks.noop_task, test_tasks.noop_task_async]: @@ -122,6 +123,7 @@ async def test_enqueue_task_async(self) -> None: self.assertEqual(result.task, task) self.assertEqual(result.args, []) self.assertEqual(result.kwargs, {}) + self.assertEqual(result.attempts, 0) def test_catches_exception(self) -> None: test_data = [ @@ -166,6 +168,7 @@ def test_catches_exception(self) -> None: self.assertEqual(result.task, task) self.assertEqual(result.args, []) self.assertEqual(result.kwargs, {}) + self.assertEqual(result.attempts, 1) def test_complex_exception(self) -> None: result = test_tasks.complex_exception.enqueue() @@ -190,6 +193,7 @@ def test_complex_exception(self) -> None: self.assertEqual(result.task, test_tasks.complex_exception) self.assertEqual(result.args, []) self.assertEqual(result.kwargs, {}) + self.assertEqual(result.attempts, 1) def test_get_result(self) -> None: result = default_task_backend.enqueue(test_tasks.noop_task, [], {}) @@ -216,6 +220,7 @@ def test_refresh_result(self) -> None: self.assertFalse(result.is_finished) self.assertIsNone(result.started_at) self.assertIsNone(result.finished_at) + self.assertEqual(result.attempts, 0) result.refresh() @@ -224,6 +229,7 @@ def test_refresh_result(self) -> None: self.assertEqual(result.status, ResultStatus.SUCCEEDED) self.assertTrue(result.is_finished) self.assertEqual(result.return_value, 42) + self.assertEqual(result.attempts, 1) def test_refresh_result_async(self) -> None: result = async_to_sync(default_task_backend.aenqueue)( @@ -236,6 +242,7 @@ def test_refresh_result_async(self) -> None: self.assertFalse(result.is_finished) self.assertIsNone(result.started_at) self.assertIsNone(result.finished_at) + self.assertEqual(result.attempts, 0) async_to_sync(result.arefresh)() @@ -244,6 +251,7 @@ def test_refresh_result_async(self) -> None: self.assertEqual(result.status, ResultStatus.SUCCEEDED) self.assertTrue(result.is_finished) self.assertEqual(result.return_value, 42) + self.assertEqual(result.attempts, 1) def test_get_missing_result(self) -> None: with self.assertRaises(ResultDoesNotExist): From 0ac596c16759e67acc18e90ccf7ea9db947a6d56 Mon Sep 17 00:00:00 2001 From: Jake Howard Date: Fri, 13 Jun 2025 17:07:30 +0100 Subject: [PATCH 4/4] Store last attempted date --- README.md | 2 ++ django_tasks/backends/database/models.py | 1 + django_tasks/backends/dummy.py | 1 + django_tasks/backends/immediate.py | 2 ++ django_tasks/backends/rq.py | 5 +++++ django_tasks/task.py | 4 ++++ tests/tests/test_database_backend.py | 9 +++++++++ tests/tests/test_dummy_backend.py | 2 ++ tests/tests/test_immediate_backend.py | 4 ++++ tests/tests/test_rq_backend.py | 8 ++++++++ 10 files changed, 38 insertions(+) diff --git a/README.md b/README.md index 0f17346..617bd9a 100644 --- a/README.md +++ b/README.md @@ -222,6 +222,8 @@ Note that currently, whilst `.errors` is a list, it will only ever contain a sin The number of times a task has been run is stored as the `.attempts` attribute. This will currently only ever be 0 or 1. +The date of the last attempt is stored as `.last_attempted_at`. + ### Backend introspecting Because `django-tasks` enables support for multiple different backends, those backends may not support all features, and it can be useful to determine this at runtime to ensure the chosen task queue meets the requirements, or to gracefully degrade functionality if it doesn't. diff --git a/django_tasks/backends/database/models.py b/django_tasks/backends/database/models.py index 6279971..032220c 100644 --- a/django_tasks/backends/database/models.py +++ b/django_tasks/backends/database/models.py @@ -171,6 +171,7 @@ def task_result(self) -> "TaskResult[T]": status=ResultStatus[self.status], enqueued_at=self.enqueued_at, started_at=self.started_at, + last_attempted_at=self.started_at, finished_at=self.finished_at, args=self.args_kwargs["args"], kwargs=self.args_kwargs["kwargs"], diff --git a/django_tasks/backends/dummy.py b/django_tasks/backends/dummy.py index c181fa7..9ae2e28 100644 --- a/django_tasks/backends/dummy.py +++ b/django_tasks/backends/dummy.py @@ -46,6 +46,7 @@ def enqueue( status=ResultStatus.READY, enqueued_at=None, started_at=None, + last_attempted_at=None, finished_at=None, args=args, kwargs=kwargs, diff --git a/django_tasks/backends/immediate.py b/django_tasks/backends/immediate.py index 5b01cdf..7cdc9a4 100644 --- a/django_tasks/backends/immediate.py +++ b/django_tasks/backends/immediate.py @@ -44,6 +44,7 @@ def _execute_task(self, task_result: TaskResult) -> None: object.__setattr__(task_result, "status", ResultStatus.RUNNING) object.__setattr__(task_result, "started_at", timezone.now()) + object.__setattr__(task_result, "last_attempted_at", timezone.now()) task_started.send(sender=type(self), task_result=task_result) try: @@ -91,6 +92,7 @@ def enqueue( status=ResultStatus.READY, enqueued_at=None, started_at=None, + last_attempted_at=None, finished_at=None, args=args, kwargs=kwargs, diff --git a/django_tasks/backends/rq.py b/django_tasks/backends/rq.py index eae2f56..a520c10 100644 --- a/django_tasks/backends/rq.py +++ b/django_tasks/backends/rq.py @@ -97,6 +97,7 @@ def into_task_result(self) -> TaskResult: status=RQ_STATUS_TO_RESULT_STATUS[self.get_status()], enqueued_at=self.enqueued_at, started_at=self.started_at, + last_attempted_at=self.started_at, finished_at=self.ended_at, args=list(self.args), kwargs=self.kwargs, @@ -119,6 +120,9 @@ def into_task_result(self) -> TaskResult: if rq_results: object.__setattr__(task_result, "_return_value", rq_results[0].return_value) + object.__setattr__( + task_result, "last_attempted_at", rq_results[0].created_at + ) return task_result @@ -178,6 +182,7 @@ def enqueue( status=ResultStatus.READY, enqueued_at=None, started_at=None, + last_attempted_at=None, finished_at=None, args=args, kwargs=kwargs, diff --git a/django_tasks/task.py b/django_tasks/task.py index 79d25bf..0cc1826 100644 --- a/django_tasks/task.py +++ b/django_tasks/task.py @@ -39,6 +39,7 @@ "_return_value", "finished_at", "started_at", + "last_attempted_at", "status", "enqueued_at", } @@ -267,6 +268,9 @@ class TaskResult(Generic[T]): finished_at: Optional[datetime] """The time this task was finished""" + last_attempted_at: Optional[datetime] + """The time this task was last attempted to be run""" + args: list """The arguments to pass to the task function""" diff --git a/tests/tests/test_database_backend.py b/tests/tests/test_database_backend.py index 376bdfe..f51c8ad 100644 --- a/tests/tests/test_database_backend.py +++ b/tests/tests/test_database_backend.py @@ -82,6 +82,7 @@ def test_enqueue_task(self) -> None: self.assertEqual(result.status, ResultStatus.READY) self.assertFalse(result.is_finished) self.assertIsNone(result.started_at) + self.assertIsNone(result.last_attempted_at) self.assertIsNone(result.finished_at) with self.assertRaisesMessage(ValueError, "Task has not finished yet"): result.return_value # noqa:B018 @@ -98,6 +99,7 @@ async def test_enqueue_task_async(self) -> None: self.assertEqual(result.status, ResultStatus.READY) self.assertFalse(result.is_finished) self.assertIsNone(result.started_at) + self.assertIsNone(result.last_attempted_at) self.assertIsNone(result.finished_at) with self.assertRaisesMessage(ValueError, "Task has not finished yet"): result.return_value # noqa:B018 @@ -137,6 +139,7 @@ def test_refresh_result(self) -> None: self.assertEqual(result.status, ResultStatus.READY) self.assertFalse(result.is_finished) self.assertIsNone(result.started_at) + self.assertIsNone(result.last_attempted_at) self.assertIsNone(result.finished_at) self.assertEqual(result.attempts, 0) @@ -144,6 +147,7 @@ def test_refresh_result(self) -> None: result.refresh() self.assertIsNotNone(result.started_at) + self.assertIsNotNone(result.last_attempted_at) self.assertIsNotNone(result.finished_at) self.assertEqual(result.status, ResultStatus.SUCCEEDED) self.assertTrue(result.is_finished) @@ -165,12 +169,14 @@ async def test_refresh_result_async(self) -> None: self.assertEqual(result.status, ResultStatus.READY) self.assertFalse(result.is_finished) self.assertIsNone(result.started_at) + self.assertIsNone(result.last_attempted_at) self.assertIsNone(result.finished_at) self.assertEqual(result.attempts, 0) await result.arefresh() self.assertIsNotNone(result.started_at) + self.assertIsNotNone(result.last_attempted_at) self.assertIsNotNone(result.finished_at) self.assertEqual(result.status, ResultStatus.SUCCEEDED) self.assertTrue(result.is_finished) @@ -474,6 +480,7 @@ def test_run_enqueued_task(self) -> None: self.assertEqual(result.attempts, 0) result.refresh() self.assertIsNotNone(result.started_at) + self.assertIsNotNone(result.last_attempted_at) self.assertIsNotNone(result.finished_at) self.assertGreaterEqual(result.started_at, result.enqueued_at) # type:ignore[arg-type,misc] self.assertGreaterEqual(result.finished_at, result.started_at) # type:ignore[arg-type,misc] @@ -540,6 +547,7 @@ def test_failing_task(self) -> None: self.assertEqual(result.status, ResultStatus.READY) result.refresh() self.assertIsNotNone(result.started_at) + self.assertIsNotNone(result.last_attempted_at) self.assertIsNotNone(result.finished_at) self.assertGreaterEqual(result.started_at, result.enqueued_at) # type: ignore @@ -568,6 +576,7 @@ def test_complex_exception(self) -> None: self.assertEqual(result.status, ResultStatus.READY) result.refresh() self.assertIsNotNone(result.started_at) + self.assertIsNotNone(result.last_attempted_at) self.assertIsNotNone(result.finished_at) self.assertGreaterEqual(result.started_at, result.enqueued_at) # type: ignore diff --git a/tests/tests/test_dummy_backend.py b/tests/tests/test_dummy_backend.py index 99618f9..cb63567 100644 --- a/tests/tests/test_dummy_backend.py +++ b/tests/tests/test_dummy_backend.py @@ -41,6 +41,7 @@ def test_enqueue_task(self) -> None: self.assertEqual(result.status, ResultStatus.READY) self.assertFalse(result.is_finished) self.assertIsNone(result.started_at) + self.assertIsNone(result.last_attempted_at) self.assertIsNone(result.finished_at) with self.assertRaisesMessage(ValueError, "Task has not finished yet"): result.return_value # noqa:B018 @@ -59,6 +60,7 @@ async def test_enqueue_task_async(self) -> None: self.assertEqual(result.status, ResultStatus.READY) self.assertFalse(result.is_finished) self.assertIsNone(result.started_at) + self.assertIsNone(result.last_attempted_at) self.assertIsNone(result.finished_at) with self.assertRaisesMessage(ValueError, "Task has not finished yet"): result.return_value # noqa:B018 diff --git a/tests/tests/test_immediate_backend.py b/tests/tests/test_immediate_backend.py index b72069c..528269a 100644 --- a/tests/tests/test_immediate_backend.py +++ b/tests/tests/test_immediate_backend.py @@ -33,6 +33,7 @@ def test_enqueue_task(self) -> None: self.assertEqual(result.status, ResultStatus.SUCCEEDED) self.assertTrue(result.is_finished) self.assertIsNotNone(result.started_at) + self.assertIsNotNone(result.last_attempted_at) self.assertIsNotNone(result.finished_at) self.assertGreaterEqual(result.started_at, result.enqueued_at) # type:ignore[arg-type, misc] self.assertGreaterEqual(result.finished_at, result.started_at) # type:ignore[arg-type, misc] @@ -50,6 +51,7 @@ async def test_enqueue_task_async(self) -> None: self.assertEqual(result.status, ResultStatus.SUCCEEDED) self.assertTrue(result.is_finished) self.assertIsNotNone(result.started_at) + self.assertIsNotNone(result.last_attempted_at) self.assertIsNotNone(result.finished_at) self.assertGreaterEqual(result.started_at, result.enqueued_at) # type:ignore[arg-type, misc] self.assertGreaterEqual(result.finished_at, result.started_at) # type:ignore[arg-type, misc] @@ -89,6 +91,7 @@ def test_catches_exception(self) -> None: result.return_value # noqa: B018 self.assertTrue(result.is_finished) self.assertIsNotNone(result.started_at) + self.assertIsNotNone(result.last_attempted_at) self.assertIsNotNone(result.finished_at) self.assertGreaterEqual(result.started_at, result.enqueued_at) # type:ignore[arg-type, misc] self.assertGreaterEqual(result.finished_at, result.started_at) # type:ignore[arg-type, misc] @@ -119,6 +122,7 @@ def test_complex_exception(self) -> None: self.assertEqual(result.status, ResultStatus.FAILED) self.assertIsNotNone(result.started_at) + self.assertIsNotNone(result.last_attempted_at) self.assertIsNotNone(result.finished_at) self.assertGreaterEqual(result.started_at, result.enqueued_at) # type:ignore[arg-type,misc] self.assertGreaterEqual(result.finished_at, result.started_at) # type:ignore[arg-type,misc] diff --git a/tests/tests/test_rq_backend.py b/tests/tests/test_rq_backend.py index 568b6b7..27298c1 100644 --- a/tests/tests/test_rq_backend.py +++ b/tests/tests/test_rq_backend.py @@ -101,6 +101,7 @@ def test_enqueue_task(self) -> None: self.assertEqual(result.status, ResultStatus.READY) self.assertFalse(result.is_finished) self.assertIsNone(result.started_at) + self.assertIsNone(result.last_attempted_at) self.assertIsNone(result.finished_at) with self.assertRaisesMessage(ValueError, "Task has not finished yet"): result.return_value # noqa:B018 @@ -117,6 +118,7 @@ async def test_enqueue_task_async(self) -> None: self.assertEqual(result.status, ResultStatus.READY) self.assertFalse(result.is_finished) self.assertIsNone(result.started_at) + self.assertIsNone(result.last_attempted_at) self.assertIsNone(result.finished_at) with self.assertRaisesMessage(ValueError, "Task has not finished yet"): result.return_value # noqa:B018 @@ -155,6 +157,7 @@ def test_catches_exception(self) -> None: result.return_value # noqa: B018 self.assertTrue(result.is_finished) self.assertIsNotNone(result.started_at) + self.assertIsNotNone(result.last_attempted_at) self.assertIsNotNone(result.finished_at) self.assertGreaterEqual(result.started_at, result.enqueued_at) # type:ignore[arg-type, misc] self.assertGreaterEqual(result.finished_at, result.started_at) # type:ignore[arg-type, misc] @@ -180,6 +183,7 @@ def test_complex_exception(self) -> None: self.assertEqual(result.status, ResultStatus.FAILED) self.assertIsNotNone(result.started_at) + self.assertIsNotNone(result.last_attempted_at) self.assertIsNotNone(result.finished_at) self.assertGreaterEqual(result.started_at, result.enqueued_at) # type:ignore[arg-type,misc] self.assertGreaterEqual(result.finished_at, result.started_at) # type:ignore[arg-type,misc] @@ -219,12 +223,14 @@ def test_refresh_result(self) -> None: self.assertEqual(result.status, ResultStatus.READY) self.assertFalse(result.is_finished) self.assertIsNone(result.started_at) + self.assertIsNone(result.last_attempted_at) self.assertIsNone(result.finished_at) self.assertEqual(result.attempts, 0) result.refresh() self.assertIsNotNone(result.started_at) + self.assertIsNotNone(result.last_attempted_at) self.assertIsNotNone(result.finished_at) self.assertEqual(result.status, ResultStatus.SUCCEEDED) self.assertTrue(result.is_finished) @@ -241,12 +247,14 @@ def test_refresh_result_async(self) -> None: self.assertEqual(result.status, ResultStatus.READY) self.assertFalse(result.is_finished) self.assertIsNone(result.started_at) + self.assertIsNone(result.last_attempted_at) self.assertIsNone(result.finished_at) self.assertEqual(result.attempts, 0) async_to_sync(result.arefresh)() self.assertIsNotNone(result.started_at) + self.assertIsNotNone(result.last_attempted_at) self.assertIsNotNone(result.finished_at) self.assertEqual(result.status, ResultStatus.SUCCEEDED) self.assertTrue(result.is_finished)