-
-
Notifications
You must be signed in to change notification settings - Fork 26
Resolve #160 -- Add picture_processed signal
#231
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: main
Are you sure you want to change the base?
Changes from 3 commits
81c6163
2621125
dbc80b3
1d6b82e
6cd32cd
cafec7e
67a476b
ed11b97
4077289
fd61329
1f7fdb9
591eb67
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,3 @@ | ||
| import django.dispatch | ||
|
|
||
| process_picture_done = django.dispatch.Signal() | ||
|
jmsmkn marked this conversation as resolved.
Outdated
|
||
| Original file line number | Diff line number | Diff line change | ||||
|---|---|---|---|---|---|---|
|
|
@@ -2,10 +2,11 @@ | |||||
|
|
||||||
| from typing import Protocol | ||||||
|
|
||||||
| from django.apps import apps | ||||||
| from django.db import transaction | ||||||
| from PIL import Image | ||||||
|
|
||||||
| from pictures import conf, utils | ||||||
| from pictures import conf, signals, utils | ||||||
|
|
||||||
|
|
||||||
| def noop(*args, **kwargs) -> None: | ||||||
|
|
@@ -19,6 +20,7 @@ def __call__( | |||||
| file_name: str, | ||||||
| new: list[tuple[str, list, dict]] | None = None, | ||||||
| old: list[tuple[str, list, dict]] | None = None, | ||||||
| field: str = "", | ||||||
|
Owner
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. All that matters for async task runners, is that the signature is JSON-serializable.
Suggested change
You can drop the default, since this will be required. And I'd prefer to keep the naming somewhat consistent. Thus, this would be the sender (sending the task).
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Making it required means that it needs to be placed before Now that we're sending along the field as the sender the Having
Owner
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Hm… but we'd always include a sender in the function call. Thus, custom processors would immediately break, right?
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. In the codebase Our problem would be on a higher level and only occur when deploying a new version. The tasks from the older version of Django pictures would have already been created with 4 kwargs (storage, filename, new (optional), old (optional)) and are sat on the message broker. With an active site it is not feasible to ensure that none of the old tasks exist on the queue when upgrading. You then deploy a new version and the workers now only accept 5 kwargs (storage, filename, sender, new (optional), old (optional)). There, Celery will reject the message as sender is not set, raise an exception and put it back on the queue as a new message. Maybe we have Celery misconfigured in some way but the worker will then pick the message up again, and repeat. We end up flooding Sentry, and the only way around it from that point is to create our own celery task that can handle both types of message on the queue. |
||||||
| ) -> None: ... | ||||||
|
|
||||||
|
|
||||||
|
|
@@ -27,6 +29,7 @@ def _process_picture( | |||||
| file_name: str, | ||||||
| new: list[tuple[str, list, dict]] | None = None, | ||||||
| old: list[tuple[str, list, dict]] | None = None, | ||||||
| field: str = "", | ||||||
| ) -> None: | ||||||
| new = new or [] | ||||||
| old = old or [] | ||||||
|
|
@@ -41,6 +44,21 @@ def _process_picture( | |||||
| picture = utils.reconstruct(*picture) | ||||||
| picture.delete() | ||||||
|
|
||||||
| if field: | ||||||
| app_label, model_name, _ = field.split(".") | ||||||
| sender = apps.get_model(app_label=app_label, model_name=model_name) | ||||||
| else: | ||||||
| sender = _process_picture | ||||||
|
|
||||||
| signals.process_picture_done.send( | ||||||
| sender=sender, | ||||||
| storage=storage.deconstruct(), | ||||||
|
jmsmkn marked this conversation as resolved.
Outdated
|
||||||
| file_name=file_name, | ||||||
| new=new, | ||||||
| old=old, | ||||||
| field=field, | ||||||
|
jmsmkn marked this conversation as resolved.
Outdated
|
||||||
| ) | ||||||
|
|
||||||
|
|
||||||
| process_picture: PictureProcessor = _process_picture | ||||||
|
|
||||||
|
|
@@ -57,21 +75,24 @@ def process_picture_with_dramatiq( | |||||
| file_name: str, | ||||||
| new: list[tuple[str, list, dict]] | None = None, | ||||||
| old: list[tuple[str, list, dict]] | None = None, | ||||||
| field: str = "", | ||||||
| ) -> None: | ||||||
| _process_picture(storage, file_name, new, old) | ||||||
| _process_picture(storage, file_name, new, old, field) | ||||||
|
|
||||||
| def process_picture( # noqa: F811 | ||||||
| storage: tuple[str, list, dict], | ||||||
| file_name: str, | ||||||
| new: list[tuple[str, list, dict]] | None = None, | ||||||
| old: list[tuple[str, list, dict]] | None = None, | ||||||
| field: str = "", | ||||||
| ) -> None: | ||||||
| transaction.on_commit( | ||||||
| lambda: process_picture_with_dramatiq.send( | ||||||
| storage=storage, | ||||||
| file_name=file_name, | ||||||
| new=new, | ||||||
| old=old, | ||||||
| field=field, | ||||||
| ) | ||||||
| ) | ||||||
|
|
||||||
|
|
@@ -91,14 +112,16 @@ def process_picture_with_celery( | |||||
| file_name: str, | ||||||
| new: list[tuple[str, list, dict]] | None = None, | ||||||
| old: list[tuple[str, list, dict]] | None = None, | ||||||
| field: str = "", | ||||||
| ) -> None: | ||||||
| _process_picture(storage, file_name, new, old) | ||||||
| _process_picture(storage, file_name, new, old, field) | ||||||
|
|
||||||
| def process_picture( # noqa: F811 | ||||||
| storage: tuple[str, list, dict], | ||||||
| file_name: str, | ||||||
| new: list[tuple[str, list, dict]] | None = None, | ||||||
| old: list[tuple[str, list, dict]] | None = None, | ||||||
| field: str = "", | ||||||
| ) -> None: | ||||||
| transaction.on_commit( | ||||||
| lambda: process_picture_with_celery.apply_async( | ||||||
|
|
@@ -107,6 +130,7 @@ def process_picture( # noqa: F811 | |||||
| file_name=file_name, | ||||||
| new=new, | ||||||
| old=old, | ||||||
| field=field, | ||||||
| ), | ||||||
| queue=conf.get_settings().QUEUE_NAME, | ||||||
| ) | ||||||
|
|
@@ -125,20 +149,23 @@ def process_picture_with_django_rq( | |||||
| file_name: str, | ||||||
| new: list[tuple[str, list, dict]] | None = None, | ||||||
| old: list[tuple[str, list, dict]] | None = None, | ||||||
| field: str = "", | ||||||
| ) -> None: | ||||||
| _process_picture(storage, file_name, new, old) | ||||||
| _process_picture(storage, file_name, new, old, field) | ||||||
|
|
||||||
| def process_picture( # noqa: F811 | ||||||
| storage: tuple[str, list, dict], | ||||||
| file_name: str, | ||||||
| new: list[tuple[str, list, dict]] | None = None, | ||||||
| old: list[tuple[str, list, dict]] | None = None, | ||||||
| field: str = "", | ||||||
| ) -> None: | ||||||
| transaction.on_commit( | ||||||
| lambda: process_picture_with_django_rq.delay( | ||||||
| storage=storage, | ||||||
| file_name=file_name, | ||||||
| new=new, | ||||||
| old=old, | ||||||
| field=field, | ||||||
| ) | ||||||
| ) | ||||||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,71 @@ | ||
| from unittest.mock import Mock | ||
|
|
||
| import pytest | ||
| from django.apps import apps | ||
| from django.dispatch import receiver | ||
|
|
||
| from pictures import signals, tasks | ||
| from tests.testapp.models import SimpleModel | ||
|
|
||
|
|
||
| @pytest.mark.django_db | ||
| def test_process_picture_sends_process_picture_done(image_upload_file): | ||
|
jmsmkn marked this conversation as resolved.
Outdated
|
||
| obj = SimpleModel.objects.create(picture=image_upload_file) | ||
|
|
||
| handler = Mock() | ||
| signals.process_picture_done.connect(handler) | ||
|
|
||
| tasks._process_picture( | ||
| obj.picture.storage.deconstruct(), | ||
| obj.picture.name, | ||
| new=[i.deconstruct() for i in obj.picture.get_picture_files_list()], | ||
| ) | ||
|
|
||
| handler.assert_called_once_with( | ||
| signal=signals.process_picture_done, | ||
| sender=tasks._process_picture, | ||
| storage=obj.picture.storage.deconstruct(), | ||
| file_name=obj.picture.name, | ||
| new=[i.deconstruct() for i in obj.picture.get_picture_files_list()], | ||
| old=[], | ||
| field="", | ||
| ) | ||
|
|
||
|
|
||
| @pytest.mark.django_db | ||
| def test_process_picture_sends_process_picture_done_on_create(image_upload_file): | ||
|
jmsmkn marked this conversation as resolved.
Outdated
|
||
| handler = Mock() | ||
| signals.process_picture_done.connect(handler) | ||
|
|
||
| obj = SimpleModel.objects.create(picture=image_upload_file) | ||
|
|
||
| handler.assert_called_once_with( | ||
| signal=signals.process_picture_done, | ||
| sender=SimpleModel, | ||
| storage=obj.picture.storage.deconstruct(), | ||
| file_name=obj.picture.name, | ||
| new=[i.deconstruct() for i in obj.picture.get_picture_files_list()], | ||
| old=[], | ||
| field="testapp.simplemodel.picture", | ||
| ) | ||
|
|
||
|
|
||
|
jmsmkn marked this conversation as resolved.
jmsmkn marked this conversation as resolved.
|
||
| @pytest.mark.django_db | ||
| def test_processed_object_found(image_upload_file): | ||
| obj = SimpleModel.objects.create() | ||
|
|
||
| found_object = None | ||
|
|
||
| @receiver(signals.process_picture_done, sender=SimpleModel) | ||
| def handler(*, file_name, field, **__): | ||
| nonlocal found_object | ||
| app_label, model_name, field_name = field.split(".") | ||
| model = apps.get_model(app_label=app_label, model_name=model_name) | ||
|
|
||
| # Users can now modify the object that process_picture_done | ||
| # corresponds to | ||
| found_object = model.objects.get(**{field_name: file_name}) | ||
|
|
||
| obj.picture.save("image.png", image_upload_file) | ||
|
|
||
| assert obj == found_object | ||
Uh oh!
There was an error while loading. Please reload this page.