use job chaining and simplify callback payload

Makefile

@@ -11,7 +11,7 @@ freeze:
 migrate:
 	./manage.py migrate
 services:
-	docker-compose up --detach celery_worker db rabbitmq redis
+	docker-compose up --detach celery_worker rabbitmq redis
 test:
 	pytest -s .
 test-ci:

README.md

@@ -39,7 +39,7 @@ BERT-serv is now running on local port 8000!
 
 Send a POST request to the `/sentiment/new/` path. A `callback_url` may be specified in
 the query parameters for asynchrous use cases and long-running sentiment analyses. This callback
-will have a JSON object in the body with an array of the new sentiment record `id`s: `'{"ids": [95, 96]}'`
+will have a JSON object in the body with an array of the new sentiment record ids: `{"ids": [95, 96]}`
 
 The body of the POST request must be a list; the strings inside will be processed synchronously.
 
@@ -67,6 +67,31 @@ curl --request GET \
   --header 'Accept: application/json'
 ```
 
+The output will look like this:
+
+```
+[
+	{
+		"created_at": "2024-08-14T04:07:15.127",
+		"label": "Positive",
+		"score": 0.9999837875366211,
+		"tags": [
+			"stock"
+		],
+		"text": "year over year growth is increasing test B"
+	},
+	{
+		"created_at": "2024-08-14T04:07:15.127",
+		"label": "Neutral",
+		"score": 0.9993372559547424,
+		"tags": [
+			"stock"
+		],
+		"text": "test text A"
+	}
+]
+```
+
 ##### Get a specific sentiment by index
 
 Add the index of the sentiment resource to the `sentiment` path:
@@ -94,7 +119,8 @@ The `/sentiment/` path accepts `page_size` and a `page` query parameters.
 
 ## Development Environment Setup
 
-The `make services` command will start all of the services besides the app. This allows for the app to be started and stopped (with `make app`) in the terminal for convenience during development.
+The `make services` command will start all of the services besides the app. This allows for the app to be started and stopped (with `make app`) in the terminal for convenience during development. Note that `make services` requires the Postgres database to be running. A database
+can be started with `make db` if one isn't running already.
 
 ### Setting up the environment
 
@@ -123,3 +149,12 @@ Notes:
 
 - `make services` requires [Docker Desktop](https://www.docker.com/products/docker-desktop/)
 - `make deps` will install dependencies via pip3 and must be run before `make app`. This can take a few minutes as the PyTorch dependencies are sizable.
+
+## Service setup
+
+The `docker-compose-services.yml` is intended to stand this service and dependencies up against an external Postgres instance.
+
+```
+# To run:
+docker-compose -f docker-compose-service.yml up
+```

docker-compose-service.yml

@@ -0,0 +1,30 @@
+# Shared database
+services:
+  web:
+    build: .
+    command: python manage.py runserver 0.0.0.0:8000
+    volumes:
+      - .:/code
+    environment:
+      ALLOWED_HOSTS: localhost
+      CELERY_HOST: localhost
+      DB_HOST: localhost
+    network_mode: host
+  rabbitmq:
+    image: rabbitmq:3.11.5-management
+    network_mode: host
+  redis:
+    image: "redis:7-alpine"
+    network_mode: host
+  celery_worker:
+    build: .
+    command: celery -A sentiment worker --pool=solo --loglevel=INFO
+    volumes:
+      - .:/code
+    environment:
+      CELERY_HOST: localhost
+      DB_HOST: localhost
+    network_mode: host
+    depends_on:
+      - rabbitmq
+      - redis

sentiment/tasks.py

@@ -37,9 +37,12 @@ def run_sentiment(sentences, tags):
     Sentiment.objects.bulk_create(sentiment_objects)
 
     # Return the IDs of the created sentiments
-    return json.dumps({'ids': [sentiment.id for sentiment in sentiment_objects]})
+    return [sentiment.id for sentiment in sentiment_objects]
 
 
 @celery.task
-def send_webhook(self, url):
-    requests.post(url, json=self)
+def send_webhook(sentiment_ids, url):
+    payload = {
+        "ids": sentiment_ids
+    }
+    requests.post(url, json=payload)

sentiment/views/createview.py

@@ -2,7 +2,7 @@
 from django.core.validators import URLValidator
 from django.http import HttpResponse
 from django.views.generic import View
-from celery import signature
+from celery import chain, signature
 import json
 import logging
 
@@ -16,8 +16,16 @@ def post(self, request, *args, **kwargs):
             text = body.get('text', [])
             tags = body.get('tags', [])
 
-            signature("sentiment.tasks.run_sentiment", args=(
-                text,tags,), link=callback_task(request)).delay()
+            callback_url = request.GET.get('callback_url')
+
+            if callback_url:
+                chain(
+                    signature("sentiment.tasks.run_sentiment", args=(text,tags,)),
+                    signature("sentiment.tasks.send_webhook", args=(callback_url,), retries=3)
+                ).delay()
+            else:
+                signature("sentiment.tasks.run_sentiment", args=(
+                    text,tags,)).delay()
 
             return HttpResponse(status=201)
         except Exception as e:
@@ -30,12 +38,3 @@ def parse_request_body(request):
         return json.loads(request.body)
     except ValueError:
         raise BadRequest("Could not parse request body as JSON.")
-
-
-def callback_task(request):
-    url = request.GET.get('callback_url')
-
-    if url:
-        return signature("sentiment.tasks.send_webhook", args=(url,), retries=3)
-
-    return None