frontend + backend SAM integration

backend/deepcell_label/blueprints.py

@@ -19,6 +19,8 @@
 from deepcell_label.label import Edit
 from deepcell_label.loaders import Loader
 from deepcell_label.models import Project
+import json
+
 
 import cv2
 import numpy as np
@@ -289,7 +291,7 @@ def test_sam_prediction():
 
     The output of this endpoint is an ndarray of 0's and 1's indicating where to draw the mask on the frontend.
     """
-    json_data = request.get_json()
+    json_data = json.loads(request.data, strict=False)
     bbox = BBox(**json_data)
 
     image_embedding, ort_session = retrieve_sam_model_data()

frontend/src/Project/Canvas/ComposeCanvas.js

@@ -7,7 +7,6 @@ import { useCanvas } from '../ProjectContext';
 const Canvas = styled('canvas')``;
 
 export const ComposeCanvas = ({ bitmaps }) => {
-  console.log(bitmaps)
   const canvas = useCanvas();
   const { sx, sy, zoom, sw, sh, scale } = useSelector(
     canvas,

frontend/src/Project/Canvas/ToolCanvas/SamCanvas.js

@@ -3,7 +3,7 @@ import { useEffect, useRef, useState } from 'react';
 import { useSam, useCanvas } from '../../ProjectContext';
 import "./styles/sam-canvas.css"
 
-const SamCanvas = ({setRunONNXCommand}) => {
+const SamCanvas = () => {
   const canvas = useCanvas();
   const width = useSelector(canvas, (state) => state.context.width);
   const height = useSelector(canvas, (state) => state.context.height);
@@ -33,13 +33,7 @@ const SamCanvas = ({setRunONNXCommand}) => {
   useEffect(() => {
     // User has selected a region for segmentation
     if (!isMouseDown && startX && startY && endX && endY) {
-        if (window.confirm("Send selected region for segmentation?")) {
-            // sam.send({ type: 'SEND_TO_API' })
-            setRunONNXCommand(true)
-            // setRunONNXCommand(false)
-        } else {
-            sam.send({ type: 'CLEAR_SELECTION' })
-        }
+        sam.send({ type: 'SEND_TO_API' })
     }
   }, [isMouseDown])
 

frontend/src/Project/Canvas/ToolCanvas/ToolCanvas.js

@@ -19,7 +19,6 @@ import SwapCanvas from './SwapCanvas';
 import ThresholdCanvas from './ThresholdCanvas';
 import WatershedCanvas from './WatershedCanvas';
 import SamCanvas from './SamCanvas';
-import SamMaskCanvas from './SamMaskCanvas';
 import { useState } from 'react';
 
 function ToolCanvas({ setBitmaps }) {
@@ -65,8 +64,7 @@ function ToolCanvas({ setBitmaps }) {
           return <WatershedCanvas setBitmaps={setBitmaps} />;
         case 'sam':
           return <>
-            <SamCanvas setRunONNXCommand={setRunONNX} />
-            <SamMaskCanvas runONNXCommand={runONNX} />
+            <SamCanvas />
           </>;
         default:
           return null;

frontend/src/Project/ProjectContext.js

@@ -300,15 +300,13 @@ export function useEditSegment() {
 }
 
 export function useEditSegmentCopy() {
-  console.log("HERE")
   const project = useProject();
   const segment = useSelector(project, (state) => {
     const tool = state.context.toolRef;
     const segment = tool.state.context.editSegmentRef;
     return segment;
   });
 
-  console.log("SEGMENT")
   return segment;
 }
 

frontend/src/Project/service/edit/segment/samMachine.js

@@ -2,27 +2,12 @@
 import { assign, Machine, send } from 'xstate';
 import { fromEventBus } from '../../eventBus';
 
-async function sendToSamAPI(ctx) {
-    const id = new URLSearchParams(window.location.search).get('projectId')
-    console.log("SHOULD SEND TO API", ctx)
-    const options = {
-        method: 'POST',
-        body: JSON.stringify(ctx),
-        'Content-Type': 'application/json',
-        };
-    const response = await fetch(`${document.location.origin}/api/sendToSam/${id}`, options)
-    const data = await response.json()
-
-    await new Promise(r => setTimeout(r, 4000))
-
-    return data
-}
-
 const createSAMMachine = (context) =>
 Machine(
     {
       invoke: [
         { src: fromEventBus('watershed', () => context.eventBuses.canvas, 'COORDINATES') },
+        { id: 'arrays', src: fromEventBus('sam', () => context.eventBuses.arrays, ['EDITED_SEGMENT']) },
       ],
       context: {
         isMouseDown: false,
@@ -50,16 +35,12 @@ Machine(
         waiting: {
             on: {
                 CLEAR_SELECTION: { target: 'done', actions: ['clearSelection']},
-                SEND_TO_API: { target: "fetching" }
+                SEND_TO_API: {target: "fetching", actions: ["sendToAPI"]}
             },
         },
         fetching: {
-            invoke: {
-                src: sendToSamAPI,
-                onDone: {
-                    target: 'done',
-                    actions: ['clearSelection']
-                },
+            on: {
+              EDITED_SEGMENT: { target: 'done', actions: ['clearSelection'] },
             }
         },
         done: {
@@ -78,6 +59,19 @@ Machine(
         setMouseIsDown: assign({ isMouseDown: true }),
         setMouseIsNotDown: assign({ isMouseDown: false }),
         clearSelection: assign({ startX: null, startY: null, endX: null, endY: null, isMouseDown: false, x: 0, y: 0 }),
+        sendToAPI: send(
+          (ctx) => ({
+            type: 'EDIT',
+            action: 'sam',
+            args: {
+              x_start: ctx.startX,
+              y_start: ctx.startY,
+              x_end: ctx.endX,
+              y_end: ctx.endY,
+            },
+          }),
+          { to: 'arrays' }
+        ),
       },
     }
   );

frontend/src/Project/service/labels/segmentApiMachine.js

@@ -40,22 +40,41 @@ async function makeEditZip(context, event) {
   return zipBlob;
 }
 
-/** Sends a label zip to the DeepCell Label API to edit. */
-async function edit(context, event) {
-  const form = new FormData();
-  const zipBlob = await makeEditZip(context, event);
-  form.append('labels', zipBlob, 'labels.zip');
-  const width = context.labeled[0].length;
-  const height = context.labeled.length;
-
+async function testSamPrediction(context, event) {
   const options = {
     method: 'POST',
-    body: form,
-    'Content-Type': 'multipart/form-data',
+    body: JSON.stringify(event.args),
+    'Content-Type': 'application/json',
   };
-  return fetch(`${document.location.origin}/api/edit`, options)
+  return fetch(`${document.location.origin}/api/testSamPrediction`, options)
     .then(checkResponseCode)
-    .then((res) => parseResponseZip(res, width, height));
+    .then((res) => res.json())
+    .then((data) => {
+      return { labeled: data.data[0], cells: context.cells}
+    })
+}
+
+/** Sends a label zip to the DeepCell Label API to edit. */
+async function edit(context, event) {
+
+  if (event.action === "sam") {
+    return testSamPrediction(context, event)
+  } else {
+    const form = new FormData();
+    const zipBlob = await makeEditZip(context, event);
+    form.append('labels', zipBlob, 'labels.zip');
+    const width = context.labeled[0].length;
+    const height = context.labeled.length;
+
+    const options = {
+      method: 'POST',
+      body: form,
+      'Content-Type': 'multipart/form-data',
+    };
+    return fetch(`${document.location.origin}/api/edit`, options)
+      .then(checkResponseCode)
+      .then((res) => parseResponseZip(res, width, height));
+  }
 }
 
 function checkResponseCode(response) {