Tasks suddenly stopped working and stuck on scheduled state

[viinxent]

Good day everyone!

We are currently having a problem with Netflix conductor, suddenly workflows stopped working and tasks are in SCHEDULED state. This only happened now we've been using Netflix conductor for months using Postgres as its persistent db. We didn't update any configuration to any of our services i.e netflix conductor, postgres, elasticsearch.

Current version we are using is 3.13.8

Also i noticed that it shows epoch on start time and end time, is this a bug?

One of the task sample, they all pretty much have same status SCHEDULED, we already tried restarting conductor itself with no avail

{
  "taskType": "HTTP",
  "status": "SCHEDULED",
  "inputData": {
    "asyncComplete": false,
    "http_request": {
      "headers": {
        "authorization": "REDACTED"
      },
      "method": "POST",
      "readTimeOut": 0,
      "body": {
        "workflow": "sf_customer_integration",
        "action": "processing"
      },
      "connectionTimeOut": 0,
      "uri": "http://svc-app-pgx.ks-devx/event/12fb4018-526e-40f5-a314-005997819fdc/logs"
    }
  },
  "referenceTaskName": "t_dev_create_event_log_processing",
  "retryCount": 0,
  "seq": 1,
  "pollCount": 0,
  "taskDefName": "t_dev_create_event_log_processing",
  "scheduledTime": 1692021151226,
  "startTime": 0,
  "endTime": 0,
  "updateTime": 0,
  "startDelayInSeconds": 0,
  "retried": false,
  "executed": false,
  "callbackFromWorker": true,
  "responseTimeoutSeconds": 0,
  "workflowInstanceId": "815f16cf-2268-4b3b-8f1f-f1c197d507f3",
  "workflowType": "wf_dev_nets2019devx_sf_customer_integration",
  "taskId": "32427222-cc80-4482-8472-830d3f7e62c4",
  "callbackAfterSeconds": 0,
  "outputData": {},
  "workflowTask": {
    "name": "t_dev_create_event_log_processing",
    "taskReferenceName": "t_dev_create_event_log_processing",
    "inputParameters": {
      "asyncComplete": false
    },
    "type": "HTTP",
    "decisionCases": {},
    "defaultCase": [],
    "forkTasks": [],
    "startDelay": 0,
    "joinOn": [],
    "optional": false,
    "taskDefinition": {
      "createTime": 1663750764071,
      "updateTime": 1663752848087,
      "createdBy": "",
      "updatedBy": "",
      "name": "t_dev_create_event_log_processing",
      "description": "Create event log (processing)",
      "retryCount": 0,
      "timeoutSeconds": 0,
      "inputKeys": [],
      "outputKeys": [],
      "timeoutPolicy": "TIME_OUT_WF",
      "retryLogic": "FIXED",
      "retryDelaySeconds": 60,
      "responseTimeoutSeconds": 600,
      "inputTemplate": {
        "http_request": {
          "method": "POST",
          "readTimeOut": 0,
          "connectionTimeOut": 0,
          "uri": "http://svc-app-pgx.ks-devx/event/${workflow.input.id}/logs",
          "body": {
            "workflow": "${workflow.input.meta.workflow}",
            "action": "processing"
          },
          "headers": {
            "authorization": "Bearer ${workflow.input.meta.token}"
          }
        }
      },
      "rateLimitPerFrequency": 0,
      "rateLimitFrequencyInSeconds": 1,
      "ownerEmail": "REDACTED",
      "backoffScaleFactor": 1
    },
    "defaultExclusiveJoinTask": [],
    "asyncComplete": false,
    "loopOver": []
  },
  "rateLimitPerFrequency": 0,
  "rateLimitFrequencyInSeconds": 1,
  "workflowPriority": 0,
  "iteration": 0,
  "subworkflowChanged": false,
  "loopOverTask": false,
  "taskDefinition": {
    "createTime": 1663750764071,
    "updateTime": 1663752848087,
    "createdBy": "",
    "updatedBy": "",
    "name": "t_dev_create_event_log_processing",
    "description": "Create event log (processing)",
    "retryCount": 0,
    "timeoutSeconds": 0,
    "inputKeys": [],
    "outputKeys": [],
    "timeoutPolicy": "TIME_OUT_WF",
    "retryLogic": "FIXED",
    "retryDelaySeconds": 60,
    "responseTimeoutSeconds": 600,
    "inputTemplate": {
      "http_request": {
        "method": "POST",
        "readTimeOut": 0,
        "connectionTimeOut": 0,
        "uri": "http://svc-app-pgx.ks-devx/event/${workflow.input.id}/logs",
        "body": {
          "workflow": "${workflow.input.meta.workflow}",
          "action": "processing"
        },
        "headers": {
          "authorization": "Bearer ${workflow.input.meta.token}"
        }
      }
    },
    "rateLimitPerFrequency": 0,
    "rateLimitFrequencyInSeconds": 1,
    "ownerEmail": "REDACTED",
    "backoffScaleFactor": 1
  },
  "queueWaitTime": 0
}

[viinxent]

As seen on the task it have connectionTimeout and readTimeout of 0 = infinite waiting, we forgot to respond to the request on our side which blocked the system workers running other tasks. Workflows/tasks piled up infinite running process that doesn't finish caused this problem. Hope this will help others.