Error in conductor-server when polling for workflows

[davidfrankoproceedy]

After upgrading to conductor-server 3.13.3 from version 3.11.3, we started getting this error. We have tried to trace it back, but we were not successful. Initially, we thought this was a problem only in production, but the same (similar) error happens in all our environments. Has anyone encountered this error and any ideas where we should be looking?

1886108407 [scheduled-task-pool-1] ERROR com.netflix.conductor.core.reconciliation.WorkflowReconciler [] - Error when polling for workflows
java.util.concurrent.ExecutionException: com.netflix.conductor.core.exception.NonTransientException: ERROR: interval field value out of range: "1683359014706 seconds"
	at java.util.concurrent.CompletableFuture.reportGet(Unknown Source) ~[?:?]
	at java.util.concurrent.CompletableFuture.get(Unknown Source) ~[?:?]
	at com.netflix.conductor.core.reconciliation.WorkflowReconciler.pollAndSweep(WorkflowReconciler.java:73) ~[conductor-core-3.13.3.jar!/:3.13.3]
	at jdk.internal.reflect.GeneratedMethodAccessor121.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source) ~[?:?]
	at java.lang.reflect.Method.invoke(Unknown Source) ~[?:?]
	at org.springframework.scheduling.support.ScheduledMethodRunnable.run(ScheduledMethodRunnable.java:84) ~[spring-context-5.3.22.jar!/:5.3.22]
	at org.springframework.scheduling.support.DelegatingErrorHandlingRunnable.run(DelegatingErrorHandlingRunnable.java:54) ~[spring-context-5.3.22.jar!/:5.3.22]
	at java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) ~[?:?]
	at java.util.concurrent.FutureTask.runAndReset(Unknown Source) ~[?:?]
	at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) ~[?:?]

	at java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) ~[?:?]
	at java.lang.Thread.run(Unknown Source) ~[?:?]
Caused by: com.netflix.conductor.core.exception.NonTransientException: ERROR: interval field value out of range: "1683359014706 seconds"
	at com.netflix.conductor.postgres.dao.PostgresBaseDAO.getWithRetriedTransactions(PostgresBaseDAO.java:147) ~[conductor-postgres-persistence-3.13.3.jar!/:3.13.3]
	at com.netflix.conductor.postgres.dao.PostgresBaseDAO.queryWithTransaction(PostgresBaseDAO.java:209) ~[conductor-postgres-persistence-3.13.3.jar!/:3.13.3]
	at com.netflix.conductor.postgres.dao.PostgresQueueDAO.setUnackTimeout(PostgresQueueDAO.java:162) ~[conductor-postgres-persistence-3.13.3.jar!/:3.13.3]
	at com.netflix.conductor.core.reconciliation.WorkflowSweeper.unack(WorkflowSweeper.java:156) ~[conductor-core-3.13.3.jar!/:3.13.3]
	at com.netflix.conductor.core.reconciliation.WorkflowSweeper.sweep(WorkflowSweeper.java:104) ~[conductor-core-3.13.3.jar!/:3.13.3]
	at com.netflix.conductor.core.reconciliation.WorkflowSweeper.sweepAsync(WorkflowSweeper.java:69) ~[conductor-core-3.13.3.jar!/:3.13.3]
	at com.netflix.conductor.core.reconciliation.WorkflowSweeper$$FastClassBySpringCGLIB$$f994448e.invoke(<generated>) ~[conductor-core-3.13.3.jar!/:3.13.3]
	at org.springframework.cglib.proxy.MethodProxy.invoke(MethodProxy.java:218) ~[spring-core-5.3.22.jar!/:5.3.22]
	at org.springframework.aop.framework.CglibAopProxy$CglibMethodInvocation.invokeJoinpoint(CglibAopProxy.java:793) ~[spring-aop-5.3.22.jar!/:5.3.22]
	at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:163) ~[spring-aop-5.3.22.jar!/:5.3.22]
	at org.springframework.aop.framework.CglibAopProxy$CglibMethodInvocation.proceed(CglibAopProxy.java:763) ~[spring-aop-5.3.22.jar!/:5.3.22]
	at org.springframework.aop.interceptor.AsyncExecutionInterceptor.lambda$invoke$0(AsyncExecutionInterceptor.java:115) ~[spring-aop-5.3.22.jar!/:5.3.22]
	at org.springframework.aop.interceptor.AsyncExecutionAspectSupport.lambda$doSubmit$3(AsyncExecutionAspectSupport.java:278) ~[spring-aop-5.3.22.jar!/:5.3.22]
	at java.util.concurrent.CompletableFuture$AsyncSupply.run(Unknown Source) ~[?:?]
	... 3 more
Caused by: com.netflix.conductor.core.exception.NonTransientException: ERROR: interval field value out of range: "1683359014706 seconds"
	at com.netflix.conductor.postgres.util.Query.executeUpdate(Query.java:284) ~[conductor-postgres-persistence-3.13.3.jar!/:3.13.3]
	at com.netflix.conductor.postgres.dao.PostgresQueueDAO.lambda$setUnackTimeout$9(PostgresQueueDAO.java:169) ~[conductor-postgres-persistence-3.13.3.jar!/:3.13.3]
	at com.netflix.conductor.postgres.dao.PostgresBaseDAO.query(PostgresBaseDAO.java:224) ~[conductor-postgres-persistence-3.13.3.jar!/:3.13.3]
	at com.netflix.conductor.postgres.dao.PostgresBaseDAO.lambda$queryWithTransaction$5(PostgresBaseDAO.java:209) ~[conductor-postgres-persistence-3.13.3.jar!/:3.13.3]
	at com.netflix.conductor.postgres.dao.PostgresBaseDAO.getWithTransaction(PostgresBaseDAO.java:121) ~[conductor-postgres-persistence-3.13.3.jar!/:3.13.3]
	at com.netflix.conductor.postgres.dao.PostgresBaseDAO.lambda$getWithRetriedTransactions$3(PostgresBaseDAO.java:145) ~[conductor-postgres-persistence-3.13.3.jar!/:3.13.3]
	at org.springframework.retry.support.RetryTemplate.doExecute(RetryTemplate.java:329) ~[spring-retry-1.3.3.jar!/:?]
	at org.springframework.retry.support.RetryTemplate.execute(RetryTemplate.java:209) ~[spring-retry-1.3.3.jar!/:?]
	at com.netflix.conductor.postgres.dao.PostgresBaseDAO.getWithRetriedTransactions(PostgresBaseDAO.java:145) ~[conductor-postgres-persistence-3.13.3.jar!/:3.13.3]
	at com.netflix.conductor.postgres.dao.PostgresBaseDAO.queryWithTransaction(PostgresBaseDAO.java:209) ~[conductor-postgres-persistence-3.13.3.jar!/:3.13.3]
	at com.netflix.conductor.postgres.dao.PostgresQueueDAO.setUnackTimeout(PostgresQueueDAO.java:162) ~[conductor-postgres-persistence-3.13.3.jar!/:3.13.3]
	at com.netflix.conductor.core.reconciliation.WorkflowSweeper.unack(WorkflowSweeper.java:156) ~[conductor-core-3.13.3.jar!/:3.13.3]
	at com.netflix.conductor.core.reconciliation.WorkflowSweeper.sweep(WorkflowSweeper.java:104) ~[conductor-core-3.13.3.jar!/:3.13.3]
	at com.netflix.conductor.core.reconciliation.WorkflowSweeper.sweepAsync(WorkflowSweeper.java:69) ~[conductor-core-3.13.3.jar!/:3.13.3]
	at com.netflix.conductor.core.reconciliation.WorkflowSweeper$$FastClassBySpringCGLIB$$f994448e.invoke(<generated>) ~[conductor-core-3.13.3.jar!/:3.13.3]
	at org.springframework.cglib.proxy.MethodProxy.invoke(MethodProxy.java:218) ~[spring-core-5.3.22.jar!/:5.3.22]
	at org.springframework.aop.framework.CglibAopProxy$CglibMethodInvocation.invokeJoinpoint(CglibAopProxy.java:793) ~[spring-aop-5.3.22.jar!/:5.3.22]
	at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:163) ~[spring-aop-5.3.22.jar!/:5.3.22]
	at org.springframework.aop.framework.CglibAopProxy$CglibMethodInvocation.proceed(CglibAopProxy.java:763) ~[spring-aop-5.3.22.jar!/:5.3.22]
	at org.springframework.aop.interceptor.AsyncExecutionInterceptor.lambda$invoke$0(AsyncExecutionInterceptor.java:115) ~[spring-aop-5.3.22.jar!/:5.3.22]
	at org.springframework.aop.interceptor.AsyncExecutionAspectSupport.lambda$doSubmit$3(AsyncExecutionAspectSupport.java:278) ~[spring-aop-5.3.22.jar!/:5.3.22]
	at java.util.concurrent.CompletableFuture$AsyncSupply.run(Unknown Source) ~[?:?]
	... 3 more
Caused by: org.postgresql.util.PSQLException: ERROR: interval field value out of range: "1683359014706 seconds"
	at org.postgresql.core.v3.QueryExecutorImpl.receiveErrorResponse(QueryExecutorImpl.java:2675) ~[postgresql-42.3.6.jar!/:42.3.6]
	at org.postgresql.core.v3.QueryExecutorImpl.processResults(QueryExecutorImpl.java:2365) ~[postgresql-42.3.6.jar!/:42.3.6]
	at org.postgresql.core.v3.QueryExecutorImpl.execute(QueryExecutorImpl.java:355) ~[postgresql-42.3.6.jar!/:42.3.6]
	at org.postgresql.jdbc.PgStatement.executeInternal(PgStatement.java:490) ~[postgresql-42.3.6.jar!/:42.3.6]
	at org.postgresql.jdbc.PgStatement.execute(PgStatement.java:408) ~[postgresql-42.3.6.jar!/:42.3.6]
	at org.postgresql.jdbc.PgPreparedStatement.executeWithFlags(PgPreparedStatement.java:167) ~[postgresql-42.3.6.jar!/:42.3.6]
	at org.postgresql.jdbc.PgPreparedStatement.executeUpdate(PgPreparedStatement.java:135) ~[postgresql-42.3.6.jar!/:42.3.6]
	at com.zaxxer.hikari.pool.ProxyPreparedStatement.executeUpdate(ProxyPreparedStatement.java:61) ~[HikariCP-4.0.3.jar!/:?]
	at com.zaxxer.hikari.pool.HikariProxyPreparedStatement.executeUpdate(HikariProxyPreparedStatement.java) ~[HikariCP-4.0.3.jar!/:?]
	at com.netflix.conductor.postgres.util.Query.executeUpdate(Query.java:275) ~[conductor-postgres-persistence-3.13.3.jar!/:3.13.3]
	at com.netflix.conductor.postgres.dao.PostgresQueueDAO.lambda$setUnackTimeout$9(PostgresQueueDAO.java:169) ~[conductor-postgres-persistence-3.13.3.jar!/:3.13.3]
	at com.netflix.conductor.postgres.dao.PostgresBaseDAO.query(PostgresBaseDAO.java:224) ~[conductor-postgres-persistence-3.13.3.jar!/:3.13.3]
	at com.netflix.conductor.postgres.dao.PostgresBaseDAO.lambda$queryWithTransaction$5(PostgresBaseDAO.java:209) ~[conductor-postgres-persistence-3.13.3.jar!/:3.13.3]
	at com.netflix.conductor.postgres.dao.PostgresBaseDAO.getWithTransaction(PostgresBaseDAO.java:121) ~[conductor-postgres-persistence-3.13.3.jar!/:3.13.3]
	at com.netflix.conductor.postgres.dao.PostgresBaseDAO.lambda$getWithRetriedTransactions$3(PostgresBaseDAO.java:145) ~[conductor-postgres-persistence-3.13.3.jar!/:3.13.3]
	at org.springframework.retry.support.RetryTemplate.doExecute(RetryTemplate.java:329) ~[spring-retry-1.3.3.jar!/:?]
	at org.springframework.retry.support.RetryTemplate.execute(RetryTemplate.java:209) ~[spring-retry-1.3.3.jar!/:?]
	at com.netflix.conductor.postgres.dao.PostgresBaseDAO.getWithRetriedTransactions(PostgresBaseDAO.java:145) ~[conductor-postgres-persistence-3.13.3.jar!/:3.13.3]
	at com.netflix.conductor.postgres.dao.PostgresBaseDAO.queryWithTransaction(PostgresBaseDAO.java:209) ~[conductor-postgres-persistence-3.13.3.jar!/:3.13.3]
	at com.netflix.conductor.postgres.dao.PostgresQueueDAO.setUnackTimeout(PostgresQueueDAO.java:162) ~[conductor-postgres-persistence-3.13.3.jar!/:3.13.3]
	at com.netflix.conductor.core.reconciliation.WorkflowSweeper.unack(WorkflowSweeper.java:156) ~[conductor-core-3.13.3.jar!/:3.13.3]
	at com.netflix.conductor.core.reconciliation.WorkflowSweeper.sweep(WorkflowSweeper.java:104) ~[conductor-core-3.13.3.jar!/:3.13.3]
	at com.netflix.conductor.core.reconciliation.WorkflowSweeper.sweepAsync(WorkflowSweeper.java:69) ~[conductor-core-3.13.3.jar!/:3.13.3]
	at com.netflix.conductor.core.reconciliation.WorkflowSweeper$$FastClassBySpringCGLIB$$f994448e.invoke(<generated>) ~[conductor-core-3.13.3.jar!/:3.13.3]
	at org.springframework.cglib.proxy.MethodProxy.invoke(MethodProxy.java:218) ~[spring-core-5.3.22.jar!/:5.3.22]
	at org.springframework.aop.framework.CglibAopProxy$CglibMethodInvocation.invokeJoinpoint(CglibAopProxy.java:793) ~[spring-aop-5.3.22.jar!/:5.3.22]
	at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:163) ~[spring-aop-5.3.22.jar!/:5.3.22]
	at org.springframework.aop.framework.CglibAopProxy$CglibMethodInvocation.proceed(CglibAopProxy.java:763) ~[spring-aop-5.3.22.jar!/:5.3.22]
	at org.springframework.aop.interceptor.AsyncExecutionInterceptor.lambda$invoke$0(AsyncExecutionInterceptor.java:115) ~[spring-aop-5.3.22.jar!/:5.3.22]
	at org.springframework.aop.interceptor.AsyncExecutionAspectSupport.lambda$doSubmit$3(AsyncExecutionAspectSupport.java:278) ~[spring-aop-5.3.22.jar!/:5.3.22]
	at java.util.concurrent.CompletableFuture$AsyncSupply.run(Unknown Source) ~[?:?]
	... 3 more