diff --git a/framework/include/systems/NonlinearSystem.h b/framework/include/systems/NonlinearSystem.h
index a561d9ce6299..a48b5912b8b8 100644
--- a/framework/include/systems/NonlinearSystem.h
+++ b/framework/include/systems/NonlinearSystem.h
@@ -100,5 +100,7 @@ class NonlinearSystem : public NonlinearSystemBase
    */
   void setupColoringFiniteDifferencedPreconditioner();
 
+  virtual bool matrixFromColoring() const override { return _use_coloring_finite_difference; }
+
   bool _use_coloring_finite_difference;
 };
diff --git a/framework/include/systems/SolverSystem.h b/framework/include/systems/SolverSystem.h
index f1deea119c9d..fc45dcd67170 100644
--- a/framework/include/systems/SolverSystem.h
+++ b/framework/include/systems/SolverSystem.h
@@ -85,6 +85,13 @@ class SolverSystem : public SystemBase
 
   /// Boolean to see if solution is invalid
   bool _solution_is_invalid;
+
+private:
+  /**
+   * Whether a system matrix is formed from coloring. This influences things like when to compute
+   * time derivatives
+   */
+  virtual bool matrixFromColoring() const { return false; }
 };
 
 inline const NumericVector<Number> * const &
diff --git a/framework/src/systems/NonlinearSystem.C b/framework/src/systems/NonlinearSystem.C
index b9422eaa8357..953ff165810c 100644
--- a/framework/src/systems/NonlinearSystem.C
+++ b/framework/src/systems/NonlinearSystem.C
@@ -232,14 +232,14 @@ NonlinearSystem::setupFiniteDifferencedPreconditioner()
 
   if (fdp->finiteDifferenceType() == "coloring")
   {
-    setupColoringFiniteDifferencedPreconditioner();
     _use_coloring_finite_difference = true;
+    setupColoringFiniteDifferencedPreconditioner();
   }
 
   else if (fdp->finiteDifferenceType() == "standard")
   {
-    setupStandardFiniteDifferencedPreconditioner();
     _use_coloring_finite_difference = false;
+    setupStandardFiniteDifferencedPreconditioner();
   }
   else
     mooseError("Unknown finite difference type");
diff --git a/framework/src/systems/SolverSystem.C b/framework/src/systems/SolverSystem.C
index 84af4ce50088..868a7f2bdc7f 100644
--- a/framework/src/systems/SolverSystem.C
+++ b/framework/src/systems/SolverSystem.C
@@ -135,7 +135,7 @@ SolverSystem::compute(const ExecFlagType type)
     compute_tds = true;
   else if (type == EXEC_NONLINEAR)
   {
-    if (_fe_problem.computingScalingJacobian())
+    if (_fe_problem.computingScalingJacobian() || matrixFromColoring())
       compute_tds = true;
   }
   else if ((type == EXEC_TIMESTEP_END) || (type == EXEC_FINAL))