Merge pull request #297 from abitmore/292-acc-his-prune

Account history: option to prune old data (#292)

libraries/app/api.cpp

@@ -493,13 +493,14 @@ namespace graphene { namespace app {
        const auto& db = *_app.chain_database();
        FC_ASSERT(limit <= 100);
        vector<operation_history_object> result;
+       const auto& stats = account(db).statistics(db);
        if( start == 0 )
-          start = account(db).statistics(db).total_ops;
+          start = stats.total_ops;
        else
-          start = min( account(db).statistics(db).total_ops, start );
+          start = min( stats.total_ops, start );
 
 
-       if( start >= stop && start > 0 && limit > 0 )
+       if( start >= stop && start > stats.removed_ops && limit > 0 )
        {
           const auto& hist_idx = db.get_index_type<account_transaction_history_index>();
           const auto& by_seq_idx = hist_idx.indices().get<by_seq>();

libraries/chain/include/graphene/chain/account_object.hpp

@@ -50,7 +50,10 @@ namespace graphene { namespace chain {
           * Keep the most recent operation as a root pointer to a linked list of the transaction history.
           */
          account_transaction_history_id_type most_recent_op;
+         /** Total operations related to this account. */
          uint32_t                            total_ops = 0;
+         /** Total operations related to this account that has been removed from the database. */
+         uint32_t                            removed_ops = 0;
 
          /**
           * When calculating votes it is necessary to know how much is stored in orders (and thus unavailable for
@@ -386,7 +389,7 @@ FC_REFLECT_DERIVED( graphene::chain::account_statistics_object,
                     (graphene::chain::object),
                     (owner)
                     (most_recent_op)
-                    (total_ops)
+                    (total_ops)(removed_ops)
                     (total_core_in_orders)
                     (lifetime_fees_paid)
                     (pending_fees)(pending_vested_fees)

libraries/chain/include/graphene/chain/operation_history_object.hpp

@@ -102,6 +102,7 @@ namespace graphene { namespace chain {
    struct by_id;
 struct by_seq;
 struct by_op;
+struct by_opid;
 typedef multi_index_container<
    account_transaction_history_object,
    indexed_by<
@@ -117,6 +118,9 @@ typedef multi_index_container<
             member< account_transaction_history_object, account_id_type, &account_transaction_history_object::account>,
             member< account_transaction_history_object, operation_history_id_type, &account_transaction_history_object::operation_id>
          >
+      >,
+      ordered_non_unique< tag<by_opid>,
+         member< account_transaction_history_object, operation_history_id_type, &account_transaction_history_object::operation_id>
       >
    >
 > account_transaction_history_multi_index_type;

libraries/plugins/account_history/account_history_plugin.cpp

@@ -66,6 +66,11 @@ class account_history_plugin_impl
       flat_set<account_id_type> _tracked_accounts;
       bool _partial_operations = false;
       primary_index< simple_index< operation_history_object > >* _oho_index;
+      uint32_t _max_ops_per_account = -1;
+   private:
+      /** add one history record, then check and remove the earliest history record */
+      void add_account_history( const account_id_type account_id, const operation_history_id_type op_id );
+
 };
 
 account_history_plugin_impl::~account_history_plugin_impl()
@@ -89,95 +94,146 @@ void account_history_plugin_impl::update_account_histories( const signed_block&
          } ) );
       };
 
-      if (_partial_operations)
+      if( !o_op.valid() || ( _max_ops_per_account == 0 && _partial_operations ) )
       {
-         if( !o_op.valid() )
-         {
-            _oho_index->use_next_id();
-            continue;
-         }
+         // Note: the 2nd and 3rd checks above are for better performance, when the db is not clean,
+         //       they will break consistency of account_stats.total_ops and removed_ops and most_recent_op
+         _oho_index->use_next_id();
+         continue;
       }
-      else
-      {
+      else if( !_partial_operations )
          // add to the operation history index
          oho = create_oho();
 
-         if( !o_op.valid() )
-         {
-            ilog( "removing failed operation with ID: ${id}", ("id", oho->id) );
-            db.remove( *oho );
-            continue;
-         }
-      }
-
       const operation_history_object& op = *o_op;
 
       // get the set of accounts this operation applies to
       flat_set<account_id_type> impacted;
       vector<authority> other;
-      operation_get_required_authorities( op.op, impacted, impacted, other );
+      operation_get_required_authorities( op.op, impacted, impacted, other ); // fee_payer is added here
 
       if( op.op.which() == operation::tag< account_create_operation >::value )
-      {
-         if (!oho.valid()) { oho = create_oho(); }
-         impacted.insert( oho->result.get<object_id_type>() );
-      }
+         impacted.insert( op.result.get<object_id_type>() );
       else
          graphene::app::operation_get_impacted_accounts( op.op, impacted );
 
       for( auto& a : other )
          for( auto& item : a.account_auths )
             impacted.insert( item.first );
 
+      // be here, either _max_ops_per_account > 0, or _partial_operations == false, or both
+      // if _partial_operations == false, oho should have been created above
+      // so the only case should be checked here is:
+      //    whether need to create oho if _max_ops_per_account > 0 and _partial_operations == true
+
       // for each operation this account applies to that is in the config link it into the history
-      if( _tracked_accounts.size() == 0 )
+      if( _tracked_accounts.size() == 0 ) // tracking all accounts
       {
+         // if tracking all accounts, when impacted is not empty (although it will always be),
+         //    still need to create oho if _max_ops_per_account > 0 and _partial_operations == true
+         //    so always need to create oho if not done
          if (!impacted.empty() && !oho.valid()) { oho = create_oho(); }
-         for( auto& account_id : impacted )
+
+         if( _max_ops_per_account > 0 )
          {
-            // we don't do index_account_keys here anymore, because
-            // that indexing now happens in observers' post_evaluate()
-
-            // add history
-            const auto& stats_obj = account_id(db).statistics(db);
-            const auto& ath = db.create<account_transaction_history_object>( [&]( account_transaction_history_object& obj ){
-                obj.operation_id = oho->id;
-                obj.account = account_id;
-                obj.sequence = stats_obj.total_ops+1;
-                obj.next = stats_obj.most_recent_op;
-            });
-            db.modify( stats_obj, [&]( account_statistics_object& obj ){
-                obj.most_recent_op = ath.id;
-                obj.total_ops = ath.sequence;
-            });
+            // Note: the check above is for better performance, when the db is not clean,
+            //       it breaks consistency of account_stats.total_ops and removed_ops and most_recent_op,
+            //       but it ensures it's safe to remove old entries in add_account_history(...)
+            for( auto& account_id : impacted )
+            {
+               // we don't do index_account_keys here anymore, because
+               // that indexing now happens in observers' post_evaluate()
+
+               // add history
+               add_account_history( account_id, oho->id );
+            }
          }
       }
-      else
+      else // tracking a subset of accounts
       {
-         for( auto account_id : _tracked_accounts )
+         // whether need to create oho if _max_ops_per_account > 0 and _partial_operations == true ?
+         // the answer: only need to create oho if a tracked account is impacted and need to save history
+
+         if( _max_ops_per_account > 0 )
          {
-            if( impacted.find( account_id ) != impacted.end() )
+            // Note: the check above is for better performance, when the db is not clean,
+            //       it breaks consistency of account_stats.total_ops and removed_ops and most_recent_op,
+            //       but it ensures it's safe to remove old entries in add_account_history(...)
+            for( auto account_id : _tracked_accounts )
             {
-               if (!oho.valid()) { oho = create_oho(); }
-               // add history
-               const auto& stats_obj = account_id(db).statistics(db);
-               const auto& ath = db.create<account_transaction_history_object>( [&]( account_transaction_history_object& obj ){
-                   obj.operation_id = oho->id;
-                   obj.account = account_id;
-                   obj.sequence = stats_obj.total_ops+1;
-                   obj.next = stats_obj.most_recent_op;
-               });
-               db.modify( stats_obj, [&]( account_statistics_object& obj ){
-                   obj.most_recent_op = ath.id;
-                   obj.total_ops = ath.sequence;
-               });
+               if( impacted.find( account_id ) != impacted.end() )
+               {
+                  if (!oho.valid()) { oho = create_oho(); }
+                  // add history
+                  add_account_history( account_id, oho->id );
+               }
             }
          }
       }
       if (_partial_operations && ! oho.valid())
          _oho_index->use_next_id();
    }
 }
+
+void account_history_plugin_impl::add_account_history( const account_id_type account_id, const operation_history_id_type op_id )
+{
+   graphene::chain::database& db = database();
+   const auto& stats_obj = account_id(db).statistics(db);
+   // add new entry
+   const auto& ath = db.create<account_transaction_history_object>( [&]( account_transaction_history_object& obj ){
+       obj.operation_id = op_id;
+       obj.account = account_id;
+       obj.sequence = stats_obj.total_ops + 1;
+       obj.next = stats_obj.most_recent_op;
+   });
+   db.modify( stats_obj, [&]( account_statistics_object& obj ){
+       obj.most_recent_op = ath.id;
+       obj.total_ops = ath.sequence;
+   });
+   // remove the earliest account history entry if too many
+   // _max_ops_per_account is guaranteed to be non-zero outside
+   if( stats_obj.total_ops - stats_obj.removed_ops > _max_ops_per_account )
+   {
+      // look for the earliest entry
+      const auto& his_idx = db.get_index_type<account_transaction_history_index>();
+      const auto& by_seq_idx = his_idx.indices().get<by_seq>();
+      auto itr = by_seq_idx.lower_bound( boost::make_tuple( account_id, 0 ) );
+      // make sure don't remove the one just added
+      if( itr != by_seq_idx.end() && itr->account == account_id && itr->id != ath.id )
+      {
+         // if found, remove the entry, and adjust account stats object
+         const auto remove_op_id = itr->operation_id;
+         const auto itr_remove = itr;
+         ++itr;
+         db.remove( *itr_remove );
+         db.modify( stats_obj, [&]( account_statistics_object& obj ){
+             obj.removed_ops = obj.removed_ops + 1;
+         });
+         // modify previous node's next pointer
+         // this should be always true, but just have a check here
+         if( itr != by_seq_idx.end() && itr->account == account_id )
+         {
+            db.modify( *itr, [&]( account_transaction_history_object& obj ){
+               obj.next = account_transaction_history_id_type();
+            });
+         }
+         // else need to modify the head pointer, but it shouldn't be true
+
+         // remove the operation history entry (1.11.x) if configured and no reference left
+         if( _partial_operations )
+         {
+            // check for references
+            const auto& by_opid_idx = his_idx.indices().get<by_opid>();
+            if( by_opid_idx.find( remove_op_id ) == by_opid_idx.end() )
+            {
+               // if no reference, remove
+               db.remove( remove_op_id(db) );
+            }
+         }
+      }
+   }
+}
+
 } // end namespace detail
 
 
@@ -207,6 +263,7 @@ void account_history_plugin::plugin_set_program_options(
    cli.add_options()
          ("track-account", boost::program_options::value<std::vector<std::string>>()->composing()->multitoken(), "Account ID to track history for (may specify multiple times)")
          ("partial-operations", boost::program_options::value<bool>(), "Keep only those operations in memory that are related to account history tracking")
+         ("max-ops-per-account", boost::program_options::value<uint32_t>(), "Maximum number of operations per account will be kept in memory")
          ;
    cfg.add(cli);
 }
@@ -221,6 +278,9 @@ void account_history_plugin::plugin_initialize(const boost::program_options::var
    if (options.count("partial-operations")) {
        my->_partial_operations = options["partial-operations"].as<bool>();
    }
+   if (options.count("max-ops-per-account")) {
+       my->_max_ops_per_account = options["max-ops-per-account"].as<uint32_t>();
+   }
 }
 
 void account_history_plugin::plugin_startup()