WIP: UI improvements to cleanup monitor

modules/admin/app/actors/cleanup/CleanupRunner.scala

@@ -1,22 +1,27 @@
 package actors.cleanup
 
-import akka.actor.{Actor, ActorLogging, ActorRef}
+import actors.Ticker
+import actors.Ticker.Tick
+import akka.actor.{Actor, ActorLogging, ActorRef, Props}
 import play.api.Configuration
 import services.data.{DataService, EventForwarder}
 import services.ingest.{Cleanup, ImportLogService, IngestService}
 
 import java.time.LocalDateTime
+import java.util.concurrent.TimeUnit
 import scala.concurrent.ExecutionContext
+import scala.concurrent.duration.{Duration, DurationInt, FiniteDuration}
 
 
 object CleanupRunner {
-  sealed trait CleanupState
+  case class Status(relinkCount: Int = 0, redirectCount: Int = 0, deleteCount: Int = 0)
 
+  sealed trait CleanupState
   case class Relinked(cleanup: Cleanup, status: Status) extends CleanupState
   case class Redirected(cleanup: Cleanup, status: Status) extends CleanupState
   case class DeleteBatch(cleanup: Cleanup, todo: Seq[String], status: Status) extends CleanupState
-  case class Status(relinkCount: Int = 0, redirectCount: Int = 0, deleteCount: Int = 0)
-  case object Done extends CleanupState
+  case class Done(secs: Duration) extends CleanupState
+
 
   case class CleanupJob(repoId: String, snapshotId: Int, jobId: String, msg: String)
 }
@@ -63,7 +68,11 @@ case class CleanupRunner(
     case r@Relinked(cleanup, newStatus) =>
       msgTo ! r
       importService.remapMovedUnits(cleanup.redirects)
-        .map(done => Redirected(cleanup, newStatus.copy(redirectCount = done)))
+        .map { _ => // NB: the API gives us a number that is the total number
+          // of redirected URLs, but we want the number of redirected items,
+          // so it's a bit of a fake here...
+          Redirected(cleanup, newStatus.copy(redirectCount = cleanup.redirects.size))
+        }
         .pipeTo(self)
 
     // Launch the delete task
@@ -73,6 +82,9 @@ case class CleanupRunner(
 
     // Delete a batch of items
     case DeleteBatch(cleanup, todo, newStatus) if todo.nonEmpty =>
+      val ticker: ActorRef = context.actorOf(Props(Ticker()))
+      ticker ! (msgTo -> "Deleting...")
+
       val (batch, rest) = todo.splitAt(batchSize)
       dataApi.batchDelete(batch, Some(job.repoId), logMsg = job.msg,
           version = true, tolerant = true, commit = true)
@@ -87,11 +99,12 @@ case class CleanupRunner(
           msgTo ! e
         }
         .pipeTo(self)
+        .onComplete(_ => ticker ! Ticker.Stop)
 
     // When there are no more items to delete, save the cleanup log and finish
     case DeleteBatch(cleanup, todo, _) if todo.isEmpty =>
       logService.saveCleanup(job.repoId, job.snapshotId, cleanup)
-        .map(_ => msgTo ! Done)
+        .map(_ => msgTo ! Done(FiniteDuration(java.time.Duration.between(time, LocalDateTime.now).toNanos, TimeUnit.NANOSECONDS)))
 
     case m =>
       msgTo ! s"Unexpected message: $m"

modules/admin/app/actors/cleanup/CleanupRunnerManager.scala

@@ -1,18 +1,21 @@
 package actors.cleanup
 
+import actors.LongRunningJob.Cancel
 import actors.Ticker.Tick
 import actors.cleanup.CleanupRunner.CleanupJob
 import akka.actor.{Actor, ActorContext, ActorLogging, ActorRef, Terminated}
+import play.api.i18n.Messages
 import services.ingest.Cleanup
 import utils.WebsocketConstants
 
 import scala.concurrent.ExecutionContext
 import scala.concurrent.duration.DurationInt
 
+
 case class CleanupRunnerManager(
   cleanupJob: CleanupJob,
   init: ActorContext => ActorRef,
-)(implicit ec: ExecutionContext) extends Actor with ActorLogging {
+)(implicit messages: Messages, ec: ExecutionContext) extends Actor with ActorLogging {
   import CleanupRunner._
 
   override def receive: Receive = {
@@ -21,7 +24,7 @@ case class CleanupRunnerManager(
       val runner = init(context)
       context.become(running(runner, Set(chan)))
       runner ! cleanupJob
-      msg(s"Starting cleanup with job id: ${cleanupJob.jobId}", Set(chan))
+      msg(Messages("cleanup.starting", cleanupJob.jobId), Set(chan))
   }
 
   def running(runner: ActorRef, subs: Set[ActorRef]): Receive = {
@@ -35,7 +38,7 @@ case class CleanupRunnerManager(
     case Terminated(actor) if actor == runner =>
       log.debug(s"Actor terminated: $actor")
       context.system.scheduler.scheduleOnce(5.seconds, self,
-        "Harvest runner unexpectedly shut down")
+        "Cleanup runner unexpectedly shut down")
 
     // Remove terminated subscribers
     case Terminated(chan) =>
@@ -44,24 +47,31 @@ case class CleanupRunnerManager(
       context.become(running(runner, subs - chan))
 
     case cleanup: Cleanup =>
-      msg(s"Relink: ${cleanup.redirects.size}, deletions: ${cleanup.deletions.size}", subs)
+      msg(Messages("cleanup.toMove", cleanup.redirects.size), subs)
+      msg(Messages("cleanup.toDelete", cleanup.deletions.size), subs)
 
-    case Relinked(_, newStatus) =>
-      msg(s"Relinked IDs: ${newStatus.relinkCount}", subs)
+    case Relinked(_, status) =>
+      msg(Messages("cleanup.relinked", status.relinkCount), subs)
 
-    case Redirected(_, newStatus) =>
-      log.info(s"Redirected IDs: ${newStatus.redirectCount}")
+    case Redirected(_, status) =>
+      msg(Messages("cleanup.redirected", status.redirectCount), subs)
 
     case Status(_, _, deleteCount) =>
-      msg(s"Deletions: $deleteCount", subs)
+      msg(Messages("cleanup.deleted", deleteCount), subs)
 
-    case Done =>
-      msg(s"${WebsocketConstants.DONE_MESSAGE}", subs)
+    case Done(d) =>
+      log.info(s"Cleanup time: ${d.toSeconds} seconds")
+      msg(Messages("cleanup.done"), subs)
+      msg(WebsocketConstants.DONE_MESSAGE, subs)
       context.stop(self)
 
     case Tick(s) =>
       msg(s, subs)
 
+    case Cancel =>
+      msg(Messages("cleanup.cancelled"), subs)
+      context.stop(self)
+
     case m =>
       msg(s"${WebsocketConstants.ERR_MESSAGE}: Unexpected message: $m", subs)
       context.stop(self)

modules/admin/app/assets/css/datasets.scss

@@ -53,12 +53,6 @@ $active-table-row: #e7f1ff;
   }
 }
 
-#batch-ops-log {
-  display: flex;
-  border: 1px solid $border-color;
-  height: 10rem;
-}
-
 .flash-notice {
   animation: highlight 1000ms ease-out;
 }
@@ -819,6 +813,12 @@ $active-table-row: #e7f1ff;
   overflow: hidden !important;
 }
 
+.modal-body .log-container {
+  display: flex;
+  border: 1px solid $border-color;
+  height: 10rem;
+}
+
 .log-container pre {
   padding: $margin-xs;
   margin: 0;

modules/admin/app/assets/js/datasets/components/_manager-snapshots.vue

@@ -2,23 +2,21 @@
 
 import MixinUtil from './_mixin-util';
 import MixinError from './_mixin-error';
-import MixinTasklog from './_mixin-tasklog.vue';
 import ModalAlert from './_modal-alert';
 import ModalSnapshotConfig from './_modal-snapshot-config';
-import PanelLogWindow from './_panel-log-window';
 import ModalCleanupConfig from './_modal-cleanup-config.vue';
 import {DatasetManagerApi} from "../api";
 import {Snapshot} from "../types";
 import {timeToRelative, displayDate} from "../common";
 
 export default {
-  mixins: [MixinUtil, MixinError, MixinTasklog],
-  components: {ModalAlert, ModalSnapshotConfig, PanelLogWindow, ModalCleanupConfig},
+  mixins: [MixinUtil, MixinError],
+  components: {ModalAlert, ModalSnapshotConfig, ModalCleanupConfig},
   props: {
     config: Object,
     api: DatasetManagerApi,
   },
-  data: function() {
+  data: function () {
     return {
       snapshots: [],
       current: null,
@@ -34,7 +32,7 @@ export default {
     }
   },
   methods: {
-    loadCleanup: async function(snapshot: Snapshot) {
+    loadCleanup: async function (snapshot: Snapshot) {
       this.loadingCleanup = true;
       try {
         this.actions = await this.api.listCleanups(snapshot.id);
@@ -45,11 +43,11 @@ export default {
         this.loadingCleanup = false;
       }
     },
-    load: async function(snapshot: Snapshot) {
+    load: async function (snapshot: Snapshot) {
       this.current = snapshot;
       await this.loadCleanup(snapshot);
     },
-    takeSnapshot: async function(notes) {
+    takeSnapshot: async function (notes) {
       this.showCreateDialog = false;
       this.inProgress = true;
       try {
@@ -61,7 +59,7 @@ export default {
         this.inProgress = false;
       }
     },
-    refresh: async function() {
+    refresh: async function () {
       this.loading = true;
       try {
         this.snapshots = await this.api.listSnapshots();
@@ -71,7 +69,7 @@ export default {
         this.loading = false;
       }
     },
-    isFiltered: function() {
+    isFiltered: function () {
       return this.filter.trim() !== "";
     },
     timeToRelative,
@@ -88,28 +86,25 @@ export default {
                            v-on:create="takeSnapshot"
                            v-on:close="showCreateDialog = false"/>
 
-    <modal-cleanup-config v-if="showCleanupDialog && cleanup !== null && current != null"
-                          v-on:close="showCleanupDialog = false"
-                          v-bind:snapshot="current"
-                          v-bind:cleanup="cleanup"
-                          v-bind:api="api"
-                          v-bind:config="config" />
-
     <div class="actions-bar">
       <div class="filter-control">
         <label class="sr-only">Filter Snapshots</label>
-        <input v-model="filter" v-bind:disabled="current || snapshots.length===0" type="text" placeholder="Filter snapshots..." class="filter-input form-control form-control-sm">
-        <i class="filtering-indicator fa fa-close fa-fw" style="cursor: pointer" v-on:click="filter = ''" v-if="isFiltered()"/>
+        <input v-model="filter" v-bind:disabled="current || snapshots.length===0" type="text"
+               placeholder="Filter snapshots..." class="filter-input form-control form-control-sm">
+        <i class="filtering-indicator fa fa-close fa-fw" style="cursor: pointer" v-on:click="filter = ''"
+           v-if="isFiltered()"/>
       </div>
 
-      <button v-if="current" v-bind:disabled="inProgress || cleanupRunning" v-on:click="current = null" class="btn btn-sm btn-default">
+      <button v-if="current" v-bind:disabled="inProgress || cleanupRunning" v-on:click="current = null"
+              class="btn btn-sm btn-default">
         <i class="fa fa-fw fa-arrow-left"></i>
         Back to list
       </button>
-      <button v-else v-bind:disabled="inProgress || cleanupRunning" v-on:click.prevent="showCreateDialog = true" class="btn btn-sm btn-info">
+      <button v-else v-bind:disabled="inProgress || cleanupRunning" v-on:click.prevent="showCreateDialog = true"
+              class="btn btn-sm btn-info">
         <i v-if="!inProgress" class="fa fa-fw fa-list"></i>
         <i v-else class="fa fa-fw fa-circle-o-notch fa-spin"></i>
-          Create Snapshot
+        Create Snapshot
       </button>
     </div>
     <p class="admin-help-notice">
@@ -121,14 +116,25 @@ export default {
 
     <div v-if="current" id="snapshot-manager-inspector">
       <h4 v-bind:title="current.created">Snapshot taken: {{ timeToRelative(current.created) }}</h4>
-      <div class="alert alert-info" v-for="[id, date] in actions">
+      <div class="alert alert-info" v-for="[id, date] in actions.slice(0, 3)">
         Cleanup run on {{ displayDate(date) }}.
       </div>
       <p v-if="loadingCleanup">
         Loading heuristic cleanup...
         <i class="fa fa-spin fa-spinner"></i>
       </p>
       <template v-else-if="cleanup !== null">
+        <keep-alive>
+          <modal-cleanup-config v-if="showCleanupDialog"
+                                v-bind:visible="showCleanupDialog"
+                                v-bind:snapshot="current"
+                                v-bind:cleanup="cleanup"
+                                v-bind:api="api"
+                                v-bind:config="config"
+                                v-on:close="showCleanupDialog = false"
+                                v-on:run-cleanup="cleanupRunning = true"
+                                v-on:cleanup-complete="cleanupRunning = false"/>
+        </keep-alive>
         <h5>Heuristic redirects: <strong>{{ cleanup.redirects.length }}</strong></h5>
         <p class="admin-help-notice">
           Inferred redirects are inferred from items pending cleanup which share local identifiers
@@ -144,14 +150,17 @@ export default {
         </p>
         <textarea readonly class="form-control" id="snapshot-manager-diff">{{ cleanup.deletions.join("\n") }}</textarea>
 
-        <button v-bind:disabled="cleanupRunning" v-on:click.prevent="showCleanupDialog = true" class="btn btn-sm btn-danger">
+        <button v-on:click.prevent="showCleanupDialog = true" class="btn btn-sm btn-secondary">
+          <i v-if="!cleanupRunning" class="fa fa-fw fa-trash"></i>
+          <i v-else class="fa fa-fw fa-circle-o-notch fa-spin"></i>
           Perform Cleanup
         </button>
       </template>
     </div>
     <div v-else-if="snapshots" class="snapshot-manager-snapshot-list">
-      <div v-for="snapshot in snapshots" v-if="!isFiltered() || (snapshot.notes && snapshot.notes.includes(this.filter) )"
-            v-on:click.stop.prevent="load(snapshot)" class="snapshot-manager-item">
+      <div v-for="snapshot in snapshots"
+           v-if="!isFiltered() || (snapshot.notes && snapshot.notes.includes(this.filter) )"
+           v-on:click.stop.prevent="load(snapshot)" class="snapshot-manager-item">
         <div v-bind:title="snapshot.created" class="snapshot-timestamp item-icon">
           <i class="fa fa-clock-o"></i>
           {{ timeToRelative(snapshot.created) }}
@@ -162,7 +171,7 @@ export default {
       </div>
     </div>
     <div v-else class="snapshot-loading-indicator">
-      <h3>Loading snapshots...</h3>
+      Loading snapshots...
       <i class="fa fa-3x fa-spinner fa-spin"></i>
     </div>
   </div>

modules/admin/app/assets/js/datasets/components/_mixin-tasklog.vue

@@ -13,6 +13,7 @@ let initialLogState = function(): object {
     jobId: null,
     cancelling: false,
     overwrite: false,
+    logDeleteLinePrefix: "Ingesting..."
   };
 };
 
@@ -31,16 +32,15 @@ export default {
 
     println: function(...msg: string[]) {
       console.debug(...msg);
-      let progPrefix = "Ingesting..."
       let line = msg.join(' ');
       if (this.overwrite) {
         this.log.write("\x1b[F");
       }
       this.log.writeln(line);
-      this.overwrite = _startsWith(line, progPrefix);
+      this.overwrite = _startsWith(line, this.logDeleteLinePrefix);
     },
 
-    monitor: async function(url: string, jobId: string, onMsg: (s: string) => any = function () {}) {
+    monitor: async function(url: string, jobId: string, onMsg: (s: string) => any = function () {}, clear: boolean = false) {
       this.jobId = jobId;
       return await new Promise(((resolve) => {
         let worker = new Worker(this.config.previewLoader);