Detect and propagate renames/moves

A rename or a move is currently seen as a delete on the old path and a
create on the new path. This means that propagating what was from
users's perspective a simple rename can require copying gigabytes of
data for big files and directories (this is in best case scenario when
local copy shortcut is used instead of transmitting those gigabytes over
a network).

Add new functionality that enables detecting renames and moves, and
propagating them without copying any data. If this is not possible (due
to conflicts, errors or user actions) then it falls back to copying, as
before. Add a new user preference to control this (defaults to "off").

This is just a shortcut (akin to copying locally instead of transmitting
over a network).

Renames/moves are detected for files and directories, and only if the
contents have not changed (for directories, "contents" means the names
and contents of all its children, recursively).

src/.depend

@@ -633,6 +633,37 @@ main.cmx : \
     remote.cmx \
     ubase/prefs.cmx \
     os.cmx
+moves.cmo : \
+    uutil.cmi \
+    ubase/util.cmi \
+    update.cmi \
+    ubase/trace.cmi \
+    ubase/safelist.cmi \
+    props.cmi \
+    ubase/prefs.cmi \
+    path.cmi \
+    os.cmi \
+    name.cmi \
+    features.cmi \
+    common.cmi \
+    moves.cmi
+moves.cmx : \
+    uutil.cmx \
+    ubase/util.cmx \
+    update.cmx \
+    ubase/trace.cmx \
+    ubase/safelist.cmx \
+    props.cmx \
+    ubase/prefs.cmx \
+    path.cmx \
+    os.cmx \
+    name.cmx \
+    features.cmx \
+    common.cmx \
+    moves.cmi
+moves.cmi : \
+    path.cmi \
+    common.cmi
 name.cmo : \
     ubase/util.cmi \
     ubase/umarshal.cmi \
@@ -846,6 +877,7 @@ recon.cmo : \
     pred.cmi \
     path.cmi \
     name.cmi \
+    moves.cmi \
     globals.cmi \
     fileinfo.cmi \
     common.cmi \
@@ -864,6 +896,7 @@ recon.cmx : \
     pred.cmx \
     path.cmx \
     name.cmx \
+    moves.cmx \
     globals.cmx \
     fileinfo.cmx \
     common.cmx \
@@ -1056,7 +1089,7 @@ test.cmo : \
     ubase/prefs.cmi \
     path.cmi \
     os.cmi \
-    lwt/lwt_util.cmi \
+    moves.cmi \
     lwt/lwt_unix.cmi \
     lwt/lwt.cmi \
     globals.cmi \
@@ -1080,7 +1113,7 @@ test.cmx : \
     ubase/prefs.cmx \
     path.cmx \
     os.cmx \
-    lwt/lwt_util.cmx \
+    moves.cmx \
     lwt/lwt_unix.cmx \
     lwt/lwt.cmx \
     globals.cmx \
@@ -1124,6 +1157,7 @@ transport.cmo : \
     ubase/prefs.cmi \
     path.cmi \
     osx.cmi \
+    moves.cmi \
     lwt/lwt_util.cmi \
     lwt/lwt.cmi \
     globals.cmi \
@@ -1140,6 +1174,7 @@ transport.cmx : \
     ubase/prefs.cmx \
     path.cmx \
     osx.cmx \
+    moves.cmx \
     lwt/lwt_util.cmx \
     lwt/lwt.cmx \
     globals.cmx \

src/Makefile.OCaml

@@ -240,7 +240,7 @@ OCAMLOBJS += \
           props.cmo fileinfo.cmo os.cmo lock.cmo clroot.cmo common.cmo \
           tree.cmo checksum.cmo transfer.cmo xferhint.cmo \
           remote.cmo external.cmo negotiate.cmo globals.cmo fswatchold.cmo \
-          fpcache.cmo update.cmo copy.cmo stasher.cmo \
+          fpcache.cmo update.cmo moves.cmo copy.cmo stasher.cmo \
 	  files.cmo sortri.cmo recon.cmo transport.cmo \
           strings.cmo uicommon.cmo uitext.cmo test.cmo
 

src/common.ml

@@ -328,9 +328,11 @@ type status =
   | `Modified
   | `PropsChanged
   | `Created
+  | `MovedOut of Path.t * replicaContent * replicaContent
+  | `MovedIn of Path.t * replicaContent * replicaContent
   | `Unchanged ]
 
-type replicaContent =
+and replicaContent =
   { typ : Fileinfo.typ;
     status : status;
     desc : Props.t;                (* Properties (for the UI) *)
@@ -390,7 +392,12 @@ let rcLength rc rc' =
   if riAction rc rc' = `SetProps then
     Uutil.Filesize.zero
   else
-    snd rc.size
+    match rc'.status with
+    | `MovedIn _ ->
+        (* A move/rename will be reverted, count its size too *)
+        Uutil.Filesize.add (snd rc.size) (snd rc'.size)
+    | _ ->
+        snd rc.size
 
 let riLength ri =
   match ri.replicas with

src/common.mli

@@ -126,15 +126,81 @@ val uc_of_compat251 : updateContent251 -> updateContent
 (*            COMMON TYPES SHARED BY RECONCILER AND TRANSPORT AGENT          *)
 (*****************************************************************************)
 
+(* `MovedOut is set as the status on the old path. In this case, the new path
+   will not have a separate difference record at all (the corresponding
+   replicaContent records for both replicas are embedded in the `MovedOut
+   status). Status of the new path is guaranteed to be not conflicting.
+   `MovedOut is a combination of `Deleted on the old path and `Created on the
+   new path. The virtual status equivalent for both paths combined is
+   `Unchanged or `PropsChanged, meaning that except for the path change (and
+   potentially the props), the file/dir contents have not changed.
+
+       (in the illustrations below, the boxes with double lines represent
+          the two replicaContents of the one difference record that is
+                  visible to the user and will be propagated)
+
+                         REPLICA A                          REPLICA B
+
+                                                on path n'
+                              /   +--------------+      +-----------------+
+         on path n           |    | p = `Created |      | q = `Unchanged  |
+  +======================+   |    +--------------+      +-----------------+
+  |                      |   /
+  | `MovedOut (n', p, q) |  <                   on path n
+  |                      |   \    +--------------+      +=================+
+  +======================+   |    |   `Deleted   |      | anything except |
+                             |    +--------------+      |    `Deleted     |
+                              \                         +=================+
+
+
+   If `MovedOut can not be set (for example, there is a conflict on the new
+   path) then `MovedIn may be set instead. `MovedOut and `MovedIn are never
+   used together on a pair of paths.
+   `MovedIn is set as the status of the new path. In this case, the old path
+   will not have a separate difference record at all (the corresponding
+   replicaContent records for both replicas are embedded in the `MovedIn
+   status). Status of the old path is guaranteed to be not conflicting.
+   `MovedIn is a combination of `Created on the new path and `Deleted on the
+   old path. The virtual status equivalent for both paths combined is
+   `Unchanged or `PropsChanged, meaning tat except for the path change (and
+   potentially the props), the file/dir contents have not changed.
+
+                         REPLICA A                          REPLICA B
+
+                                                on path n
+                              /   +--------------+      +================+
+         on path n           |    |   `Created   |      |    anything    |
+  +======================+   |    +--------------+      +================+
+  |                      |   /
+  | `MovedIn  (n', p, q) |  <
+  |                      |   \                  on path n'
+  +======================+   |    +--------------+      +----------------+
+                             |    | p = `Deleted |      | q = `Unchanged |
+                              \   +--------------+      +----------------+
+
+  (Usually the status of replica b on path n will not be `Unchanged, because
+   then `MovedOut will be used instead. It can be `Unchanged if typ is not
+   `ABSENT. In other words, `Create in replica b is overwriting something.)
+
+   Note that even though path for only one replica is recorded in `MovedOut/
+   `MovedIn, it will not cause trouble when in case insensitive mode on a
+   case sensitive filesystem (commit 005a53075b998dba27eeff74a1fc8f9d73558fb8
+   for details). Correct path translation is done by [Update.translatePath]. *)
 type status =
   [ `Deleted
   | `Modified
   | `PropsChanged
   | `Created
+  | `MovedOut of Path.t          (* new path *)
+              * replicaContent   (* rc of new path, this replica *)
+              * replicaContent   (* rc of new path, other replica *)
+  | `MovedIn of Path.t           (* old path *)
+              * replicaContent   (* rc of old path, this replica *)
+              * replicaContent   (* rc of old path, other replica *)
   | `Unchanged ]
 
 (* Variable name prefix: "rc" *)
-type replicaContent =
+and replicaContent =
   { typ : Fileinfo.typ;
     status : status;
     desc : Props.t;                (* Properties (for the UI) *)