Add ZipStore storage backend.

This commit adds a ZipStore storage backend as described in the
specification zarr-developers/zarr-specs#311 .
Note that the implementation loads the entire zip archive into memory so
care must be taken to ensure the zip archive is not too big to fit into
the machine's memory. To use a ZipStore impelementation that does not
load the archive into memory see `examples/zipstore.ml`.

dune-project

@@ -30,6 +30,7 @@
      (and (>= 4.14.0)))
    (yojson (>= 1.6.0))
    (stdint (>= 0.7.2))
+   (zipc (>= 0.2.0))
    (checkseum (>= 0.4.0))
    (odoc :with-doc)
    (ounit2 :with-test)

examples/dune

@@ -4,12 +4,6 @@
  (ocamlopt_flags (:standard -O3))
  (libraries zarr-eio camlzip))
 
-(executable
- (name inmemory_zipstore)
- (modules inmemory_zipstore)
- (ocamlopt_flags (:standard -O3))
- (libraries zarr-lwt zipc))
-
 (executable
  (name picos_fs_store)
  (modules picos_fs_store)

zarr-eio/src/storage.ml

@@ -4,6 +4,12 @@ module MemoryStore = struct
   let create = M.create
 end
 
+module ZipStore = struct
+  module Z = Zarr.Zip.Make(Deferred)
+  include Zarr.Storage.Make(Z)
+  let with_open = Z.with_open
+end
+
 module FilesystemStore = struct
   module FS = struct
     module Deferred = Deferred

zarr-eio/src/storage.mli

@@ -1,6 +1,9 @@
 (** An in-memory storage backend for Zarr V3 hierarchy. *)
 module MemoryStore : sig include Zarr.Memory.S with type 'a Deferred.t = 'a end
 
+(** An Eio-aware Zip file storage backend for a Zarr v3 hierarchy. *)
+module ZipStore : sig include Zarr.Zip.S with type 'a Deferred.t = 'a end
+
 module FilesystemStore : sig
   (** A local filesystem storage backend for a Zarr V3 hierarchy. *)
 

zarr-eio/test/test_eio.ml

@@ -143,6 +143,11 @@ let _ =
         (Zarr.Storage.Not_a_filesystem_store fn)
         (fun () -> FilesystemStore.open_store ~env fn);
 
+      (* test with non-existant archive *)
+      let zpath = tmp_dir ^ ".zip" in
+      ZipStore.with_open `Read_write zpath (fun z -> test_storage (module ZipStore) z);
+      (* test just opening the now exisitant archive created by the previous test. *)
+      ZipStore.with_open `Read_only zpath (fun _ -> ZipStore.Deferred.return_unit);
       test_storage (module MemoryStore) @@ MemoryStore.create ();
       test_storage (module FilesystemStore) s)
   ])

zarr-lwt/src/storage.ml

@@ -4,6 +4,12 @@ module MemoryStore = struct
   let create = M.create
 end
 
+module ZipStore = struct
+  module Z = Zarr.Zip.Make(Deferred)
+  include Zarr.Storage.Make(Z)
+  let with_open = Z.with_open
+end
+
 module FilesystemStore = struct
   module FS = struct
     module Deferred = Deferred

zarr-lwt/src/storage.mli

@@ -1,6 +1,9 @@
 (** An in-memory storage backend for Zarr V3 hierarchy. *)
 module MemoryStore : sig include Zarr.Memory.S with type 'a Deferred.t = 'a Lwt.t end
 
+(** An Lwt-aware Zip file storage backend for a Zarr v3 hierarchy. *)
+module ZipStore : sig include Zarr.Zip.S with type 'a Deferred.t = 'a Lwt.t end
+
 module FilesystemStore : sig
   (** A local filesystem storage backend for a Zarr V3 hierarchy. *)
 

zarr-lwt/test/test_lwt.ml

@@ -141,8 +141,11 @@ let _ =
         (Zarr.Storage.Not_a_filesystem_store fn)
         (fun () -> FilesystemStore.open_store fn);
 
-      Lwt_main.run @@
-      Lwt.join 
-        [test_storage (module MemoryStore) @@ MemoryStore.create ()
+      let zpath = tmp_dir ^ ".zip" in
+      Lwt_main.run @@ Lwt.join 
+        [ZipStore.with_open `Read_write zpath (fun z -> test_storage (module ZipStore) z)
+         (* test just opening the now exisitant archive created by the previous test. *)
+        ;ZipStore.with_open `Read_only zpath (fun _ -> ZipStore.Deferred.return_unit)
+        ;test_storage (module MemoryStore) @@ MemoryStore.create ()
         ;test_storage (module FilesystemStore) s])
 ])

zarr-sync/src/storage.ml

@@ -4,6 +4,12 @@ module MemoryStore = struct
   let create = M.create
 end
 
+module ZipStore = struct
+  module Z = Zarr.Zip.Make(Deferred)
+  include Zarr.Storage.Make(Z)
+  let with_open = Z.with_open
+end
+
 module FilesystemStore = struct
   module F = struct
     module Deferred = Deferred

zarr-sync/src/storage.mli

@@ -1,6 +1,9 @@
 (** An in-memory storage backend for Zarr V3 hierarchy. *)
 module MemoryStore : sig include Zarr.Memory.S with type 'a Deferred.t = 'a end
 
+(** A blocking I/O Zip file storage backend for a Zarr v3 hierarchy. *)
+module ZipStore : sig include Zarr.Zip.S with type 'a Deferred.t = 'a end
+
 module FilesystemStore : sig
   (** A local filesystem storage backend for a Zarr V3 hierarchy. *)
 

zarr-sync/test/test_sync.ml

@@ -203,6 +203,11 @@ let _ =
         (Zarr.Storage.Not_a_filesystem_store fn)
         (fun () -> FilesystemStore.open_store fn);
 
+      (* test with non-existant archive *)
+      let zpath = tmp_dir ^ ".zip" in
+      ZipStore.with_open `Read_write zpath (fun z -> test_storage (module ZipStore) z);
+      (* test just opening the now exisitant archive created by the previous test. *)
+      ZipStore.with_open `Read_only zpath (fun _ -> ZipStore.Deferred.return_unit);
       test_storage (module MemoryStore) @@ MemoryStore.create ();
       test_storage (module FilesystemStore) s)
   ])

zarr.opam

@@ -18,6 +18,7 @@ depends: [
   "ocaml" {>= "4.14.0"}
   "yojson" {>= "1.6.0"}
   "stdint" {>= "0.7.2"}
+  "zipc" {>= "0.2.0"}
   "checkseum" {>= "0.4.0"}
   "odoc" {with-doc}
   "ounit2" {with-test}

zarr/src/dune

@@ -5,6 +5,7 @@
    yojson
    bytesrw.zstd
    bytesrw.zlib
+   zipc
    stdint
    checkseum)
  (ocamlopt_flags