perf: fast copy (#209)

* perf: fast copy

* fix(api): swagger comment

api/cache/file_cache.go

@@ -67,6 +67,17 @@ func (c *FileCache) Refresh(id string) (model.File, error) {
 	return res, nil
 }
 
+func (c *FileCache) RefreshWithExisting(file model.File, userID string) (model.File, error) {
+	err := c.fileRepo.PopulateModelFieldsForUser([]model.File{file}, userID)
+	if err != nil {
+		return nil, err
+	}
+	if err = c.Set(file); err != nil {
+		return nil, err
+	}
+	return file, nil
+}
+
 func (c *FileCache) Delete(id string) error {
 	if err := c.redis.Delete(c.keyPrefix + id); err != nil {
 		return nil

api/docs/swagger.yaml

@@ -98,15 +98,6 @@ definitions:
       size:
         type: integer
     type: object
-  router.FileCopyOptions:
-    properties:
-      ids:
-        items:
-          type: string
-        type: array
-    required:
-    - ids
-    type: object
   router.FileDeleteOptions:
     properties:
       ids:
@@ -273,6 +264,33 @@ definitions:
       workspaceId:
         type: string
     type: object
+  service.FileCopyManyOptions:
+    properties:
+      sourceIds:
+        items:
+          type: string
+        type: array
+      targetId:
+        type: string
+    required:
+    - sourceIds
+    - targetId
+    type: object
+  service.FileCopyManyResult:
+    properties:
+      failed:
+        items:
+          type: string
+        type: array
+      new:
+        items:
+          type: string
+        type: array
+      succeeded:
+        items:
+          type: string
+        type: array
+    type: object
   service.FileList:
     properties:
       data:
@@ -950,22 +968,21 @@ paths:
       summary: Patch
       tags:
       - Files
-  /files/{id}/copy:
+  /files/{id}/copy/{targetId}:
     post:
-      description: Copy
-      operationId: files_copy
+      description: Copy One
+      operationId: files_copy_one
       parameters:
       - description: ID
         in: path
         name: id
         required: true
         type: string
-      - description: Body
-        in: body
-        name: body
+      - description: Target ID
+        in: path
+        name: targetId
         required: true
-        schema:
-          $ref: '#/definitions/router.FileCopyOptions'
+        type: string
       produces:
       - application/json
       responses:
@@ -977,7 +994,7 @@ paths:
           description: Internal Server Error
           schema:
             $ref: '#/definitions/errorpkg.ErrorResponse'
-      summary: Copy
+      summary: Copy One
       tags:
       - Files
   /files/{id}/count:
@@ -1433,6 +1450,35 @@ paths:
       summary: Get User Permissions
       tags:
       - Files
+  /files/copy:
+    post:
+      description: Copy Many
+      operationId: files_copy_many
+      parameters:
+      - description: Body
+        in: body
+        name: body
+        required: true
+        schema:
+          $ref: '#/definitions/service.FileCopyManyOptions'
+      produces:
+      - application/json
+      responses:
+        "200":
+          description: OK
+          schema:
+            $ref: '#/definitions/service.FileCopyManyResult'
+        "404":
+          description: Not Found
+          schema:
+            $ref: '#/definitions/errorpkg.ErrorResponse'
+        "500":
+          description: Internal Server Error
+          schema:
+            $ref: '#/definitions/errorpkg.ErrorResponse'
+      summary: Copy Many
+      tags:
+      - Files
   /files/create_from_s3:
     post:
       consumes:

api/helper/filename.go

@@ -0,0 +1,25 @@
+// Copyright 2023 Anass Bouassaba.
+//
+// Use of this software is governed by the Business Source License
+// included in the file licenses/BSL.txt.
+//
+// As of the Change Date specified in that file, in accordance with
+// the Business Source License, use of this software will be governed
+// by the GNU Affero General Public License v3.0 only, included in the file
+// licenses/AGPL.txt.
+
+package helper
+
+import (
+	"fmt"
+	"path/filepath"
+)
+
+func UniqueFilename(name string) string {
+	return fmt.Sprintf("%s %s%s", FilenameWithoutExtension(name), NewID(), filepath.Ext(name))
+}
+
+func FilenameWithoutExtension(name string) string {
+	withExt := filepath.Base(name)
+	return withExt[0 : len(withExt)-len(filepath.Ext(name))]
+}

api/model/file_model.go

@@ -34,6 +34,8 @@ type File interface {
 	SetName(string)
 	SetText(*string)
 	SetSnapshotID(*string)
+	SetUserPermissions([]CoreUserPermission)
+	SetGroupPermissions([]CoreGroupPermission)
 	SetCreateTime(string)
 	SetUpdateTime(*string)
 }

api/repo/file_repo.go

@@ -27,6 +27,7 @@ type FileRepo interface {
 	FindChildren(id string) ([]model.File, error)
 	FindPath(id string) ([]model.File, error)
 	FindTree(id string) ([]model.File, error)
+	FindTreeIDs(id string) ([]string, error)
 	Count() (int64, error)
 	GetIDsByWorkspace(workspaceID string) ([]string, error)
 	GetIDsBySnapshot(snapshotID string) ([]string, error)
@@ -43,6 +44,7 @@ type FileRepo interface {
 	RevokeUserPermission(tree []model.File, userID string) error
 	GrantGroupPermission(id string, groupID string, permission string) error
 	RevokeGroupPermission(tree []model.File, groupID string) error
+	PopulateModelFieldsForUser(files []model.File, userID string) error
 }
 
 func NewFileRepo() FileRepo {
@@ -162,6 +164,20 @@ func (f *fileEntity) SetSnapshotID(snapshotID *string) {
 	f.SnapshotID = snapshotID
 }
 
+func (f *fileEntity) SetUserPermissions(permissions []model.CoreUserPermission) {
+	f.UserPermissions = make([]*UserPermissionValue, len(permissions))
+	for i, p := range permissions {
+		f.UserPermissions[i] = p.(*UserPermissionValue)
+	}
+}
+
+func (f *fileEntity) SetGroupPermissions(permissions []model.CoreGroupPermission) {
+	f.GroupPermissions = make([]*GroupPermissionValue, len(permissions))
+	for i, p := range permissions {
+		f.GroupPermissions[i] = p.(*GroupPermissionValue)
+	}
+}
+
 func (f *fileEntity) SetCreateTime(createTime string) {
 	f.CreateTime = createTime
 }
@@ -301,6 +317,28 @@ func (repo *fileRepo) FindTree(id string) ([]model.File, error) {
 	return res, nil
 }
 
+func (repo *fileRepo) FindTreeIDs(id string) ([]string, error) {
+	type Value struct {
+		Result string
+	}
+	var values []Value
+	db := repo.db.
+		Raw(`WITH RECURSIVE rec (id, parent_id, create_time) AS
+             (SELECT f.id, f.parent_id, f.create_time FROM file f WHERE f.id = ?
+             UNION SELECT f.id, f.parent_id, f.create_time FROM rec, file f WHERE f.parent_id = rec.id)
+             SELECT rec.id as result FROM rec ORDER BY create_time ASC`,
+			id).
+		Scan(&values)
+	if db.Error != nil {
+		return nil, db.Error
+	}
+	res := []string{}
+	for _, v := range values {
+		res = append(res, v.Result)
+	}
+	return res, nil
+}
+
 func (repo *fileRepo) Count() (int64, error) {
 	type Result struct {
 		Result int64
@@ -589,6 +627,19 @@ func (repo *fileRepo) RevokeGroupPermission(tree []model.File, groupID string) e
 	return nil
 }
 
+func (repo *fileRepo) PopulateModelFieldsForUser(files []model.File, userID string) error {
+	for _, f := range files {
+		userPermissions := make([]model.CoreUserPermission, 0)
+		userPermissions = append(userPermissions, &UserPermissionValue{
+			UserID: userID,
+			Value:  model.PermissionOwner,
+		})
+		f.SetUserPermissions(userPermissions)
+		f.SetGroupPermissions(make([]model.CoreGroupPermission, 0))
+	}
+	return nil
+}
+
 func (repo *fileRepo) populateModelFields(entities []*fileEntity) error {
 	for _, f := range entities {
 		f.UserPermissions = make([]*UserPermissionValue, 0)

api/repo/snapshot_repo.go

@@ -36,6 +36,7 @@ type SnapshotRepo interface {
 	Delete(id string) error
 	Update(id string, opts SnapshotUpdateOptions) error
 	MapWithFile(id string, fileID string) error
+	BulkMapWithFile(entities []*SnapshotFileEntity, chunkSize int) error
 	DeleteMappingsForFile(fileID string) error
 	DeleteAllDangling() error
 	GetLatestVersionForFile(fileID string) (int64, error)
@@ -349,6 +350,21 @@ func (s *snapshotEntity) GetUpdateTime() *string {
 	return s.UpdateTime
 }
 
+type SnapshotFileEntity struct {
+	SnapshotID string `gorm:"column:snapshot_id"`
+	FileID     string `gorm:"column:file_id"`
+	CreateTime string `gorm:"column:create_time"`
+}
+
+func (*SnapshotFileEntity) TableName() string {
+	return "snapshot_file"
+}
+
+func (s *SnapshotFileEntity) BeforeCreate(*gorm.DB) (err error) {
+	s.CreateTime = helper.NewTimestamp()
+	return nil
+}
+
 type snapshotRepo struct {
 	db *gorm.DB
 }
@@ -492,6 +508,13 @@ func (repo *snapshotRepo) MapWithFile(id string, fileID string) error {
 	return nil
 }
 
+func (repo *snapshotRepo) BulkMapWithFile(entities []*SnapshotFileEntity, chunkSize int) error {
+	if db := repo.db.CreateInBatches(entities, chunkSize); db.Error != nil {
+		return db.Error
+	}
+	return nil
+}
+
 func (repo *snapshotRepo) DeleteMappingsForFile(fileID string) error {
 	if db := repo.db.Exec("DELETE FROM snapshot_file WHERE file_id = ?", fileID); db.Error != nil {
 		return db.Error

api/router/file_router.go

@@ -62,6 +62,7 @@ func NewFileRouter() *FileRouter {
 func (r *FileRouter) AppendRoutes(g fiber.Router) {
 	g.Post("/", r.Create)
 	g.Get("/list", r.ListByPath)
+	g.Post("/copy", r.CopyMany)
 	g.Get("/", r.GetByPath)
 	g.Delete("/", r.Delete)
 	g.Get("/:id", r.Get)
@@ -71,7 +72,7 @@ func (r *FileRouter) AppendRoutes(g fiber.Router) {
 	g.Get("/:id/path", r.GetPath)
 	g.Post("/:id/move", r.Move)
 	g.Patch("/:id/name", r.PatchName)
-	g.Post("/:id/copy", r.Copy)
+	g.Post("/:id/copy/:targetId", r.CopyOne)
 	g.Get("/:id/size", r.GetSize)
 	g.Post("/grant_user_permission", r.GrantUserPermission)
 	g.Post("/revoke_user_permission", r.RevokeUserPermission)
@@ -427,28 +428,49 @@ type FileCopyOptions struct {
 	IDs []string `json:"ids" validate:"required"`
 }
 
-// Copy godoc
+// CopyOne godoc
 //
-//	@Summary		Copy
-//	@Description	Copy
+//	@Summary		Copy One
+//	@Description	Copy One
 //	@Tags			Files
-//	@Id				files_copy
+//	@Id				files_copy_one
 //	@Produce		json
-//	@Param			id		path		string			true	"ID"
-//	@Param			body	body		FileCopyOptions	true	"Body"
+//	@Param			id			path		string	true	"ID"
+//	@Param			targetId	path		string	true	"Target ID"
+//	@Failure		404			{object}	errorpkg.ErrorResponse
+//	@Failure		500			{object}	errorpkg.ErrorResponse
+//	@Router			/files/{id}/copy/{targetId} [post]
+func (r *FileRouter) CopyOne(c *fiber.Ctx) error {
+	userID := GetUserID(c)
+	res, err := r.fileSvc.CopyOne(c.Params("id"), c.Params("targetId"), userID)
+	if err != nil {
+		return err
+	}
+	return c.JSON(res)
+}
+
+// CopyMany godoc
+//
+//	@Summary		Copy Many
+//	@Description	Copy Many
+//	@Tags			Files
+//	@Id				files_copy_many
+//	@Produce		json
+//	@Param			body	body		service.FileCopyManyOptions	true	"Body"
+//	@Success		200		{object}	service.FileCopyManyResult
 //	@Failure		404		{object}	errorpkg.ErrorResponse
 //	@Failure		500		{object}	errorpkg.ErrorResponse
-//	@Router			/files/{id}/copy [post]
-func (r *FileRouter) Copy(c *fiber.Ctx) error {
+//	@Router			/files/copy [post]
+func (r *FileRouter) CopyMany(c *fiber.Ctx) error {
 	userID := GetUserID(c)
-	opts := new(FileCopyOptions)
+	opts := new(service.FileCopyManyOptions)
 	if err := c.BodyParser(opts); err != nil {
 		return err
 	}
 	if err := validator.New().Struct(opts); err != nil {
 		return errorpkg.NewRequestBodyValidationError(err)
 	}
-	res, err := r.fileSvc.Copy(c.Params("id"), opts.IDs, userID)
+	res, err := r.fileSvc.CopyMany(*opts, userID)
 	if err != nil {
 		return err
 	}