Skip to content

Commit

Permalink
feat: add arg to force update object , and a command to commit object
Browse files Browse the repository at this point in the history
  • Loading branch information
hunjixin committed Mar 17, 2024
1 parent fac84ce commit 9aa219e
Show file tree
Hide file tree
Showing 15 changed files with 398 additions and 141 deletions.
17 changes: 17 additions & 0 deletions api/custom_response.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ package api
import (
"encoding/json"
"errors"
"fmt"
"net/http"

"github.com/GitDataAI/jiaozifs/auth"
Expand Down Expand Up @@ -67,6 +68,13 @@ func (response *JiaozifsResponse) Error(err error) {
return
}

var codeErr ErrCode
if errors.As(err, &codeErr) {
response.WriteHeader(int(codeErr))
_, _ = response.Write([]byte(err.Error()))
return
}

response.WriteHeader(http.StatusInternalServerError)
_, _ = response.Write([]byte(err.Error()))
}
Expand All @@ -89,3 +97,12 @@ func (response *JiaozifsResponse) String(msg string, code ...int) {
func (response *JiaozifsResponse) Code(code int) {
response.WriteHeader(code)
}

type ErrCode int

func NewErrCode(code int) ErrCode {
return ErrCode(code)

Check warning on line 104 in api/custom_response.go

View check run for this annotation

Codecov / codecov/patch

api/custom_response.go#L103-L104

Added lines #L103 - L104 were not covered by tests
}
func (err ErrCode) Error() string {
return fmt.Sprintf("code %d msg %s", err, http.StatusText(int(err)))
}
11 changes: 11 additions & 0 deletions api/custom_response_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -77,6 +77,17 @@ func TestJiaozifsResponse(t *testing.T) {
jzResp.Error(fmt.Errorf("mock"))
})

t.Run("err code", func(t *testing.T) {
ctrl := gomock.NewController(t)
resp := NewMockResponseWriter(ctrl)
jzResp := JiaozifsResponse{resp}

resp.EXPECT().WriteHeader(http.StatusConflict)
resp.EXPECT().Write([]byte("mock code 409 msg Conflict"))

jzResp.Error(fmt.Errorf("mock %w", ErrCode(http.StatusConflict)))
})

t.Run("error not found", func(t *testing.T) {
ctrl := gomock.NewController(t)
resp := NewMockResponseWriter(ctrl)
Expand Down
215 changes: 121 additions & 94 deletions api/jiaozifs.gen.go

Large diffs are not rendered by default.

9 changes: 9 additions & 0 deletions api/swagger.yml
Original file line number Diff line number Diff line change
Expand Up @@ -1239,6 +1239,13 @@ paths:
tags:
- objects
operationId: uploadObject
parameters:
- in: query
name: isReplace
description: indicate to replace existing object or not
allowEmptyValue: true
schema:
type: boolean
x-validation-exclude-body: true
requestBody:
content:
Expand All @@ -1265,6 +1272,8 @@ paths:
description: ValidationError
401:
description: Unauthorized ValidationError
409:
description: Resource Conflict
403:
description: Forbidden
404:
Expand Down
28 changes: 14 additions & 14 deletions cmd/helper.go
Original file line number Diff line number Diff line change
@@ -1,24 +1,24 @@
package cmd

import (
"io"
"net/http"

"github.com/GitDataAI/jiaozifs/api"
"github.com/GitDataAI/jiaozifs/config"
"github.com/spf13/cobra"
)

func GetDefaultClient() (*api.Client, error) {
swagger, err := api.GetSwagger()
if err != nil {
return nil, err
}
func GetClient(cmd *cobra.Command) (*api.Client, error) {
url := cmd.Flags().Lookup("url").Value.String()
ak := cmd.Flags().Lookup("ak").Value.String()
sk := cmd.Flags().Lookup("sk").Value.String()
return api.NewClient(url, api.AkSkOption(ak, sk))

Check warning on line 15 in cmd/helper.go

View check run for this annotation

Codecov / codecov/patch

cmd/helper.go#L11-L15

Added lines #L11 - L15 were not covered by tests
}

//get runtime version
cfg, err := config.LoadConfig(cfgFile)
if err != nil {
return nil, err
}
basePath, err := swagger.Servers[0].BasePath()
func tryLogError(resp *http.Response) string {
bodyContent, err := io.ReadAll(resp.Body)

Check warning on line 19 in cmd/helper.go

View check run for this annotation

Codecov / codecov/patch

cmd/helper.go#L18-L19

Added lines #L18 - L19 were not covered by tests
if err != nil {
return nil, err
return ""

Check warning on line 21 in cmd/helper.go

View check run for this annotation

Codecov / codecov/patch

cmd/helper.go#L21

Added line #L21 was not covered by tests
}
return api.NewClient(cfg.API.Listen + basePath)
return string(bodyContent)
}
4 changes: 4 additions & 0 deletions cmd/root.go
Original file line number Diff line number Diff line change
Expand Up @@ -35,4 +35,8 @@ func init() {
_ = viper.BindPFlag("api.listen", rootCmd.PersistentFlags().Lookup("listen"))
_ = viper.BindPFlag("config", rootCmd.PersistentFlags().Lookup("config"))
_ = viper.BindPFlag("log.level", rootCmd.PersistentFlags().Lookup("log-level"))

uploadCmd.PersistentFlags().String("url", "https://127.0.0.1:34913", "url")
uploadCmd.PersistentFlags().String("ak", "", "access key")
uploadCmd.PersistentFlags().String("sk", "", "secret key")
}
139 changes: 139 additions & 0 deletions cmd/uploadfiles.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,139 @@
package cmd

import (
"errors"
"fmt"
"io/fs"
"net/http"
"os"
path2 "path"
"path/filepath"
"strings"

"github.com/GitDataAI/jiaozifs/utils"

"github.com/GitDataAI/jiaozifs/api"

"github.com/spf13/cobra"
)

// versionCmd represents the version command
var uploadCmd = &cobra.Command{
Use: "upload",
Short: "upload files to server",
RunE: func(cmd *cobra.Command, _ []string) error {
client, err := GetClient(cmd)
if err != nil {
return err
}

path, err := cmd.Flags().GetString("path")
if err != nil {
return err
}
path = path2.Clean(path)
if len(path) == 0 {
return errors.New("path must be set")
}

Check warning on line 37 in cmd/uploadfiles.go

View check run for this annotation

Codecov / codecov/patch

cmd/uploadfiles.go#L30-L37

Added lines #L30 - L37 were not covered by tests

owner, err := cmd.Flags().GetString("owner")
if err != nil {
return err
}
if len(owner) == 0 {
return errors.New("owner must be set")
}

Check warning on line 45 in cmd/uploadfiles.go

View check run for this annotation

Codecov / codecov/patch

cmd/uploadfiles.go#L39-L45

Added lines #L39 - L45 were not covered by tests

repo, err := cmd.Flags().GetString("repo")
if err != nil {
return err
}
if len(owner) == 0 || len(repo) == 0 {
return errors.New("owner and repo must be set")
}

Check warning on line 53 in cmd/uploadfiles.go

View check run for this annotation

Codecov / codecov/patch

cmd/uploadfiles.go#L47-L53

Added lines #L47 - L53 were not covered by tests

refName, err := cmd.Flags().GetString("refName")
if err != nil {
return err
}
if len(refName) == 0 {
return errors.New("refName must be set")
}

Check warning on line 61 in cmd/uploadfiles.go

View check run for this annotation

Codecov / codecov/patch

cmd/uploadfiles.go#L55-L61

Added lines #L55 - L61 were not covered by tests

uploadPath, err := cmd.Flags().GetString("uploadPath")
if err != nil {
return err
}
uploadPath = path2.Clean(uploadPath)
if len(uploadPath) == 0 {
uploadPath = "/"
}

Check warning on line 70 in cmd/uploadfiles.go

View check run for this annotation

Codecov / codecov/patch

cmd/uploadfiles.go#L63-L70

Added lines #L63 - L70 were not covered by tests

if len(path) == 0 {
return errors.New("path not set")
}

Check warning on line 74 in cmd/uploadfiles.go

View check run for this annotation

Codecov / codecov/patch

cmd/uploadfiles.go#L72-L74

Added lines #L72 - L74 were not covered by tests

st, err := os.Stat(path)
if err != nil {
return err
}

Check warning on line 79 in cmd/uploadfiles.go

View check run for this annotation

Codecov / codecov/patch

cmd/uploadfiles.go#L76-L79

Added lines #L76 - L79 were not covered by tests

var files []string
if st.IsDir() {
err = filepath.Walk(path, func(path string, info fs.FileInfo, _ error) error {
if info.IsDir() {
return nil
}
files = append(files, path)
return nil

Check warning on line 88 in cmd/uploadfiles.go

View check run for this annotation

Codecov / codecov/patch

cmd/uploadfiles.go#L81-L88

Added lines #L81 - L88 were not covered by tests
})
if err != nil {
return err
}

Check warning on line 92 in cmd/uploadfiles.go

View check run for this annotation

Codecov / codecov/patch

cmd/uploadfiles.go#L90-L92

Added lines #L90 - L92 were not covered by tests
}

fmt.Printf("Files dected, %d files need to be uploaded\n", len(files))
_, err = client.GetWip(cmd.Context(), owner, repo, &api.GetWipParams{RefName: refName})
if err != nil {
return err
}

Check warning on line 99 in cmd/uploadfiles.go

View check run for this annotation

Codecov / codecov/patch

cmd/uploadfiles.go#L95-L99

Added lines #L95 - L99 were not covered by tests

basename := filepath.Base(path)
for _, file := range files {
fs, err := os.Open(file)
if err != nil {
return err
}
relativePath := strings.Replace(file, path, "", 1)
destPath := path2.Join(uploadPath, basename, relativePath)

resp, err := client.UploadObjectWithBody(cmd.Context(), owner, repo, &api.UploadObjectParams{
RefName: refName,
// Path relative to the ref
Path: destPath,
IsReplace: utils.Bool(true),
}, "application/json", fs)
if err != nil {
return err
}

Check warning on line 118 in cmd/uploadfiles.go

View check run for this annotation

Codecov / codecov/patch

cmd/uploadfiles.go#L101-L118

Added lines #L101 - L118 were not covered by tests

if resp.StatusCode == http.StatusCreated || resp.StatusCode == http.StatusOK {
fmt.Println("Upload file success ", file, " dest path", destPath)
continue

Check warning on line 122 in cmd/uploadfiles.go

View check run for this annotation

Codecov / codecov/patch

cmd/uploadfiles.go#L120-L122

Added lines #L120 - L122 were not covered by tests
}
return fmt.Errorf("upload file failed %d, %s", resp.StatusCode, tryLogError(resp))

Check warning on line 124 in cmd/uploadfiles.go

View check run for this annotation

Codecov / codecov/patch

cmd/uploadfiles.go#L124

Added line #L124 was not covered by tests
}
return nil

Check warning on line 126 in cmd/uploadfiles.go

View check run for this annotation

Codecov / codecov/patch

cmd/uploadfiles.go#L126

Added line #L126 was not covered by tests
},
}

func init() {
rootCmd.AddCommand(uploadCmd)

uploadCmd.Flags().String("path", "", "path of files to upload")
uploadCmd.Flags().String("owner", "", "owner")
uploadCmd.Flags().String("repo", "", "repo")
uploadCmd.Flags().String("refName", "main", "branch name")
uploadCmd.Flags().String("uploadPath", "", "path to save in server")
uploadCmd.Flags().Bool("replace", true, "path to save in server")
}
2 changes: 1 addition & 1 deletion cmd/version.go
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ var versionCmd = &cobra.Command{
fmt.Println("Version ", version.UserVersion())
fmt.Println("API Version ", swagger.Info.Version)

client, err := GetDefaultClient()
client, err := GetClient(cmd)

Check warning on line 24 in cmd/version.go

View check run for this annotation

Codecov / codecov/patch

cmd/version.go#L24

Added line #L24 was not covered by tests
if err != nil {
return err
}
Expand Down
24 changes: 23 additions & 1 deletion controller/object_ctl.go
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package controller

import (
"bytes"
"context"
"errors"
"fmt"
Expand Down Expand Up @@ -388,7 +389,28 @@ func (oct ObjectController) UploadObject(ctx context.Context, w *api.JiaozifsRes

path := versionmgr.CleanPath(params.Path)
err = oct.Repo.Transaction(ctx, func(dRepo models.IRepo) error {
err = workTree.AddLeaf(ctx, path, blob)
oldData, _, err := workTree.FindBlob(ctx, path)
if err != nil && !errors.Is(err, versionmgr.ErrPathNotFound) {
return err
}

Check warning on line 395 in controller/object_ctl.go

View check run for this annotation

Codecov / codecov/patch

controller/object_ctl.go#L394-L395

Added lines #L394 - L395 were not covered by tests
if oldData == nil {
err = workTree.AddLeaf(ctx, path, blob)
if err != nil {
return err
}

Check warning on line 400 in controller/object_ctl.go

View check run for this annotation

Codecov / codecov/patch

controller/object_ctl.go#L399-L400

Added lines #L399 - L400 were not covered by tests
return dRepo.WipRepo().UpdateByID(ctx, models.NewUpdateWipParams(workRepo.CurWip().ID).SetCurrentTree(workTree.Root().Hash()))
}

if bytes.Equal(oldData.CheckSum, blob.CheckSum) {
return nil
}

if !utils.BoolValue(params.IsReplace) {
return fmt.Errorf("object exit %w", api.ErrCode(http.StatusConflict))
}

//allow to update
err = workTree.ReplaceLeaf(ctx, path, blob)

Check warning on line 413 in controller/object_ctl.go

View check run for this annotation

Codecov / codecov/patch

controller/object_ctl.go#L413

Added line #L413 was not covered by tests
if err != nil {
return err
}
Expand Down
18 changes: 9 additions & 9 deletions integrationtest/commit_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -26,9 +26,9 @@ func GetEntriesInRefSpec(ctx context.Context, urlStr string) func(c convey.C) {
_ = createRepo(ctx, client, repoName, false)
_ = createBranch(ctx, client, userName, repoName, "main", branchName)
_ = createWip(ctx, client, userName, repoName, branchName)
_ = uploadObject(ctx, client, userName, repoName, branchName, "m.dat")
_ = uploadObject(ctx, client, userName, repoName, branchName, "g/x.dat")
_ = uploadObject(ctx, client, userName, repoName, branchName, "g/m.dat")
_ = uploadObject(ctx, client, userName, repoName, branchName, "m.dat", true)
_ = uploadObject(ctx, client, userName, repoName, branchName, "g/x.dat", true)
_ = uploadObject(ctx, client, userName, repoName, branchName, "g/m.dat", true)
})

c.Convey("get wip entries", func(c convey.C) {
Expand Down Expand Up @@ -230,8 +230,8 @@ func GetEntriesInRefSpec(ctx context.Context, urlStr string) func(c convey.C) {

c.Convey("prepare data for commit test", func(_ convey.C) {
createWip(ctx, client, userName, repoName, "main")
uploadObject(ctx, client, userName, repoName, "main", "a.dat") //delete\
uploadObject(ctx, client, userName, repoName, "main", "g/m.dat") //modify
uploadObject(ctx, client, userName, repoName, "main", "a.dat", true) //delete\
uploadObject(ctx, client, userName, repoName, "main", "g/m.dat", true) //modify
_ = commitWip(ctx, client, userName, repoName, "main", "test")
})

Expand Down Expand Up @@ -389,21 +389,21 @@ func GetCommitChangesSpec(ctx context.Context, urlStr string) func(c convey.C) {
loginAndSwitch(ctx, client, userName, false)
createRepo(ctx, client, repoName, false)
createWip(ctx, client, userName, repoName, "main")
uploadObject(ctx, client, userName, repoName, "main", "m.dat")
uploadObject(ctx, client, userName, repoName, "main", "m.dat", true)
_ = commitWip(ctx, client, userName, repoName, "main", "test")

uploadObject(ctx, client, userName, repoName, "main", "g/x.dat")
uploadObject(ctx, client, userName, repoName, "main", "g/x.dat", true)
_ = commitWip(ctx, client, userName, repoName, "main", "test")

//delete
deleteObject(ctx, client, userName, repoName, "main", "g/x.dat")

//modify
deleteObject(ctx, client, userName, repoName, "main", "m.dat")
uploadObject(ctx, client, userName, repoName, "main", "m.dat")
uploadObject(ctx, client, userName, repoName, "main", "m.dat", true)

//insert
uploadObject(ctx, client, userName, repoName, "main", "g/m.dat")
uploadObject(ctx, client, userName, repoName, "main", "g/m.dat", true)
_ = commitWip(ctx, client, userName, repoName, "main", "test")

})
Expand Down
11 changes: 6 additions & 5 deletions integrationtest/helper_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ func InitCmd(ctx context.Context, jzHome string, listen string, db string) error
buf := new(bytes.Buffer)
cmd.RootCmd().SetOut(buf)
cmd.RootCmd().SetErr(buf)
cmd.RootCmd().SetArgs([]string{"init", "--listen", listen, "--db_debug", "true", "--db", db,
cmd.RootCmd().SetArgs([]string{"init", "--listen", listen, "--db_debug", "false", "--db", db,
"--config", fmt.Sprintf("%s/config.toml", jzHome), "--bs_path", fmt.Sprintf("%s/blockstore", jzHome)})

return cmd.RootCmd().ExecuteContext(ctx)
Expand Down Expand Up @@ -171,11 +171,12 @@ func createRepo(ctx context.Context, client *api.Client, repoName string, visibl
return result.JSON201
}

func uploadObject(ctx context.Context, client *api.Client, user string, repoName string, refName string, path string) *api.ObjectStats { //nolint
func uploadObject(ctx context.Context, client *api.Client, user string, repoName string, refName string, path string, replace bool) *api.ObjectStats { //nolint
resp, err := client.UploadObjectWithBody(ctx, user, repoName, &api.UploadObjectParams{
RefName: refName,
Path: path,
}, "application/octet-stream", io.LimitReader(rand.Reader, 50))
RefName: refName,
Path: path,
IsReplace: utils.Bool(replace),
}, "application/octet-stream", io.LimitReader(rand.Reader, 100))
convey.So(err, convey.ShouldBeNil)
convey.So(resp.StatusCode, convey.ShouldEqual, http.StatusCreated)

Expand Down
Loading

0 comments on commit 9aa219e

Please sign in to comment.