diff --git a/_data/bulkinsert.csv b/_data/bulkinsert.csv new file mode 100644 index 0000000..f65991f --- /dev/null +++ b/_data/bulkinsert.csv @@ -0,0 +1,11 @@ +id,name +1,item1 +2,item2 +3,item3 +4,item4 +5,item5 +6,item6 +7,item7 +8,item8 +9,item9 +10,item10 diff --git a/loader.go b/loader.go index 9ebae64..3e18188 100644 --- a/loader.go +++ b/loader.go @@ -22,8 +22,9 @@ type FixtureLoader struct { delete bool bulkInsert bool // Load Option - table string - format string + table string + format string + bulkInsertLimit int } // Option is set load option @@ -41,9 +42,9 @@ const ( ) var ( - baseNameRegexp *regexp.Regexp - formatRegexp *regexp.Regexp - bulkInsertLimit = 2000 + baseNameRegexp *regexp.Regexp + formatRegexp *regexp.Regexp + defaultBulkInsertLimit = 2000 ) func init() { @@ -97,6 +98,17 @@ func BulkInsert(bulk bool) Option { } } +// BulkInsertLimit is sets the rows limit of one bulkInsert, which is enabled when bulkInsert is true +func BulkInsertLimit(bulkInsertLimit int) Option { + return func(f *FixtureLoader) error { + if bulkInsertLimit == 0 { + return errors.New("error `bulkInsertLimit` is not allowed 0 value") + } + f.bulkInsertLimit = bulkInsertLimit + return nil + } +} + // Table set insert table name func Table(table string) Option { return func(f *FixtureLoader) error { @@ -120,8 +132,9 @@ func New(db *sql.DB, driver string, options ...Option) (FixtureLoader, error) { txManager := txmanager.NewDB(db) fl := FixtureLoader{ - txManager: txManager, - driver: driver, + txManager: txManager, + driver: driver, + bulkInsertLimit: defaultBulkInsertLimit, } for _, option := range options { @@ -245,11 +258,9 @@ func (fl FixtureLoader) loadFixtureFromData(data Data, options ...Option) error } if f.bulkInsert { - count := 0 - for _, value := range rows { + for i, value := range rows { builder = builder.Values(value...) - count++ - if count > bulkInsertLimit { + if (i+1)%f.bulkInsertLimit == 0 { query, args, err = builder.ToSql() if err != nil { break @@ -259,12 +270,14 @@ func (fl FixtureLoader) loadFixtureFromData(data Data, options ...Option) error if err != nil { break } - count = 0 builder = squirrel.Insert(quote(f.table)).Columns(quotedColumns...) } } - query, args, err = builder.ToSql() - _, err = tx.Exec(query, args...) + + if len(rows)%f.bulkInsertLimit != 0 { + query, args, err = builder.ToSql() + _, err = tx.Exec(query, args...) + } } else { for _, value := range rows { query, args, err = builder.Values(value...).ToSql() diff --git a/loader_test.go b/loader_test.go index 643f2ea..9249ea6 100644 --- a/loader_test.go +++ b/loader_test.go @@ -49,8 +49,9 @@ func TestNew(t *testing.T) { }, Output: Output{ Loader: FixtureLoader{ - txManager: txmanager.NewDB(nil), - driver: MySQL, + txManager: txmanager.NewDB(nil), + driver: MySQL, + bulkInsertLimit: defaultBulkInsertLimit, }, Error: nil, }, @@ -65,9 +66,10 @@ func TestNew(t *testing.T) { }, Output: Output{ Loader: FixtureLoader{ - txManager: txmanager.NewDB(nil), - driver: MySQL, - update: true, + txManager: txmanager.NewDB(nil), + driver: MySQL, + update: true, + bulkInsertLimit: defaultBulkInsertLimit, }, Error: nil, }, @@ -82,9 +84,10 @@ func TestNew(t *testing.T) { }, Output: Output{ Loader: FixtureLoader{ - txManager: txmanager.NewDB(nil), - driver: MySQL, - delete: true, + txManager: txmanager.NewDB(nil), + driver: MySQL, + delete: true, + bulkInsertLimit: defaultBulkInsertLimit, }, Error: nil, }, @@ -99,9 +102,10 @@ func TestNew(t *testing.T) { }, Output: Output{ Loader: FixtureLoader{ - txManager: txmanager.NewDB(nil), - driver: "sqlite", - delete: true, + txManager: txmanager.NewDB(nil), + driver: "sqlite", + delete: true, + bulkInsertLimit: defaultBulkInsertLimit, }, Error: nil, }, @@ -117,10 +121,11 @@ func TestNew(t *testing.T) { }, Output: Output{ Loader: FixtureLoader{ - txManager: txmanager.NewDB(nil), - driver: MySQL, - update: true, - bulkInsert: true, + txManager: txmanager.NewDB(nil), + driver: MySQL, + update: true, + bulkInsert: true, + bulkInsertLimit: defaultBulkInsertLimit, }, Error: nil, }, @@ -150,6 +155,35 @@ func TestNew(t *testing.T) { Error: fmt.Errorf("error `update` and `ignore` are exclusive option"), }, }, + Test{ + Title: "success: set bulkInsertLimit option", + Input: Input{ + Driver: MySQL, + Options: []Option{ + BulkInsertLimit(100), + }, + }, + Output: Output{ + Loader: FixtureLoader{ + txManager: txmanager.NewDB(nil), + driver: MySQL, + bulkInsertLimit: 100, + }, + Error: nil, + }, + }, + Test{ + Title: "error: set bulkInsertLimit option of zero", + Input: Input{ + Driver: MySQL, + Options: []Option{ + BulkInsertLimit(0), + }, + }, + Output: Output{ + Error: errors.New("error `bulkInsertLimit` is not allowed 0 value"), + }, + }, } for _, test := range tests { @@ -286,6 +320,30 @@ func TestLoadFixrure(t *testing.T) { }, Output: []item{}, }, + Test{ + Title: "load csv with bulkInsert/bulkInsertLimit option/rows number is the multiple of bulkInsertLimit", + Input: Input{ + File: "_data/bulkinsert.csv", + Options: []Option{ + Delete(true), + BulkInsert(true), + BulkInsertLimit(2), + Table("item"), + }, + }, + Output: []item{ + item{id: 1, name: "item1"}, + item{id: 2, name: "item2"}, + item{id: 3, name: "item3"}, + item{id: 4, name: "item4"}, + item{id: 5, name: "item5"}, + item{id: 6, name: "item6"}, + item{id: 7, name: "item7"}, + item{id: 8, name: "item8"}, + item{id: 9, name: "item9"}, + item{id: 10, name: "item10"}, + }, + }, } fl, err := New(db, MySQL)