diff --git a/src/backend/booster/bk_dist/common/file/file.go b/src/backend/booster/bk_dist/common/file/file.go index c6b9b1f6..b43e6231 100644 --- a/src/backend/booster/bk_dist/common/file/file.go +++ b/src/backend/booster/bk_dist/common/file/file.go @@ -82,16 +82,33 @@ func GetFileInfoByEnumDir(fp string) *Info { } } +type FileType int + +const ( + Unknown FileType = 0 + + RealFile FileType = 10 + RealDir FileType = 11 + LinkFile FileType = 12 + LinkDir FileType = 13 +) + // Info describe the os.FileInfo and handle some actions type Info struct { filePath string LinkTarget string + FileType FileType // info and err are return from os.Stat info os.FileInfo err error } +// Key return the uniq key of this file info +func (i *Info) Key() string { + return i.filePath +} + // Path return the file path func (i *Info) Path() string { return i.filePath @@ -224,6 +241,7 @@ func GetFileInfo(fs []string, mustexisted bool, notdir bool, statbysearchdir boo tempf := notf try := 0 maxtry := 10 + var oldi *Info for { var i *Info if statbysearchdir { @@ -231,6 +249,7 @@ func GetFileInfo(fs []string, mustexisted bool, notdir bool, statbysearchdir boo } else { i = Lstat(tempf) } + i.FileType = RealFile tempis[tempf] = i try++ @@ -279,6 +298,30 @@ func GetFileInfo(fs []string, mustexisted bool, notdir bool, statbysearchdir boo continue } + // 根据当前文件属性，给上一次的文件属性赋值 + if loopagain { // 需要等链接的属性 + i.FileType = LinkFile // 先假设是指向普通文件的链接 + if oldi != nil { + oldi.FileType = LinkFile + blog.Infof("common util: set %s to LinkFile by assume", oldi.filePath) + } + } else { + if i.Basic().IsDir() { + i.FileType = RealDir + if oldi != nil { + oldi.FileType = LinkDir + blog.Infof("common util: set %s to LinkDir", oldi.filePath) + } + } else { + i.FileType = RealFile + if oldi != nil { + oldi.FileType = LinkFile + blog.Infof("common util: set %s to LinkFile", oldi.filePath) + } + } + } + oldi = i + is = append(is, i) if !loopagain { @@ -298,3 +341,21 @@ func GetFileInfo(fs []string, mustexisted bool, notdir bool, statbysearchdir boo return is, nil } + +func Uniq(input []*Info) []*Info { + if input == nil { + return input + } + + newarr := make([]*Info, 0, len(input)/2) + tempMap := make(map[string]struct{}, len(newarr)) + for _, v := range input { + if _, ok := tempMap[v.Key()]; !ok { + tempMap[v.Key()] = struct{}{} + newarr = append(newarr, v) + } + } + + return newarr + +} diff --git a/src/backend/booster/bk_dist/common/sdk/worker.go b/src/backend/booster/bk_dist/common/sdk/worker.go index 1d9e0294..6c0c73c5 100644 --- a/src/backend/booster/bk_dist/common/sdk/worker.go +++ b/src/backend/booster/bk_dist/common/sdk/worker.go @@ -5,6 +5,7 @@ import ( "net" "os" + "github.com/TencentBlueKing/bk-turbo/src/backend/booster/bk_dist/common/file" dcFile "github.com/TencentBlueKing/bk-turbo/src/backend/booster/bk_dist/common/file" "github.com/TencentBlueKing/bk-turbo/src/backend/booster/bk_dist/common/protocol" dcProtocol "github.com/TencentBlueKing/bk-turbo/src/backend/booster/bk_dist/common/protocol" @@ -186,6 +187,17 @@ type BKSlotRspAck struct { } func GetPriority(i *dcFile.Info) FileDescPriority { + switch i.FileType { + case file.RealDir: + return RealDirPriority + case file.LinkDir: + return LinkDirPriority + case file.RealFile: + return RealFilePriority + case file.LinkFile: + return LinkFilePriority + } + isLink := i.Basic().Mode()&os.ModeSymlink != 0 if !isLink { if i.Basic().IsDir() { diff --git a/src/backend/booster/bk_dist/controller/pkg/manager/remote/mgr.go b/src/backend/booster/bk_dist/controller/pkg/manager/remote/mgr.go index 2e054d18..2c51bca2 100644 --- a/src/backend/booster/bk_dist/controller/pkg/manager/remote/mgr.go +++ b/src/backend/booster/bk_dist/controller/pkg/manager/remote/mgr.go @@ -665,7 +665,7 @@ func (m *Mgr) ensureFilesWithPriority( fileDetails []*types.FilesDetails) ([]string, error) { // 刷新优先级，windows的先不实现 - if runtime.GOOS != "windows" { + if runtime.GOOS != "windows" && runtime.GOOS != "darwin" { freshPriority(fileDetails) for _, v := range fileDetails { blog.Debugf("remote: after fresh Priority, file:%+v", *v) diff --git a/src/backend/booster/bk_dist/controller/pkg/manager/remote/utils.go b/src/backend/booster/bk_dist/controller/pkg/manager/remote/utils.go index 9ee30ce7..39dba418 100644 --- a/src/backend/booster/bk_dist/controller/pkg/manager/remote/utils.go +++ b/src/backend/booster/bk_dist/controller/pkg/manager/remote/utils.go @@ -204,6 +204,7 @@ func calculateDependencies(fileDetails []*types.FilesDetails) [][]int { return dependencies } +// 判断s2是否是s1的子串 func isSubString(s1, s2 string) bool { return len(s1) > len(s2) && strings.HasPrefix(s1, s2) && @@ -211,37 +212,54 @@ func isSubString(s1, s2 string) bool { } // dirDepend 检查 s1的目录 是否依赖 s2的目录 +// 优化规则：只有s2是指向目录的链接时，才需要判断依赖关系； +// 因为普通目录（非链接）不影响远端路径的生成；而指向目录的链接，会在保存依赖时提炼出来 func dirDepend(s1, s2 *types.FilesDetails) bool { - is1File := s1.File.Priority == sdk.RealFilePriority || s1.File.Priority == sdk.LinkFilePriority - is2File := s2.File.Priority == sdk.RealFilePriority || s2.File.Priority == sdk.LinkFilePriority - // 如果s1是文件，s2是目录 - if is1File { - if is2File { // 如果s1是文件，s2是文件 - if isSubString(s1.File.Targetrelativepath, s2.File.Targetrelativepath) { - return true - } - } else { // 如果s1是文件，s2是目录 - if isSubString(s1.File.FilePath, s2.File.FilePath) { - return true - } - } - } else { - if is2File { // 如果s1是目录，s2是文件 - if isSubString(s1.File.FilePath, s2.File.Targetrelativepath) { - return true - } - } else { // 如果s1是目录，s2是目录 - if isSubString(s1.File.FilePath, s2.File.FilePath) { - return true - } - } + if s2.File.Priority != sdk.LinkDirPriority { + return false + } + + if isSubString(s1.File.FilePath, s2.File.FilePath) { + return true } return false + + // is1File := s1.File.Priority == sdk.RealFilePriority || s1.File.Priority == sdk.LinkFilePriority + // is2File := s2.File.Priority == sdk.RealFilePriority || s2.File.Priority == sdk.LinkFilePriority + + // // 如果s1是文件，s2是目录 + // if is1File { + // if is2File { // 如果s1是文件，s2是文件 + // if isSubString(s1.File.Targetrelativepath, s2.File.Targetrelativepath) { + // return true + // } + // } else { // 如果s1是文件，s2是目录 + // if isSubString(s1.File.FilePath, s2.File.FilePath) { + // return true + // } + // } + // } else { + // if is2File { // 如果s1是目录，s2是文件 + // if isSubString(s1.File.FilePath, s2.File.Targetrelativepath) { + // return true + // } + // } else { // 如果s1是目录，s2是目录 + // if isSubString(s1.File.FilePath, s2.File.FilePath) { + // return true + // } + // } + // } + + // return false } // linkDepend 检查 s1 是否链接到了 s2 func linkDepend(s1, s2 *types.FilesDetails) bool { + if s1.File.Priority != sdk.LinkDirPriority && s1.File.Priority != sdk.LinkFilePriority { + return false + } + if s1.File.LinkTarget != "" && s1.File.LinkTarget == s2.File.FilePath { return true } @@ -334,9 +352,11 @@ func printLeftDepend(fileDetails []*types.FilesDetails, dependencies [][]int) { if fileDetails[i].File.Priority < 0 { dependfiles := []string{} for _, v := range dependencies[i] { - dependPriority := int(fileDetails[v].File.Priority) - if dependPriority < 0 { - dependfiles = append(dependfiles, fileDetails[v].File.FilePath) + if v >= 0 { + dependPriority := int(fileDetails[v].File.Priority) + if dependPriority < 0 { + dependfiles = append(dependfiles, fileDetails[v].File.FilePath) + } } } blog.Warnf("remote util: after max try, %s wait %s", diff --git a/src/backend/booster/bk_dist/handler/cc/handler.go b/src/backend/booster/bk_dist/handler/cc/handler.go index 0e8ca25b..eac7300c 100644 --- a/src/backend/booster/bk_dist/handler/cc/handler.go +++ b/src/backend/booster/bk_dist/handler/cc/handler.go @@ -223,14 +223,15 @@ func (cc *TaskCC) FinalExecute(args []string) { // GetFilterRules add file send filter func (cc *TaskCC) GetFilterRules() ([]dcSDK.FilterRuleItem, error) { - return []dcSDK.FilterRuleItem{ - { - Rule: dcSDK.FilterRuleFileSuffix, - Operator: dcSDK.FilterRuleOperatorEqual, - Standard: ".gch", - HandleType: dcSDK.FilterRuleHandleAllDistribution, - }, - }, nil + // return []dcSDK.FilterRuleItem{ + // { + // Rule: dcSDK.FilterRuleFileSuffix, + // Operator: dcSDK.FilterRuleOperatorEqual, + // Standard: ".gch", + // HandleType: dcSDK.FilterRuleHandleAllDistribution, + // }, + // }, nil + return nil, nil } func (cc *TaskCC) analyzeIncludes(dependf string) ([]*dcFile.Info, error) { @@ -679,8 +680,6 @@ func (cc *TaskCC) trypumpwithcache(command []string) (*dcSDK.BKDistCommand, erro tstart = tend if err == nil { - blog.Infof("cc: parse command,got total %d includes files", len(includes)) - // add pch file as input if pchfile != "" { // includes = append(includes, pchfile) @@ -699,6 +698,10 @@ func (cc *TaskCC) trypumpwithcache(command []string) (*dcSDK.BKDistCommand, erro } } + oldlen := len(includes) + includes = dcFile.Uniq(includes) + blog.Infof("cc: parse command,got total %d uniq %d includes files", oldlen, len(includes)) + inputFiles := []dcSDK.FileDesc{} // priority := dcSDK.MaxFileDescPriority for _, f := range includes { diff --git a/src/backend/booster/bk_dist/handler/ue4/cc/handler.go b/src/backend/booster/bk_dist/handler/ue4/cc/handler.go index f214e0ce..c72c4f38 100644 --- a/src/backend/booster/bk_dist/handler/ue4/cc/handler.go +++ b/src/backend/booster/bk_dist/handler/ue4/cc/handler.go @@ -225,14 +225,15 @@ func (cc *TaskCC) FinalExecute(args []string) { // GetFilterRules add file send filter func (cc *TaskCC) GetFilterRules() ([]dcSDK.FilterRuleItem, error) { - return []dcSDK.FilterRuleItem{ - { - Rule: dcSDK.FilterRuleFileSuffix, - Operator: dcSDK.FilterRuleOperatorEqual, - Standard: ".gch", - HandleType: dcSDK.FilterRuleHandleAllDistribution, - }, - }, nil + // return []dcSDK.FilterRuleItem{ + // { + // Rule: dcSDK.FilterRuleFileSuffix, + // Operator: dcSDK.FilterRuleOperatorEqual, + // Standard: ".gch", + // HandleType: dcSDK.FilterRuleHandleAllDistribution, + // }, + // }, nil + return nil, nil } func (cc *TaskCC) getIncludeExe() (string, error) { @@ -816,8 +817,6 @@ func (cc *TaskCC) trypump(command []string) (*dcSDK.BKDistCommand, error, error) // } if err == nil { - blog.Infof("cc: parse command,got total %d includes files", len(includes)) - // add pch file as input if pchfile != "" { // includes = append(includes, pchfile) @@ -836,6 +835,10 @@ func (cc *TaskCC) trypump(command []string) (*dcSDK.BKDistCommand, error, error) } } + oldlen := len(includes) + includes = dcFile.Uniq(includes) + blog.Infof("cc: parse command,got total %d uniq %d includes files", oldlen, len(includes)) + inputFiles := []dcSDK.FileDesc{} // priority := dcSDK.MaxFileDescPriority for _, f := range includes { diff --git a/src/backend/booster/bk_dist/handler/ue4/cl/handler.go b/src/backend/booster/bk_dist/handler/ue4/cl/handler.go index d51af311..c6542992 100644 --- a/src/backend/booster/bk_dist/handler/ue4/cl/handler.go +++ b/src/backend/booster/bk_dist/handler/ue4/cl/handler.go @@ -339,13 +339,14 @@ func (cl *TaskCL) FinalExecute(args []string) { // GetFilterRules add file send filter func (cl *TaskCL) GetFilterRules() ([]dcSDK.FilterRuleItem, error) { - return []dcSDK.FilterRuleItem{ - { - Rule: dcSDK.FilterRuleFileSuffix, - Operator: dcSDK.FilterRuleOperatorEqual, - Standard: ".pch", - }, - }, nil + // return []dcSDK.FilterRuleItem{ + // { + // Rule: dcSDK.FilterRuleFileSuffix, + // Operator: dcSDK.FilterRuleOperatorEqual, + // Standard: ".pch", + // }, + // }, nil + return nil, nil } func (cl *TaskCL) getIncludeExe() (string, error) { @@ -719,8 +720,6 @@ func (cl *TaskCL) trypump(command []string) (*dcSDK.BKDistCommand, error, error) tstart = tend if err == nil { - blog.Infof("cl: parse command,got total %d includes files", len(includes)) - // add pch file as input if pchfile != "" { // includes = append(includes, pchfile) @@ -739,6 +738,10 @@ func (cl *TaskCL) trypump(command []string) (*dcSDK.BKDistCommand, error, error) } } + oldlen := len(includes) + includes = dcFile.Uniq(includes) + blog.Infof("cc: parse command,got total %d uniq %d includes files", oldlen, len(includes)) + inputFiles := []dcSDK.FileDesc{} // priority := dcSDK.MaxFileDescPriority for _, f := range includes { diff --git a/src/backend/booster/bk_dist/handler/ue4/clfilter/handler.go b/src/backend/booster/bk_dist/handler/ue4/clfilter/handler.go index 39b7aafb..d286ac57 100644 --- a/src/backend/booster/bk_dist/handler/ue4/clfilter/handler.go +++ b/src/backend/booster/bk_dist/handler/ue4/clfilter/handler.go @@ -187,13 +187,14 @@ func (cf *TaskCLFilter) FinalExecute(args []string) { // GetFilterRules add file send filter func (cf *TaskCLFilter) GetFilterRules() ([]dcSDK.FilterRuleItem, error) { - return []dcSDK.FilterRuleItem{ - { - Rule: dcSDK.FilterRuleFileSuffix, - Operator: dcSDK.FilterRuleOperatorEqual, - Standard: ".pch", - }, - }, nil + // return []dcSDK.FilterRuleItem{ + // { + // Rule: dcSDK.FilterRuleFileSuffix, + // Operator: dcSDK.FilterRuleOperatorEqual, + // Standard: ".pch", + // }, + // }, nil + return nil, nil } func (cf *TaskCLFilter) preExecute(command []string) (*dcSDK.BKDistCommand, dcType.BKDistCommonError) { diff --git a/src/backend/booster/bk_dist/handler/winclangcl/handler.go b/src/backend/booster/bk_dist/handler/winclangcl/handler.go index 04108a49..5c3b415a 100644 --- a/src/backend/booster/bk_dist/handler/winclangcl/handler.go +++ b/src/backend/booster/bk_dist/handler/winclangcl/handler.go @@ -182,14 +182,15 @@ func (cc *WinClangCl) FinalExecute(args []string) { // GetFilterRules add file send filter func (cc *WinClangCl) GetFilterRules() ([]dcSDK.FilterRuleItem, error) { - return []dcSDK.FilterRuleItem{ - { - Rule: dcSDK.FilterRuleFileSuffix, - Operator: dcSDK.FilterRuleOperatorEqual, - Standard: ".gch", - HandleType: dcSDK.FilterRuleHandleAllDistribution, - }, - }, nil + // return []dcSDK.FilterRuleItem{ + // { + // Rule: dcSDK.FilterRuleFileSuffix, + // Operator: dcSDK.FilterRuleOperatorEqual, + // Standard: ".gch", + // HandleType: dcSDK.FilterRuleHandleAllDistribution, + // }, + // }, nil + return nil, nil } func (cc *WinClangCl) GetPreloadConfig(config dcType.BoosterConfig) (*dcSDK.PreloadConfig, error) {