Skip to content
This repository has been archived by the owner on Feb 27, 2023. It is now read-only.

Commit

Permalink
refactor: make common/util to be specific pkgs
Browse files Browse the repository at this point in the history 
Signed-off-by: Allen Sun <[email protected]>
  • Loading branch information
@allencloud
allencloud committed Aug 2, 2019
1 parent b7bce11 commit 892ad7a
Show file tree
Hide file tree
Showing 120 changed files with 778 additions and 638 deletions.
83 changes: 45 additions & 38 deletions cmd/dfdaemon/app/init.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,11 +13,12 @@ import (
"syscall"
"time"

"github.com/dragonflyoss/Dragonfly/common/dflog"
dferr "github.com/dragonflyoss/Dragonfly/common/errors"
"github.com/dragonflyoss/Dragonfly/common/util"
"github.com/dragonflyoss/Dragonfly/dfdaemon/config"
"github.com/dragonflyoss/Dragonfly/dfdaemon/constant"
"github.com/dragonflyoss/Dragonfly/pkg/dflog"
"github.com/dragonflyoss/Dragonfly/pkg/errortypes"
"github.com/dragonflyoss/Dragonfly/pkg/fileutils"
statutil "github.com/dragonflyoss/Dragonfly/pkg/stat"

"github.com/pkg/errors"
"github.com/sirupsen/logrus"
Expand All @@ -39,7 +40,7 @@ func initDfdaemon(cfg config.Properties) error {
}

if err := os.MkdirAll(cfg.DFRepo, 0755); err != nil {
return dferr.Newf(
return errortypes.Newf(
constant.CodeExitRepoCreateFail,
"ensure local repo %s exists", cfg.DFRepo,
)
Expand Down Expand Up @@ -75,51 +76,57 @@ func initLogger(cfg config.Properties) error {
return errors.Wrap(err, "init log file")
}

if logFile, ok := (logrus.StandardLogger().Out).(*os.File); ok {
go func(logFile *os.File) {
logrus.Infof("rotate %s every 60 seconds", logFilePath)
ticker := time.NewTicker(60 * time.Second)
for range ticker.C {
if err := rotateLog(logFile); err != nil {
logrus.Errorf("failed to rotate log %s: %v", logFile.Name(), err)
}
}
}(logFile)
logFile, ok := (logrus.StandardLogger().Out).(*os.File)
if !ok {
return nil
}
go func(logFile *os.File) {
logrus.Infof("rotate %s every 60 seconds", logFilePath)
ticker := time.NewTicker(60 * time.Second)
for range ticker.C {
if err := rotateLog(logFile); err != nil {
logrus.Errorf("failed to rotate log %s: %v", logFile.Name(), err)
}
}
}(logFile)

return nil
}

// rotateLog truncates the logs file by a certain amount bytes.
func rotateLog(logFile *os.File) error {
stat, err := logFile.Stat()
fStat, err := logFile.Stat()
if err != nil {
return err
}
logSizeLimit := int64(20 * 1024 * 1024)

if fStat.Size() <= logSizeLimit {
return nil
}

// if it exceeds the 20MB limitation
if stat.Size() > logSizeLimit {
log.SetOutput(ioutil.Discard)
// make sure set the output of log back to logFile when error be raised.
defer log.SetOutput(logFile)
logFile.Sync()
truncateSize := logSizeLimit/2 - 1
mem, err := syscall.Mmap(int(logFile.Fd()), 0, int(stat.Size()),
syscall.PROT_READ|syscall.PROT_WRITE, syscall.MAP_SHARED)
if err != nil {
return err
}
copy(mem[0:], mem[truncateSize:])
if err := syscall.Munmap(mem); err != nil {
return err
}
if err := logFile.Truncate(stat.Size() - truncateSize); err != nil {
return err
}
if _, err := logFile.Seek(truncateSize, 0); err != nil {
return err
}
log.SetOutput(ioutil.Discard)
// make sure set the output of log back to logFile when error be raised.
defer log.SetOutput(logFile)
logFile.Sync()
truncateSize := logSizeLimit/2 - 1
mem, err := syscall.Mmap(int(logFile.Fd()), 0, int(fStat.Size()),
syscall.PROT_READ|syscall.PROT_WRITE, syscall.MAP_SHARED)
if err != nil {
return err
}
copy(mem[0:], mem[truncateSize:])
if err := syscall.Munmap(mem); err != nil {
return err
}
if err := logFile.Truncate(fStat.Size() - truncateSize); err != nil {
return err
}
if _, err := logFile.Seek(truncateSize, 0); err != nil {
return err
}

return nil
}

Expand All @@ -146,13 +153,13 @@ func cleanLocalRepo(dfpath string) {
return nil
}
// get the last access time
statT, ok := util.GetSys(info)
statT, ok := fileutils.GetSys(info)
if !ok {
logrus.Warnf("ignore %s: failed to get last access time", path)
return nil
}
// if the last access time is 1 hour ago
if time.Since(util.Atime(statT)) > time.Hour {
if time.Since(statutil.Atime(statT)) > time.Hour {
if err := os.Remove(path); err == nil {
logrus.Infof("remove file:%s success", path)
} else {
Expand Down
6 changes: 3 additions & 3 deletions cmd/dfdaemon/app/root.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,11 +23,11 @@ import (
"path/filepath"
"reflect"

dferr "github.com/dragonflyoss/Dragonfly/common/errors"
"github.com/dragonflyoss/Dragonfly/common/util"
"github.com/dragonflyoss/Dragonfly/dfdaemon"
"github.com/dragonflyoss/Dragonfly/dfdaemon/config"
"github.com/dragonflyoss/Dragonfly/dfdaemon/constant"
dferr "github.com/dragonflyoss/Dragonfly/pkg/errortypes"
"github.com/dragonflyoss/Dragonfly/pkg/netutils"

"github.com/mitchellh/mapstructure"
"github.com/pkg/errors"
Expand Down Expand Up @@ -93,7 +93,7 @@ func init() {
rf.String("localrepo", filepath.Join(os.Getenv("HOME"), ".small-dragonfly/dfdaemon/data/"), "temp output dir of dfdaemon")
rf.String("callsystem", "com_ops_dragonfly", "caller name")
rf.String("dfpath", defaultDfgetPath, "dfget path")
rf.String("ratelimit", util.NetLimit(), "net speed limit,format:xxxM/K")
rf.String("ratelimit", netutils.NetLimit(), "net speed limit,format:xxxM/K")
rf.String("urlfilter", "Signature&Expires&OSSAccessKeyId", "filter specified url fields")
rf.Bool("notbs", true, "not try back source to download if throw exception")
rf.StringSlice("node", nil, "specify the addresses(host:port) of supernodes that will be passed to dfget.")
Expand Down
33 changes: 14 additions & 19 deletions cmd/dfget/app/root.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,14 +24,14 @@ import (
"strings"
"time"

"github.com/dragonflyoss/Dragonfly/common/dflog"
"github.com/dragonflyoss/Dragonfly/common/errors"
cutil "github.com/dragonflyoss/Dragonfly/common/util"
"github.com/dragonflyoss/Dragonfly/dfget/config"
"github.com/dragonflyoss/Dragonfly/dfget/core"
"github.com/dragonflyoss/Dragonfly/dfget/util"
"github.com/dragonflyoss/Dragonfly/pkg/dflog"
"github.com/dragonflyoss/Dragonfly/pkg/errortypes"
"github.com/dragonflyoss/Dragonfly/pkg/printer"
"github.com/dragonflyoss/Dragonfly/pkg/stringutils"

errHandler "github.com/pkg/errors"
"github.com/pkg/errors"
"github.com/sirupsen/logrus"
"github.com/spf13/cobra"
)
Expand Down Expand Up @@ -72,38 +72,33 @@ func init() {
func runDfget() error {
// initialize logger
if err := initClientLog(); err != nil {
util.Printer.Println(fmt.Sprintf("init log error: %v", err))
return err
}

if err := transParams(); err != nil {
util.Printer.Println(err.Error())
return err
}

// get config from property files
initProperties()

if err := handleNodes(); err != nil {
util.Printer.Println(err.Error())
return err
}

if err := checkParameters(); err != nil {
util.Printer.Println(err.Error())
return err
}
logrus.Infof("get cmd params:%q", os.Args)

if err := config.AssertConfig(cfg); err != nil {
util.Printer.Println(fmt.Sprintf("assert context error: %v", err))
return err
return errors.Wrap(err, "failed to assert context")
}
logrus.Infof("get init config:%v", cfg)

// enter the core process
err := core.Start(cfg)
util.Printer.Println(resultMsg(cfg, time.Now(), err))
printer.Println(resultMsg(cfg, time.Now(), err))
if err != nil {
os.Exit(err.Code)
}
Expand All @@ -112,7 +107,7 @@ func runDfget() error {

func checkParameters() error {
if len(os.Args) < 2 {
return errors.New(-1, "Please use the command 'help' to show the help information.")
return errortypes.New(-1, "Please use the command 'help' to show the help information.")
}
return nil
}
Expand Down Expand Up @@ -157,15 +152,15 @@ func transParams() error {

var err error
if cfg.LocalLimit, err = transLimit(localLimit); err != nil {
return errHandler.Wrapf(errors.ErrConvertFailed, "locallimit: %v", err)
return errors.Wrapf(errortypes.ErrConvertFailed, "locallimit: %v", err)
}

if cfg.MinRate, err = transLimit(minRate); err != nil {
return errHandler.Wrapf(errors.ErrConvertFailed, "minrate: %v", err)
return errors.Wrapf(errortypes.ErrConvertFailed, "minrate: %v", err)
}

if cfg.TotalLimit, err = transLimit(totalLimit); err != nil {
return errHandler.Wrapf(errors.ErrConvertFailed, "totallimit: %v", err)
return errors.Wrapf(errortypes.ErrConvertFailed, "totallimit: %v", err)
}

return nil
Expand Down Expand Up @@ -255,7 +250,7 @@ func initFlags() {

// Helper functions.
func transLimit(limit string) (int, error) {
if cutil.IsEmptyStr(limit) {
if stringutils.IsEmptyStr(limit) {
return 0, nil
}
l := len(limit)
Expand All @@ -277,7 +272,7 @@ func transLimit(limit string) (int, error) {
}

func transFilter(filter string) []string {
if cutil.IsEmptyStr(filter) {
if stringutils.IsEmptyStr(filter) {
return nil
}
return strings.Split(filter, "&")
Expand All @@ -298,7 +293,7 @@ func handleNodes() error {
return nil
}

func resultMsg(cfg *config.Config, end time.Time, e *errors.DfError) string {
func resultMsg(cfg *config.Config, end time.Time, e *errortypes.DfError) string {
if e != nil {
return fmt.Sprintf("download FAIL(%d) cost:%.3fs length:%d reason:%d error:%v",
e.Code, end.Sub(cfg.StartTime).Seconds(), cfg.RV.FileLength,
Expand Down
8 changes: 4 additions & 4 deletions cmd/dfget/app/root_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,9 +27,9 @@ import (
"testing"
"time"

"github.com/dragonflyoss/Dragonfly/common/errors"
"github.com/dragonflyoss/Dragonfly/common/util"
"github.com/dragonflyoss/Dragonfly/dfget/config"
"github.com/dragonflyoss/Dragonfly/pkg/errortypes"
"github.com/dragonflyoss/Dragonfly/pkg/stringutils"

"github.com/sirupsen/logrus"
"github.com/stretchr/testify/suite"
Expand Down Expand Up @@ -117,7 +117,7 @@ func (suit *dfgetSuit) Test_transLimit() {
for k, v := range cases {
i, e := transLimit(k)
suit.Equal(i, v.i)
if util.IsEmptyStr(v.err) {
if stringutils.IsEmptyStr(v.err) {
suit.Nil(e)
} else {
suit.NotNil(e)
Expand Down Expand Up @@ -157,7 +157,7 @@ func (suit *dfgetSuit) TestResultMsg() {
msg := resultMsg(cfg, end, nil)
suit.Equal(msg, "download SUCCESS(0) cost:0.100s length:-1 reason:0")

msg = resultMsg(cfg, end, errors.New(1, "TestFail"))
msg = resultMsg(cfg, end, errortypes.New(1, "TestFail"))
suit.Equal(msg, "download FAIL(1) cost:0.100s length:-1 reason:0 error:"+
`{"Code":1,"Msg":"TestFail"}`)
}
Expand Down
2 changes: 1 addition & 1 deletion cmd/dfget/app/server.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,9 +20,9 @@ import (
"fmt"
"path"

"github.com/dragonflyoss/Dragonfly/common/dflog"
"github.com/dragonflyoss/Dragonfly/dfget/config"
"github.com/dragonflyoss/Dragonfly/dfget/core/uploader"
"github.com/dragonflyoss/Dragonfly/pkg/dflog"

"github.com/spf13/cobra"
)
Expand Down
19 changes: 10 additions & 9 deletions cmd/supernode/app/root.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,15 +22,16 @@ import (
"path"
"reflect"

"github.com/dragonflyoss/Dragonfly/pkg/dflog"
"github.com/dragonflyoss/Dragonfly/pkg/errortypes"
"github.com/dragonflyoss/Dragonfly/pkg/netutils"
"github.com/dragonflyoss/Dragonfly/pkg/stringutils"
"github.com/dragonflyoss/Dragonfly/supernode/config"
"github.com/dragonflyoss/Dragonfly/supernode/daemon"

"github.com/pkg/errors"
"github.com/sirupsen/logrus"
"github.com/spf13/cobra"

"github.com/dragonflyoss/Dragonfly/common/dflog"
errorType "github.com/dragonflyoss/Dragonfly/common/errors"
cutil "github.com/dragonflyoss/Dragonfly/common/util"
"github.com/dragonflyoss/Dragonfly/supernode/config"
"github.com/dragonflyoss/Dragonfly/supernode/daemon"
)

var (
Expand Down Expand Up @@ -115,7 +116,7 @@ func runSuperNode() error {
}

// set supernode advertise ip
if cutil.IsEmptyStr(cfg.AdvertiseIP) {
if stringutils.IsEmptyStr(cfg.AdvertiseIP) {
if err := setAdvertiseIP(); err != nil {
return err
}
Expand Down Expand Up @@ -173,9 +174,9 @@ func initConfig() error {

func setAdvertiseIP() error {
// use the first non-loop address if the AdvertiseIP is empty
ipList, err := cutil.GetAllIPs()
ipList, err := netutils.GetAllIPs()
if err != nil {
return errors.Wrapf(errorType.ErrSystemError, "failed to get ip list: %v", err)
return errors.Wrapf(errortypes.ErrSystemError, "failed to get ip list: %v", err)
}
if len(ipList) == 0 {
logrus.Debugf("get empty system's unicast interface addresses")
Expand Down
2 changes: 1 addition & 1 deletion dfdaemon/config/config.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,8 +25,8 @@ import (
"path/filepath"
"regexp"

dferr "github.com/dragonflyoss/Dragonfly/common/errors"
"github.com/dragonflyoss/Dragonfly/dfdaemon/constant"
dferr "github.com/dragonflyoss/Dragonfly/pkg/errortypes"

"github.com/pkg/errors"
"github.com/spf13/afero"
Expand Down
6 changes: 3 additions & 3 deletions dfdaemon/config/config_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,8 +26,8 @@ import (
"testing"
"time"

dferr "github.com/dragonflyoss/Dragonfly/common/errors"
"github.com/dragonflyoss/Dragonfly/dfdaemon/constant"
"github.com/dragonflyoss/Dragonfly/pkg/errortypes"

"github.com/pkg/errors"
"github.com/spf13/afero"
Expand Down Expand Up @@ -62,7 +62,7 @@ func (ts *configTestSuite) TestValidatePort() {
c.Port = p
err := c.Validate()
r.NotNil(err)
de, ok := err.(*dferr.DfError)
de, ok := err.(*errortypes.DfError)
r.True(ok)
r.Equal(constant.CodeExitPortInvalid, de.Code)
}
Expand Down Expand Up @@ -367,7 +367,7 @@ func (m *yamlM) Marshal(d interface{}) ([]byte, error) { return yaml.Marsha
func (m *yamlM) Unmarshal(text []byte, d interface{}) error { return yaml.Unmarshal(text, d) }

func getCode(err error) int {
if de, ok := err.(*dferr.DfError); ok {
if de, ok := err.(*errortypes.DfError); ok {
return de.Code
}
return 0
Expand Down
Loading

0 comments on commit 892ad7a

Please sign in to comment.