Skip to content

Commit

Permalink
✨ Change namespaces created_at field type (#239)
Browse files Browse the repository at this point in the history 
* ✨ Change namespaces created_at field type

* ✨ Add index for all tables

* ✨ Change created_at field type
  • Loading branch information
@tosone
tosone authored Nov 24, 2023
1 parent b173b0c commit 033b745
Show file tree
Hide file tree
Showing 93 changed files with 1,423 additions and 918 deletions.
10 changes: 6 additions & 4 deletions go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,10 +11,10 @@ require (
github.com/aliyun/aliyun-oss-go-sdk v2.2.9+incompatible
github.com/anchore/syft v0.97.1
github.com/aquasecurity/trivy v0.47.0
github.com/aws/aws-sdk-go v1.48.2
github.com/aws/aws-sdk-go v1.48.3
github.com/bytedance/json v0.0.0-20190516032711-0d89175f1949
github.com/caarlos0/env/v9 v9.0.0
github.com/casbin/casbin/v2 v2.77.2
github.com/casbin/casbin/v2 v2.79.0
github.com/casbin/gorm-adapter/v3 v3.20.0
github.com/deckarep/golang-set/v2 v2.4.0
github.com/distribution/distribution/v3 v3.0.0-20231016181039-1d410148efe6
Expand Down Expand Up @@ -85,7 +85,6 @@ require (
dario.cat/mergo v1.0.0 // indirect
github.com/AdaLogics/go-fuzz-headers v0.0.0-20230811130428-ced1acdcaa24 // indirect
github.com/AdamKorcz/go-118-fuzz-build v0.0.0-20230306123547-8075edf89bb0 // indirect
github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible // indirect
github.com/KyleBanks/depth v1.2.1 // indirect
github.com/Masterminds/goutils v1.1.1 // indirect
github.com/Masterminds/semver/v3 v3.2.1 // indirect
Expand All @@ -106,6 +105,7 @@ require (
github.com/aquasecurity/trivy-db v0.0.0-20231005141211-4fc651f7ac8d // indirect
github.com/becheran/wildmatch-go v1.0.0 // indirect
github.com/bmatcuk/doublestar/v4 v4.6.1 // indirect
github.com/casbin/govaluate v1.1.0 // indirect
github.com/cespare/xxhash/v2 v2.2.0 // indirect
github.com/clbanning/mxj v1.8.4 // indirect
github.com/cloudflare/circl v1.3.3 // indirect
Expand Down Expand Up @@ -294,11 +294,13 @@ require (
k8s.io/klog/v2 v2.100.1 // indirect
k8s.io/kube-openapi v0.0.0-20230816210353-14e408962443 // indirect
k8s.io/utils v0.0.0-20230726121419-3b25d923346b // indirect
modernc.org/libc v1.29.0 // indirect
modernc.org/libc v1.34.9 // indirect
modernc.org/mathutil v1.6.0 // indirect
modernc.org/memory v1.7.2 // indirect
modernc.org/sqlite v1.27.0 // indirect
sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect
sigs.k8s.io/structured-merge-diff/v4 v4.3.0 // indirect
sigs.k8s.io/yaml v1.3.0 // indirect
)

replace gorm.io/plugin/soft_delete => github.com/go-sigma/soft_delete v0.0.0-20231124084503-fb6a66078e2b
21 changes: 10 additions & 11 deletions go.sum
View file
Original file line number Diff line number Diff line change
Expand Up @@ -72,8 +72,6 @@ github.com/DataDog/datadog-go v3.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3
github.com/DmitriyVTitov/size v1.5.0/go.mod h1:le6rNI4CoLQV1b9gzp1+3d7hMAD/uu2QcJ+aYbNgiU0=
github.com/IBM/sarama v1.42.1 h1:wugyWa15TDEHh2kvq2gAy1IHLjEjuYOYgXz/ruC/OSQ=
github.com/IBM/sarama v1.42.1/go.mod h1:Xxho9HkHd4K/MDUo/T/sOqwtX/17D33++E9Wib6hUdQ=
github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible h1:1G1pk05UrOh0NlF1oeaaix1x8XzrfjIDK47TY0Zehcw=
github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0=
github.com/KyleBanks/depth v1.2.1 h1:5h8fQADFrWtarTdtDudMmGsC7GPbOAu6RVB3ffsVFHc=
github.com/KyleBanks/depth v1.2.1/go.mod h1:jzSb9d0L43HxTQfT+oSA1EEp2q+ne2uh6XgeJcm8brE=
github.com/Masterminds/goutils v1.1.1 h1:5nUrii3FMTL5diU80unEVvNevw1nH4+ZV4DSLVJLSYI=
Expand Down Expand Up @@ -147,8 +145,8 @@ github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj
github.com/armon/go-radix v1.0.0/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio=
github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs=
github.com/aws/aws-sdk-go v1.48.2 h1:Lf7+Y4WmHB0AQLRQZA46diSwDa+LWbwY6IGaYoCVtTc=
github.com/aws/aws-sdk-go v1.48.2/go.mod h1:LF8svs817+Nz+DmiMQKTO3ubZ/6IaTpq3TjupRn3Eqk=
github.com/aws/aws-sdk-go v1.48.3 h1:btYjT+opVFxUbRz+qSCjJe07cdX82BHmMX/FXYmoL7g=
github.com/aws/aws-sdk-go v1.48.3/go.mod h1:LF8svs817+Nz+DmiMQKTO3ubZ/6IaTpq3TjupRn3Eqk=
github.com/becheran/wildmatch-go v1.0.0 h1:mE3dGGkTmpKtT4Z+88t8RStG40yN9T+kFEGj2PZFSzA=
github.com/becheran/wildmatch-go v1.0.0/go.mod h1:gbMvj0NtVdJ15Mg/mH9uxk2R1QCistMyU7d9KFzroX4=
github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
Expand All @@ -168,10 +166,12 @@ github.com/bytedance/json v0.0.0-20190516032711-0d89175f1949 h1:JG8x1j2Jvu4e1Jwz
github.com/bytedance/json v0.0.0-20190516032711-0d89175f1949/go.mod h1:sP1wlZn6XebMh0e4IPCNmVKCwQIXK4CDjUsQUbi0Ewo=
github.com/caarlos0/env/v9 v9.0.0 h1:SI6JNsOA+y5gj9njpgybykATIylrRMklbs5ch6wO6pc=
github.com/caarlos0/env/v9 v9.0.0/go.mod h1:ye5mlCVMYh6tZ+vCgrs/B95sj88cg5Tlnc0XIzgZ020=
github.com/casbin/casbin/v2 v2.77.2 h1:yQinn/w9x8AswiwqwtrXz93VU48R1aYTXdHEx4RI3jM=
github.com/casbin/casbin/v2 v2.77.2/go.mod h1:mzGx0hYW9/ksOSpw3wNjk3NRAroq5VMFYUQ6G43iGPk=
github.com/casbin/casbin/v2 v2.79.0 h1:6/Q1Ud/fiJ1S999Aa1vT3XqwhNKXvsa9Q3oeCnH0bUU=
github.com/casbin/casbin/v2 v2.79.0/go.mod h1:jX8uoN4veP85O/n2674r2qtfSXI6myvxW85f6TH50fw=
github.com/casbin/gorm-adapter/v3 v3.20.0 h1:VpGKTlL56xIkhNUOC07bnzwjA/xqfVOAbkt6sniVxMo=
github.com/casbin/gorm-adapter/v3 v3.20.0/go.mod h1:pvTTuyP2Es8VPHLyUssGtvOb3ETYD2tG7TfT5K8X2Sg=
github.com/casbin/govaluate v1.1.0 h1:6xdCWIpE9CwHdZhlVQW+froUrCsjb6/ZYNcXODfLT+E=
github.com/casbin/govaluate v1.1.0/go.mod h1:G/UnbIjZk/0uMNaLwZZmFQrR72tYRZWQkO70si/iR7A=
github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
github.com/census-instrumentation/opencensus-proto v0.3.0/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
Expand Down Expand Up @@ -377,6 +377,8 @@ github.com/go-redsync/redsync/v4 v4.11.0 h1:OPEcAxHBb95EzfwCKWM93ksOwHd5bTce2BD4
github.com/go-redsync/redsync/v4 v4.11.0/go.mod h1:ZfayzutkgeBmEmBlUR3j+rF6kN44UUGtEdfzhBFZTPc=
github.com/go-resty/resty/v2 v2.10.0 h1:Qla4W/+TMmv0fOeeRqzEpXPLfTUnR5HZ1+lGs+CkiCo=
github.com/go-resty/resty/v2 v2.10.0/go.mod h1:iiP/OpA0CkcL3IGt1O0+/SIItFUbkkyw5BGXiVdTu+A=
github.com/go-sigma/soft_delete v0.0.0-20231124084503-fb6a66078e2b h1:dP1itc+/9pJ/0ku3ntc8Rti0w+zuslJfOAnMNyv3N7I=
github.com/go-sigma/soft_delete v0.0.0-20231124084503-fb6a66078e2b/go.mod h1:Zv7vQctOJTGOsJ/bWgrN1n3od0GBAZgnLjEx+cApLGk=
github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
github.com/go-sql-driver/mysql v1.7.0/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI=
github.com/go-sql-driver/mysql v1.7.1 h1:lUIinVbN1DY0xBg0eMOzmmtGoHwWBbvnWubQUrtU8EI=
Expand Down Expand Up @@ -998,7 +1000,6 @@ github.com/tencentyun/cos-go-sdk-v5 v0.7.45 h1:5/ZGOv846tP6+2X7w//8QjLgH2KcUK+Hc
github.com/tencentyun/cos-go-sdk-v5 v0.7.45/go.mod h1:DH9US8nB+AJXqwu/AMOrCFN1COv3dpytXuJWHgdg7kE=
github.com/therootcompany/xz v1.0.1 h1:CmOtsn1CbtmyYiusbfmhmkpAAETj0wBIH6kCYaX+xzw=
github.com/therootcompany/xz v1.0.1/go.mod h1:3K3UH1yCKgBneZYhuQUvJ9HPD19UEXEI0BWbMn8qNMY=
github.com/tidwall/gjson v1.14.4/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
github.com/tidwall/gjson v1.17.0 h1:/Jocvlh98kcTfpN2+JzGQWQcqrPQwDrVEMApx/M5ZwM=
github.com/tidwall/gjson v1.17.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA=
Expand Down Expand Up @@ -1648,8 +1649,6 @@ gorm.io/hints v1.1.2 h1:b5j0kwk5p4+3BtDtYqqfY+ATSxjj+6ptPgVveuynn9o=
gorm.io/hints v1.1.2/go.mod h1:/ARdpUHAtyEMCh5NNi3tI7FsGh+Cj/MIUlvNxCNCFWg=
gorm.io/plugin/dbresolver v1.5.0 h1:XVHLxh775eP0CqVh3vcfJtYqja3uFl5Wr3cKlY8jgDY=
gorm.io/plugin/dbresolver v1.5.0/go.mod h1:l4Cn87EHLEYuqUncpEeTC2tTJQkjngPSD+lo8hIvcT0=
gorm.io/plugin/soft_delete v1.2.1 h1:qx9D/c4Xu6w5KT8LviX8DgLcB9hkKl6JC9f44Tj7cGU=
gorm.io/plugin/soft_delete v1.2.1/go.mod h1:Zv7vQctOJTGOsJ/bWgrN1n3od0GBAZgnLjEx+cApLGk=
gotest.tools/v3 v3.0.3 h1:4AuOwCGf4lLR9u3YOe2awrHygurzhO/HeQ6laiA6Sx0=
gotest.tools/v3 v3.0.3/go.mod h1:Z7Lb0S5l+klDB31fvDQX8ss/FlKDxtlFlw3Oa8Ymbl8=
honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
Expand All @@ -1671,8 +1670,8 @@ k8s.io/kube-openapi v0.0.0-20230816210353-14e408962443 h1:CAIciCnJnSOQxPd0xvpV6J
k8s.io/kube-openapi v0.0.0-20230816210353-14e408962443/go.mod h1:wZK2AVp1uHCp4VamDVgBP2COHZjqD1T68Rf0CM3YjSM=
k8s.io/utils v0.0.0-20230726121419-3b25d923346b h1:sgn3ZU783SCgtaSJjpcVVlRqd6GSnlTLKgpAAttJvpI=
k8s.io/utils v0.0.0-20230726121419-3b25d923346b/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0=
modernc.org/libc v1.29.0 h1:tTFRFq69YKCF2QyGNuRUQxKBm1uZZLubf6Cjh/pVHXs=
modernc.org/libc v1.29.0/go.mod h1:DaG/4Q3LRRdqpiLyP0C2m1B8ZMGkQ+cCgOIjEtQlYhQ=
modernc.org/libc v1.34.9 h1:yhQGs5jsWHJIU7jY6nCe8GJw27j+z6xYi0eorwPpGgI=
modernc.org/libc v1.34.9/go.mod h1:YAXkAZ8ktnkCKaN9sw/UDeUVkGYJ/YquGO4FTi5nmHE=
modernc.org/mathutil v1.6.0 h1:fRe9+AmYlaej+64JsEEhoWuAYBkOtQiMEU7n/XgfYi4=
modernc.org/mathutil v1.6.0/go.mod h1:Ui5Q9q1TR2gFm0AQRqQUaBWFLAhQpCwNcuhBOSedWPo=
modernc.org/memory v1.7.2 h1:Klh90S215mmH8c9gO98QxQFsY+W451E8AnzjoE2ee1E=
Expand Down
7 changes: 4 additions & 3 deletions pkg/daemon/gc/gc_repository.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -175,9 +175,10 @@ func (g gcRepository) deleteRepositoryCheck() {
log.Error().Err(err).Int64("RepositoryID", task.Repository.ID).Msg("Get repository by id failed")
continue
}
if !repositoryObj.UpdatedAt.Before(time.Now().Add(-1 * 24 * time.Duration(task.Runner.Rule.RetentionDay) * time.Hour)) {
continue
}
log.Info().Interface("repo", repositoryObj).Send()
// if !repositoryObj.UpdatedAt.Before(time.Now().Add(-1 * 24 * time.Duration(task.Runner.Rule.RetentionDay) * time.Hour)) {
// continue
// }
}
g.deleteRepositoryChan <- task
}
Expand Down
20 changes: 10 additions & 10 deletions pkg/dal/dal.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -111,26 +111,20 @@ func Initialize() error {
return nil
}

// func connectRedis() error {
// redisOpt, err := redis.ParseURL(viper.GetString("redis.url"))
// if err != nil {
// return err
// }
// RedisCli = redis.NewClient(redisOpt)
// return nil
// }

func connectMysql() (string, error) {
host := viper.GetString("database.mysql.host")
port := viper.GetString("database.mysql.port")
user := viper.GetString("database.mysql.user")
password := viper.GetString("database.mysql.password")
dbname := viper.GetString("database.mysql.dbname")

dsn := fmt.Sprintf("%s:%s@tcp(%s:%s)/%s?charset=utf8mb4&parseTime=True&loc=Local", user, password, host, port, dbname)
dsn := fmt.Sprintf("%s:%s@tcp(%s:%s)/%s?charset=utf8mb4&parseTime=True&loc=UTC", user, password, host, port, dbname)
log.Debug().Str("dsn", dsn).Msg("Connect to mysql database")

db, err := gorm.Open(mysql.Open(dsn), &gorm.Config{
NowFunc: func() time.Time {
return time.Now().UTC()
},
Logger: logger.ZLogger{},
})
if err != nil {
Expand All @@ -152,6 +146,9 @@ func connectPostgres() (string, error) {

dsn := fmt.Sprintf("host=%s port=%s user=%s dbname=%s password=%s sslmode=%s", host, port, user, dbname, password, sslmode)
db, err := gorm.Open(postgres.Open(dsn), &gorm.Config{
NowFunc: func() time.Time {
return time.Now().UTC()
},
Logger: logger.ZLogger{},
})
if err != nil {
Expand All @@ -169,6 +166,9 @@ func connectSqlite3() (string, error) {
dbname := viper.GetString("database.sqlite3.path")

db, err := gorm.Open(sqlite.Open(dbname), &gorm.Config{
NowFunc: func() time.Time {
return time.Now().UTC()
},
Logger: logger.ZLogger{},
})
if err != nil {
Expand Down
2 changes: 1 addition & 1 deletion pkg/dal/dao/artifact.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -123,7 +123,7 @@ func (s *artifactService) Create(ctx context.Context, artifact *models.Artifact)
func (s *artifactService) FindWithLastPull(ctx context.Context, repositoryID int64, before time.Time, limit, last int64) ([]*models.Artifact, error) {
return s.tx.Artifact.WithContext(ctx).
Where(s.tx.Artifact.LastPull.Lt(sql.NullTime{Valid: true, Time: before})).
Or(s.tx.Artifact.LastPull.IsNull(), s.tx.Artifact.UpdatedAt.Lt(before)).
Or(s.tx.Artifact.LastPull.IsNull(), s.tx.Artifact.UpdatedAt.Lt(before.UTC().UnixMilli())).
Where(s.tx.Artifact.ID.Gt(last), s.tx.Artifact.RepositoryID.Eq(repositoryID)).
Limit(int(limit)).Find()
}
Expand Down
2 changes: 1 addition & 1 deletion pkg/dal/dao/blob.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -87,7 +87,7 @@ func (s *blobService) Create(ctx context.Context, blob *models.Blob) error {
func (s *blobService) FindWithLastPull(ctx context.Context, before time.Time, last, limit int64) ([]*models.Blob, error) {
return s.tx.Blob.WithContext(ctx).
Where(s.tx.Blob.LastPull.Lt(sql.NullTime{Valid: true, Time: before})).
Or(s.tx.Blob.LastPull.IsNull(), s.tx.Blob.UpdatedAt.Lt(before)).
Or(s.tx.Blob.LastPull.IsNull(), s.tx.Blob.UpdatedAt.Lt(before.UTC().UnixMilli())).
Where(s.tx.Blob.ID.Gt(last)).Find()
}

Expand Down
2 changes: 1 addition & 1 deletion pkg/dal/dao/tag.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -161,7 +161,7 @@ func (s *tagService) FindWithQuantityCursor(ctx context.Context, repositoryID in
func (s *tagService) FindWithDayCursor(ctx context.Context, repositoryID int64, day, limit int, last int64) ([]*models.Tag, error) {
q := s.tx.Tag.WithContext(ctx).Where(s.tx.Tag.RepositoryID.Eq(repositoryID)).Order(s.tx.Tag.UpdatedAt.Desc())
if last == 0 {
q = q.Where(s.tx.Tag.UpdatedAt.Gt(time.Now().Add(time.Hour * 24 * time.Duration(day))))
q = q.Where(s.tx.Tag.UpdatedAt.Gt(time.Now().Add(time.Hour * 24 * time.Duration(day)).UTC().UnixMilli()))
} else {
q = q.Where(s.tx.Tag.ID.Gt(last))
}
Expand Down
Loading

0 comments on commit 033b745

Please sign in to comment.