Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

✨ Add raw results to cron scans #1741

Merged
merged 12 commits into from
Mar 19, 2022
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 1 addition & 2 deletions checks/binary_artifact.go
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ const CheckBinaryArtifacts string = "Binary-Artifacts"

//nolint
func init() {
var supportedRequestTypes = []checker.RequestType{
supportedRequestTypes := []checker.RequestType{
checker.FileBased,
checker.CommitBased,
}
Expand All @@ -47,7 +47,6 @@ func BinaryArtifacts(c *checker.CheckRequest) checker.CheckResult {
// Return raw results.
if c.RawResults != nil {
c.RawResults.BinaryArtifactResults = rawData
return checker.CheckResult{}
}

// Return the score evaluation.
Expand Down
1 change: 0 additions & 1 deletion checks/branch_protection.go
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,6 @@ func BranchProtection(c *checker.CheckRequest) checker.CheckResult {
// Return raw results.
if c.RawResults != nil {
c.RawResults.BranchProtectionResults = rawData
return checker.CheckResult{}
}

// Return the score evaluation.
Expand Down
1 change: 0 additions & 1 deletion checks/code_review.go
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,6 @@ func CodeReview(c *checker.CheckRequest) checker.CheckResult {
// Return raw results.
if c.RawResults != nil {
c.RawResults.CodeReviewResults = rawData
return checker.CheckResult{}
}

// Return the score evaluation.
Expand Down
3 changes: 1 addition & 2 deletions checks/dependency_update_tool.go
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ const CheckDependencyUpdateTool = "Dependency-Update-Tool"

//nolint
func init() {
var supportedRequestTypes = []checker.RequestType{
supportedRequestTypes := []checker.RequestType{
checker.FileBased,
}
if err := registerCheck(CheckDependencyUpdateTool, DependencyUpdateTool, supportedRequestTypes); err != nil {
Expand All @@ -46,7 +46,6 @@ func DependencyUpdateTool(c *checker.CheckRequest) checker.CheckResult {
// Return raw results.
if c.RawResults != nil {
c.RawResults.DependencyUpdateToolResults = rawData
return checker.CheckResult{}
}

// Return the score evaluation.
Expand Down
1 change: 0 additions & 1 deletion checks/security_policy.go
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,6 @@ func SecurityPolicy(c *checker.CheckRequest) checker.CheckResult {
// Set the raw results.
if c.RawResults != nil {
c.RawResults.SecurityPolicyResults = rawData
return checker.CheckResult{}
}

return evaluation.SecurityPolicy(CheckSecurityPolicy, c.Dlogger, &rawData)
Expand Down
1 change: 0 additions & 1 deletion checks/vulnerabilities.go
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,6 @@ func Vulnerabilities(c *checker.CheckRequest) checker.CheckResult {
// Set the raw results.
if c.RawResults != nil {
c.RawResults.VulnerabilitiesResults = rawData
return checker.CheckResult{}
}

return evaluation.Vulnerabilities(CheckVulnerabilities, c.Dlogger, &rawData)
Expand Down
1 change: 0 additions & 1 deletion cmd/root.go
Original file line number Diff line number Diff line change
Expand Up @@ -128,7 +128,6 @@ func rootCmd(o *options.Options) {
ctx,
repoURI,
o.Commit,
o.Format == options.FormatRaw,
enabledChecks,
repoClient,
ossFuzzRepoClient,
Expand Down
2 changes: 1 addition & 1 deletion cmd/serve.go
Original file line number Diff line number Diff line change
Expand Up @@ -69,7 +69,7 @@ func serveCmd(o *options.Options) *cobra.Command {
defer ossFuzzRepoClient.Close()
ciiClient := clients.DefaultCIIBestPracticesClient()
repoResult, err := pkg.RunScorecards(
ctx, repo, clients.HeadSHA /*commitSHA*/, false /*raw*/, checks.AllChecks, repoClient,
ctx, repo, clients.HeadSHA /*commitSHA*/, checks.AllChecks, repoClient,
ossFuzzRepoClient, ciiClient, vulnsClient)
if err != nil {
logger.Error(err, "running enabled scorecard checks on repo")
Expand Down
20 changes: 19 additions & 1 deletion cron/config/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -47,9 +47,12 @@ const (
metricExporter string = "SCORECARD_METRIC_EXPORTER"
ciiDataBucketURL string = "SCORECARD_CII_DATA_BUCKET_URL"
blacklistedChecks string = "SCORECARD_BLACKLISTED_CHECKS"

// v2 results.
bigqueryTableV2 string = "SCORECARD_BIGQUERY_TABLEV2"
resultDataBucketURLV2 string = "SCORECARD_DATA_BUCKET_URLV2"
// Raw results.
rawBigqueryTable string = "RAW_SCORECARD_BIGQUERY_TABLE"
rawResultDataBucketURL string = "RAW_SCORECARD_DATA_BUCKET_URL"
)

var (
Expand Down Expand Up @@ -78,6 +81,9 @@ type config struct {
// UPGRADEv2: to remove.
ResultDataBucketURLV2 string `yaml:"result-data-bucket-url-v2"`
BigQueryTableV2 string `yaml:"bigquery-table-v2"`
// Raw results.
RawResultDataBucketURL string `yaml:"raw-result-data-bucket-url"`
RawBigQueryTable string `yaml:"raw-bigquery-table"`
}

func getParsedConfigFromFile(byteValue []byte) (config, error) {
Expand Down Expand Up @@ -196,6 +202,18 @@ func GetResultDataBucketURLV2() (string, error) {
return getStringConfigValue(resultDataBucketURLV2, configYAML, "ResultDataBucketURLV2", "result-data-bucket-url-v2")
}

// GetRawBigQueryTable returns the table name to transfer cron job results.
func GetRawBigQueryTable() (string, error) {
return getStringConfigValue(rawBigqueryTable, configYAML,
"RawBigQueryTable", "raw-bigquery-table")
}

// GetRawResultDataBucketURL returns the bucketURL for storing cron job's raw results.
func GetRawResultDataBucketURL() (string, error) {
return getStringConfigValue(rawResultDataBucketURL, configYAML,
"RawResultDataBucketURL", "raw-result-data-bucket-url")
}

// GetShardSize returns the shard_size for the cron job.
func GetShardSize() (int, error) {
return getIntConfigValue(shardSize, configYAML, "ShardSize", "shard-size")
Expand Down
3 changes: 3 additions & 0 deletions cron/config/config.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -29,3 +29,6 @@ metric-exporter: stackdriver
# UPGRADEv2: to remove.
result-data-bucket-url-v2: gs://ossf-scorecard-data2
bigquery-table-v2: scorecard-v2
# Raw results.
raw-result-data-bucket-url: gs://ossf-scorecard-rawdata
raw-bigquery-table: scorecard-rawdata
9 changes: 7 additions & 2 deletions cron/config/config_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,9 @@ const (
// UPGRADEv2: to remove.
prodBucketV2 = "gs://ossf-scorecard-data2"
prodBigQueryTableV2 = "scorecard-v2"
// Raw results.
prodRawBucket = "gs://ossf-scorecard-rawdata"
prodRawBigQueryTable = "scorecard-rawdata"
)

func getByteValueFromFile(filename string) ([]byte, error) {
Expand Down Expand Up @@ -73,8 +76,10 @@ func TestYAMLParsing(t *testing.T) {
ShardSize: prodShardSize,
MetricExporter: prodMetricExporter,
// UPGRADEv2: to remove.
ResultDataBucketURLV2: prodBucketV2,
BigQueryTableV2: prodBigQueryTableV2,
ResultDataBucketURLV2: prodBucketV2,
BigQueryTableV2: prodBigQueryTableV2,
RawResultDataBucketURL: prodRawBucket,
RawBigQueryTable: prodRawBigQueryTable,
},
},

Expand Down
19 changes: 18 additions & 1 deletion cron/controller/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,8 @@ import (
var headSHA = clients.HeadSHA

func publishToRepoRequestTopic(iter data.Iterator, topicPublisher pubsub.Publisher,
shardSize int, datetime time.Time) (int32, error) {
shardSize int, datetime time.Time,
) (int32, error) {
var shardNum int32
request := data.ScorecardBatchRequest{
JobTime: timestamppb.New(datetime),
Expand Down Expand Up @@ -122,6 +123,11 @@ func main() {
panic(err)
}

rawBucket, err := config.GetRawResultDataBucketURL()
if err != nil {
panic(err)
}

shardNum, err := publishToRepoRequestTopic(reader, topicPublisher, shardSize, t)
if err != nil {
panic(err)
Expand Down Expand Up @@ -154,4 +160,15 @@ func main() {
if err != nil {
panic(fmt.Errorf("error writing to BlobStore2: %w", err))
}

// Raw data.
*metadata.ShardLoc = rawBucket + "/" + data.GetBlobFilename("", t)
metadataJSON, err = protojson.Marshal(&metadata)
if err != nil {
panic(fmt.Errorf("error during protojson.Marshal raw: %w", err))
}
err = data.WriteToBlobStore(ctx, rawBucket, data.GetShardMetadataFilename(t), metadataJSON)
if err != nil {
panic(fmt.Errorf("error writing to BlobStore raw: %w", err))
}
}
Loading