Skip to content
This repository has been archived by the owner on Jul 12, 2023. It is now read-only.

Commit

Permalink
adjust symptom onset handling (#1131)
Browse files Browse the repository at this point in the history 
* adjust symptom onset handling

* test failures

* updated scoring

* add missing setting

* review comments
  • Loading branch information
@mikehelmick
mikehelmick authored Nov 10, 2020
1 parent 34ef7b6 commit b3ac176
Show file tree
Hide file tree
Showing 11 changed files with 466 additions and 202 deletions.
14 changes: 7 additions & 7 deletions internal/generate/config.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,14 +45,14 @@ type Config struct {
MaxSameStartIntervalKeys uint `env:"MAX_SAME_START_INTERVAL_KEYS, default=2"`
SimulateSameDayRelease bool `env:"SIMULATE_SAME_DAY_RELEASE, default=false"`
MaxIntervalAge time.Duration `env:"MAX_INTERVAL_AGE_ON_PUBLISH, default=360h"`
MaxMagnitudeSymptomOnsetDays uint `env:"MAX_SYMPTOM_ONSET_DAYS, default=21"`
MaxMagnitudeSymptomOnsetDays uint `env:"MAX_SYMPTOM_ONSET_DAYS, default=14"`
MaxSypmtomOnsetReportDays uint `env:"MAX_VALID_SYMPOTOM_ONSET_REPORT_DAYS, default=28"`
CreatedAtTruncateWindow time.Duration `env:"TRUNCATE_WINDOW, default=1h"`
DefaultRegion string `env:"DEFAULT_REGION, default=US"`
ChanceOfKeyRevision int `env:"CHANCE_OF_KEY_REVISION, default=30"` // 0-100 are valid values.
ChanceOfTraveler int `env:"CHANCE_OF_TRAVELER, default=20"` // 0-100 are valid values
KeyRevisionDelay time.Duration `env:"KEY_REVISION_DELAY, default=2h"` // key revision will be forward dates this amount.
UseDefaultSymptomOnset bool `env:"USE_DEFAULT_SYMPTOM_ONSET_DAYS, default=true"`
SymptomOnsetDays uint `env:"DEFAULT_SYMPTOM_ONSET_DAYS, default=10"`
SymptomOnsetDaysAgo uint `env:"DEFAULT_SYMPTOM_ONSET_DAYS_AGO, default=4"`
}

func (c *Config) MaxExposureKeys() uint {
Expand All @@ -75,12 +75,12 @@ func (c *Config) MaxSymptomOnsetDays() uint {
return c.MaxMagnitudeSymptomOnsetDays
}

func (c *Config) UseDefaultSymptomOnsetDays() bool {
return c.UseDefaultSymptomOnset
func (c *Config) MaxValidSymptomOnsetReportDays() uint {
return c.MaxSypmtomOnsetReportDays
}

func (c *Config) DefaultSymptomOnsetDays() int32 {
return int32(c.SymptomOnsetDays)
func (c *Config) DefaultSymptomOnsetDaysAgo() uint {
return c.SymptomOnsetDaysAgo
}

func (c *Config) DebugReleaseSameDayKeys() bool {
Expand Down
25 changes: 13 additions & 12 deletions internal/integration/helpers.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -207,18 +207,19 @@ func NewTestServer(tb testing.TB) (*serverenv.ServerEnv, *Client) {
}
mux.Handle("/key-rotation/", http.StripPrefix("/key-rotation", rotationServer.Routes(ctx)))

// Publish
publishConfig := &publish.Config{
RevisionToken: revConfig,
MaxKeysOnPublish: 15,
MaxSameStartIntervalKeys: 2,
MaxIntervalAge: 360 * time.Hour,
CreatedAtTruncateWindow: 1 * time.Second,
ReleaseSameDayKeys: true,
RevisionKeyCacheDuration: time.Second,
}

publishHandler, err := publish.NewHandler(ctx, publishConfig, env)
// Parse the config to load default values.
publishConfig := publish.Config{}
envconfig.ProcessWith(ctx, &publishConfig, envconfig.OsLookuper())
// Make overrides.
publishConfig.RevisionToken = revConfig
publishConfig.MaxKeysOnPublish = 15
publishConfig.MaxSameStartIntervalKeys = 2
publishConfig.MaxIntervalAge = 360 * time.Hour
publishConfig.CreatedAtTruncateWindow = time.Second
publishConfig.ReleaseSameDayKeys = true
publishConfig.RevisionKeyCacheDuration = time.Second

publishHandler, err := publish.NewHandler(ctx, &publishConfig, env)
if err != nil {
tb.Fatal(err)
}
Expand Down
5 changes: 5 additions & 0 deletions internal/integration/integration_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -417,6 +417,9 @@ func getExposures(db *database.DB, criteria publishdb.IterateExposuresCriteria)
// output).
func exportedKeysFrom(tb testing.TB, keys []verifyapi.ExposureKey) []*export.TemporaryExposureKey {
s := make([]*export.TemporaryExposureKey, len(keys))
sort.Slice(keys, func(i, j int) bool {
return keys[i].IntervalNumber >= keys[j].IntervalNumber
})
for i, key := range keys {
decoded, err := base64util.DecodeString(key.Key)
if err != nil {
Expand All @@ -428,6 +431,8 @@ func exportedKeysFrom(tb testing.TB, keys []verifyapi.ExposureKey) []*export.Tem
TransmissionRiskLevel: proto.Int32(int32(key.TransmissionRisk)),
RollingStartIntervalNumber: proto.Int32(key.IntervalNumber),
ReportType: export.TemporaryExposureKey_CONFIRMED_TEST.Enum(),
// Keys are generated 1 day ago and then -1 day for each additional.
DaysSinceOnsetOfSymptoms: proto.Int32(int32(3 - i)),
}
}

Expand Down
49 changes: 36 additions & 13 deletions internal/publish/config.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
package publish

import (
"fmt"
"time"

"github.com/google/exposure-notifications-server/internal/authorizedapp"
Expand All @@ -27,6 +28,7 @@ import (
"github.com/google/exposure-notifications-server/pkg/keys"
"github.com/google/exposure-notifications-server/pkg/observability"
"github.com/google/exposure-notifications-server/pkg/secrets"
"github.com/hashicorp/go-multierror"
)

// Compile-time check to assert this config matches requirements.
Expand All @@ -51,15 +53,21 @@ type Config struct {
Port string `env:"PORT, default=8080"`
MaxKeysOnPublish uint `env:"MAX_KEYS_ON_PUBLISH, default=30"`
// Provides compatibility w/ 1.5 release.
MaxSameStartIntervalKeys uint `env:"MAX_SAME_START_INTERVAL_KEYS, default=3"`
MaxIntervalAge time.Duration `env:"MAX_INTERVAL_AGE_ON_PUBLISH, default=360h"`
MaxMagnitudeSymptomOnsetDays uint `env:"MAX_SYMPTOM_ONSET_DAYS, default=10"`
CreatedAtTruncateWindow time.Duration `env:"TRUNCATE_WINDOW, default=1h"`

// If set, TEKs that arrive without a days since symptom onset (i.e. no symptom onset date)
// will be set to the default symptom onset days.
UseDefaultSymptomOnset bool `env:"USE_DEFAULT_SYMPTOM_ONSET_DAYS, default=true"`
SymptomOnsetDays uint `env:"DEFAULT_SYMPTOM_ONSET_DAYS, default=10"`
MaxSameStartIntervalKeys uint `env:"MAX_SAME_START_INTERVAL_KEYS, default=3"`
MaxIntervalAge time.Duration `env:"MAX_INTERVAL_AGE_ON_PUBLISH, default=360h"`
CreatedAtTruncateWindow time.Duration `env:"TRUNCATE_WINDOW, default=1h"`

// Symptom onset settings.
// Maximum valid range. TEKs presneted with values outside this range, but still "reasonable" will not be saved.
MaxMagnitudeSymptomOnsetDays uint `env:"MAX_SYMPTOM_ONSET_DAYS, default=14"`
// MaxValidSypmtomOnsetReportDays indicates how many days would be considered
// a valid symptom onset report (-val..+val). Anything outside
// that range would be subject to the default symptom onset flags (see below).
MaxSypmtomOnsetReportDays uint `env:"MAX_VALID_SYMPOTOM_ONSET_REPORT_DAYS, default=28"`

// TEKs that arrive without a days since symptom onset (i.e. no symptom onset date),
// then the upload date minus DEFAULT_SYMPTOM_ONSET_DAYS_AGO is used.
SymptomOnsetDaysAgo uint `env:"DEFAULT_SYMPTOM_ONSET_DAYS_AGO, default=4"`

ResponsePaddingMinBytes int64 `env:"RESPONSE_PADDING_MIN_BYTES, default=1024"`
ResponsePaddingRange int64 `env:"RESPONSE_PADDING_RANGE, default=1024"`
Expand Down Expand Up @@ -91,6 +99,21 @@ type Config struct {
DebugLogBadCertificates bool `env:"DEBUG_LOG_BAD_CERTIFICATES"`
}

func (c *Config) Validate() error {
var result *multierror.Error

if c.MaxMagnitudeSymptomOnsetDays == 0 {
result = multierror.Append(result,
fmt.Errorf("env var `MAX_SYMPTOM_ONSET_DAYS` must be > 0, got: %v", c.MaxMagnitudeSymptomOnsetDays))
}
if c.MaxSypmtomOnsetReportDays == 0 {
result = multierror.Append(result,
fmt.Errorf("env var `MAX_VALID_SYMPOTOM_ONSET_REPORT_DAYS` must be > 0, got: %v", c.MaxSypmtomOnsetReportDays))
}

return result.ErrorOrNil()
}

func (c *Config) MaxExposureKeys() uint {
return c.MaxKeysOnPublish
}
Expand All @@ -111,12 +134,12 @@ func (c *Config) MaxSymptomOnsetDays() uint {
return c.MaxMagnitudeSymptomOnsetDays
}

func (c *Config) UseDefaultSymptomOnsetDays() bool {
return c.UseDefaultSymptomOnset
func (c *Config) MaxValidSymptomOnsetReportDays() uint {
return c.MaxSypmtomOnsetReportDays
}

func (c *Config) DefaultSymptomOnsetDays() int32 {
return int32(c.SymptomOnsetDays)
func (c *Config) DefaultSymptomOnsetDaysAgo() uint {
return c.SymptomOnsetDaysAgo
}

func (c *Config) DebugReleaseSameDayKeys() bool {
Expand Down
90 changes: 51 additions & 39 deletions internal/publish/model/exposure_model.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ import (
"github.com/google/exposure-notifications-server/internal/verification"
"github.com/google/exposure-notifications-server/pkg/base64util"
"github.com/google/exposure-notifications-server/pkg/logging"
"github.com/google/exposure-notifications-server/pkg/timeutils"
"github.com/hashicorp/go-multierror"

verifyapi "github.com/google/exposure-notifications-server/pkg/api/v1"
Expand Down Expand Up @@ -379,21 +380,21 @@ type TransformerConfig interface {
MaxIntervalStartAge() time.Duration
TruncateWindow() time.Duration
MaxSymptomOnsetDays() uint
UseDefaultSymptomOnsetDays() bool
DefaultSymptomOnsetDays() int32
MaxValidSymptomOnsetReportDays() uint
DefaultSymptomOnsetDaysAgo() uint
DebugReleaseSameDayKeys() bool
}

// Transformer represents a configured Publish -> Exposure[] transformer.
type Transformer struct {
maxExposureKeys int // Overall maximum number of keys.
maxSameDayKeys int // Number of keys that are allowed to have the same start interval.
maxIntervalStartAge time.Duration // How many intervals old does this server accept?
truncateWindow time.Duration
maxSymptomOnsetDays float64 // to avoid casting in comparisons
useDefaultSymptomOnsetDays bool
defaultSymptomOnsetDays int32
debugReleaseSameDay bool // If true, still valid keys are not embargoed.
maxExposureKeys int // Overall maximum number of keys.
maxSameDayKeys int // Number of keys that are allowed to have the same start interval.
maxIntervalStartAge time.Duration // How many intervals old does this server accept?
truncateWindow time.Duration
maxSymptomOnsetDays float64 // to avoid casting in comparisons
maxValidSymptomOnsetReportDays uint
defaultSymptomOnsetDaysAgo uint
debugReleaseSameDay bool // If true, still valid keys are not embargoed.
}

// NewTransformer creates a transformer for turning publish API requests into
Expand All @@ -407,14 +408,14 @@ func NewTransformer(config TransformerConfig) (*Transformer, error) {
return nil, fmt.Errorf("maxSameDayKeys must be >= 1, got %v", config.MaxSameDayKeys())
}
return &Transformer{
maxExposureKeys: int(config.MaxExposureKeys()),
maxSameDayKeys: int(config.MaxSameDayKeys()),
maxIntervalStartAge: config.MaxIntervalStartAge(),
truncateWindow: config.TruncateWindow(),
maxSymptomOnsetDays: float64(config.MaxSymptomOnsetDays()),
useDefaultSymptomOnsetDays: config.UseDefaultSymptomOnsetDays(),
defaultSymptomOnsetDays: config.DefaultSymptomOnsetDays(),
debugReleaseSameDay: config.DebugReleaseSameDayKeys(),
maxExposureKeys: int(config.MaxExposureKeys()),
maxSameDayKeys: int(config.MaxSameDayKeys()),
maxIntervalStartAge: config.MaxIntervalStartAge(),
truncateWindow: config.TruncateWindow(),
maxSymptomOnsetDays: float64(config.MaxSymptomOnsetDays()),
maxValidSymptomOnsetReportDays: config.MaxValidSymptomOnsetReportDays(),
defaultSymptomOnsetDaysAgo: config.DefaultSymptomOnsetDaysAgo(),
debugReleaseSameDay: config.DebugReleaseSameDayKeys(),
}, nil
}

Expand Down Expand Up @@ -559,11 +560,33 @@ func (t *Transformer) TransformPublish(ctx context.Context, inData *verifyapi.Pu
BatchWindow: t.truncateWindow,
}

onsetInterval := inData.SymptomOnsetInterval
// If the symtom onset interval provided on publish is too old to be relevant
// and one was provided in the verification certificate, take that one.
if onsetInterval < settings.MinStartInterval && claims != nil && claims.SymptomOnsetInterval > 0 {
onsetInterval = int32(claims.SymptomOnsetInterval)
// For validating key timing information, can't be newer than now.
currentInterval := IntervalNumber(batchTime)
// For validating the passed in symptom interval, relative to current time.
minSymptomInterval := IntervalNumber(
timeutils.UTCMidnight(timeutils.SubtractDays(batchTime, t.maxValidSymptomOnsetReportDays)))

// Base level, assume there is no symptom onset interval present.
onsetInterval := int32(0)
if pubInt := inData.SymptomOnsetInterval; pubInt < currentInterval && pubInt >= minSymptomInterval {
onsetInterval = pubInt
} else if claims != nil {
if vcInt := int32(claims.SymptomOnsetInterval); vcInt < currentInterval && vcInt >= minSymptomInterval {
// If the symtom onset interval provided on publish is too old to be relevant
// and one was provided in the verification certificate, take that one.
onsetInterval = int32(claims.SymptomOnsetInterval)
}
}
// If we reach this point, and onsetInterval is 0 OR if the onset interval
// is "unreasonable" then we default the onsetInterval to 4 (*configurable)
// days ago to approximate symptom onset.
//
// There are launched applications using this sever that rely on this
// behavior - that are passing invalid symotom onset interviews, those
// are screened about above when the onsetInterval is set.
if daysSince := math.Abs(float64(DaysFromSymptomOnset(onsetInterval, currentInterval))); onsetInterval == 0 || daysSince > float64(t.maxValidSymptomOnsetReportDays) {
logger.Debugw("defaulting days since symptom onset")
onsetInterval = IntervalNumber(timeutils.SubtractDays(batchTime, t.defaultSymptomOnsetDaysAgo))
}

// Regions are a multi-value property, uppercase them for storage.
Expand Down Expand Up @@ -597,29 +620,18 @@ func (t *Transformer) TransformPublish(ctx context.Context, inData *verifyapi.Pu
// Set days since onset, either from the API or from the verified claims (see above).
if onsetInterval > 0 {
daysSince := DaysFromSymptomOnset(onsetInterval, exposure.IntervalNumber)
// Check if the magnitude of this value is too large. If it is too large,
// we won't want to set a days since symptom onset value on the TEK
// itself, but we do want to warn the application developer that this
// value (not TEK) was dropped.
//
// There are launched applications using this sever that rely on this
// behavior.
//
// Note that previously this returned an error, but this broke the iOS
// implementation since it is unable to handle partial success. As such,
// it was converted to a warning that's a separate field in the API
// response.
if abs := math.Abs(float64(daysSince)); abs > t.maxSymptomOnsetDays {
logger.Debugw("setting days since symptom onset to null on key due to symptom onset magnitude too high", "daysSince", daysSince)
transformWarnings = append(transformWarnings, fmt.Sprintf("key %d symptom onset is too large, %v > %v - saving without days since symptom onset", i, abs, t.maxSymptomOnsetDays))
} else {
// The value is within acceptable range, save it.
exposure.SetDaysSinceSymptomOnset(daysSince)
transformWarnings = append(transformWarnings, fmt.Sprintf("key %d symptom onset is too large, %v > %v - saving without this key", i, abs, t.maxSymptomOnsetDays))
continue
}
}
// See if a default symptom onset days should be applied.
if exposure.DaysSinceSymptomOnset == nil && t.useDefaultSymptomOnsetDays {
exposure.SetDaysSinceSymptomOnset(t.defaultSymptomOnsetDays)

// The value is within acceptable range, save it.
exposure.SetDaysSinceSymptomOnset(daysSince)
}

exposure.Traveler = inData.Traveler
Expand Down
Loading

0 comments on commit b3ac176

Please sign in to comment.