Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add sampler API, use in SDK tracer #225

Merged
merged 25 commits into from
Oct 24, 2019
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 14 additions & 2 deletions .flake8
Original file line number Diff line number Diff line change
@@ -1,3 +1,15 @@
[flake8]
ignore = E501,W503,E203
exclude = .svn,CVS,.bzr,.hg,.git,__pycache__,.tox,ext/opentelemetry-ext-jaeger/src/opentelemetry/ext/jaeger/gen/,ext/opentelemetry-ext-jaeger/build/*
ignore =
E501 # line too long, defer to black
F401 # unused import, defer to pylint
W503 # allow line breaks after binary ops, not after
exclude =
.bzr
.git
.hg
.svn
.tox
CVS
__pycache__
ext/opentelemetry-ext-jaeger/src/opentelemetry/ext/jaeger/gen/
ext/opentelemetry-ext-jaeger/build/*
3 changes: 2 additions & 1 deletion .pylintrc
Original file line number Diff line number Diff line change
Expand Up @@ -68,7 +68,8 @@ disable=missing-docstring,
ungrouped-imports, # Leave this up to isort
wrong-import-order, # Leave this up to isort
bad-continuation, # Leave this up to black
line-too-long # Leave this up to black
line-too-long, # Leave this up to black
exec-used

# Enable the message, report, category or checker with the given id(s). You can
# either give multiple identifier separated by comma (,) or put this option
Expand Down
2 changes: 1 addition & 1 deletion docs/conf.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@
# -- Project information -----------------------------------------------------

project = "OpenTelemetry"
copyright = "2019, OpenTelemetry Authors"
copyright = "2019, OpenTelemetry Authors" # pylint: disable=redefined-builtin
author = "OpenTelemetry Authors"


Expand Down
17 changes: 12 additions & 5 deletions opentelemetry-api/src/opentelemetry/trace/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -253,8 +253,8 @@ def __exit__(
class TraceOptions(int):
"""A bitmask that represents options specific to the trace.

The only supported option is the "recorded" flag (``0x01``). If set, this
flag indicates that the trace may have been recorded upstream.
The only supported option is the "sampled" flag (``0x01``). If set, this
flag indicates that the trace may have been sampled upstream.

See the `W3C Trace Context - Traceparent`_ spec for details.

Expand All @@ -263,12 +263,16 @@ class TraceOptions(int):
"""

DEFAULT = 0x00
RECORDED = 0x01
SAMPLED = 0x01

@classmethod
def get_default(cls) -> "TraceOptions":
return cls(cls.DEFAULT)

@property
def sampled(self) -> bool:
return bool(self & TraceOptions.SAMPLED)


DEFAULT_TRACE_OPTIONS = TraceOptions.get_default()

Expand Down Expand Up @@ -309,8 +313,8 @@ class SpanContext:
Args:
trace_id: The ID of the trace that this span belongs to.
span_id: This span's ID.
options: Trace options to propagate.
state: Tracing-system-specific info to propagate.
trace_options: Trace options to propagate.
trace_state: Tracing-system-specific info to propagate.
"""

def __init__(
Expand Down Expand Up @@ -363,6 +367,9 @@ def __init__(self, context: "SpanContext") -> None:
def get_context(self) -> "SpanContext":
return self._context

def is_recording_events(self) -> bool:
c24t marked this conversation as resolved.
Show resolved Hide resolved
return False


INVALID_SPAN_ID = 0x0000000000000000
INVALID_TRACE_ID = 0x00000000000000000000000000000000
Expand Down
125 changes: 125 additions & 0 deletions opentelemetry-api/src/opentelemetry/trace/sampling.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,125 @@
# Copyright 2019, OpenTelemetry Authors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import abc
from typing import Dict, Mapping, Optional, Sequence

# pylint: disable=unused-import
from opentelemetry.trace import Link, SpanContext
from opentelemetry.util.types import AttributeValue


class Decision:
"""A sampling decision as applied to a newly-created Span.

Args:
sampled: Whether the `Span` should be sampled.
attributes: Attributes to add to the `Span`.
"""

def __repr__(self) -> str:
return "{}({}, attributes={})".format(
type(self).__name__, str(self.sampled), str(self.attributes)
)

def __init__(
self,
sampled: bool = False,
attributes: Mapping[str, "AttributeValue"] = None,
) -> None:
self.sampled = sampled # type: bool
if attributes is None:
self.attributes = {} # type: Dict[str, "AttributeValue"]
else:
self.attributes = dict(attributes)


class Sampler(abc.ABC):
@abc.abstractmethod
def should_sample(
self,
parent_context: Optional["SpanContext"],
trace_id: int,
span_id: int,
name: str,
links: Sequence["Link"] = (),
) -> "Decision":
pass


class StaticSampler(Sampler):
"""Sampler that always returns the same decision."""

def __init__(self, decision: "Decision"):
self._decision = decision

def should_sample(
self,
parent_context: Optional["SpanContext"],
trace_id: int,
span_id: int,
name: str,
links: Sequence["Link"] = (),
) -> "Decision":
return self._decision


class ProbabilitySampler(Sampler):
def __init__(self, rate: float):
self._rate = rate
self._bound = self.get_bound_for_rate(self._rate)

# The sampler checks the last 8 bytes of the trace ID to decide whether to
# sample a given trace.
CHECK_BYTES = 0xFFFFFFFFFFFFFFFF
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

According to the W3 spec, you should use the high ("left") 8 bytes instead:

Many unique identification generation algorithms create IDs where one part of the value is constant (often time- or host-based), and the other part is a randomly generated value. Because tracing systems may make sampling decisions based on the value of trace-id, for increased interoperability vendors MUST keep the random part of trace-id ID on the left side.

But I wonder if we can find a more robust way to maybe randomly mix some bits here and there together.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I considered this but decided to stick to the OC behavior (or at least the intended behavior: this also fixes a rounding/OBO bug). FWIW checking the high bytes also seems more correct to me, but in practice -- if people are using short trace IDs in the wild -- sampling every request seems worse than sampling based on the non-random part.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think you're right about this. I suggested it as a change to the spec in open-telemetry/opentelemetry-specification#331.

Another fun benefit of checking bytes high-to-low is that the sampling decision should be mostly consistent between samplers that check different numbers of bytes. Unlike checking low-to-high where every incremental bit is effectively another coin toss. Ultimately we'll probably just put this number in the spec, but it's a neat property in any case. :D

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Let's revisit this once open-telemetry/opentelemetry-specification#331 and w3c/trace-context#344 are resolved.


@classmethod
def get_bound_for_rate(cls, rate: float) -> int:
return round(rate * (cls.CHECK_BYTES + 1))
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If rate is less than 0.0000000000000000000271, bound will be rounded to 0 which means it never gets sampled.
Do we consider to round up or this is just too pedantic?

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Actually it is 0.0000000000000000000813 since bound is rounded to 1 (and all-zero trace id is illegal according to the TraceContext spec). 😄

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I used round intentionally here for what I thought were pedantic reasons: this way each of the 2^64 sampling rate "buckets" are split exactly in half:

            r <= 2^-65     : 0
    2^-65 < r <= 3 * 2^-65 : 1
3 * 2^-65 < r <= 5 * 2^-65 : 2

and etc.

I think if you pass in a sampling rate <= 2^-65 it's more correct for us to make the actual rate 0 than 2^-64.

This doesn't exactly work because float precision is worse than 2^-64 for values close to 1, but it works near 0 and is at least theoretically correct (or should be, there may still be bugs here).

all-zero trace id is illegal according to the TraceContext spec.

Oh that's interesting, so the trace ID space is [0x1, 0xf...f], not [0x0, 0xf...f]. So would you expect this line to be:

return 1 + round(rate * cls.CHECK_BYTES)

I think that'd give us the same behavior as above, but always sample trace ID 0x0.

Should we have special handling for 0x0 trace IDs?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm also happy to leave this as-is and treat 0x0 as a regular ID here, but treat it differently at span creation time. Up to you. :D

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I don't have any preference here as I cannot think of a scenario where people would need such low but non-zero sample rate (unless someone is instrumenting SETI@home). 😆

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Something I missed above: there are and 2^64 - 1 trace IDs besides 0x0 that end in 0x0000000000000000. If we want to always/never sample the all-zero trace ID we have to make that decision before masking.


@property
def rate(self) -> float:
return self._rate

@rate.setter
def rate(self, new_rate: float) -> None:
self._rate = new_rate
self._bound = self.get_bound_for_rate(self._rate)

@property
def bound(self) -> int:
return self._bound

def should_sample(
self,
parent_context: Optional["SpanContext"],
trace_id: int,
span_id: int,
name: str,
links: Sequence["Link"] = (),
) -> "Decision":
if parent_context is not None:
return Decision(parent_context.trace_options.sampled)

return Decision(trace_id & self.CHECK_BYTES < self.bound)


# Samplers that ignore the parent sampling decision and never/always sample.
ALWAYS_OFF = StaticSampler(Decision(False))
ALWAYS_ON = StaticSampler(Decision(True))

# Samplers that respect the parent sampling decision, but otherwise
# never/always sample.
DEFAULT_OFF = ProbabilitySampler(0.0)
DEFAULT_ON = ProbabilitySampler(1.0)
Loading