Skip to content

Commit

Permalink
feat!: switch to MOJO output
Browse files Browse the repository at this point in the history 
We make MOJO the output format for Echion.
  • Loading branch information
@P403n1x87
P403n1x87 committed Oct 31, 2023
1 parent 0121b28 commit 9b7cdda
Show file tree
Hide file tree
Showing 8 changed files with 204 additions and 43 deletions.
16 changes: 14 additions & 2 deletions echion/coremodule.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -168,8 +168,20 @@ _sampler()
return;
}

output << "# mode: " << (cpu ? "cpu" : "wall") << std::endl;
output << "# interval: " << interval << std::endl;
mojo_header();

mojo_metadata("mode", (cpu ? "cpu" : "wall"));
mojo_metadata("interval", interval);
mojo_metadata("sampler", "echion");

// DEV: Workaround for the austin-python library: we send an empty sample
// to set the PID. We also map the key value 0 to the empty string, to
// support task name frames.
mojo_stack(pid, 0, "");
mojo_string_event(0, "");
mojo_string_event(1, "<invalid>");
mojo_string_event(2, "<unknown>");
mojo_metric_time(0);

while (running)
{
Expand Down
19 changes: 13 additions & 6 deletions echion/frame.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,16 +23,16 @@
#include <libunwind.h>

#include <echion/cache.h>
#include <echion/mojo.h>
#include <echion/strings.h>
#include <echion/vm.h>

#define MOJO_INT32 ((uintptr_t)(1 << (6 + 7 * 3)) - 1)

class Frame
{
public:
typedef std::reference_wrapper<Frame> Ref;
typedef std::unique_ptr<Frame> Ptr;
using Ref = std::reference_wrapper<Frame>;
using Ptr = std::unique_ptr<Frame>;
using Key = uintptr_t;

class Error : public std::exception
{
Expand All @@ -52,6 +52,7 @@ class Frame
}
};

Key cache_key = 0;
StringTable::Key filename = 0;
StringTable::Key name = 0;

Expand Down Expand Up @@ -107,7 +108,7 @@ class Frame
private:
void infer_location(PyCodeObject *, int);

static inline uintptr_t key(PyCodeObject *code, int lasti)
static inline Key key(PyCodeObject *code, int lasti)
{
return (((uintptr_t)(((uintptr_t)code) & MOJO_INT32) << 16) | lasti);
}
Expand Down Expand Up @@ -323,7 +324,7 @@ Frame &Frame::get(PyCodeObject *code_addr, int lasti)
if (copy_type(code_addr, code))
return INVALID_FRAME;

uintptr_t frame_key = Frame::key(code_addr, lasti);
auto frame_key = Frame::key(code_addr, lasti);

try
{
Expand All @@ -334,7 +335,9 @@ Frame &Frame::get(PyCodeObject *code_addr, int lasti)
try
{
auto new_frame = std::make_unique<Frame>(&code, lasti);
new_frame->cache_key = frame_key;
auto &f = *new_frame;
mojo_frame(frame_key, new_frame);
frame_cache->store(frame_key, std::move(new_frame));
return f;
}
Expand Down Expand Up @@ -362,7 +365,9 @@ Frame &Frame::get(unw_cursor_t &cursor)
try
{
auto frame = std::make_unique<Frame>(cursor, pc);
frame->cache_key = frame_key;
auto &f = *frame;
mojo_frame(frame_key, frame);
frame_cache->store(frame_key, std::move(frame));
return f;
}
Expand All @@ -383,7 +388,9 @@ Frame &Frame::get(StringTable::Key name)
catch (LRUCache<uintptr_t, Frame>::LookupError &)
{
auto frame = std::make_unique<Frame>(name);
frame->cache_key = frame_key;
auto &f = *frame;
mojo_frame(frame_key, frame);
frame_cache->store(frame_key, std::move(frame));
return f;
}
Expand Down
140 changes: 140 additions & 0 deletions echion/mojo.h
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,140 @@
// This file is part of "echion" which is released under MIT.
//
// Copyright (c) 2023 Gabriele N. Tornetta <[email protected]>.

#pragma once

#include <echion/config.h>

#define MOJO_VERSION 3

enum
{
MOJO_RESERVED,
MOJO_METADATA,
MOJO_STACK,
MOJO_FRAME,
MOJO_FRAME_INVALID,
MOJO_FRAME_REF,
MOJO_FRAME_KERNEL,
MOJO_GC,
MOJO_IDLE,
MOJO_METRIC_TIME,
MOJO_METRIC_MEMORY,
MOJO_STRING,
MOJO_STRING_REF,
MOJO_MAX,
};

#if defined __arm__
typedef unsigned long mojo_int_t;
#else
typedef unsigned long long mojo_int_t;
#endif

// Bitmask to ensure that we encode at most 4 bytes for an integer.
#define MOJO_INT32 ((mojo_int_t)(1 << (6 + 7 * 3)) - 1)

// Primitives

#define mojo_event(event) \
{ \
output.put((char)event); \
}

#define mojo_string(string) \
output << string; \
output.put('\0');

static inline void
mojo_integer(mojo_int_t integer, int sign)
{
unsigned char byte = integer & 0x3f;
if (sign)
byte |= 0x40;

integer >>= 6;
if (integer)
byte |= 0x80;

output.put(byte);

while (integer)
{
byte = integer & 0x7f;
integer >>= 7;
if (integer)
byte |= 0x80;
output.put(byte);
}
}

// We expect the least significant bits to be varied enough to provide a valid
// key. This way we can keep the size of references to a maximum of 4 bytes.
#define mojo_ref(integer) (mojo_integer(MOJO_INT32 & ((mojo_int_t)integer), 0))

// Mojo events

#define mojo_header() \
{ \
output << "MOJ"; \
mojo_integer(MOJO_VERSION, 0); \
output.flush(); \
}

#define mojo_metadata(label, value) \
mojo_event(MOJO_METADATA); \
mojo_string(label); \
mojo_string(value);

#define mojo_stack(pid, iid, tid) \
mojo_event(MOJO_STACK); \
mojo_integer(pid, 0); \
mojo_integer(iid, 0); \
output << std::hex << tid; \
output.put('\0');

#define mojo_frame(key, frame) \
mojo_event(MOJO_FRAME); \
mojo_integer(frame->cache_key, 0); \
mojo_ref(frame->filename); \
mojo_ref(frame->name); \
mojo_integer(frame->location.line, 0); \
mojo_integer(frame->location.line_end, 0); \
mojo_integer(frame->location.column, 0); \
mojo_integer(frame->location.column_end, 0);

static inline void
mojo_frame_ref(mojo_int_t key)
{
if (key == 0)
{
mojo_event(MOJO_FRAME_INVALID);
}
else
{
mojo_event(MOJO_FRAME_REF);
mojo_integer(key, 0);
}
}

#define mojo_frame_kernel(scope) \
mojo_event(MOJO_FRAME_KERNEL); \
mojo_string(scope);

#define mojo_metric_time(value) \
mojo_event(MOJO_METRIC_TIME); \
mojo_integer(value, 0);

#define mojo_metric_memory(value) \
mojo_event(MOJO_METRIC_MEMORY); \
mojo_integer(value < 0 ? -value : value, value < 0);

#define mojo_string_event(key, string) \
mojo_event(MOJO_STRING); \
mojo_ref(key); \
mojo_string(string);

#define mojo_string_ref(key) \
mojo_event(MOJO_STRING_REF); \
mojo_ref(key);
2 changes: 1 addition & 1 deletion echion/stacks.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ class FrameStack : public std::deque<Frame::Ref>
// This is a shim frame so we skip it.
continue;
#endif
(*it).get().render(output);
mojo_frame_ref((*it).get().cache_key);
}
}
};
Expand Down
12 changes: 6 additions & 6 deletions echion/strings.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -95,7 +95,6 @@ class StringTable : public std::unordered_map<uintptr_t, std::string>

if (this->find(k) == this->end())
{
// TODO: Emit MOJO string signal
try
{
#if PY_VERSION_HEX >= 0x030c0000
Expand All @@ -108,6 +107,7 @@ class StringTable : public std::unordered_map<uintptr_t, std::string>
auto str = pyunicode_to_utf8(s);
#endif
this->emplace(k, str);
mojo_string_event(k, str);
}
catch (StringError &)
{
Expand All @@ -125,12 +125,12 @@ class StringTable : public std::unordered_map<uintptr_t, std::string>

if (this->find(k) == this->end())
{
// TODO: Emit MOJO string signal
try
{
auto s = std::string(32, '\0');
std::snprintf((char *)s.c_str(), 32, "native@%p", (void *)k);
this->emplace(k, s);
char buffer[32] = {0};
std::snprintf(buffer, 32, "native@%p", (void *)k);
this->emplace(k, buffer);
mojo_string_event(k, buffer);
}
catch (StringError &)
{
Expand All @@ -152,7 +152,6 @@ class StringTable : public std::unordered_map<uintptr_t, std::string>

if (this->find(k) == this->end())
{
// TODO: Emit MOJO string signal
unw_word_t offset; // Ignored. All the information is in the PC anyway.
char sym[256];
if (unw_get_proc_name(&cursor, sym, sizeof(sym), &offset))
Expand All @@ -171,6 +170,7 @@ class StringTable : public std::unordered_map<uintptr_t, std::string>
}

this->emplace(k, name);
mojo_string_event(k, name);

if (demangled)
std::free(demangled);
Expand Down
8 changes: 4 additions & 4 deletions echion/threads.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -337,7 +337,7 @@ void ThreadInfo::sample(int64_t iid, PyThreadState *tstate, microsecond_t delta)
if (current_tasks.empty())
{
// Print the PID and thread name
output << "P" << pid << ";T" << iid << ":" << name;
mojo_stack(pid, iid, name);

// Print the stack
if (native)
Expand All @@ -349,13 +349,13 @@ void ThreadInfo::sample(int64_t iid, PyThreadState *tstate, microsecond_t delta)
python_stack.render(output);

// Print the metric
output << " " << delta << std::endl;
mojo_metric_time(delta);
}
else
{
for (auto &task_stack : current_tasks)
{
output << "P" << pid << ";T" << iid << ":" << name;
mojo_stack(pid, iid, name);

if (native)
{
Expand All @@ -367,7 +367,7 @@ void ThreadInfo::sample(int64_t iid, PyThreadState *tstate, microsecond_t delta)
else
task_stack->render(output);

output << " " << delta << std::endl;
mojo_metric_time(delta);
}

current_tasks.clear();
Expand Down
2 changes: 1 addition & 1 deletion pyproject.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@ template = "tests"
dependencies = [
"pytest>=5.4.2",
"pytest-cov>=2.8.1",
"austin-python",
"austin-python~=1.7",
"bytecode",
]

Expand Down
Loading

0 comments on commit 9b7cdda

Please sign in to comment.