Skip to content

Commit

Permalink
Fixed #70 - Nucleus auto-recovers from non-finite outputs.
Browse files Browse the repository at this point in the history
Nucleus now checks every quarter of a second or so for non-finite output.
If found, Nucleus resets itself and turns the OUT knob bright pink
for one second to indicate a problem.

I see this problem only rarely, but it's annoying because I have to
close and open the patch again, or reset Nucleus (which loses all
my knob settings).

Now the recovery is automatic and prompt!
The user can keep fiddling around until the rebooting stops.
  • Loading branch information
cosinekitty committed Feb 11, 2024
1 parent 6241328 commit bbc9735
Show file tree
Hide file tree
Showing 6 changed files with 128 additions and 5 deletions.
10 changes: 10 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,16 @@
<th align="left">Notes</th>
</tr>

<tr valign="top">
<td align="center">11 Feb 2024</td>
<td align="center">2.4.1</td>
<td align="left">
<ul>
<li>Nucleus <a href="https://github.com/cosinekitty/sapphire/issues/30">issue #30</a>: watch out for infinite/NAN input. If it happens, reset the internal state and keep going.</li>
</ul>
</td>
</tr>

<tr valign="top">
<td align="center">7 Feb 2024</td>
<td align="center">2.4.0</td>
Expand Down
2 changes: 1 addition & 1 deletion plugin.json
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
{
"slug": "CosineKitty-Sapphire",
"name": "Sapphire",
"version": "2.4.0",
"version": "2.4.1",
"license": "GPL-3.0-or-later",
"brand": "Sapphire",
"author": "Don Cross",
Expand Down
46 changes: 42 additions & 4 deletions src/nucleus.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
#include "sapphire_widget.hpp"
#include "nucleus_engine.hpp"
#include "nucleus_init.hpp"
#include "nucleus_reset.hpp"
#include "nucleus_panel.hpp"
#include "tricorder.hpp"

Expand Down Expand Up @@ -85,11 +86,13 @@ namespace Sapphire
struct NucleusModule : Module
{
NucleusEngine engine{NUM_PARTICLES};
CrashChecker crashChecker;
AgcLevelQuantity *agcLevelQuantity{};
bool enableLimiterWarning = true;
int tricorderOutputIndex = 1; // 1..4: which output row to send to Tricorder
Tricorder::Communicator communicator;
bool resetTricorder{};
int recoveryCountdown = 0;

NucleusModule()
: communicator(*this)
Expand Down Expand Up @@ -316,6 +319,29 @@ namespace Sapphire
// the actual output stream (not simulated physical time).
engine.update(speed * args.sampleTime, halflife, args.sampleRate, gain);

#if 0
// Use this code to verify correct functioning of the recovery countdown / CODE BLUE logic.
static int evil;
if (++evil == 100000)
{
evil = 0;
engine.particle(4).pos[0] = NAN;
}
#endif

// Look for NAN/infinite outputs, and reset the engine as needed (every 10K samples, that is).
if (crashChecker.check(engine))
{
// We just "rebooted" the engine due to invalid output.
// Make the output knob glow "code blue" for a little while.
recoveryCountdown = static_cast<int>(args.sampleRate);
}
else
{
if (recoveryCountdown > 0)
--recoveryCountdown;
}

// Let the audio/cv toggle pushbutton light reflect its button state.
lights[AUDIO_MODE_BUTTON_LIGHT].setBrightness(isEnabledAudioMode() ? 1.0f : 0.0f);

Expand Down Expand Up @@ -390,10 +416,22 @@ namespace Sapphire
{
if (layer == 1)
{
// Update the warning light state dynamically.
// Turn on the warning when the AGC is limiting the output.
double distortion = nucleusModule ? nucleusModule->engine.getAgcDistortion() : 0.0;
color = warningColor(distortion);
if (nucleusModule != nullptr)
{
// Update the warning light state dynamically.
if (nucleusModule->recoveryCountdown > 0)
{
// The Nucleus engine just "rebooted" due to non-finite output.
// Show a CODE BLUE knob.
color = nvgRGBA(0xff, 0x00, 0xff, 0xb0);
}
else
{
// Turn on the warning when the AGC is limiting the output.
double distortion = nucleusModule->engine.getAgcDistortion();
color = warningColor(distortion);
}
}
}
LightWidget::drawLayer(args, layer);
}
Expand Down
21 changes: 21 additions & 0 deletions src/nucleus_engine.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,13 @@ namespace Sapphire
PhysicsVector vel;
PhysicsVector force;
float mass = 1.0e-3f;

bool isFinite() const
{
// We don't check `force` because it is a temporary part of the calculation.
// Any problems in `force` will show up in `pos` and `vel`.
return pos.isFinite3d() && vel.isFinite3d();
}
};

using NucleusDcRejectFilter = StagedFilter<float, 3>;
Expand Down Expand Up @@ -195,6 +202,20 @@ namespace Sapphire
// For example, the caller might want to call SetMinimumEnergy(engine) after calling this function.
}

void resetAfterCrash() // called when infinite/NAN output is detected, to pop back into the finite world
{
filtersNeedReset = true;
agc.initialize();

const int n = static_cast<int>(numParticles());
for (int i = 0; i < n; ++i)
for (int k = 0; k < 3; ++k)
output(i, k) = 0;

// The caller is responsible for resetting particle states.
// For example, the caller might want to call SetMinimumEnergy(engine) after calling this function.
}

void enableFixedOversample(int n)
{
fixedOversample = std::max(1, n);
Expand Down
48 changes: 48 additions & 0 deletions src/nucleus_reset.hpp
Original file line number Diff line number Diff line change
@@ -0,0 +1,48 @@
#pragma once
#include "nucleus_engine.hpp"
#include "nucleus_reset.hpp"

namespace Sapphire
{
namespace Nucleus
{
class CrashChecker
{
private:
const int interval = 10000;
int countdown = 0;

public:
bool check(NucleusEngine& engine)
{
bool crashed = false;
if (countdown > 0)
{
--countdown;
}
else
{
countdown = interval;

const int nparticles = static_cast<int>(engine.numParticles());
for (int i = 0; i < nparticles; ++i)
{
if (!engine.particle(i).isFinite())
crashed = true;

for (int k = 0; k < 3; ++k)
if (!std::isfinite(engine.output(i, k)))
crashed = true;
}

if (crashed)
{
engine.resetAfterCrash();
SetMinimumEnergy(engine);
}
}
return crashed;
}
};
}
}
6 changes: 6 additions & 0 deletions src/sapphire_simd.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,12 @@ namespace Sapphire
float& operator[](int i) { return s[i]; }
const float& operator[](int i) const { return s[i]; }
static PhysicsVector zero() { return PhysicsVector{}; }

bool isFinite3d() const
{
// Check for finite 3D vector, optimized by ignoring the fourth component that we don't care about.
return std::isfinite(s[0]) && std::isfinite(s[1]) && std::isfinite(s[2]);
}
};

inline PhysicsVector operator + (const PhysicsVector& a, const PhysicsVector& b)
Expand Down

0 comments on commit bbc9735

Please sign in to comment.