Merge pull request #2 from tillahoffmann/profile-timeouts

Profile timeouts

README.md

@@ -1,6 +1,6 @@
 # Reproduction materials for "Scalable Gaussian Process Inference with Stan" [![Reproduction Materials](https://github.com/tillahoffmann/gptools-reproduction-material/actions/workflows/main.yaml/badge.svg)](https://github.com/tillahoffmann/gptools-reproduction-material/actions/workflows/main.yaml)
 
-This repository comprises code and data to reproduce the results and figures in the accompanying manuscript [Scalable Gaussian Process Inference with Stan](https://doi.org/10.48550/arXiv.2301.08836).
+This repository comprises code and data to reproduce the results and figures in the accompanying manuscript [Scalable Gaussian Process Inference with Stan](https://doi.org/10.48550/arXiv.2301.08836). The source code of the package `gptools-stan` used by these reproduction materials is available [here](https://github.com/onnela-lab/gptools) and can be installed using `pip install gptools-stan`.
 
 ## Setting up a computing environment
 

profile/profile.md

@@ -20,14 +20,20 @@ import matplotlib as mpl
 from matplotlib import pyplot as plt
 import numpy as np
 import pickle
-from gptools.stan.profile import PARAMETERIZATIONS, LOG10_NOISE_SCALES, SIZES
 import os
 import pandas as pd
 import re
 from scipy import stats
 import types
 from pathlib import Path
 
+# Same as in `recipe.py`.
+SIZES = 16 * 2 ** np.arange(11)
+LOG10_NOISE_SCALES = np.linspace(-1, 1, 7)
+PARAMETERIZATIONS = [
+    "graph_centered", "graph_non_centered", "fourier_centered", "fourier_non_centered",
+    "standard_centered", "standard_non_centered"
+]
 
 mpl.style.use("../jss.mplstyle")
 fig_width, fig_height = mpl.rcParams["figure.figsize"]
@@ -70,7 +76,7 @@ def load_lps(filename: str) -> np.ndarray:
     errors = []
     for fit, eta, data in zip(result["fits"], result["etas"], result["data"]):
         if fit is None:
-            mses.append(np.nan)
+            errors.append(np.nan)
             continue
         test_idx = np.setdiff1d(1 + np.arange(size), data["observed_idx"]) - 1
         if not test_idx.size:

profile/run_profile.py

@@ -1,5 +1,4 @@
 import argparse
-import colorama
 from gptools.stan import compile_model
 from gptools.util import Timer
 from gptools.util.fft import transform_irfft
@@ -101,17 +100,16 @@ def __main__(args: Optional[List[str]] = None) -> None:
                         "data": data,
                         "seed": args.seed,
                     }
-                    remaining = args.timeout - total_timer.duration
                     if args.method == "sample":
                         iter_warmup = args.iter_warmup or args.iter_sampling
                         fit = call_with_timeout(
-                            remaining, model.sample, iter_sampling=args.iter_sampling, chains=1,
+                            args.timeout, model.sample, iter_sampling=args.iter_sampling, chains=1,
                             threads_per_chain=1, show_progress=args.show_progress,
                             iter_warmup=iter_warmup, **kwargs,
                         )
                     elif args.method == "variational":
                         fit = call_with_timeout(
-                            remaining, model.variational, output_samples=args.iter_sampling,
+                            args.timeout, model.variational, output_samples=args.iter_sampling,
                             require_converged=not args.ignore_converged, **kwargs,
                         )
                     else:  # pragma: no cover
@@ -162,11 +160,6 @@ def __main__(args: Optional[List[str]] = None) -> None:
     if args.method == "sample" and args.show_diagnostics:
         print(fit.diagnose())
 
-    # Complain if the timeout was exceeded by more than a second.
-    if args.timeout is not None and total_timer.duration > args.timeout + 1:
-        print(f"{colorama.Back.RED}TOTAL DURATION EXCEEDED TIMEOUT: {total_timer.duration:.1f} > "
-              f"{args.timeout:.1f}{colorama.Back.RESET}")
-
 
 if __name__ == "__main__":
     __main__()

requirements.in

@@ -1,5 +1,5 @@
 cook-build
-gptools-stan>=0.1.4
+gptools-stan>=0.1.5
 gptools-util>=0.1.1
 matplotlib
 jupyter

requirements.txt

@@ -74,12 +74,10 @@ fonttools==4.42.1
     # via matplotlib
 fqdn==1.5.1
     # via jsonschema
-gptools-stan==0.1.4
+gptools-stan==0.1.5
     # via -r requirements.in
 gptools-util==0.1.1
-    # via
-    #   -r requirements.in
-    #   gptools-stan
+    # via -r requirements.in
 idna==3.4
     # via
     #   anyio