Merge pull request #1226 from mozilla-services/feat/1045

feat: switch from snaek to milksnake

.travis.yml

@@ -2,12 +2,15 @@ language: python
 cache:
   directories:
     - $HOME/.cargo
-    - cargo
     - autopush_rs/target
     - $HOME/.cache/pip
 sudo: required
 dist: trusty
 
+env:
+  global:
+    - WITH_RUST=release
+
 matrix:
   include:
     - python: 2.7
@@ -20,10 +23,6 @@ matrix:
   allow_failures:
     - env: TOXENV=py36-mypy WITH_RUST=false
 
-before_install:
-# https://github.com/travis-ci/travis-ci/issues/7940
-- sudo rm -f /etc/boto.cfg
-
 install:
 - ${DDB:+make ddb}
 - pip install tox ${CODECOV:+codecov}

Dockerfile

@@ -12,7 +12,7 @@ RUN \
     apt-get install -y -qq libexpat1-dev gcc libssl-dev libffi-dev libjemalloc1 && \
     curl https://sh.rustup.rs | sh -s -- -y && \
     make clean && \
-    pip install -r requirements.txt && \
+    WITH_RUST=release pip install -r requirements.txt && \
     pypy setup.py develop
 
 ENTRYPOINT ["/app/entrypoint.sh"]

Dockerfile.python27

@@ -11,7 +11,7 @@ RUN \
     apt-get install -y -qq libexpat1-dev gcc libssl-dev libffi-dev && \
     curl https://sh.rustup.rs | sh -s -- -y && \
     make clean && \
-    pip install -r requirements.txt && \
+    WITH_RUST=release pip install -r requirements.txt && \
     python setup.py develop
 
 CMD ["autopush"]

autopush_rs/Cargo.toml

@@ -4,8 +4,12 @@ version = "0.1.0"
 authors = ["Alex Crichton <[email protected]>"]
 
 [lib]
+name = "autopush"
 crate-type = ["cdylib"]
 
+[build-dependencies]
+cbindgen = "0.6.0"
+
 [dependencies]
 base64 = "0.9.1"
 bytes = "0.4.6"
@@ -36,7 +40,8 @@ rusoto_credential = "0.11.0"
 rusoto_dynamodb = "0.32.0"
 sentry = "0.2.0"
 serde = "1.0.45"
-serde_derive = "1.0.45"
+# Pinned to 1.0.21 due to cbindgen
+serde_derive = "1.0.21"
 serde_dynamodb = "0.1.2"
 serde_json = "1.0.13"
 # slog: Use this first version for debug builds

autopush_rs/build.rs

@@ -0,0 +1,16 @@
+//! Generate autopush.h via cbindgen
+extern crate cbindgen;
+
+use std::env;
+
+fn main() {
+    let crate_dir = env::var("CARGO_MANIFEST_DIR").expect("CARGO_MANIFEST_DIR undefined");
+    let pkg_name = env::var("CARGO_PKG_NAME").expect("CARGO_PKG_NAME undefined");
+    let target = format!("{}/target/{}.h", crate_dir, pkg_name);
+    cbindgen::Builder::new()
+        .with_crate(crate_dir)
+        .with_language(cbindgen::Language::C)
+        .generate()
+        .expect("cbindgen unable to generate bindings")
+        .write_to_file(target.as_str());
+}

autopush_rs/src/call.rs

@@ -28,7 +28,6 @@ use rt::{self, AutopushError, UnwindGuard};
 use protocol;
 use server::Server;
 
-#[repr(C)]
 pub struct AutopushPythonCall {
     inner: UnwindGuard<Inner>,
 }

autopush_rs/src/queue.rs

@@ -11,13 +11,12 @@ use std::sync::Mutex;
 use call::{AutopushPythonCall, PythonCall};
 use rt::{self, AutopushError};
 
-#[repr(C)]
 pub struct AutopushQueue {
-    tx: Mutex<Sender>,
+    tx: Mutex<AutopushSender>,
     rx: Mutex<Option<mpsc::Receiver<Option<PythonCall>>>>,
 }
 
-pub type Sender = mpsc::Sender<Option<PythonCall>>;
+pub type AutopushSender = mpsc::Sender<Option<PythonCall>>;
 
 fn _assert_kinds() {
     fn _assert<T: Send + Sync>() {}
@@ -72,7 +71,7 @@ pub extern "C" fn autopush_queue_free(queue: *mut AutopushQueue) {
 }
 
 impl AutopushQueue {
-    pub fn tx(&self) -> Sender {
+    pub fn tx(&self) -> AutopushSender {
         self.tx.lock().unwrap().clone()
     }
 }

autopush_rs/src/server/mod.rs

@@ -59,7 +59,6 @@ mod webpush_io;
 
 const UAHEADER: &str = "User-Agent";
 
-#[repr(C)]
 pub struct AutopushServer {
     inner: UnwindGuard<AutopushServerInner>,
 }
@@ -105,7 +104,7 @@ pub struct Server {
     pub ddb: DynamoStorage,
     open_connections: Cell<u32>,
     tls_acceptor: Option<SslAcceptor>,
-    pub tx: queue::Sender,
+    pub tx: queue::AutopushSender,
     pub opts: Arc<ServerOptions>,
     pub handle: Handle,
     pub metrics: StatsdClient,
@@ -291,7 +290,7 @@ impl Server {
     /// This will spawn a new server with the `opts` specified, spinning up a
     /// separate thread for the tokio reactor. The returned ShutdownHandles can
     /// be used to interact with it (e.g. shut it down).
-    fn start(opts: &Arc<ServerOptions>, tx: queue::Sender) -> Result<Vec<ShutdownHandle>> {
+    fn start(opts: &Arc<ServerOptions>, tx: queue::AutopushSender) -> Result<Vec<ShutdownHandle>> {
         let mut shutdown_handles = vec![];
         if let Some(handle) = Server::start_sentry()? {
             shutdown_handles.push(handle);
@@ -366,7 +365,7 @@ impl Server {
         Ok(Some(ShutdownHandle(donetx, thread)))
     }
 
-    fn new(opts: &Arc<ServerOptions>, tx: queue::Sender) -> Result<(Rc<Server>, Core)> {
+    fn new(opts: &Arc<ServerOptions>, tx: queue::AutopushSender) -> Result<(Rc<Server>, Core)> {
         let core = Core::new()?;
         let broadcaster = if let Some(ref megaphone_url) = opts.megaphone_api_url {
             let megaphone_token = opts.megaphone_api_token

requirements.txt

@@ -1,4 +1,3 @@
--e git+https://github.com/mitsuhiko/snaek.git@2b14b8b010a9486af0f298b4ad4c73dc1ceff9d6#egg=snaek
 -e git+https://github.com/habnabit/txstatsd.git@157ef85fbdeafe23865c7c4e176237ffcb3c3f1f#egg=txStatsD-master
 apns==2.0.1
 asn1crypto==0.24.0        # via cryptography

setup.py

@@ -10,18 +10,38 @@
 with io.open(os.path.join(here, 'CHANGELOG.md'), encoding='utf8') as f:
     CHANGES = f.read()
 
-WITH_RUST = os.environ.get('WITH_RUST', 'true').lower() not in ('false', '0')
+WITH_RUST = os.environ.get('WITH_RUST', 'true')
 
 extra_options = {
     "packages": find_packages(),
 }
-if WITH_RUST:
+if WITH_RUST.lower() not in ('false', '0'):
+    def build_native(spec):
+        cmd = ['cargo', 'build']
+        in_path = 'target/debug'
+        if WITH_RUST.lower() == 'release':
+            cmd.append('--release')
+            in_path = 'target/release'
+
+        build = spec.add_external_build(
+            cmd=cmd,
+            path='./autopush_rs'
+        )
+
+        spec.add_cffi_module(
+            module_path='autopush_rs._native',
+            dylib=lambda: build.find_dylib('autopush', in_path=in_path),
+            header_filename=lambda: build.find_header(
+                'autopush.h', in_path='target'),
+            rtld_flags=['NOW', 'NODELETE']
+        )
+
     extra_options.update(
-        setup_requires=['snaek'],
-        install_requires=['snaek'],
-        snaek_rust_modules=[
-            ('autopush_rs._native', 'autopush_rs/'),
-        ],
+        setup_requires=['milksnake'],
+        install_requires=['milksnake'],
+        milksnake_tasks=[
+            build_native
+        ]
     )
 
 

tox.ini

@@ -4,7 +4,7 @@ envlist = py27,pypy,flake8,py36-mypy
 [testenv]
 deps = -rtest-requirements.txt
 usedevelop = True
-passenv = SKIP_INTEGRATION AWS_SHARED_CREDENTIALS_FILE BOTO_CONFIG
+passenv = SKIP_INTEGRATION WITH_RUST
 commands =
     nosetests {posargs} autopush
 install_command = pip install --pre {opts} {packages}