Skip to content
This repository has been archived by the owner on Apr 26, 2024. It is now read-only.

Check if Rust lib needs rebuilding. #13759

Merged
merged 8 commits into from
Sep 12, 2022
Merged
Show file tree
Hide file tree
Changes from 6 commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions changelog.d/13759.misc
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
Add a check for editable installs if the Rust library needs rebuilding.
4 changes: 4 additions & 0 deletions rust/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -19,3 +19,7 @@ name = "synapse.synapse_rust"

[dependencies]
pyo3 = { version = "0.16.5", features = ["extension-module", "macros", "abi3", "abi3-py37"] }

[build-dependencies]
blake2 = "0.10.4"
hex = "0.4.3"
49 changes: 49 additions & 0 deletions rust/build.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,49 @@
//! This build script is calculates the hash of all files in the `src/`
//! directory and adds it as an environment variable during build time.
erikjohnston marked this conversation as resolved.
Show resolved Hide resolved
//!
//! This is used so that the python code can detect when the built native module
//! does not match the source in-tree, helping to detect the case where the
//! source has been updated but the library hasn't been rebuilt.

use std::path::PathBuf;

use blake2::{Blake2b512, Digest};

fn main() -> Result<(), std::io::Error> {
erikjohnston marked this conversation as resolved.
Show resolved Hide resolved
let mut dirs = vec![PathBuf::from("src")];

let mut paths = Vec::new();
while let Some(path) = dirs.pop() {
let mut entries = std::fs::read_dir(path)?
.map(|res| res.map(|e| e.path()))
.collect::<Result<Vec<_>, std::io::Error>>()?;

entries.sort();

let mut new_dirs = Vec::new();

for entry in entries {
if entry.is_dir() {
new_dirs.push(entry);
} else {
paths.push(entry.to_str().expect("valid rust paths").to_string());
}
}

dirs.append(&mut new_dirs);
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

As someone who has written fewer than 10 lines of Rust, why can't we append to dirs directly inside the loop?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Woops, this was a holdover from a previous iteration of the code. Will move it inside!

}

paths.sort();

let mut hasher = Blake2b512::new();

for path in paths {
let bytes = std::fs::read(path)?;
hasher.update(bytes);
}

let hex_digest = hex::encode(hasher.finalize());
println!("cargo:rustc-env=SYNAPSE_RUST_DIGEST={hex_digest}");

Ok(())
}
10 changes: 9 additions & 1 deletion rust/src/lib.rs
Original file line number Diff line number Diff line change
@@ -1,5 +1,13 @@
use pyo3::prelude::*;

/// Returns the hash of all the rust source files at the time it was compiled.
///
/// Used by python to detect if the rust library is outdated.
#[pyfunction]
fn get_rust_file_digest() -> &'static str {
env!("SYNAPSE_RUST_DIGEST")
}

/// Formats the sum of two numbers as string.
#[pyfunction]
#[pyo3(text_signature = "(a, b, /)")]
Expand All @@ -11,6 +19,6 @@ fn sum_as_string(a: usize, b: usize) -> PyResult<String> {
#[pymodule]
fn synapse_rust(_py: Python<'_>, m: &PyModule) -> PyResult<()> {
m.add_function(wrap_pyfunction!(sum_as_string, m)?)?;

m.add_function(wrap_pyfunction!(get_rust_file_digest, m)?)?;
Ok(())
}
1 change: 1 addition & 0 deletions stubs/synapse/synapse_rust.pyi
Original file line number Diff line number Diff line change
@@ -1 +1,2 @@
def sum_as_string(a: int, b: int) -> str: ...
def get_rust_file_digest() -> str: ...
5 changes: 5 additions & 0 deletions synapse/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,8 @@
import os
import sys

from synapse.util.rust import check_rust_lib_up_to_date

# Check that we're not running on an unsupported Python version.
if sys.version_info < (3, 7):
print("Synapse requires Python 3.7 or above.")
Expand Down Expand Up @@ -78,3 +80,6 @@
from synapse.util.patch_inline_callbacks import do_patch

do_patch()


check_rust_lib_up_to_date()
84 changes: 84 additions & 0 deletions synapse/util/rust.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,84 @@
# Copyright 2022 The Matrix.org Foundation C.I.C.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import os
import sys
from hashlib import blake2b

import synapse
from synapse.synapse_rust import get_rust_file_digest


def check_rust_lib_up_to_date() -> None:
"""For editable installs check if the rust library is outdated and needs to
be rebuilt.
"""

if not _dist_is_editable():
return

synapse_dir = os.path.dirname(synapse.__file__)
synapse_root = os.path.abspath(os.path.join(synapse_dir, ".."))

# Double check we've not gone into site-packages...
if os.path.basename(synapse_root) == "site-packages":
return

# ... and it looks like the root of a python project.
if not os.path.exists("pyproject.toml"):
return

# Get the hash of all Rust source files
hash = _hash_rust_files_in_directory(os.path.join(synapse_root, "rust", "src"))

if hash != get_rust_file_digest():
raise Exception("Rust module outdated. Please rebuild using `poetry install`")


def _hash_rust_files_in_directory(directory: str) -> str:
"""Get the hash of all files in a directory (recursively)"""

directory = os.path.abspath(directory)

paths = []

dirs = [directory]
while dirs:
dir = dirs.pop()
with os.scandir(dir) as d:
for entry in d:
if entry.is_dir():
dirs.append(entry.path)
else:
paths.append(entry.path)

# We sort to make sure that we get a consistent and well-defined ordering.
paths.sort()

hasher = blake2b()

for path in paths:
with open(os.path.join(directory, path), "rb") as f:
hasher.update(f.read())

return hasher.hexdigest()


def _dist_is_editable() -> bool:
"""Is distribution an editable install?"""
for path_item in sys.path:
egg_link = os.path.join(path_item, "matrix-synapse.egg-link")
if os.path.isfile(egg_link):
return True
return False