Skip to content

Commit

Permalink
Rewrite index API code and fill in defaults for any failure
Browse files Browse the repository at this point in the history
  • Loading branch information
@Nemo157
Nemo157 authored and Joshua Nelson committed Jun 18, 2020
1 parent 28af151 commit 2a22207
Show file tree
Hide file tree
Showing 5 changed files with 92 additions and 143 deletions.
2 changes: 2 additions & 0 deletions Cargo.lock
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

4 changes: 2 additions & 2 deletions Cargo.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,8 +13,8 @@ log = "0.4"
regex = "1"
structopt = "0.3"
crates-index-diff = "7"
reqwest = { version = "0.10.6", features = ["blocking"] } # TODO: Remove blocking when async is ready
semver = "0.9"
reqwest = { version = "0.10.6", features = ["blocking", "json"] } # TODO: Remove blocking when async is ready
semver = { version = "0.9", features = ["serde"] }
slug = "=0.1.1"
env_logger = "0.7"
r2d2 = "0.8"
Expand Down
14 changes: 1 addition & 13 deletions src/docbuilder/rustwide_builder.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,6 @@ use crate::db::file::add_path_into_database;
use crate::db::{add_build_into_database, add_package_into_database, connect_db};
use crate::docbuilder::{crates::crates_from_path, Limits};
use crate::error::Result;
use crate::index::api::RegistryCrateData;
use crate::storage::CompressionAlgorithms;
use crate::utils::{copy_doc_dir, parse_rustc_version, CargoMetadata};
use failure::ResultExt;
Expand Down Expand Up @@ -389,17 +388,6 @@ impl RustwideBuilder {
} else {
crate::web::metrics::NON_LIBRARY_BUILDS.inc();
}
let registry_data = if let Some(api) = doc_builder.index.api() {
api.get_crate_data(name, version)?
} else {
// If the index has no API, we insert empty data
RegistryCrateData {
release_time: chrono::Utc::now(),
yanked: false,
downloads: 0,
owners: vec![],
}
};
let release_id = add_package_into_database(
&conn,
res.cargo_metadata.root(),
Expand All @@ -408,7 +396,7 @@ impl RustwideBuilder {
&res.target,
files_list,
successful_targets,
&registry_data,
&doc_builder.index.api().get_crate_data(name, version),
has_docs,
has_examples,
algs,
Expand Down
201 changes: 83 additions & 118 deletions src/index/api.rs
View file
Original file line number Diff line number Diff line change
@@ -1,10 +1,9 @@
use std::io::Read;

use chrono::{DateTime, Utc};
use failure::err_msg;
use log::info;
use reqwest::header::{HeaderValue, ACCEPT, USER_AGENT};
use semver::Version;
use serde_json::Value;
use serde::Deserialize;
use url::Url;

use crate::error::Result;
Expand All @@ -16,7 +15,7 @@ const APP_USER_AGENT: &str = concat!(
);

pub(crate) struct Api {
api_base: Url,
api_base: Option<Url>,
client: reqwest::blocking::Client,
}

Expand All @@ -35,7 +34,7 @@ pub(crate) struct CrateOwner {
}

impl Api {
pub(super) fn new(api_base: &Url) -> Result<Self> {
pub(super) fn new(api_base: Option<Url>) -> Result<Self> {
let headers = vec![
(USER_AGENT, HeaderValue::from_static(APP_USER_AGENT)),
(ACCEPT, HeaderValue::from_static("application/json")),
Expand All @@ -47,23 +46,34 @@ impl Api {
.default_headers(headers)
.build()?;

Ok(Self {
api_base: api_base.clone(),
client,
})
Ok(Self { api_base, client })
}

fn api_base(&self) -> Result<Url> {
self.api_base
.clone()
.ok_or_else(|| err_msg("index is missing an api base url"))
}

pub(crate) fn get_crate_data(&self, name: &str, version: &str) -> Result<RegistryCrateData> {
let (release_time, yanked, downloads) =
self.get_release_time_yanked_downloads(name, version)?;
let owners = self.get_owners(name)?;
pub(crate) fn get_crate_data(&self, name: &str, version: &str) -> RegistryCrateData {
let (release_time, yanked, downloads) = self
.get_release_time_yanked_downloads(name, version)
.unwrap_or_else(|err| {
info!("Failed to get crate data for {}-{}: {}", name, version, err);
(Utc::now(), false, 0)
});

let owners = self.get_owners(name).unwrap_or_else(|err| {
info!("Failed to get owners for {}-{}: {}", name, version, err);
Vec::new()
});

Ok(RegistryCrateData {
RegistryCrateData {
release_time,
yanked,
downloads,
owners,
})
}
}

/// Get release_time, yanked and downloads from the registry's API
Expand All @@ -73,126 +83,81 @@ impl Api {
version: &str,
) -> Result<(DateTime<Utc>, bool, i32)> {
let url = {
let mut url = self.api_base.clone();
let mut url = self.api_base()?;
url.path_segments_mut()
.map_err(|()| err_msg("Invalid API url"))?
.extend(&["api", "v1", "crates", name, "versions"]);
url
};
// FIXME: There is probably better way to do this
// and so many unwraps...
// TODO: When reqwest is upgraded remove the `as_str` here
let mut res = self.client.get(url.as_str()).send()?;
let mut body = String::new();
res.read_to_string(&mut body).unwrap();
let json: Value = serde_json::from_str(&body[..])?;
let versions = json
.as_object()
.and_then(|o| o.get("versions"))
.and_then(|v| v.as_array())
.ok_or_else(|| err_msg("Not a JSON object"))?;

let (mut release_time, mut yanked, mut downloads) = (None, None, None);

for version_data in versions {
let version_data = version_data
.as_object()
.ok_or_else(|| err_msg("Not a JSON object"))?;
let version_num = version_data
.get("num")
.and_then(|v| v.as_str())
.ok_or_else(|| err_msg("Not a JSON object"))?;

if Version::parse(version_num)?.to_string() == version {
let release_time_raw = version_data
.get("created_at")
.and_then(|c| c.as_str())
.ok_or_else(|| err_msg("Not a JSON object"))?;

release_time = Some(
DateTime::parse_from_str(release_time_raw, "%Y-%m-%dT%H:%M:%S%.f%:z")?
.with_timezone(&Utc),
);

yanked = Some(
version_data
.get("yanked")
.and_then(|c| c.as_bool())
.ok_or_else(|| err_msg("Not a JSON object"))?,
);

downloads = Some(
version_data
.get("downloads")
.and_then(|c| c.as_i64())
.ok_or_else(|| err_msg("Not a JSON object"))? as i32,
);

break;
}

#[derive(Deserialize)]
struct Response {
versions: Vec<VersionData>,
}

#[derive(Deserialize)]
struct VersionData {
num: Version,
#[serde(default = "Utc::now")]
created_at: DateTime<Utc>,
#[serde(default)]
yanked: bool,
#[serde(default)]
downloads: i32,
}

Ok((
release_time.unwrap_or_else(Utc::now),
yanked.unwrap_or(false),
downloads.unwrap_or(0),
))
let response: Response = self.client.get(url).send()?.error_for_status()?.json()?;

let version = Version::parse(version)?;
let version = response
.versions
.into_iter()
.find(|data| data.num == version)
.ok_or_else(|| err_msg("Could not find version in response"))?;

Ok((version.created_at, version.yanked, version.downloads))
}

/// Fetch owners from the registry's API
fn get_owners(&self, name: &str) -> Result<Vec<CrateOwner>> {
let url = {
let mut url = self.api_base.clone();
let mut url = self.api_base()?;
url.path_segments_mut()
.map_err(|()| err_msg("Invalid API url"))?
.extend(&["api", "v1", "crates", name, "owners"]);
url
};
// TODO: When reqwest is upgraded remove the `as_str` here
let mut res = self.client.get(url.as_str()).send()?;
// FIXME: There is probably better way to do this
// and so many unwraps...
let mut body = String::new();
res.read_to_string(&mut body).unwrap();
let json: Value = serde_json::from_str(&body[..])?;

let owners = json
.as_object()
.and_then(|j| j.get("users"))
.and_then(|j| j.as_array());

let result = if let Some(owners) = owners {
owners
.iter()
.filter_map(|owner| {
fn extract<'a>(owner: &'a Value, field: &str) -> &'a str {
owner
.as_object()
.and_then(|o| o.get(field))
.and_then(|o| o.as_str())
.unwrap_or_default()
}

let avatar = extract(owner, "avatar");
let email = extract(owner, "email");
let login = extract(owner, "login");
let name = extract(owner, "name");

if login.is_empty() {
return None;
}

Some(CrateOwner {
avatar: avatar.to_string(),
email: email.to_string(),
login: login.to_string(),
name: name.to_string(),
})
})
.collect()
} else {
Vec::new()
};

#[derive(Deserialize)]
struct Response {
users: Vec<OwnerData>,
}

#[derive(Deserialize)]
struct OwnerData {
#[serde(default)]
avatar: String,
#[serde(default)]
email: String,
#[serde(default)]
login: String,
#[serde(default)]
name: String,
}

let response: Response = self.client.get(url).send()?.error_for_status()?.json()?;

let result = response
.users
.into_iter()
.filter(|data| !data.login.is_empty())
.map(|data| CrateOwner {
avatar: data.avatar,
email: data.email,
login: data.login,
name: data.name,
})
.collect();

Ok(result)
}
Expand Down
14 changes: 4 additions & 10 deletions src/index/mod.rs
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,5 @@
use std::path::{Path, PathBuf};

use log::info;
use url::Url;

use self::api::Api;
Expand All @@ -11,7 +10,7 @@ pub(crate) mod api;
pub(crate) struct Index {
diff: crates_index_diff::Index,
path: PathBuf,
api: Option<Api>,
api: Api,
}

#[derive(serde::Deserialize, Clone)]
Expand Down Expand Up @@ -44,21 +43,16 @@ impl Index {
let path = path.as_ref().to_owned();
let diff = crates_index_diff::Index::from_path_or_cloned(&path)?;
let config = load_config(diff.repository())?;
let api = if let Some(api_base) = &config.api {
Some(Api::new(api_base)?)
} else {
info!("Cannot load registry data as index is missing an api base url");
None
};
let api = Api::new(config.api)?;
Ok(Self { diff, path, api })
}

pub(crate) fn diff(&self) -> &crates_index_diff::Index {
&self.diff
}

pub(crate) fn api(&self) -> Option<&Api> {
self.api.as_ref()
pub(crate) fn api(&self) -> &Api {
&self.api
}
}

Expand Down

0 comments on commit 2a22207

Please sign in to comment.