Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Better error handling when doing Deep Copy of a Feature Service #160

Merged
merged 5 commits into from
Dec 16, 2016
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 16 additions & 1 deletion src/css/styles.css
Original file line number Diff line number Diff line change
Expand Up @@ -104,8 +104,23 @@ body {
font-size: .8em;
}


.message {
display: inline-block;
float: right;
height: 20px;
width: auto;
}

.messages {
display: inline;
float: left;
margin-right: 10px;
opacity: 0.7;
}

.harvester {
display: block;
display: inline;
float: right;
height: 20px;
width: 20px;
Expand Down
148 changes: 109 additions & 39 deletions src/js/main.js
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,7 @@ require([
portal.portalUrl = portalUrl;
portal.version()
.then(function(data) {
console.log("API v" + data.currentVersion);
console.info("API v" + data.currentVersion);
jquery(".alert-danger.alert-dismissable").remove();
jquery(el).next().addClass("glyphicon-ok");
})
Expand All @@ -68,7 +68,7 @@ require([
portal.withCredentials = true;
portal.version()
.then(function(data) {
console.log("API v" + data.currentVersion);
console.info("API v" + data.currentVersion);
jquery(".alert-danger.alert-dismissable").remove();
jquery(el).next().addClass("glyphicon-ok");
jquery(checkbox).trigger("click");
Expand All @@ -79,7 +79,7 @@ require([
portal.version().then(function(data) {
// It worked so keep enterprise auth but turn jsonp back off.
portal.jsonp = false;
console.log("API v" + data.currentVersion);
console.info("API v" + data.currentVersion);
jquery(".alert-danger.alert-dismissable").remove();
jquery(el).next().addClass("glyphicon-ok");
}).catch(function() {
Expand Down Expand Up @@ -1053,17 +1053,22 @@ require([
var serviceDescription = item[0].serviceDescription;
var layers = serviceDescription.layers;

// Preserve the icon on the cloned button.
// Preserve the icon and label on the cloned button.
var clone = jquery("#" + id + "_clone");
var span = jquery("#" + id + "_clone > span");
jquery("#" + id + "_clone").text(name);
jquery("#" + id + "_clone").prepend(span);

clone.text(name);
clone.prepend(span);
clone.addClass("btn-info");
clone.append("<div class='message'><p class='messages'></p></div>");

var messages = jquery("#" + id + "_clone").find(".messages");
serviceDescription.name = name;
var serviceDefinition = serviceDescription;
delete serviceDefinition.layers;
messages.text("creating service");
messages.after("<img src='css/grid.svg' class='harvester'/>");
destinationPortal.createService(destinationPortal.username, folder, JSON.stringify(serviceDefinition)).then(function(service) {
var clone = jquery("#" + id + "_clone");
clone.addClass("btn-info");
clone.append("<img src='css/grid.svg' class='harvester'/>");
clone.attr("data-id", service.itemId);
clone.attr("data-portal", destinationPortal.portalUrl);

Expand All @@ -1072,27 +1077,80 @@ require([

// Update the new item's tags to make it easier to trace its origins.
var newTags = description.tags;
newTags.push("source-" + description.id);
destinationPortal.updateDescription(destinationPortal.username, service.itemId, folder, JSON.stringify({
tags: newTags
}));
newTags.push("sourceId-" + description.id);
newTags.push("copied with ago-assistant");
destinationPortal.updateDescription(destinationPortal.username, service.itemId, folder, JSON.stringify({tags: newTags}));

portal.serviceLayers(description.url)
.then(function(definition) {
/*
* Force in the spatial reference.
* Don't know why this is necessary, but if you
* don't then any geometries not in 102100 end up
* on Null Island.
*/
var layerCount = definition.layers.length;
var layerJobs = {};
var layerSummary = {};
// var totalRecords = 0; // Keep track of the total records for the entire service.
// var totalAdded = 0; // Keep track of the total successfully added records.
var reportResult = function(layerId) {
// Check if the current layer's requests have all finished.
// Using 'attempted' handles both successes and failures.
if (layerJobs[layerId].attempted >= layerJobs[layerId].recordCount) {
layerSummary[layerId] = layerJobs[layerId];
delete layerJobs[layerId];
}

// Check if all layers have completed.
if (Object.keys(layerJobs).length === 0) {
var errors = false;
console.info("Copy summary for " + name);
Object.keys(layerSummary).forEach(function(k) {
// Check for errors and log to the console.
var layer = layerSummary[k];
if (layer.added !== layer.recordCount) {
errors = true;
console.warn(k + " (" + layer.name + "): Added " + layer.added.toLocaleString([]) + "/" + layer.recordCount.toLocaleString([]) + " records");
} else {
console.info(k + " (" + layer.name + "): Added " + layer.added.toLocaleString([]) + "/" + layer.recordCount.toLocaleString([]) + " records");
}
});

clone.find("img").remove();
clone.removeClass("btn-info");
if (errors) {
clone.addClass("btn-warning");
messages.text("Incomplete--check console");
} else {
clone.addClass("btn-success");
messages.text("Copy OK");
}
}
};

jquery.each(definition.layers, function(i, layer) {

// Set up an object to track the copy status for this layer.
layerJobs[layer.id] = {name: layer.name, recordCount: 0, attempted: 0, added: 0};

/*
* Force in the spatial reference.
* Don't know why this is necessary, but if you
* don't then any geometries not in 102100 end up
* on Null Island.
*/
layer.adminLayerInfo = {
geometryField: {
name: "Shape",
srid: 102100
}
};

/*
* Clear out the layer's indexes.
* This prevents occasional critical errors on the addToServiceDefinition call.
* The indexes will automatically be created when the new service is published.
*/
layer.indexes = [];
});


messages.text("updating definition");
destinationPortal.addToServiceDefinition(service.serviceurl, JSON.stringify(definition))
.then(function(response) {
if (!("error" in response)) {
Expand All @@ -1101,46 +1159,58 @@ require([
portal.layerRecordCount(description.url, layerId)
.then(function(records) {
var offset = 0;

layerJobs[layerId].recordCount = records.count;
// Set the count manually in weird cases where maxRecordCount is negative.
var count = definition.layers[layerId].maxRecordCount < 1 ? 1000 : definition.layers[layerId].maxRecordCount;
var added = 0;
var x = 1; // eslint-disable-line no-unused-vars
while (offset <= records.count) {
x++;
messages.text("harvesting data");
portal.harvestRecords(description.url, layerId, offset, count)
// the linter doesn't like anonymous callback functions within loops
/* eslint-disable no-loop-func */
.then(function(serviceData) {
messages.text("adding features for " + layerCount + " layers");
destinationPortal.addFeatures(service.serviceurl, layerId, JSON.stringify(serviceData.features))
.then(function() {
added += count;
if (added >= records.count) {
jquery("#" + id + "_clone > img").remove();
jquery("#" + id + "_clone").removeClass("btn-info");
jquery("#" + id + "_clone").addClass("btn-success");
}
.then(function(result) {
layerJobs[layerId].attempted += serviceData.features.length;
layerJobs[layerId].added += result.addResults.length;
reportResult(layerId);
})
.catch(function() { // Catch on addFeatures.
layerJobs[layerId].attempted += serviceData.features.length;
reportResult(layerId);
});
})
.catch(function() { // Catch on harvestRecords.
messages.text("Incomplete—check console");
console.info("Errors creating service " + name);
console.info("Failed to retrieve all records.");
});
/* eslint-enable no-loop-func */
offset += count;
}
});
});
} else {
jquery("#" + id + "_clone > img").remove();
jquery("#" + id + "_clone").removeClass("btn-info");
jquery("#" + id + "_clone").addClass("btn-danger");
var message = response.error.message;
showCopyError(id, message);
clone.find("img").remove();
clone.removeClass("btn-info");
clone.addClass("btn-danger");
messages.text("Failed—check console");
console.info("Copy summary for " + name);
console.warn(response.error.message);
response.error.details.forEach(function(detail) {
console.warn(detail);
});
}
})
.catch(function() {
jquery("#" + id + "_clone > img").remove();
jquery("#" + id + "_clone").removeClass("btn-info");
jquery("#" + id + "_clone").addClass("btn-danger");
var message = "Something went wrong.";
showCopyError(id, message);
.catch(function() { // Catch on addToServiceDefinition.
clone.find("img").remove();
clone.removeClass("btn-info");
clone.addClass("btn-danger");
messages.text("Failed—check console");
console.info("Errors creating service " + name);
console.warn("Failed to create the service.");
});
});
});
Expand Down
9 changes: 5 additions & 4 deletions src/js/portal/content/updateDescription.js
Original file line number Diff line number Diff line change
Expand Up @@ -11,14 +11,15 @@ export function updateDescription(username, id, folder, description) {
* This is necessary because some of the item descriptions (e.g. tags and extent)
* are returned as arrays, but the POST operation expects comma separated strings.
*/
for (let [key, value] of description) {
let payload = JSON.parse(description);
for (let key of Object.keys(payload)) {
let value = payload[key];
if (value === null) {
description[key] = "";
payload[key] = "";
} else if (value instanceof Array) {
description[key] = value.toString();
payload[key] = value.toString();
}
}
let payload = JSON.parse(description);
payload.token = portal.token;
payload.f = "json";
let options = {
Expand Down
20 changes: 20 additions & 0 deletions src/js/portal/request.js
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,8 @@ function get(url, parameters, options) {

// Resolve the promise with the response.
resolve(response);
} else if (xhr.readyState === 4 && xhr.status == 500) {
reject(Error(xhr));
}
});

Expand All @@ -27,6 +29,14 @@ function get(url, parameters, options) {
});

xhr.open("GET", `${url}?${serialize(parameters)}`);

// Reject the request after 120 seconds.
xhr.timeout = 120000;
xhr.ontimeout = function() {
console.log("timeout");
reject(Error(xhr));
};

xhr.send();
});

Expand Down Expand Up @@ -54,6 +64,8 @@ function post(url, data, options) {

// Resolve the promise with the response.
resolve(response);
} else if (xhr.readyState === 4 && xhr.status == 500) {
reject(Error(xhr));
}
});

Expand All @@ -62,6 +74,14 @@ function post(url, data, options) {
});

xhr.open("POST", url);

// Reject the request after 120 seconds.
xhr.timeout = 120000;
xhr.ontimeout = function() {
console.log("timeout");
reject(Error(xhr));
};

xhr.setRequestHeader("content-type", "application/x-www-form-urlencoded");
xhr.send(serialize(data));
});
Expand Down