Skip to content

Commit

Permalink
Feature/hydro storage (electricitymaps#331)
Browse files Browse the repository at this point in the history
  • Loading branch information
@corradio
corradio authored Jan 30, 2017
1 parent 7320314 commit cd65aab
Show file tree
Hide file tree
Showing 8 changed files with 269 additions and 171 deletions.
6 changes: 6 additions & 0 deletions feeder/migrate_db.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,4 +35,10 @@ def migrate(db, validate_production):
# except:
# print 'Warning: row %s did not pass validation' % row['_id']
# print row
# ** 2017-01-28 Add storage
for row in col_production.find({'countryCode': 'FR', 'consumption': {'$exists': True}}):
print 'Migrating %s' % row['datetime']
row['storage'] = row['consumption']
del row['consumption']
col_production.update_one({'_id': row['_id']}, {'$set': row}, upsert=False)
print 'Migration done.'
50 changes: 30 additions & 20 deletions feeder/parsers/ENTSOE.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -169,26 +169,28 @@ def parse_production(xml_text):
if not xml_text: return None
soup = BeautifulSoup(xml_text, 'html.parser')
# Get all points
quantities = []
productions = []
storages = []
datetimes = []
for timeseries in soup.find_all('timeseries'):
resolution = timeseries.find_all('resolution')[0].contents[0]
datetime_start = arrow.get(timeseries.find_all('start')[0].contents[0])
is_production = len(timeseries.find_all('inBiddingZone_Domain.mRID'.lower())) > 0
for entry in timeseries.find_all('point'):
quantity = float(entry.find_all('quantity')[0].contents[0])
# If this is not a production, then it is storage (consumption)
if not is_production: quantity *= -1
position = int(entry.find_all('position')[0].contents[0])
datetime = datetime_from_position(datetime_start, position, resolution)
# Find out whether or not we should update the net production
try:
i = datetimes.index(datetime)
quantities[i] += quantity
if is_production:
productions[i] = quantity
else:
storages[i] = quantity
except ValueError: # Not in list
quantities.append(quantity)
datetimes.append(datetime)
return quantities, datetimes
productions.append(quantity if is_production else 0)
storages.append(quantity if not is_production else 0)
return productions, storages, datetimes

def parse_exchange(xml_text, is_import, quantities=None, datetimes=None):
if not xml_text: return None
Expand Down Expand Up @@ -256,6 +258,10 @@ def get_hydro(values):
values.get('Hydro Run-of-river and poundage', 0) + \
values.get('Hydro Water Reservoir', 0)

def get_hydro_storage(storage_values):
if 'Hydro Pumped Storage' in storage_values:
return max(0, storage_values.get('Hydro Pumped Storage', 0))

def get_oil(values):
if 'Fossil Oil' in values or 'Fossil Oil shale' in values:
value = values.get('Fossil Oil', 0) + values.get('Fossil Oil shale', 0)
Expand Down Expand Up @@ -300,9 +306,9 @@ def fetch_production(country_code, session=None):
for k in ENTSOE_PARAMETER_DESC.keys():
parsed = parse_production(query_production(k, domain, session))
if parsed:
quantities, datetimes = parsed
for i in range(len(quantities)):
production_hashmap[datetimes[i]][k] = quantities[i]
productions, storages, datetimes = parsed
for i in range(len(datetimes)):
production_hashmap[datetimes[i]][k] = (productions[i], storages[i])


# Take the last production date that is present for all parameters
Expand All @@ -314,21 +320,25 @@ def fetch_production(country_code, session=None):
production_dates)
production_date = production_dates[production_dates_with_counts.index(max(production_dates_with_counts))]

values = {ENTSOE_PARAMETER_DESC[k]: v for k, v in production_hashmap[production_date].iteritems()}
production_values = {ENTSOE_PARAMETER_DESC[k]: v[0] for k, v in production_hashmap[production_date].iteritems()}
storage_values = {ENTSOE_PARAMETER_DESC[k]: v[1] for k, v in production_hashmap[production_date].iteritems()}

data = {
'countryCode': country_code,
'datetime': production_date.datetime,
'production': {
'biomass': values.get('Biomass', None),
'coal': get_coal(values),
'gas': get_gas(values),
'hydro': get_hydro(values),
'nuclear': values.get('Nuclear', None),
'oil': get_oil(values),
'solar': values.get('Solar', None),
'wind': get_wind(values),
'unknown': get_unknown(values)
'biomass': production_values.get('Biomass', None),
'coal': get_coal(production_values),
'gas': get_gas(production_values),
'hydro': get_hydro(production_values),
'nuclear': production_values.get('Nuclear', None),
'oil': get_oil(production_values),
'solar': production_values.get('Solar', None),
'wind': get_wind(production_values),
'unknown': get_unknown(production_values)
},
'storage': {
'hydro': get_hydro_storage(storage_values),
},
'source': 'entsoe.eu'
}
Expand Down
4 changes: 2 additions & 2 deletions feeder/parsers/FR.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ def fetch_production(country_code='FR', session=None):
data = {
'countryCode': country_code,
'production': {},
'consumption': {},
'storage': {},
'source': 'rte-france.com',
}
for item in mixtr.getchildren():
Expand All @@ -37,7 +37,7 @@ def fetch_production(country_code='FR', session=None):
if key in MAP_GENERATION:
data['production'][MAP_GENERATION[key]] = float(value.text)
elif key in MAP_STORAGE:
data['consumption'][MAP_STORAGE[key]] = float(value.text)
data['storage'][MAP_STORAGE[key]] = -1 * float(value.text)

data['datetime'] = arrow.get(arrow.get(obj[1].text).datetime,
'Europe/Paris').replace(minutes=+(int(value.attrib['periode']) * 15.0)).datetime
Expand Down
92 changes: 68 additions & 24 deletions web/app/countrytable.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,8 @@ var d3 = require('d3');
var moment = require('moment');

function CountryTable(selector, co2Color) {
var that = this;

this.root = d3.select(selector);
this.co2Color = co2Color;

Expand All @@ -13,7 +15,7 @@ function CountryTable(selector, co2Color) {
// Constants
this.ROW_HEIGHT = 10;
this.RECT_OPACITY = 0.8;
this.LABEL_MAX_WIDTH = 60;
this.LABEL_MAX_WIDTH = 80;
this.PADDING_X = 5; this.PADDING_Y = 5; // Inner paddings
this.FLAG_SIZE_MULTIPLIER = 3;
this.TEXT_ADJUST_Y = 9; // To align properly on a line
Expand All @@ -26,9 +28,29 @@ function CountryTable(selector, co2Color) {
'gas': '#bb2f51',
'coal': '#ac8c35',
'oil': '#867d66',
'unknown': 'gray'
'unknown': 'lightgray'
};
this.STORAGE_COLORS = {
'hydro storage': this.PRODUCTION_COLORS['hydro']
}
this.PRODUCTION_MODES = d3.keys(this.PRODUCTION_COLORS);
this.STORAGE_MODES = d3.keys(this.STORAGE_COLORS);
this.MODES = [];
// Display order is defined here
[
'wind',
'solar',
'hydro',
'hydro storage',
'biomass',
'nuclear',
'gas',
'coal',
'oil',
'unknown'
].forEach(function(k) {
that.MODES.push({'mode': k, 'isStorage': k.indexOf('storage') != -1});
});

// State
this._displayByEmissions = false;
Expand All @@ -54,23 +76,20 @@ CountryTable.prototype.render = function() {

// ** Production labels and rects **
var gNewRow = this.productionRoot.selectAll('.row')
.data(this.PRODUCTION_MODES)
.data(this.MODES)
.enter()
.append('g')
.attr('class', 'row')
.attr('transform', function(d, i) {
return 'translate(0,' + (i * (that.ROW_HEIGHT + that.PADDING_Y)) + ')';
});
gNewRow.append('text')
.text(function(d) { return d; })
.text(function(d) { return d.mode })
.attr('transform', 'translate(0, ' + this.TEXT_ADJUST_Y + ')');
gNewRow.append('rect')
.attr('class', 'capacity')
.attr('height', this.ROW_HEIGHT)
.attr('fill', function (d) { return that.PRODUCTION_COLORS[d]; })
.attr('fill-opacity', 0.2)
.attr('stroke', function (d) { return that.PRODUCTION_COLORS[d]; })
.attr('stroke-width', 1.0)
.attr('fill-opacity', 0.4)
.attr('opacity', 0.3)
.attr('shape-rendering', 'crispEdges');
gNewRow.append('rect')
Expand Down Expand Up @@ -134,7 +153,7 @@ CountryTable.prototype.onProductionMouseMove = function(arg) {

CountryTable.prototype.resize = function() {
this.headerHeight = 2 * this.ROW_HEIGHT;
this.productionHeight = this.PRODUCTION_MODES.length * (this.ROW_HEIGHT + this.PADDING_Y);
this.productionHeight = this.MODES.length * (this.ROW_HEIGHT + this.PADDING_Y);
this.exchangeHeight = (!this._data) ? 0 : d3.entries(this._exchangeData).length * (this.ROW_HEIGHT + this.PADDING_Y);

this.yProduction = this.headerHeight + this.ROW_HEIGHT;
Expand Down Expand Up @@ -178,24 +197,33 @@ CountryTable.prototype.data = function(arg) {
});

// Construct a list having each production in the same order as
// `this.PRODUCTION_MODES`
var sortedProductionData = this.PRODUCTION_MODES.map(function (d) {
var footprint = that._data.productionCo2Intensities ?
that._data.productionCo2Intensities[d] : undefined;
var production = arg.production ? arg.production[d] : undefined;
// `PRODUCTION_MODES` merged with `STORAGE_MODES`
var sortedProductionData = this.MODES.map(function (d) {
var footprint = !d.isStorage ?
that._data.productionCo2Intensities ?
that._data.productionCo2Intensities[d.mode] :
undefined :
0;
var production = !d.isStorage ? (that._data.production || {})[d.mode] : undefined;
var storage = d.isStorage ? (that._data.storage || {})[d.mode.replace(' storage', '')] : undefined;
var capacity = !d.isStorage ? (that._data.capacity || {})[d.mode] : undefined;
return {
production: production,
capacity: arg.capacity ? arg.capacity[d] : undefined,
mode: d,
storage: storage,
isStorage: d.isStorage,
capacity: capacity,
mode: d.mode,
gCo2eqPerkWh: footprint,
gCo2eqPerH: footprint * 1000.0 * production
gCo2eqPerH: footprint * 1000.0 * Math.max(production, 0)
};
});

// update scales
this.powerScale
.domain([
-this._data.maxExport || 0,
Math.min(
-this._data.maxExport || 0,
-this._data.maxStorage || 0),
Math.max(
this._data.maxCapacity || 0,
this._data.maxProduction || 0,
Expand Down Expand Up @@ -246,6 +274,9 @@ CountryTable.prototype.data = function(arg) {

this.gPowerAxis
.transition()
// TODO: We should offset by just one pixel because it looks better when
// the rectangles don't start exactly on the axis...
// But we should also handle "negative" rects
.attr('transform', 'translate(' + (this.powerScale.range()[0] + this.LABEL_MAX_WIDTH) + ', 24)')
.call(this.axis);
this.gPowerAxis.selectAll('.tick text')
Expand Down Expand Up @@ -286,7 +317,7 @@ CountryTable.prototype.data = function(arg) {
.transition()
.attr('x', that.LABEL_MAX_WIDTH + that.powerScale(0))
.attr('width', function (d) {
return (d.capacity == null || d.production == null) ? 0 : (that.powerScale(d.capacity) - that.powerScale(0));
return d.capacity !== undefined ? (that.powerScale(d.capacity) - that.powerScale(0)) : 0;
})
.on('end', function () { d3.select(this).style('display', 'block'); });
// Add event handlers
Expand Down Expand Up @@ -315,7 +346,9 @@ CountryTable.prototype.data = function(arg) {
// color by Co2 Intensity
// return that.co2Color(that._data.productionCo2Intensities[d.mode, that._data.countryCode]);
// color by production mode
return that.PRODUCTION_COLORS[d.mode];
return d.isStorage ?
that.STORAGE_COLORS[d.mode] :
that.PRODUCTION_COLORS[d.mode];
})
.attr('x', that.LABEL_MAX_WIDTH + that.co2Scale(0))
.attr('width', function (d) {
Expand All @@ -324,16 +357,27 @@ CountryTable.prototype.data = function(arg) {
else
selection.select('rect.production')
.transition()
.attr('fill', function (d) { return that.PRODUCTION_COLORS[d.mode]; })
.attr('x', that.LABEL_MAX_WIDTH + that.powerScale(0))
.attr('fill', function (d) {
return d.isStorage ?
that.STORAGE_COLORS[d.mode] :
that.PRODUCTION_COLORS[d.mode];
})
.attr('x', function (d) {
var value = (!d.isStorage) ? d.production : -1 * d.storage;
return that.LABEL_MAX_WIDTH + ((value == undefined || !isFinite(value)) ? that.powerScale(0) : that.powerScale(Math.min(0, value)));
})
.attr('width', function (d) {
return d.production === undefined ? 0 : (that.powerScale(d.production) - that.powerScale(0));
var value = d.production != undefined ? d.production : -1 * d.storage;
return (value == undefined || !isFinite(value)) ? 0 : Math.abs(that.powerScale(value) - that.powerScale(0));
});
selection.select('text.unknown')
.transition()
.attr('x', that.LABEL_MAX_WIDTH + (that._displayByEmissions ? that.co2Scale(0) : that.powerScale(0)))
.style('display', function (d) {
return d.capacity != 0 && d.mode != 'unknown' && (d.production === undefined || d.production === null) ? 'block' : 'none';
return (d.capacity == undefined || d.capacity > 0) &&
d.mode != 'unknown' &&
(d.isStorage ? d.storage == undefined : d.production == undefined) ?
'block' : 'none';
});

// Construct exchanges
Expand Down
Loading

0 comments on commit cd65aab

Please sign in to comment.