Skip to content

Commit

Permalink
enhance the industry scripts and adapt the fuel aggregation
Browse files Browse the repository at this point in the history
  • Loading branch information
@hazemakhalek
hazemakhalek committed Jun 18, 2024
1 parent 31a3354 commit e870f90
Show file tree
Hide file tree
Showing 4 changed files with 139 additions and 115 deletions.
39 changes: 39 additions & 0 deletions data/unsd_transactions.csv
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
Transaction;clean_name
consumption not elsewhere specified (industry);other
consumption by food and tobacco ;food and tobacco
consumption by non-metallic minerals ;non-metallic minerals
consumption by non-ferrous metals;non-ferrous metals
consumption by non-metallic minerals;non-metallic minerals
consumption by iron and steel;iron and steel
consumption by paper, pulp and print;paper pulp and print
consumption by food and tobacco;food and tobacco
consumption by chemical and petrochemical;chemical and petrochemical
consumption by machinery;machinery
consumption by textile and leather;textile and leather
consumption by construction;construction
consumption by mining and quarrying;mining and quarrying
consumption by transport equipment ;transport equipment
consumption by non-ferrous metals ;non-ferrous metals
consumption by wood and wood products ;wood and wood products
consumption by machinery ;machinery
consumption by mining and quarrying ;mining and quarrying
consumption by construction ;construction
consumption by textile and leather ;textile and leather
consumption by chemical and petrochemical industry;chemical and petrochemical
consumption by industries not elsewhere specified;other
consumption by non-ferrous metals industry;non-ferrous metals
consumption by non-metallic minerals industry;non-metallic minerals
consumption by mining and quarrying industry;mining and quarrying
consumption by food, beverage and tobacco industry;food and tobacco
consumption by iron and steel industry;iron and steel
consumption by transport equipment industry;transport equipment
consumption by machinery industry;machinery
consumption by wood and wood products industry;wood and wood products
consumption by construction industry;construction
consumption by wood and wood products;wood and wood products
consumption by transport equipment;transport equipment
consumption by food and tobacco industry;food and tobacco
consumption by textile and leather industry;textile and leather
consumption by other manufacturing, construction and non-fuel;other
consumption by chemical and petrochemicalindustry;chemical and petrochemical
consumption by chemical industry;chemical and petrochemical
38 changes: 27 additions & 11 deletions scripts/build_base_energy_totals.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -291,13 +291,19 @@ def calc_sector(sector):
4,
)
elif sector == "other energy":

if snakemake.config["sector"]["coal"]["shift_to_elec"]:
condition = (
df_sector.Commodity
== "Electricity" | df_sector.Commodity.isin(coal_fuels)
)
else:
condition = df_sector.Commodity == "Electricity"

energy_totals_base.at[country, "other electricity"] = round(
df_sector[
(df_sector.Commodity == "Electricity")
| df_sector.Commodity.isin(coal_fuels)
].Quantity_TWh.sum(),
4,
df_sector[condition].Quantity_TWh.sum(), 4
)

energy_totals_base.at[country, "other oil"] = round(
df_sector[df_sector.Commodity.isin(oil_fuels)].Quantity_TWh.sum(), 4
)
Expand All @@ -316,13 +322,21 @@ def calc_sector(sector):
4,
)
elif sector == "non energy use":


if snakemake.config["sector"]["coal"]["shift_to_elec"]:
condition = (
df_sector.Commodity
== "Electricity" | df_sector.Commodity.isin(coal_fuels)
)
else:
condition = df_sector.Commodity == "Electricity"

energy_totals_base.at[country, "non energy electricity"] = round(
df_sector[
(df_sector.Commodity == "Electricity")
| df_sector.Commodity.isin(coal_fuels)
].Quantity_TWh.sum(),
4,
df_sector[condition].Quantity_TWh.sum(), 4
)


energy_totals_base.at[country, "non energy oil"] = round(
df_sector[df_sector.Commodity.isin(oil_fuels)].Quantity_TWh.sum(), 4
)
Expand Down Expand Up @@ -445,9 +459,11 @@ def calc_sector(sector):

other_energy = [
"consumption not elsewhere specified (other)",
"consumption not elsewhere specified (other)"
"Consumption not elsewhere specified (other)",
"Consumption by other consumers not elsewhere specified",
"consumption by other consumers not elsewhere specified",

]

# non_energy = ['non energy uses', 'non-energy uses', 'consumption for non-energy uses', 'Consumption for non-energy uses', 'non-energy use']
Expand All @@ -470,4 +486,4 @@ def calc_sector(sector):
calc_sector(sector)

# Export the base energy totals file
energy_totals_base.to_csv(snakemake.output.energy_totals_base)
energy_totals_base.to_csv(snakemake.output.energy_totals_base)
86 changes: 15 additions & 71 deletions scripts/build_base_industry_totals.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
import pandas as pd
from helpers import aggregate_fuels, get_conv_factors
from prepare_sector_network import get
import re

# def calc_industry_base(df):

Expand Down Expand Up @@ -60,25 +61,14 @@ def create_industry_base_totals(df):
columns="Transaction", index=["country", "carrier"]
).fillna(0.0)
industry_totals_base = industry_totals_base.droplevel(level=0, axis=1)
industry_totals_base["other"] = 0
#industry_totals_base["other"] = 0

if include_other:
if not include_other:
# Loop through the columns in the list and sum them if they exist
print(
"other industries are included, check thoroughly as data sometimes is not accurate"
)
for col_name in other_list:
if col_name in industry_totals_base.columns:
industry_totals_base["other"] += industry_totals_base[col_name]
industry_totals_base.drop(col_name, axis=1, inplace=True)
else:
print(
"other industries are included, check thoroughly as data sometimes is not accurate"
)
industry_totals_base.drop(
columns=[col for col in other_list if col in industry_totals_base.columns],
inplace=True,
"unspecified industries are not included, check thoroughly as values sometimes significant for some countries"
)
industry_totals_base.drop("other", axis=1)

industry_totals_base = industry_totals_base.rename(
columns={"paper, pulp and print": "paper pulp and print"}
Expand Down Expand Up @@ -112,8 +102,8 @@ def create_industry_base_totals(df):

year = snakemake.config["demand_data"]["base_year"]
countries = snakemake.config["countries"]
# countries = ["DE", "US", "EG", "MA", "UA", "UK"]
# countries = ["EG", "BH"]

transactions_from_list = snakemake.config["industry"]

investment_year = int(snakemake.wildcards.planning_horizons)
demand_sc = snakemake.wildcards.demand
Expand All @@ -122,44 +112,10 @@ def create_industry_base_totals(df):
)
include_other = snakemake.config["demand_data"]["other_industries"]

industry_list = [
"iron and steel",
"chemical and petrochemical",
"non-ferrous metals",
"non-metallic minerals",
"transport equipment",
"machinery",
"mining and quarrying",
"food and tobacco",
"paper, pulp and printing",
"paper, pulp and print",
"wood and wood products",
"textile and leather",
"construction",
"not elsewhere specified (industry)",
"other manuf., const. and non-fuel ind.",
"other manuf., const. and non-fuel min. ind.",
]
other_list = [
"other manuf., const. and non-fuel ind.",
"other manuf., const. and non-fuel min. ind.",
]
transaction = pd.read_csv("/nimble/home/haz43975/pes_paper/uptodate/pypsa-earth-sec/data/unsd_transactions.csv", sep=";")

clean_industry_list = [
"iron and steel",
"chemical and petrochemical",
"non-ferrous metals",
"non-metallic minerals",
"transport equipment",
"machinery",
"mining and quarrying",
"food and tobacco",
"paper pulp and print",
"wood and wood products",
"textile and leather",
"construction",
"other",
]
renaming_dit = transaction.set_index('Transaction')['clean_name'].to_dict()
clean_industry_list = list(transaction.clean_name.unique())

unsd_path = (
os.path.dirname(snakemake.input["energy_totals_base"]) + "/demand/unsd/data/"
Expand All @@ -178,6 +134,8 @@ def create_industry_base_totals(df):
" - ", expand=True
)

df = df[df.Commodity != 'Other bituminous coal'] #dropping problematic column leading to double counting

# Remove fill na in Transaction column
df["Transaction"] = df["Transaction"].fillna("NA")
df["Transaction"] = df["Transaction"].str.lower()
Expand Down Expand Up @@ -231,28 +189,14 @@ def create_industry_base_totals(df):

# Filter for the year and country
df_yr = df[df.Year == year]
df_yr = df_yr[df_yr.country.isin(countries)]
# df_yr = df_yr[df_yr["Commodity - Transaction"].str.contains("industry")]
df_yr = df_yr[
df_yr["Transaction"].str.lower().str.contains("|".join(industry_list))
]

df_yr = df_yr[
~df_yr.Transaction.str.lower().str.contains(
"consumption by manufacturing, construction and non-fuel"
)
]
df_yr["Transaction"] = df_yr["Transaction"].str.replace(" industry", "")

df_yr["Transaction"] = df_yr["Transaction"].str.rstrip()
df_yr["Transaction"] = df_yr["Transaction"].str.replace("consumption by ", "")
df_yr = df_yr[df_yr.Transaction.isin(transaction.Transaction)]

df_yr["Transaction"] = df_yr['Transaction'].map(renaming_dit)

# df_yr[df_yr['Commodity - Transaction'].str.lower().str.contains('|'.join(industry_list))]
# df_yr "consumption by manufacturing, construction and non-fuel"
# Create the industry totals file
industry_totals_base = create_industry_base_totals(df_yr)

# Export the industry totals dataframe
industry_totals_base.to_csv(snakemake.output["base_industry_totals"])

# print("end")
91 changes: 58 additions & 33 deletions scripts/helpers.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -625,65 +625,90 @@ def get_conv_factors(sector):
"Paraffin waxes": 0.01117,
"Ethane": 0.01289,
"Oil shale": 0.00247,
"Other kerosene": 0.01216,

}
return fuels_conv_toTWh


def aggregate_fuels(sector):
gas_fuels = [
"Blast Furnace Gas",
"Natural gas (including LNG)",
"Natural Gas (including LNG)",
"Natural gas liquids",
"Natural gas (including LNG)",#
"Natural Gas (including LNG)",#
]

oil_fuels = [
"Motor Gasoline",
"Liquefied petroleum gas (LPG)",
"Liquified Petroleum Gas (LPG)",
"Fuel oil",
"Kerosene-type Jet Fuel",
"Conventional crude oil",
"Crude petroleum",
"Motor Gasoline",##
"Liquefied petroleum gas (LPG)",##
"Liquified Petroleum Gas (LPG)",##
"Fuel oil",##
"Kerosene-type Jet Fuel",##
"Conventional crude oil", #
"Crude petroleum",##
"Lubricants",
"Naphtha",
"Gas Oil/ Diesel Oil",
"Petroleum coke",
"Petroleum Coke",
"Patent fuel",
"Ethane",
"Bitumen",
"Refinery gas",
"Naphtha",##
"Gas Oil/ Diesel Oil",##
"Petroleum coke",##
"Petroleum Coke",##
"Ethane",##
"Bitumen",##
"Refinery gas",##
'Additives and Oxygenates',#
'Refinery Gas', ##
'Aviation gasoline',##
'Gasoline-type jet fuel', ##
'Paraffin waxes',##
"Natural gas liquids",#
"Other kerosene",

]


biomass_fuels = [
"Bagasse",
"Fuelwood",
"Bagasse",#
"Fuelwood",#
"Biogases",
"Biogasoline",
"Biodiesel",
"Charcoal",
"Black Liquor",
"Biogasoline",#
"Biodiesel",#
"Charcoal",#
"Black Liquor",#
"Bio jet kerosene",#
"Animal waste",#
"Industrial Waste",#
"Industrial waste",
"Municipal Wastes",#
"Vegetal waste"
]

coal_fuels = [
"Anthracite",
"Brown coal",
"Brown coal briquettes",
"Brown coal",#
"Brown coal briquettes",#
"Coke oven coke",
"Coke-oven coke",
"Coke Oven Coke",
"Hard coal",
"Hrad coal",
"Coking coal",
"Hard coal", #
"Hrad coal", #
"Other bituminous coal",
"Sub-bituminous coal",
"Coking coal",
"Coke Oven Gas",
"Coke Oven Gas",##
"Gas Coke",
"Gasworks Gas",
"Lignite",
"Peat",
"Gasworks Gas",##
"Lignite",#
"Peat",#
"Peat products",
'Coal Tar', ##
'Brown Coal Briquettes', ##
'Gas coke',
'Peat Products',
'Oil shale',#
'Oil Shale',#
'Coal coke', ##
'Patent fuel', ##
'Blast Furnace Gas', ##
'Recovered gases', ##
]

electricity = ["Electricity"]
Expand Down

0 comments on commit e870f90

Please sign in to comment.