Skip to content

Commit

Permalink
Merge branch 'bugfix/csv2rec' into maintenance/3.0.x
Browse files Browse the repository at this point in the history
  • Loading branch information
@barronh
barronh committed Jul 10, 2019
2 parents 4e4f92b + 245b816 commit 1840228
Show file tree
Hide file tree
Showing 7 changed files with 155 additions and 13 deletions.
2 changes: 1 addition & 1 deletion setup.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@ def find_data():
data = find_data()
MAJOR = 3
MINOR = 0
MICRO = 0
MICRO = 2
ISRELEASED = False
VERSION = '%d.%d.%d' % (MAJOR, MINOR, MICRO)

Expand Down
38 changes: 30 additions & 8 deletions src/PseudoNetCDF/cmaqfiles/profile.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,11 +2,10 @@
__all__ = ['bcon_profile', 'icon_profile']
from PseudoNetCDF import PseudoNetCDFFile, PseudoNetCDFVariables
from ._ioapi import ioapi_base
from matplotlib.mlab import csv2rec
try:
from StringIO import StringIO
from StringIO import StringIO as BytesIO
except ImportError:
from io import StringIO
from io import BytesIO
from datetime import datetime
import numpy as np

Expand All @@ -22,7 +21,22 @@ def _getunit(varkey):
else:
unit = 'ppmV'
return unit.ljust(16)


def _profile2dict(lines, fieldnames):
import re

data = np.recfromtxt(
BytesIO(
bytes(
re.sub(r'[ ]+"', '"', '\n'.join(lines)).replace('"', ''),
encoding='ascii')
),
delimiter=' ', names=fieldnames,
converters=dict(names=lambda x: x.strip()),
)
return data


class icon_profile(ioapi_base):
def __init__(self, path):
lines = open(path).read().split('\n')
Expand All @@ -38,7 +52,11 @@ def __init__(self, path):
ends = [s + nspc for s in starts]
keys = ['all']
fieldnames = ('name',) + tuple(['s%f' % i for i in sigmas])
data = dict([(k, csv2rec(StringIO('\n'.join(lines[s:e])), delimiter = ' ', names = fieldnames, converterd = dict(names = lambda x: str(x).strip()))) for k, s, e in zip(keys, starts, ends)])
data = dict([
(k, _profile2dict(lines[s:e], fieldnames))
for k, s, e in zip(keys, starts, ends)
])

profile_spcs = np.char.strip(data[keys[0]]['name'])
data_type = data[keys[0]].dtype
data_shape = data[keys[0]].shape
Expand All @@ -58,7 +76,7 @@ def __init__(self, path):
raise IOError('File is corrupt or inconsistent')
varlist = []
for a in data['all']:
varkey = a[0].strip()
varkey = a[0].strip().decode()
self.createVariable(varkey, 'f', ('LAY',), units = _getunit(varkey), values = np.array([tuple(a)[1:]])[0].astype('f'), long_name = varkey.ljust(16), var_desc = varkey.ljust(16))
varlist.append(varkey.ljust(16))
self.NVARS = len(varlist)
Expand Down Expand Up @@ -114,7 +132,11 @@ def __init__(self, path):
ends = [s + 1 + nspc for s in starts]
keys = [lines[s].strip().lower() for s in starts]
fieldnames = ('name',) + tuple(['s%f' % i for i in sigmas])
data = dict([(k, csv2rec(StringIO('\n'.join(lines[s+1:e])), delimiter = ' ', names = fieldnames, converterd = dict(names = lambda x: str(x).strip()))) for k, s, e in zip(keys, starts, ends)])
data = dict([
(k, _profile2dict(lines[s + 1:e], fieldnames))
for k, s, e in zip(keys, starts, ends)
])

profile_spcs = np.char.strip(data[keys[0]]['name'])
data_type = data[keys[0]].dtype
data_shape = data[keys[0]].shape
Expand All @@ -136,7 +158,7 @@ def __init__(self, path):
varlist = []
for w, s, e, n in zip(data['west'], data['south'], data['east'], data['north']):
assert(w[0] == s[0] and e[0] == n[0] and n[0] == s[0])
varkey = w[0].strip()
varkey = w[0].strip().decode()
self.createVariable(varkey, 'f', ('LAY', 'south_east_north_west'), units = _getunit(varkey), values = np.array([(lambda x: tuple(x)[1:])(_v) for _v in[s, e, n, w]]).T.astype('f'), long_name = varkey.ljust(16), var_desc = varkey.ljust(16))
varlist.append(varkey.ljust(16))
self.NVARS = len(varlist)
Expand Down
5 changes: 2 additions & 3 deletions src/PseudoNetCDF/geoschemfiles/_planelog.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,13 +51,12 @@ def __init__(self, pathlike):
if __name__ == '__main__':
import sys
bfile1 = flightlogs(sys.argv[1:])
from matplotlib.mlab import prctile
for label, key in [('O3', 'O3[:]'), ('NO2', 'NO2[:]')]:
bvar = eval(key, None, bfile1.variables)
b2var = eval(key, None, bfile1.variables)
assert((bvar == b2var).all())
print('\n%s (BASE: %6.2f)' % (label, bvar.mean()), file = sys.stdout)
print('\n BASE:', sep = '', file = sys.stdout)
prctile(bvar, np.ma.arange(.1, 1., .1)* 100).tofile(sys.stdout, sep = ', ', format = '%6.2f')
percentile(bvar, np.ma.arange(.1, 1., .1)* 100).tofile(sys.stdout, sep = ', ', format = '%6.2f')
print('', file = sys.stdout)


75 changes: 75 additions & 0 deletions src/PseudoNetCDF/test/test_cmaqfiles.py
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,75 @@
import unittest
import numpy as np


np_all_close = np.testing.assert_allclose

class ProfileTest(unittest.TestCase):
def setUp(self):
pass

def testIcon(self):
from ..cmaqfiles import icon_profile
from ..testcase import cmaqfiles_paths
icon_file = icon_profile(cmaqfiles_paths['icon_profile'])
itestvals = {
"O3": [
3.5E-02, 3.5E-02, 4.E-02, 5.E-02, 6.E-02, 7.E-02],
"ASO4I": [
4.810E-03, 4.810E-03, 3.207E-03, 3.207E-03, 6.413E-04,
3.207E-04
],
"NUMATKN": [
1.478E+09, 1.470E+09, 9.774E+08, 9.655E+08, 1.920E+08,
9.584E+07
],
"SRFACC": [
1.491E-05, 1.373E-05, 8.694E-06, 6.815E-06, 1.190E-06,
5.697E-07
]
}
test = self.assertEqual
for testk, testvals in itestvals.items():
vals = icon_file.variables[testk]
test(True, np.allclose(vals, testvals))


def testBcon(self):
from ..cmaqfiles import bcon_profile
from ..testcase import cmaqfiles_paths
btestvals = {
"O3": [
3.000E-02, 3.500E-02, 4.000E-02, 5.000E-02, 6.000E-02,
7.000E-02, 3.000E-02, 3.500E-02, 4.000E-02, 5.000E-02,
6.000E-02, 7.000E-02, 3.500E-02, 3.500E-02, 4.000E-02,
5.000E-02, 6.000E-02, 7.000E-02, 3.500E-02, 4.000E-02,
4.500E-02, 5.000E-02, 6.000E-02, 7.000E-02
],
"ASO4I": [
6.413E-03, 6.413E-03, 6.413E-03, 3.207E-03, 6.413E-04,
3.207E-04, 6.413E-03, 6.413E-03, 6.413E-03, 6.413E-03,
6.413E-04, 3.207E-04, 4.810E-03, 4.810E-03, 3.207E-03,
3.207E-03, 6.413E-04, 3.207E-04, 9.620E-03, 6.413E-03,
6.413E-03, 3.207E-03, 6.413E-04, 3.207E-04
],
"NUMATKN": [
1.957E+09, 1.949E+09, 1.936E+09, 9.655E+08, 1.920E+08,
9.584E+07, 1.957E+09, 1.949E+09, 1.936E+09, 1.924E+09,
1.920E+08, 9.584E+07, 1.478E+09, 1.470E+09, 9.774E+08,
9.655E+08, 1.920E+08, 9.584E+07, 2.915E+09, 1.949E+09,
1.936E+09, 9.655E+08, 1.920E+08, 9.584E+07
],
"SRFACC": [
1.776E-05, 1.658E-05, 1.439E-05, 6.815E-06, 1.190E-06,
5.697E-07, 1.776E-05, 1.658E-05, 1.439E-05, 1.251E-05,
1.190E-06, 5.697E-07, 1.491E-05, 1.373E-05, 8.694E-06,
6.815E-06, 1.190E-06, 5.697E-07, 2.346E-05, 1.658E-05,
1.439E-05, 6.815E-06, 1.190E-06, 5.697E-07
],
}
bcon_file = bcon_profile(cmaqfiles_paths['bcon_profile'])
test = self.assertEqual
for testk, testvals in btestvals.items():
vals = bcon_file.variables[testk]
test(True, np.allclose(vals[:].T.ravel(), testvals))

12 changes: 11 additions & 1 deletion src/PseudoNetCDF/testcase/__init__.py
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,17 @@
__all__ = ['camxfiles_paths', 'net_balance_paths', 'geoschemfiles_paths', 'icarttfiles_paths', 'all_paths', 'self_described_paths']
__all__ = ['camxfiles_paths', 'net_balance_paths', 'geoschemfiles_paths', 'icarttfiles_paths', 'all_paths', 'self_described_paths',
'cmaqfiles_paths'
]

from os.path import join, realpath, abspath

cmaqfiles_paths = dict(
icon_profile='cmaqfiles/profiles/test.icon_profile',
bcon_profile='cmaqfiles/profiles/test.bcon_profile',
)

for key, val in cmaqfiles_paths.items():
cmaqfiles_paths[key] = abspath(join(*__path__ + val.split('/')))

camxfiles_paths = dict(wind = 'camxfiles/wind/test.wind',
landuse = 'camxfiles/landuse/test.landuse',
temperature = 'camxfiles/temperature/test.temperature',
Expand Down
26 changes: 26 additions & 0 deletions src/PseudoNetCDF/testcase/cmaqfiles/profiles/test.bcon_profile
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
CB05 gas mechanism with AERO5 species and toxics
bcon profile data converted from RADM2 gas mechanism species
layers and defined sigma levels are listed below.
6 4 1.00 0.98 0.93 0.84 0.60 0.30 0.00
2011-08-10
North
"O3 " 3.500E-02 3.500E-02 4.000E-02 5.000E-02 6.000E-02 7.000E-02
"ASO4I " 4.810E-03 4.810E-03 3.207E-03 3.207E-03 6.413E-04 3.207E-04
"NUMATKN " 1.478E+09 1.470E+09 9.774E+08 9.655E+08 1.920E+08 9.584E+07
"SRFACC " 1.491E-05 1.373E-05 8.694E-06 6.815E-06 1.190E-06 5.697E-07
East
"O3 " 3.000E-02 3.500E-02 4.000E-02 5.000E-02 6.000E-02 7.000E-02
"ASO4I " 6.413E-03 6.413E-03 6.413E-03 6.413E-03 6.413E-04 3.207E-04
"NUMATKN " 1.957E+09 1.949E+09 1.936E+09 1.924E+09 1.920E+08 9.584E+07
"SRFACC " 1.776E-05 1.658E-05 1.439E-05 1.251E-05 1.190E-06 5.697E-07
South
"O3 " 3.000E-02 3.500E-02 4.000E-02 5.000E-02 6.000E-02 7.000E-02
"ASO4I " 6.413E-03 6.413E-03 6.413E-03 3.207E-03 6.413E-04 3.207E-04
"NUMATKN " 1.957E+09 1.949E+09 1.936E+09 9.655E+08 1.920E+08 9.584E+07
"SRFACC " 1.776E-05 1.658E-05 1.439E-05 6.815E-06 1.190E-06 5.697E-07
West
"O3 " 3.500E-02 4.000E-02 4.500E-02 5.000E-02 6.000E-02 7.000E-02
"ASO4I " 9.620E-03 6.413E-03 6.413E-03 3.207E-03 6.413E-04 3.207E-04
"NUMATKN " 2.915E+09 1.949E+09 1.936E+09 9.655E+08 1.920E+08 9.584E+07
"SRFACC " 2.346E-05 1.658E-05 1.439E-05 6.815E-06 1.190E-06 5.697E-07

10 changes: 10 additions & 0 deletions src/PseudoNetCDF/testcase/cmaqfiles/profiles/test.icon_profile
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
CB05 gas mechanism with AERO5 species and toxics
icon profile data converted from RADM2 gas mechanism species
number of layers, number of species, terrain-following sigma levels:
6 4 1.00 0.98 0.93 0.84 0.60 0.30 0.00
2011-08-10
"O3 " 3.500E-02 3.500E-02 4.000E-02 5.000E-02 6.000E-02 7.000E-02
"ASO4I " 4.810E-03 4.810E-03 3.207E-03 3.207E-03 6.413E-04 3.207E-04
"NUMATKN " 1.478E+09 1.470E+09 9.774E+08 9.655E+08 1.920E+08 9.584E+07
"SRFACC " 1.491E-05 1.373E-05 8.694E-06 6.815E-06 1.190E-06 5.697E-07

0 comments on commit 1840228

Please sign in to comment.