Skip to content

Commit

Permalink
lookup support set mode
Browse files Browse the repository at this point in the history 
add test case for lookup - dict
  • Loading branch information
@wjo1212
wjo1212 committed Jan 18, 2019
1 parent accfad1 commit 7cc6abb
Show file tree
Hide file tree
Showing 2 changed files with 44 additions and 6 deletions.
12 changes: 7 additions & 5 deletions aliyun/log/etl_core/trans_comp/trans_lookup.py
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
from .trans_base import trans_comp_base
from .trans_base import trans_comp_check_mdoe_base
import six
import os
from ..exceptions import SettingError
Expand Down Expand Up @@ -60,10 +60,12 @@ def __contains__(self, item):
return item in self.data or self.default is not None


class trans_comp_lookup(trans_comp_base):
class trans_comp_lookup(trans_comp_check_mdoe_base):
EXTERNAL_CACHE = {}

def __init__(self, data, output_fields, sep=',', quote='"', lstrip=True, case_insensitive=True, headers=None):
def __init__(self, data, output_fields, sep=',', quote='"', lstrip=True, case_insensitive=True, headers=None, mode=None):
super(trans_comp_lookup, self).__init__(mode=mode)

data = self._u(data)
output_fields = self._u(output_fields)
sep = sep
Expand Down Expand Up @@ -164,7 +166,7 @@ def __call__(self, event, inpt):
# set output value
for f_n in self.output_fields:
if f_v in self.data:
event[f_n] = self.data[f_v]
self.set(event, f_n, self.data[f_v])
else:
logger.info(u'trans_comp_lookup: value {0} not exit in lookup {1}'.format(f_v, self.data))
else:
Expand Down Expand Up @@ -206,7 +208,7 @@ def __call__(self, event, inpt):

for f, f_new in six.iteritems(self.output_fields):
if f in row:
event[f_new] = row[f]
self.set(event, f_new, row[f])
else:
logger.info(u"trans_comp_lookup: field {0} doesn't exit in lookup row {1}".format(f, row))

Expand Down
38 changes: 37 additions & 1 deletion tests/etl_test/test_case.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -546,7 +546,43 @@ def test_lookup_dict():
# multiple inputs
assert t((["pro", "protocol"], LOOKUP({"http": "tcp", "dns": "udp", "https": "tcp"}, "type")))({'data': '123', "pro": "http"}) == {'data': '123', "pro": "http", "type": "tcp"}
assert t((["pro", "protocol"], LOOKUP({"http": "tcp", "dns": "udp", "https": "tcp"}, "type")))({'data': '123', "protocol": "http"}) == {'data': '123', "protocol": "http", "type": "tcp"}
assert t((["pro", "protocol"], LOOKUP({"http": "tcp", "dns": "udp", "https": "tcp"}, "type")))({'data': '123', "pro": "dns", "protocol": "http"}) == {'data': '123', "pro": "dns", "protocol": "http", "type": "tcp"}
assert t((["pro", "protocol"], LOOKUP({"http": "tcp", "dns": "udp", "https": "tcp"}, "type", mode='overwrite')))({'data': '123', "pro": "dns", "protocol": "http"}) == {'data': '123', "pro": "dns", "protocol": "http", "type": "tcp"}

####
## Mode
# mode - default - src empty
assert t( ("pro", LOOKUP({"1": "TCP", "2": "UDP", "3": "HTTP"}, "protocol") ) )({'data': '123', "pro": "1", "protocol": ""}) == {'data': '123', "pro": "1", "protocol": "TCP"}
assert t( ("pro", LOOKUP({"1": "TCP", "2": "UDP", "3": "HTTP"}, "protocol", mode="overwrite") ) )({'data': '123', "pro": "1", "protocol": ""}) == {'data': '123', "pro": "1", "protocol": "TCP"}
assert t( ("pro", LOOKUP({"1": "TCP", "2": "UDP", "3": "HTTP"}, "protocol", mode="overwrite-auto") ) )({'data': '123', "pro": "1", "protocol": ""}) == {'data': '123', "pro": "1", "protocol": "TCP"}
assert t( ("pro", LOOKUP({"1": "TCP", "2": "UDP", "3": "HTTP"}, "protocol", mode="add") ) )({'data': '123', "pro": "1", "protocol": ""}) == {'data': '123', "pro": "1", "protocol": ""}
assert t( ("pro", LOOKUP({"1": "TCP", "2": "UDP", "3": "HTTP"}, "protocol", mode="add-auto") ) )({'data': '123', "pro": "1", "protocol": ""}) == {'data': '123', "pro": "1", "protocol": ""}
assert t( ("pro", LOOKUP({"1": "TCP", "2": "UDP", "3": "HTTP"}, "protocol", mode="fill") ) )({'data': '123', "pro": "1", "protocol": ""}) == {'data': '123', "pro": "1", "protocol": "TCP"}

# mode - default - src non-exit
assert t( ("pro", LOOKUP({"1": "TCP", "2": "UDP", "3": "HTTP"}, "protocol", mode="overwrite") ) )({'data': '123', "pro": "1"}) == {'data': '123', "pro": "1", "protocol": "TCP"}
assert t( ("pro", LOOKUP({"1": "TCP", "2": "UDP", "3": "HTTP"}, "protocol", mode="overwrite-auto") ) )({'data': '123', "pro": "1"}) == {'data': '123', "pro": "1", "protocol": "TCP"}
assert t( ("pro", LOOKUP({"1": "TCP", "2": "UDP", "3": "HTTP"}, "protocol", mode="add") ) )({'data': '123', "pro": "1"}) == {'data': '123', "pro": "1", "protocol": "TCP"}
assert t( ("pro", LOOKUP({"1": "TCP", "2": "UDP", "3": "HTTP"}, "protocol", mode="add-auto") ) )({'data': '123', "pro": "1"}) == {'data': '123', "pro": "1", "protocol": "TCP"}
assert t( ("pro", LOOKUP({"1": "TCP", "2": "UDP", "3": "HTTP"}, "protocol", mode="fill") ) )({'data': '123', "pro": "1"}) == {'data': '123', "pro": "1", "protocol": "TCP"}

# mode - default - src non-empty
assert t( ("pro", LOOKUP({"1": "TCP", "2": "UDP", "3": "HTTP"}, "protocol") ) )({'data': '123', "pro": "1", "protocol": "SNMP"}) == {'data': '123', "pro": "1", "protocol": "SNMP"}
assert t( ("pro", LOOKUP({"1": "TCP", "2": "UDP", "3": "HTTP"}, "protocol", mode="overwrite") ) )({'data': '123', "pro": "1", "protocol": "SNMP"}) == {'data': '123', "pro": "1", "protocol": "TCP"}
assert t( ("pro", LOOKUP({"1": "TCP", "2": "UDP", "3": "HTTP"}, "protocol", mode="overwrite-auto") ) )({'data': '123', "pro": "1", "protocol": "SNMP"}) == {'data': '123', "pro": "1", "protocol": "TCP"}
assert t( ("pro", LOOKUP({"1": "TCP", "2": "UDP", "3": "HTTP"}, "protocol", mode="add") ) )({'data': '123', "pro": "1", "protocol": "SNMP"}) == {'data': '123', "pro": "1", "protocol": "SNMP"}
assert t( ("pro", LOOKUP({"1": "TCP", "2": "UDP", "3": "HTTP"}, "protocol", mode="add-auto") ) )({'data': '123', "pro": "1", "protocol": "SNMP"}) == {'data': '123', "pro": "1", "protocol": "SNMP"}
assert t( ("pro", LOOKUP({"1": "TCP", "2": "UDP", "3": "HTTP"}, "protocol", mode="fill") ) )({'data': '123', "pro": "1", "protocol": "SNMP"}) == {'data': '123', "pro": "1", "protocol": "SNMP"}

# mode - default - dest empty
assert t( ("pro", LOOKUP({"1": "", "2": "UDP", "3": "HTTP"}, "protocol") ) )({'data': '123', "pro": "1", "protocol": "SNMP"}) == {'data': '123', "pro": "1", "protocol": "SNMP"}
assert t( ("pro", LOOKUP({"1": "", "2": "UDP", "3": "HTTP"}, "protocol", mode="overwrite") ) )({'data': '123', "pro": "1", "protocol": "SNMP"}) == {'data': '123', "pro": "1", "protocol": ""}
assert t( ("pro", LOOKUP({"1": "", "2": "UDP", "3": "HTTP"}, "protocol", mode="overwrite-auto") ) )({'data': '123', "pro": "1", "protocol": "SNMP"}) == {'data': '123', "pro": "1", "protocol": "SNMP"}
assert t( ("pro", LOOKUP({"1": "", "2": "UDP", "3": "HTTP"}, "protocol", mode="add") ) )({'data': '123', "pro": "1", "protocol": "SNMP"}) == {'data': '123', "pro": "1", "protocol": "SNMP"}
assert t( ("pro", LOOKUP({"1": "", "2": "UDP", "3": "HTTP"}, "protocol", mode="add-auto") ) )({'data': '123', "pro": "1", "protocol": "SNMP"}) == {'data': '123', "pro": "1", "protocol": "SNMP"}
assert t( ("pro", LOOKUP({"1": "", "2": "UDP", "3": "HTTP"}, "protocol", mode="fill") ) )({'data': '123', "pro": "1", "protocol": "SNMP"}) == {'data': '123', "pro": "1", "protocol": "SNMP"}





import atexit
Expand Down

0 comments on commit 7cc6abb

Please sign in to comment.