diff --git a/Gemfile.lock b/Gemfile.lock index 67d4a1b..cba5b8b 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,7 +1,7 @@ PATH remote: . specs: - powertrack (1.1.1) + powertrack (1.2.0) em-http-request (~> 1.1) eventmachine (~> 1.0) exponential-backoff (~> 0.0.2) diff --git a/History.txt b/History.txt index c5229b2..4472e27 100644 --- a/History.txt +++ b/History.txt @@ -1,3 +1,9 @@ +v1.2.0 +------ + + * Support PowerTrack v2 + - Rule validator and Replay v2 not supported yet + v1.1.1 ------ diff --git a/README.md b/README.md index 7c47e0c..6db9d32 100644 --- a/README.md +++ b/README.md @@ -102,14 +102,14 @@ Backfill is a feature provided by GNIP to avoid losing activities when being disconnected. It automatically resends the messages sent on the stream for the last 5 minutes when reconnecting. -Provide a (numerical) client id as the last (but optional) argument of the -PowerTrack::Stream constructor to enable this feature. +Provide a (numerical) client id by setting the ```:client_id``` option when +building a ```PowerTrack::Stream``` object to enable this feature. ## Replay Replay is a feature provided by GNIP to recover lost activities over the last 5 days. The Replay stream lives aside the realtime stream and is activated -by setting the ```:replay``` option to true when building a ```PowerTrack::Stream``` +by setting the ```:replay``` option to ```true``` when building a ```PowerTrack::Stream``` object. Once Replay is activated, you use the stream as previously, starting by @@ -128,6 +128,32 @@ replaying the same timeframe again and again when GNIP is unstable. All the errors that come from PowerTrack are defined through an ad-hoc exception class hierarchy. See ```lib/powertrack/errors.rb```. +## PowerTrack v2 + +The library provides early support for PowerTrack API version 2. Please read +[PowerTrack API v2](http://support.gnip.com/apis/powertrack2.0/index.html) and +the [Migration Guide](http://support.gnip.com/apis/powertrack2.0/transition.html) +for details about this new major release. + +Set the ```:v2```option to ```true``` when building a ```PowerTrack::Stream``` +object to enable this feature. The library uses v1 by default. + +Everything should work the same for v2 as for v1 except + +o ```PowerTrack::Stream.add_rule``` and ```PowerTrack::Stream.delete_rule``` + returns a status instead of nil +o The Backfill feature is configured by the ```:backfill_minutes``` option passed + to the ```PowerTrack::Stream.track``` method instead of passing a ```:client_id``` + option to the ```PowerTrack::Stream``` initializer (which is simply ignored + when v2 is turned on). The new option specifies a number of minutes of backfill + data to receive. +o The Replay feature still uses v1 even if you explicitly turn v2 on. Support + for [Replay v2](http://support.gnip.com/apis/replay2.0/api_reference.html) is + planned but not scheduled yet. + +Finally, PowerTrack v2 has a new endpoint for rule validation that is not +supported by this library yet. + ## Credits The ```powertrack``` gem heavily relies on *EventMachine* and the *em-http-request* diff --git a/TODO.md b/TODO.md index b01acfa..133c322 100644 --- a/TODO.md +++ b/TODO.md @@ -60,7 +60,7 @@ See [Data format](http://support.gnip.com/sources/twitter/data_format.html) * _[DONE]_ Support Original output format * _[DONE]_ Support Activity Stream output format * _[DONE]_ Support raw format -* + * _[OUT]_ Manage retweets. See [Identifying and Understanding retweets](http://support.gnip.com/articles/identifying-and-understanding-retweets.html) @@ -71,8 +71,8 @@ See [Managing disconnections](http://support.gnip.com/articles/disconnections-ex * _[DONE]_ Reconnect after disconnect. See [Disconnections & Reconnecting](http://support.gnip.com/apis/consuming_streaming_data.html#Disconnections) * _[DONE]_ Reconnect using an exponential backoff pattern. -* _[DONE]_ Support Backfill -* Support Replay +* _[DONE]_ Support Backfill (v1) +* _[DONE]_ Support Replay (v1) * Reconnect when there's a GNIP server issue signaled by the 503 HTTP response status ## Other features @@ -80,4 +80,24 @@ See [Managing disconnections](http://support.gnip.com/articles/disconnections-ex * _[DONE]_ Support test and development streams * _[DONE]_ Support Replay mode (5-days back history) * Support status dashboard -* Support Historical Powertrack +* Support Historical PowerTrack + +## PowerTrack v2 +See [Migration Guide](http://support.gnip.com/apis/powertrack2.0/transition.html) +and [PowerTrack API v2](http://support.gnip.com/apis/powertrack2.0/index.html). + +* _[DONE]_ Support both v1 and v2 with the same interface/class +* _[DONE]_ Support new endpoint URLs +* Support rule validator +* Support new operators and quoted tweet filtering. + Double-check with tests that the gem does not prevent their usage +* _[DONE]_ Support new backfill behavior + * _[DONE]_ Support fixed backfill period used at first connection + * _[DONE]_ Support fixed backfill period used at each reconnect + * Support dynamic backfill period at each reconnect, calibrated according to + the number of minutes the stream was disconnected. Emit a warning if the + stream was disconnected more than 5 minutes (tweets were probably lost) +* _[DONE]_ Use HTTP POST verb (instead of DELETE) for rule deletions +* _[DONE]_ Fallback to v1 when Replay mode wants to use v2. Emit a warning. +* Support Replay v2 + [Replay API 2.0 Reference](http://support.gnip.com/apis/replay2.0/api_reference.html) diff --git a/lib/powertrack/rules/rule.rb b/lib/powertrack/rules/rule.rb index 48e8386..f39196c 100644 --- a/lib/powertrack/rules/rule.rb +++ b/lib/powertrack/rules/rule.rb @@ -19,17 +19,32 @@ class Rule # The maximum number of negative terms in a single rule value MAX_NEGATIVE_TERMS = 50 - attr_reader :value, :tag, :error - - # Builds a new rule based on a value and an optional tag. + # The default rule features + DEFAULT_RULE_FEATURES = { + # no id by default + id: nil, + # no tag by default + tag: nil, + # long determined by value length + long: nil + }.freeze + + attr_reader :value, :id, :tag, :error + + # Builds a new rule based on a value and some optional features + # (:id, :tag, :long). + # # By default, the constructor assesses if it's a long rule or not # based on the length of the value. But the 'long' feature can be - # explicitly specified with the third parameter. - def initialize(value, tag=nil, long=nil) + # explicitly specified with the :long feature. + def initialize(value, features=nil) @value = value || '' - @tag = tag + features = DEFAULT_RULE_FEATURES.merge(features || {}) + @tag = features[:tag] + @id = features[:id] # check if long is a boolean - @long = long == !!long ? long : @value.size > MAX_STD_RULE_VALUE_LENGTH + _long = features[:long] + @long = _long == !!_long ? _long : @value.size > MAX_STD_RULE_VALUE_LENGTH @error = nil end @@ -70,6 +85,7 @@ def to_json(options={}) def to_hash res = {:value => @value} res[:tag] = @tag unless @tag.nil? + res[:id] = @id unless @id.nil? res end diff --git a/lib/powertrack/rules/string_extension.rb b/lib/powertrack/rules/string_extension.rb index 1c7f4be..7e66c92 100644 --- a/lib/powertrack/rules/string_extension.rb +++ b/lib/powertrack/rules/string_extension.rb @@ -3,7 +3,7 @@ # Extend core String class with a rule transformer class String # Returns a PowerTrace::Rule instance based on the value of the string. - def to_pwtk_rule(tag=nil, long=nil) - PowerTrack::Rule.new(self, tag, long) + def to_pwtk_rule(features=nil) + PowerTrack::Rule.new(self, features) end end diff --git a/lib/powertrack/streaming/data_buffer.rb b/lib/powertrack/streaming/data_buffer.rb index 100df9a..857642a 100644 --- a/lib/powertrack/streaming/data_buffer.rb +++ b/lib/powertrack/streaming/data_buffer.rb @@ -5,7 +5,7 @@ module PowerTrack class DataBuffer # The pattern used by GNIP PowerTrack to delimitate a single message. - MESSAGE_PATTERN = /^([^\r]*)\r\n/m + MESSAGE_PATTERN = /^([^\r]*)\r\n/m.freeze # Builds a new data buffer. def initialize diff --git a/lib/powertrack/streaming/retrier.rb b/lib/powertrack/streaming/retrier.rb index 1fdf35e..e383cdf 100644 --- a/lib/powertrack/streaming/retrier.rb +++ b/lib/powertrack/streaming/retrier.rb @@ -22,7 +22,7 @@ class Retrier max_elapsed_time: DEFAULT_MAX_ELAPSED_TIME, multiplier: DEFAULT_INTERVAL_MULTIPLIER, randomize_factor: DEFAULT_RANDOMIZE_FACTOR - } + }.freeze # Builds a retrier that will retry a maximum retries number of times. def initialize(max_retries, options=nil) diff --git a/lib/powertrack/streaming/stream.rb b/lib/powertrack/streaming/stream.rb index 1c69955..4c646e0 100644 --- a/lib/powertrack/streaming/stream.rb +++ b/lib/powertrack/streaming/stream.rb @@ -18,7 +18,12 @@ class Stream include VoidLogger::LoggerMixin # The format of the URLs to connect to the various stream services - FEATURE_URL_FORMAT = "https://%s:%s/accounts/%s/publishers/%s/%s/track/%s%s.json".freeze + FEATURE_URL_FORMAT = { + # [ hostname, account, source, mode, label, feature ] + v1: "https://%s.gnip.com/accounts/%s/publishers/%s/%s/track/%s%s.json".freeze, + # [ hostname, feature, account, source, label, sub-feature ] + v2: "https://gnip-%s.twitter.com/%s/powertrack/accounts/%s/publishers/%s/%s%s.json".freeze + }.freeze # The default timeout on a connection to PowerTrack. Can be overriden per call. DEFAULT_CONNECTION_TIMEOUT = 30 @@ -29,24 +34,28 @@ class Stream # The default options for using the stream. DEFAULT_STREAM_OPTIONS = { + # enable PowerTrack v2 API (using v1 by default) + v2: false, + # override the default connection timeout connect_timeout: DEFAULT_CONNECTION_TIMEOUT, + # override the default inactivity timeout inactivity_timeout: DEFAULT_INACTIVITY_TIMEOUT, - # use a client id if you want to leverage the Backfill feature + # use a client id if you want to leverage the Backfill feature in v1 client_id: nil, # enable the replay mode to get activities over the last 5 days # see http://support.gnip.com/apis/replay/api_reference.html replay: false - } + }.freeze DEFAULT_OK_RESPONSE_STATUS = 200 # The patterns used to identify the various types of message received from GNIP # everything else is an activity - HEARTBEAT_MESSAGE_PATTERN = /\A\s*\z/ - SYSTEM_MESSAGE_PATTERN = /\A\s*\{\s*"(info|warn|error)":/mi + HEARTBEAT_MESSAGE_PATTERN = /\A\s*\z/.freeze + SYSTEM_MESSAGE_PATTERN = /\A\s*\{\s*"(info|warn|error)":/mi.freeze # The format used to send UTC timestamps in Replay mode - REPLAY_TIMESTAMP_FORMAT = '%Y%m%d%H%M' + REPLAY_TIMESTAMP_FORMAT = '%Y%m%d%H%M'.freeze attr_reader :username, :account_name, :data_source, :label @@ -57,9 +66,12 @@ def initialize(username, password, account_name, data_source, label, options=nil @data_source = data_source @label = label @options = DEFAULT_STREAM_OPTIONS.merge(options || {}) - @client_id = @options[:client_id] @replay = !!@options[:replay] + @client_id = @options[:client_id] @stream_mode = @replay ? 'replay' : 'streams' + + # force v1 if Replay activated + @v2 = !@replay && !!@options[:v2] end # Adds many rules to your PowerTrack stream’s ruleset. @@ -69,7 +81,9 @@ def initialize(username, password, account_name, data_source, label, options=nil # See http://support.gnip.com/apis/powertrack/api_reference.html#AddRules def add_rules(*rules) # flatten the rules in case it was provided as an array - make_rules_request(:post, body: MultiJson.encode('rules' => rules.flatten), ok: 201) + make_rules_request(:post, + body: MultiJson.encode('rules' => rules.flatten), + ok: 201) end # Removes the specified rules from the stream. @@ -78,8 +92,14 @@ def add_rules(*rules) # # See http://support.gnip.com/apis/powertrack/api_reference.html#DeleteRules def delete_rules(*rules) + # v2 does not use DELETE anymore + delete_verb = @v2 ? :post : :delete # flatten the rules in case it was provided as an array - make_rules_request(:delete, body: MultiJson.encode('rules' => rules.flatten)) + delete_options = { body: MultiJson.encode('rules' => rules.flatten) } + # v2 uses a query parameter + delete_options[:query] = { '_method' => 'delete' } if @v2 + + make_rules_request(delete_verb, delete_options) end DEFAULT_LIST_RULES_OPTIONS = { @@ -103,7 +123,9 @@ def list_rules(options=nil) res.is_a?(Hash) && (rules = res['rules']).is_a?(Array) && rules.all? { |rule| rule.is_a?(Hash) && rule.key?('value') } - rules.map { |rule| PowerTrack::Rule.new(rule['value'], rule['tag']) } + rules.map do |rule| + PowerTrack::Rule.new(rule['value'], tag: rule['tag'], id: rule['id']) + end else res end @@ -125,6 +147,8 @@ def list_rules(options=nil) from: nil, # the ending date to which the activities will be recovered (replay mode only) to: nil, + # specify a number of minutes to leverage the Backfill feature (v2 only) + backfill_minutes: nil, # called for each message received, except heartbeats on_message: nil, # called for each activity received @@ -140,6 +164,8 @@ def list_rules(options=nil) # Establishes a persistent connection to the PowerTrack data stream, # through which the social data will be delivered. # + # Manages reconnections when being disconnected. + # # GET /track/:stream # # See http://support.gnip.com/apis/powertrack/api_reference.html#Stream @@ -151,30 +177,31 @@ def track(options=nil) private - # Returns the fully-qualified domain name of a GNIP PowerTrack server - # based on a hostname. - def gnip_server_name(hostname) - "%s.gnip.com" % [ hostname ] - end - - # Returns the port used by GNIP PowerTrack servers. - def gnip_server_port - '443' - end - # Returns the URL of the stream for a given feature. - def feature_url(hostname, feature=nil) - feature = feature ? "/#{feature}" : '' - _url = FEATURE_URL_FORMAT % - [ gnip_server_name(hostname), - gnip_server_port, - @account_name, - @data_source, - @stream_mode, - @label, - feature ] - - _url += "?client=#{@client_id}" if @client_id + def feature_url(hostname, feature=nil, sub_feature=nil) + _url = nil + if @v2 + feature ||= hostname + sub_feature = sub_feature ? "/#{sub_feature}" : '' + _url = FEATURE_URL_FORMAT[:v2] % + [ hostname, + feature, + @account_name, + @data_source, + @label, + sub_feature ] + else + feature = feature ? "/#{feature}" : '' + _url = FEATURE_URL_FORMAT[:v1] % + [ hostname, + @account_name, + @data_source, + @stream_mode, + @label, + feature ] + + _url += "?client=#{@client_id}" if @client_id + end _url end @@ -198,8 +225,8 @@ def connection_headers end # Opens a new connection to GNIP PowerTrack. - def connect(hostname, feature=nil) - url = feature_url(hostname, feature) + def connect(hostname, feature=nil, sub_feature=nil) + url = feature_url(hostname, feature, sub_feature) logger.debug("Connecting to '#{url}' with headers #{connection_headers}...") EventMachine::HttpRequest.new(url, connection_headers) end @@ -264,8 +291,9 @@ def handle_api_response(status, error, body, ok=DEFAULT_OK_RESPONSE_STATUS) DEFAULT_RULES_REQUEST_OPTIONS = { ok: DEFAULT_OK_RESPONSE_STATUS, headers: {}, + query: {}, body: nil - } + }.freeze # Makes a rules-related request with a specific HTTP verb and a few options. # Returns the response if successful or an exception if the request failed. @@ -279,6 +307,7 @@ def make_rules_request(verb, options=nil) con = connect('api', 'rules') http = con.setup_request(verb, head: rules_req_headers.merge(options[:headers]), + query: options[:query], body: options[:body]) http.errback do @@ -315,10 +344,10 @@ def track_req_headers(compressed) .merge(gzip_compressed_header(compressed)) end - # Connects to the /track endpoint and manages reconnections when being - # disconnected. + # Connects to the /track endpoint. def track_once(options, retrier) logger.info "Starting tracker for retry ##{retrier.retries}..." + backfill_minutes = options[:backfill_minutes] stop_timeout = options[:stop_timeout] on_heartbeat = options[:on_heartbeat] on_message = options[:on_message] @@ -336,7 +365,10 @@ def track_once(options, retrier) EM.run do logger.info "Starting the reactor..." con = connect('stream') - get_opts = { head: track_req_headers(options[:compressed]) } + get_opts = { + head: track_req_headers(options[:compressed]), + query: {} + } # add a timeframe in replay mode if @replay @@ -346,14 +378,18 @@ def track_once(options, retrier) # stop 30 minutes ago by default to = options[:to] || (now - 30*60) - get_opts[:query] = { + get_opts[:query].merge!({ 'fromDate' => from.utc.strftime(REPLAY_TIMESTAMP_FORMAT), 'toDate' => to.utc.strftime(REPLAY_TIMESTAMP_FORMAT) - } + }) logger.info "Replay mode enabled from '#{from}' to '#{to}'" end + if @v2 && backfill_minutes + get_opts[:query]['backfillMinutes'] = backfill_minutes + end + http = con.get(get_opts) # polls to see if the connection should be closed diff --git a/lib/powertrack/version.rb b/lib/powertrack/version.rb index c732c8a..1cd7916 100644 --- a/lib/powertrack/version.rb +++ b/lib/powertrack/version.rb @@ -1,3 +1,3 @@ module PowerTrack - VERSION = '1.1.1' + VERSION = '1.2.0'.freeze end diff --git a/test/minitest_helper.rb b/test/minitest_helper.rb index dca7a3f..108048c 100644 --- a/test/minitest_helper.rb +++ b/test/minitest_helper.rb @@ -30,13 +30,14 @@ def powertrack_config end # Returns a brand-new stream based on the config found in test/powertrack.yml. - def new_stream(replay=false) + def new_stream(v2=false, replay=false) PowerTrack::Stream.new( powertrack_config[:username], powertrack_config[:password], powertrack_config[:account_name], powertrack_config[:data_source], - replay ? 'prod' : powertrack_config[:stream_label], - replay: replay) + replay ? 'prod' : (v2 ? 'prod2' : powertrack_config[:stream_label]), + replay: replay, + v2: v2) end end diff --git a/test/test_manage_rules.rb b/test/test_manage_rules.rb index 67af2ea..1a7178f 100644 --- a/test/test_manage_rules.rb +++ b/test/test_manage_rules.rb @@ -4,35 +4,78 @@ class TestManageRules < Minitest::Test - def test_add_then_delete_a_single_rule - add_then_delete_a_single_rule(false) + def test_add_then_delete_a_single_rule_v1 + add_then_delete_a_single_rule(false, false) + end + + def test_add_then_delete_a_single_rule_v2 + add_then_delete_a_single_rule(true, false) end def test_add_then_delete_a_single_rule_in_replay_mode - add_then_delete_a_single_rule(true) + add_then_delete_a_single_rule(false, true) end - def add_then_delete_a_single_rule(replay) - stream = new_stream(replay) + def add_then_delete_a_single_rule(v2, replay) + stream = new_stream(v2, replay) - rule = PowerTrack::Rule.new('coke') - assert rule.valid? + # add a logger + stream.logger = Logger.new(STDERR) + + new_rule = PowerTrack::Rule.new('coke') + assert new_rule.valid? pre_existing_rules = stream.list_rules + $stderr.puts pre_existing_rules.inspect assert pre_existing_rules.is_a?(Array) + assert pre_existing_rules.all? { |rule| !rule.id.nil? } if v2 + + already_in = pre_existing_rules.any? { |rule| new_rule == rule } - assert_nil stream.add_rule(rule) + res = stream.add_rule(new_rule) - rules_after_addition = stream.list_rules(false) + if v2 + assert res.is_a?(Hash) + assert res['summary'].is_a?(Hash) + + if already_in + assert_equal 0, res['summary']['created'] + assert_equal 1, res['summary']['not_created'] + else + assert_equal 1, res['summary']['created'] + assert_equal 0, res['summary']['not_created'] + end + else + assert_nil res + end + + rules_after_addition = stream.list_rules assert rules_after_addition.is_a?(Array) - assert_equal pre_existing_rules.size + 1, rules_after_addition.size - assert [ rule ], rules_after_addition - pre_existing_rules + assert rules_after_addition.all? { |rule| !rule.id.nil? } if v2 + + if already_in + assert_equal pre_existing_rules.size, rules_after_addition.size + assert [], rules_after_addition - pre_existing_rules + else + assert_equal pre_existing_rules.size + 1, rules_after_addition.size + assert [ new_rule ], rules_after_addition - pre_existing_rules + end + + res = stream.delete_rules(new_rule) - assert_nil stream.delete_rules(rule) + if v2 + assert res.is_a?(Hash) + assert res['summary'].is_a?(Hash) + assert_equal 1, res['summary']['deleted'] + assert_equal 0, res['summary']['not_deleted'] + else + assert_nil res + end rules_after_removal = stream.list_rules assert rules_after_removal.is_a?(Array) assert_equal rules_after_addition.size - 1, rules_after_removal.size assert_equal [], rules_after_removal - rules_after_addition + assert rules_after_removal.all? { |rule| !rule.id.nil? } if v2 end end diff --git a/test/test_rule.rb b/test/test_rule.rb index 4586f9f..20e019f 100644 --- a/test/test_rule.rb +++ b/test/test_rule.rb @@ -12,7 +12,7 @@ def test_valid_rule assert rule.valid? assert_nil rule.error - rule = PowerTrack::Rule.new('pepsi', 'soda', true) + rule = PowerTrack::Rule.new('pepsi', tag: 'soda', long: true) assert_equal 'pepsi', rule.value assert_equal 'soda', rule.tag assert rule.long? @@ -22,12 +22,12 @@ def test_valid_rule def test_too_long_tag long_tag = 'a' * PowerTrack::Rule::MAX_TAG_LENGTH - rule = PowerTrack::Rule.new('coke', long_tag, false) + rule = PowerTrack::Rule.new('coke', tag: long_tag, long: false) assert rule.valid? assert_nil rule.error long_tag = 'b' * 2 * PowerTrack::Rule::MAX_TAG_LENGTH - rule = PowerTrack::Rule.new('coke', long_tag, true) + rule = PowerTrack::Rule.new('coke', tag: long_tag, long: true) assert !rule.valid? assert_match /too long tag/i, rule.error end @@ -38,13 +38,13 @@ def test_too_long_value assert rule.valid? long_val = 'c' * PowerTrack::Rule::MAX_LONG_RULE_VALUE_LENGTH - rule = long_val.to_pwtk_rule(nil, false) + rule = long_val.to_pwtk_rule(long: false) assert !rule.valid? assert_match /too long value/i, rule.error assert long_val.to_pwtk_rule.valid? - assert long_val.to_pwtk_rule(nil, true).valid? + assert long_val.to_pwtk_rule(long: true).valid? very_long_val = 'rrr' * PowerTrack::Rule::MAX_LONG_RULE_VALUE_LENGTH rule = very_long_val.to_pwtk_rule @@ -59,18 +59,18 @@ def test_too_many_positive_terms assert rule.valid? assert_nil rule.error - long_rule = PowerTrack::Rule.new(phrase, nil, true) + long_rule = PowerTrack::Rule.new(phrase, long: true) assert long_rule.long? assert long_rule.valid? assert_nil long_rule.error phrase = ([ 'coke' ] * (2 * PowerTrack::Rule::MAX_POSITIVE_TERMS)).join(' ') - rule = PowerTrack::Rule.new(phrase, nil, false) + rule = PowerTrack::Rule.new(phrase, long: false) assert !rule.long? assert !rule.valid? assert_match /too many positive terms/i, rule.error - long_rule = PowerTrack::Rule.new(phrase, nil, true) + long_rule = PowerTrack::Rule.new(phrase, long: true) assert long_rule.long? assert long_rule.valid? assert_nil long_rule.error @@ -93,7 +93,7 @@ def test_too_many_negative_terms assert rule.valid? assert_nil rule.error - long_rule = PowerTrack::Rule.new(phrase, nil, true) + long_rule = PowerTrack::Rule.new(phrase, long: true) assert long_rule.long? assert long_rule.valid? assert_nil long_rule.error @@ -104,7 +104,7 @@ def test_too_many_negative_terms assert !rule.valid? assert_match /too many negative terms/i, rule.error - long_rule = PowerTrack::Rule.new(phrase, nil, true) + long_rule = PowerTrack::Rule.new(phrase, long: true) assert long_rule.long? assert long_rule.valid? assert_nil long_rule.error @@ -125,7 +125,7 @@ def test_to_hash_and_json assert_equal MultiJson.encode(res), rule.to_json res[:tag] = 'soda' - rule = PowerTrack::Rule.new(res[:value], res[:tag], true) + rule = PowerTrack::Rule.new(res[:value], tag: res[:tag], long: true) assert_equal res, rule.to_hash assert_equal MultiJson.encode(res), rule.to_json end @@ -141,9 +141,9 @@ def test_double_quote_jsonification def test_hash short_rule = PowerTrack::Rule.new('coke') - not_long_rule = PowerTrack::Rule.new('coke', nil, false) - false_long_rule = PowerTrack::Rule.new('coke', nil, true) - short_rule_with_tag = PowerTrack::Rule.new('coke', 'soda') + not_long_rule = PowerTrack::Rule.new('coke', long: false) + false_long_rule = PowerTrack::Rule.new('coke', long: true) + short_rule_with_tag = PowerTrack::Rule.new('coke', tag: 'soda') assert short_rule == not_long_rule assert_equal short_rule, not_long_rule @@ -158,6 +158,6 @@ def test_hash assert_equal 2, h[short_rule] assert_equal h[short_rule], h[not_long_rule] assert_equal 4, h[short_rule_with_tag] - assert_nil h[PowerTrack::Rule.new('pepsi', 'soda')] + assert_nil h[PowerTrack::Rule.new('pepsi', tag: 'soda')] end end diff --git a/test/test_track_stream.rb b/test/test_track_stream.rb index 5e5197f..2c00849 100644 --- a/test/test_track_stream.rb +++ b/test/test_track_stream.rb @@ -4,28 +4,46 @@ class TestTrackStream < Minitest::Test - def test_track_realtime_stream - track_simple_stream(false) + def test_track_realtime_stream_v1 + track_simple_stream(false, false) end - def test_track_replay_stream - track_simple_stream(true) + def test_track_realtime_stream_v2 + track_simple_stream(true, false) end - def track_simple_stream(replay) - stream = new_stream(replay) + def test_track_replay_stream_v1 + track_simple_stream(false, true) + end + +# def test_track_replay_stream_v2 +# track_simple_stream(true, true) +# end + + def track_simple_stream(v2, replay) + stream = new_stream(v2, replay) # add a logger stream.logger = Logger.new(STDERR) - rule = PowerTrack::Rule.new('ny OR nyc OR #nyc OR new york') - assert rule.valid? + new_rule = PowerTrack::Rule.new('ny OR nyc OR #nyc OR new york') + assert new_rule.valid? begin - assert_nil stream.add_rule(rule) + res = stream.add_rule(new_rule) + + if v2 + assert res.is_a?(Hash) + assert res['summary'].is_a?(Hash) + else + assert_nil res + end + rules_after_addition = stream.list_rules assert rules_after_addition.is_a?(Array) assert rules_after_addition.size > 0 + assert rules_after_addition.any? { |rule| rule == new_rule } + assert rules_after_addition.all? { |rule| !rule.id.nil? } if v2 heartbeats = 0 received = 0 @@ -87,7 +105,8 @@ def track_simple_stream(replay) assert (ended_at - started_at) >= delay end - assert heartbeats > 0, 'No heartbeat received' + # heartbeats only sent every 10 minutes in v2... + assert heartbeats > 0, 'No heartbeat received' unless v2 puts "#{heartbeats} heartbeats received" assert received > 0, 'No message received so far' @@ -95,8 +114,19 @@ def track_simple_stream(replay) assert tweeted > 0, 'No tweet received so far' puts "#{tweeted} tweets received" + rescue + p $! ensure - assert_nil stream.delete_rules(rule) + res = stream.delete_rules(new_rule) + + if v2 + assert res.is_a?(Hash) + assert res['summary'].is_a?(Hash) + assert_equal 1, res['summary']['deleted'] + assert_equal 0, res['summary']['not_deleted'] + else + assert_nil res + end end end end