v1.3.0: More PowerTrack v2 features

* Support Replay v2
* Support specificities of rules v2

Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    powertrack (1.2.0)
+    powertrack (1.3.0)
       em-http-request (~> 1.1)
       eventmachine (~> 1.0)
       exponential-backoff (~> 0.0.2)

History.txt

@@ -1,3 +1,9 @@
+v1.3.0
+------
+
+  * Support Replay v2
+  * Support specificities of rules v2 in PowerTrack::Rule class
+
 v1.2.0
 ------
 

README.md

@@ -147,9 +147,14 @@ o The Backfill feature is configured by the ```:backfill_minutes``` option passe
   option to the ```PowerTrack::Stream``` initializer (which is simply ignored
   when v2 is turned on). The new option specifies a number of minutes of backfill
   data to receive.
-o The Replay feature still uses v1 even if you explicitly turn v2 on. Support
-  for [Replay v2](http://support.gnip.com/apis/replay2.0/api_reference.html) is
-  planned but not scheduled yet.
+o A v2 ```PowerTrack::Rule``` instance (initialized by passing the ```v2: true```
+  feature to the constructor) has a few specificities described in
+  [Migrating PowerTrack Rules from Version 1.0 to 2.0](http://support.gnip.com/articles/migrating-powertrack-rules.html).
+
+  In particular,
+  o it is always long (accepting up to 2048 characters),
+  o it has no limits on the number of positive or negative terms used,
+  o it forbids the usage of *AND*, *or* and *NOT* logical phrases.
 
 Finally, PowerTrack v2 has a new endpoint for rule validation that is not
 supported by this library yet.

TODO.md

@@ -99,5 +99,5 @@ and [PowerTrack API v2](http://support.gnip.com/apis/powertrack2.0/index.html).
     stream was disconnected more than 5 minutes (tweets were probably lost)
 * _[DONE]_ Use HTTP POST verb (instead of DELETE) for rule deletions
 * _[DONE]_ Fallback to v1 when Replay mode wants to use v2. Emit a warning.
-* Support Replay v2
+* _[DONE]_ Support Replay v2
   [Replay API 2.0 Reference](http://support.gnip.com/apis/replay2.0/api_reference.html)

lib/powertrack/errors.rb

@@ -73,6 +73,14 @@ def initialize(message, body)
     end
   end
 
+  # Invalid Endpoint. This occurs when your client tries to connect to an
+  # endpoint URL that does not exist.
+  class InvalidEndpoint < PredefinedStatusPowerTrackError
+    def initialize(message, body)
+      super(404, message, body)
+    end
+  end
+
   # Generally, this occurs where your client fails to properly include the
   # headers to accept gzip encoding from the stream, but can occur in other
   # circumstances as well.

lib/powertrack/rules/rule.rb

@@ -19,32 +19,46 @@ class Rule
     # The maximum number of negative terms in a single rule value
     MAX_NEGATIVE_TERMS = 50
 
+    # The maximum size of the HTTP body accepted by PowerTrack /rules calls (in bytes)
+    # 1 MB for v1, 5MB for v2
+    MAX_RULES_BODY_SIZE = {
+      v1: 1024**2,
+      v2: 5*1024**2
+    }
+
     # The default rule features
     DEFAULT_RULE_FEATURES = {
       # no id by default
       id: nil,
       # no tag by default
       tag: nil,
       # long determined by value length
-      long: nil
+      long: nil,
+      # v1 by default
+      v2: false
     }.freeze
 
     attr_reader :value, :id, :tag, :error
 
     # Builds a new rule based on a value and some optional features
-    # (:id, :tag, :long).
+    # (:id, :tag, :long, :v2).
     #
     # By default, the constructor assesses if it's a long rule or not
     # based on the length of the value. But the 'long' feature can be
-    # explicitly specified with the :long feature.
+    # explicitly specified with the :long feature. Finally, if :v2 is
+    # true the rule is always considered long.
     def initialize(value, features=nil)
       @value = value || ''
       features = DEFAULT_RULE_FEATURES.merge(features || {})
       @tag = features[:tag]
       @id = features[:id]
+      # only accept boolean values
+      _v2 = features[:v2]
+      @v2 = (_v2 == !!_v2) ? _v2 : false
       # check if long is a boolean
       _long = features[:long]
-      @long = _long == !!_long ? _long : @value.size > MAX_STD_RULE_VALUE_LENGTH
+      # v2 rules are always long
+      @long = (@v2 ? true : (_long == !!_long ? _long : @value.size > MAX_STD_RULE_VALUE_LENGTH))
       @error = nil
     end
 
@@ -53,19 +67,39 @@ def long?
       @long
     end
 
+    # Returns true if the rule is v2.
+    def v2?
+      @v2
+    end
+
     # Returns true if the rule is valid, false otherwise. The validation error
     # can be through the error method.
     def valid?
       # reset error
       @error = nil
 
-      [ :too_long_value?,
-        :too_many_positive_terms?,
-        :too_many_negative_terms?,
+      validation_rules = [
+        :too_long_value?,
         :contains_empty_source?,
         :contains_negated_or?,
-        :too_long_tag? ].each do |validator|
+        :too_long_tag?
+      ]
+
+      if @v2
+        validation_rules += [
+          :contains_explicit_and?,
+          :contains_lowercase_or?,
+          :contains_explicit_not?
+        ]
+      else
+        # no more restriction on the number of positive and negative terms in v2
+        validation_rules += [
+          :too_many_positive_terms?,
+          :too_many_negative_terms?
+        ]
+      end
 
+      validation_rules.each do |validator|
         # stop when 1 validator fails
         if self.send(validator)
           @error = validator.to_s.gsub(/_/, ' ').gsub(/\?/, '').capitalize
@@ -129,6 +163,21 @@ def contains_negated_or?
       !@value[/\-\w+ OR/].nil? || !@value[/OR \-\w+/].nil?
     end
 
+    # Does the rule value contain a forbidden AND ?
+    def contains_explicit_and?
+      !@value[/ AND /].nil?
+    end
+
+    # Does the rule value contain a forbidden lowercase or ?
+    def contains_lowercase_or?
+      !@value[/ or /].nil?
+    end
+
+    # Does the rule value contain a forbidden NOT ?
+    def contains_explicit_not?
+      !@value[/(^| )NOT /].nil?
+    end
+
     # Does the rule value contain too many positive terms ?
     def too_many_positive_terms?
       return false if long?

lib/powertrack/streaming/api.rb

@@ -21,6 +21,7 @@ def add_rule(rule)
     # Removes the specified rules from the stream.
     #
     # <tt>DELETE /rules</tt>
+    # <tt>POST /rules?method=_delete</tt>
     #
     # See http://support.gnip.com/apis/powertrack/api_reference.html#DeleteRules
     def delete_rules(*rules)
@@ -30,6 +31,7 @@ def delete_rules(*rules)
     # Removes the specified rule from the stream.
     #
     # <tt>DELETE /rules</tt>
+    # <tt>POST /rules?method=_delete</tt>
     #
     # See http://support.gnip.com/apis/powertrack/api_reference.html#DeleteRules
     def delete_rule(rule)

lib/powertrack/streaming/stream.rb

@@ -21,8 +21,8 @@ class Stream
     FEATURE_URL_FORMAT = {
       # [ hostname, account, source, mode, label, feature ]
       v1: "https://%s.gnip.com/accounts/%s/publishers/%s/%s/track/%s%s.json".freeze,
-      # [ hostname, feature, account, source, label, sub-feature ]
-      v2: "https://gnip-%s.twitter.com/%s/powertrack/accounts/%s/publishers/%s/%s%s.json".freeze
+      # [ hostname, domain, feature, stream type, account, source, label, sub-feature ]
+      v2: "https://gnip-%s.%s.com/%s/%s/accounts/%s/publishers/%s/%s%s.json".freeze
     }.freeze
 
     # The default timeout on a connection to PowerTrack. Can be overriden per call.
@@ -68,10 +68,7 @@ def initialize(username, password, account_name, data_source, label, options=nil
       @options = DEFAULT_STREAM_OPTIONS.merge(options || {})
       @replay = !!@options[:replay]
       @client_id = @options[:client_id]
-      @stream_mode = @replay ? 'replay' : 'streams'
-
-      # force v1 if Replay activated
-      @v2 = !@replay && !!@options[:v2]
+      @v2 = !!@options[:v2]
     end
 
     # Adds many rules to your PowerTrack stream’s ruleset.
@@ -181,22 +178,30 @@ def track(options=nil)
     def feature_url(hostname, feature=nil, sub_feature=nil)
       _url = nil
       if @v2
-        feature ||= hostname
+        feature ||= @replay ? 'replay' : hostname
         sub_feature = sub_feature ? "/#{sub_feature}" : ''
+        stream_type = (feature == 'rules' && @replay ? 'powertrack-replay' : 'powertrack')
+        # replay streaming is on gnip.com while replay rules are on twitter.com...
+        domain = (feature == 'replay' && @replay  ? 'gnip' : 'twitter')
+
         _url = FEATURE_URL_FORMAT[:v2] %
                 [ hostname,
+                  domain,
                   feature,
+                  stream_type,
                   @account_name,
                   @data_source,
                   @label,
                   sub_feature ]
       else
         feature = feature ? "/#{feature}" : ''
+        mode = @replay ? 'replay' : 'streams'
+
         _url = FEATURE_URL_FORMAT[:v1] %
                 [ hostname,
                   @account_name,
                   @data_source,
-                  @stream_mode,
+                  mode,
                   @label,
                   feature ]
 

lib/powertrack/version.rb

@@ -1,3 +1,3 @@
 module PowerTrack
-  VERSION = '1.2.0'.freeze
+  VERSION = '1.3.0'.freeze
 end

test/minitest_helper.rb

@@ -36,7 +36,7 @@ def new_stream(v2=false, replay=false)
       powertrack_config[:password],
       powertrack_config[:account_name],
       powertrack_config[:data_source],
-      replay ? 'prod' : (v2 ? 'prod2' : powertrack_config[:stream_label]),
+      replay ? 'prod' : powertrack_config[:stream_label],
       replay: replay,
       v2: v2)
   end

test/test_manage_rules.rb

@@ -12,10 +12,14 @@ def test_add_then_delete_a_single_rule_v2
     add_then_delete_a_single_rule(true, false)
   end
 
-  def test_add_then_delete_a_single_rule_in_replay_mode
+  def test_add_then_delete_a_single_rule_in_replay_mode_v1
     add_then_delete_a_single_rule(false, true)
   end
 
+  def test_add_then_delete_a_single_rule_in_replay_mode_v2
+    add_then_delete_a_single_rule(true, true)
+  end
+
   def add_then_delete_a_single_rule(v2, replay)
     stream = new_stream(v2, replay)