Merge pull request #58 from redouane59/develop

V1.8
redouane59 · Sep 5, 2020 · 66cfb8c · 66cfb8c
2 parents 14fce2c + 0b0a70c
commit 66cfb8c
Show file tree

Hide file tree

Showing 9 changed files with 81 additions and 53 deletions.
diff --git a/README.md b/README.md
@@ -15,7 +15,7 @@ In your pom.xml, add the following dependency :
 <dependency>
   <groupId>com.github.redouane59.twitter</groupId>
   <artifactId>twittered</artifactId>
-  <version>1.7</version>
+  <version>1.8</version>
 </dependency>
 ```
 In order to use your own developer credentials, you have several options :

diff --git a/pom.xml b/pom.xml
@@ -6,7 +6,7 @@
 
   <groupId>com.github.redouane59.twitter</groupId>
   <artifactId>twittered</artifactId>
-  <version>1.7</version>
+  <version>1.8</version>
 
   <name>twittered</name>
   <description>java client for twitter API</description>

diff --git a/src/main/java/com/github/redouane59/twitter/ITwitterClientV1.java b/src/main/java/com/github/redouane59/twitter/ITwitterClientV1.java
@@ -156,9 +156,10 @@ public interface ITwitterClientV1 {
    * @param query the search query
    * @param fromDate the start date
    * @param toDate the end date
+   * @param envName name of the premium environment. See https://developer.twitter.com/en/account/environments
    * @return a list of tweets
    */
-  List<Tweet> searchForTweetsWithin30days(String query, LocalDateTime fromDate, LocalDateTime toDate);
+  List<Tweet> searchForTweetsWithin30days(String query, LocalDateTime fromDate, LocalDateTime toDate, String envName);
 
   /**
    * Search historic tweets calling https://api.twitter.com/1.1/tweets/search/fullarchive/dev.json Your development environment name should be "dev".
@@ -167,10 +168,11 @@ public interface ITwitterClientV1 {
    * @param query the search query
    * @param fromDate the start date
    * @param toDate the end date
+   * @param envName name of the premium environment. See https://developer.twitter.com/en/account/environments
    * @return a list of tweets
    */
-  
-  List<Tweet> searchForTweetsArchive(String query, LocalDateTime fromDate, LocalDateTime toDate);
+
+  List<Tweet> searchForTweetsArchive(String query, LocalDateTime fromDate, LocalDateTime toDate, String envName);
 
   /**
    * Get token and secret token (oAuth1) calling https://api.twitter.com/oauth/request_token

diff --git a/src/main/java/com/github/redouane59/twitter/ITwitterClientV2.java b/src/main/java/com/github/redouane59/twitter/ITwitterClientV2.java
@@ -145,5 +145,10 @@ public interface ITwitterClientV2 {
    */
   String getBearerToken();
 
+  /**
+   * Stream about 1% of all tweets calling https://api.twitter.com/2/tweets/sample/stream
+   */
+  void startSampledStream(Consumer<Tweet> consumer);
+
 }
 
diff --git a/src/main/java/com/github/redouane59/twitter/TwitterClient.java b/src/main/java/com/github/redouane59/twitter/TwitterClient.java
@@ -375,7 +375,7 @@ public List<Tweet> searchForTweetsWithin7days(String query, LocalDateTime fromDa
       Optional<TweetSearchResponseV2> tweetSearchV2DTO = this.requestHelperV2.getRequestWithParameters(
           URLHelper.SEARCH_TWEET_7_DAYS_URL, parameters, TweetSearchResponseV2.class);
       if (tweetSearchV2DTO.isEmpty() || tweetSearchV2DTO.get().getData() == null) {
-        LOGGER.error("empty response on searchForTweetsWithin7days");
+        LOGGER.warn("empty response on searchForTweetsWithin7days");
         break;
       }
       result.addAll(tweetSearchV2DTO.get().getData());
@@ -414,28 +414,28 @@ public TweetSearchResponse searchForTweetsWithin7days(String query, LocalDateTim
     Optional<TweetSearchResponseV2> tweetSearchV2DTO = this.requestHelperV2.getRequestWithParameters(
         URLHelper.SEARCH_TWEET_7_DAYS_URL, parameters, TweetSearchResponseV2.class);
     if (tweetSearchV2DTO.isEmpty() || tweetSearchV2DTO.get().getData() == null) {
-      LOGGER.error("empty response on searchForTweetsWithin7days");
+      LOGGER.warn("empty response on searchForTweetsWithin7days");
       return new TweetSearchResponse(new ArrayList<>(), null);
     }
     List<Tweet> result = new ArrayList<>(tweetSearchV2DTO.get().getData());
     return new TweetSearchResponse(result, tweetSearchV2DTO.get().getMeta().getNextToken());
   }
 
   @Override
-  public List<Tweet> searchForTweetsWithin30days(String query, LocalDateTime fromDate, LocalDateTime toDate) {
+  public List<Tweet> searchForTweetsWithin30days(String query, LocalDateTime fromDate, LocalDateTime toDate, String envName) {
     int                 count      = 100;
     Map<String, String> parameters = new HashMap<>();
     parameters.put("query", query);
-    parameters.put("max_results", String.valueOf(count));
+    parameters.put("maxResults", String.valueOf(count));
     parameters.put("fromDate", ConverterHelper.getStringFromDate(fromDate));
     parameters.put("toDate", ConverterHelper.getStringFromDate(toDate));
     String      next;
     List<Tweet> result = new ArrayList<>();
     do {
       Optional<TweetSearchResponseV1> tweetSearchV1DTO = this.requestHelperV2.getRequestWithParameters(
-          URLHelper.SEARCH_TWEET_30_DAYS_URL, parameters, TweetSearchResponseV1.class);
-      if (tweetSearchV1DTO.isEmpty()) {
-        LOGGER.error("empty response on searchForTweetsWithin30days");
+          urlHelper.getSearchTweet30DaysUrl(envName), parameters, TweetSearchResponseV1.class);
+      if (tweetSearchV1DTO.isEmpty() || tweetSearchV1DTO.get().getResults() == null) {
+        LOGGER.warn("empty response on searchForTweetsWithin30days");
         break;
       }
       result.addAll(tweetSearchV1DTO.get().getResults());
@@ -447,7 +447,7 @@ public List<Tweet> searchForTweetsWithin30days(String query, LocalDateTime fromD
   }
 
   @Override
-  public List<Tweet> searchForTweetsArchive(String query, LocalDateTime fromDate, LocalDateTime toDate) {
+  public List<Tweet> searchForTweetsArchive(String query, LocalDateTime fromDate, LocalDateTime toDate, String envName) {
     int                 count      = 100;
     Map<String, String> parameters = new HashMap<>();
     parameters.put("query", query);
@@ -458,7 +458,7 @@ public List<Tweet> searchForTweetsArchive(String query, LocalDateTime fromDate,
     List<Tweet> result = new ArrayList<>();
     do {
       Optional<TweetSearchResponseV1> tweetSearchV1DTO = this.requestHelperV2.getRequestWithParameters(
-          URLHelper.SEARCH_TWEET_FULL_ARCHIVE_URL, parameters, TweetSearchResponseV1.class);
+          urlHelper.getSearchTweetFullArchiveUrl(envName), parameters, TweetSearchResponseV1.class);
       if (tweetSearchV1DTO.isEmpty()) {
         LOGGER.error("empty response on searchForTweetsArchive");
         break;
@@ -509,6 +509,12 @@ public StreamMeta deleteFilteredStreamRule(String ruleValue) {
     return result.getMeta();
   }
 
+  @Override
+  public void startSampledStream(Consumer<Tweet> consumer) {
+    String url = this.urlHelper.getSampledStreamUrl();
+    this.requestHelperV2.getAsyncRequest(url, consumer);
+  }
+
   @Override
   public List<TweetV1> readTwitterDataFile(File file) throws IOException {
     SimpleModule module = new SimpleModule();

diff --git a/src/main/java/com/github/redouane59/twitter/helpers/RequestHelperV2.java b/src/main/java/com/github/redouane59/twitter/helpers/RequestHelperV2.java
@@ -53,8 +53,9 @@ public <T> Optional<T> getRequestWithParameters(String url, Map<String, String>
       } else if (response.code() == 401) {
         LOGGER.info("Error 401, user may be private");
         return Optional.empty();
+      } else if (response.code() < 200 || response.code() > 299) {
+        LOGGER.error("(POST) Error calling " + url + " " + stringResponse + " - " + response.code());
       }
-      LOGGER.info(stringResponse);
       result = TwitterClient.OBJECT_MAPPER.readValue(stringResponse, classType);
     } catch (Exception e) {
       LOGGER.error(e.getMessage());

diff --git a/src/main/java/com/github/redouane59/twitter/helpers/URLHelper.java b/src/main/java/com/github/redouane59/twitter/helpers/URLHelper.java
@@ -30,10 +30,10 @@ public class URLHelper {
   private static final String USERS                   = "/users";
   private static final String TWEETS                  = "/tweets";
   private static final String SEARCH                  = "/search";
+  private static final String SAMPLE                  = "/sample";
   private static final String STREAM                  = "/stream";
   private static final String THIRTY_DAYS             = "/30day";
   private static final String FULL_ARCHIVE            = "/fullarchive";
-  private static final String DEV_ENV_NAME            = "/dev"; // @todo config
   private static final String ACCOUNT_ACTIVITY        = "/account_activity/all";
   private static final String WEBHOOKS                = "/webhooks";
   private static final String USER_ID                 = "user_id";
@@ -57,13 +57,22 @@ public class URLHelper {
   public static final  String LAST_TWEET_LIST_URL     = ROOT_URL_V1 + STATUSES + USER_TIMELINE;
   public static final  String RATE_LIMIT_URL          = ROOT_URL_V1 + "/application/rate_limit_status.json";
   ;
-  public static final String SEARCH_TWEET_30_DAYS_URL      = ROOT_URL_V1 + TWEETS + SEARCH + THIRTY_DAYS + DEV_ENV_NAME + JSON;
-  public static final String SEARCH_TWEET_FULL_ARCHIVE_URL = ROOT_URL_V1 + TWEETS + SEARCH + FULL_ARCHIVE + DEV_ENV_NAME + JSON;
-  public static final String SEARCH_TWEET_STANDARD_URL     = ROOT_URL_V1 + SEARCH + TWEETS + JSON;
-  public static final String LIVE_EVENT_URL                = ROOT_URL_V1 + ACCOUNT_ACTIVITY + DEV_ENV_NAME + WEBHOOKS + JSON;
-  public static final String SEARCH_TWEET_7_DAYS_URL       = ROOT_URL_V2 + TWEETS + SEARCH + "/recent";
-  public static final String GET_BEARER_TOKEN_URL          = "https://api.twitter.com/oauth2/token";
-  public static final String GET_OAUTH1_TOKEN_URL          = "https://api.twitter.com/oauth/request_token";
+  public static final String SEARCH_TWEET_STANDARD_URL = ROOT_URL_V1 + SEARCH + TWEETS + JSON;
+  public static final String SEARCH_TWEET_7_DAYS_URL   = ROOT_URL_V2 + TWEETS + SEARCH + "/recent";
+  public static final String GET_BEARER_TOKEN_URL      = "https://api.twitter.com/oauth2/token";
+  public static final String GET_OAUTH1_TOKEN_URL      = "https://api.twitter.com/oauth/request_token";
+
+  public String getSearchTweet30DaysUrl(String envName) {
+    return ROOT_URL_V1 + TWEETS + SEARCH + THIRTY_DAYS + "/" + envName + JSON;
+  }
+
+  public String getSearchTweetFullArchiveUrl(String envName) {
+    return ROOT_URL_V1 + TWEETS + SEARCH + FULL_ARCHIVE + "/" + envName + JSON;
+  }
+
+  public String getLiveEventUrl(String envName) {
+    return ROOT_URL_V1 + ACCOUNT_ACTIVITY + "/" + envName + WEBHOOKS + JSON;
+  }
 
   public String getFollowUrl(String userId) {
     return ROOT_URL_V1 +
@@ -289,4 +298,7 @@ public String getFilteredStreamUrl() {
     return ROOT_URL_V2 + TWEETS + SEARCH + STREAM;
   }
 
+  public String getSampledStreamUrl() {
+    return ROOT_URL_V2 + TWEETS + SAMPLE + STREAM;
+  }
 }
diff --git a/src/test/java/com/github/redouane59/twitter/nrt/ITwitterClientV1Test.java b/src/test/java/com/github/redouane59/twitter/nrt/ITwitterClientV1Test.java
@@ -175,14 +175,15 @@ public void testGetFavorites() {
     assertTrue(favorites.size() > count);
   }
 
-    /*
-    @Test
-    public void testSearchTweets30days(){
-        LocalDateTime startDate = DateUtils.truncate(new Date(),Calendar.MONTH);
-        LocalDateTime endDate = DateUtils.addDays(startDate, 1);
-        List<ITweet> result = twitterClient.searchForTweetsWithin30days("@RedTheOne -RT",startDate, endDate);
-        assertTrue(result.size()>0);
-    }
+  @Test
+  public void testSearchTweets30days() {
+    List<Tweet>
+        result =
+        twitterClient.searchForTweetsWithin30days("@RedTheOne -RT", LocalDateTime.of(2020, 9, 1, 0, 0), LocalDateTime.of(2020, 9, 3, 0, 0), "30days");
+    assertTrue(result.size() > 0);
+  }
+
+        /*
 
     @Test
     public void testSearchTweetsArchive(){

diff --git a/src/test/java/com/github/redouane59/twitter/unit/UrlHelperTest.java b/src/test/java/com/github/redouane59/twitter/unit/UrlHelperTest.java
@@ -103,20 +103,6 @@ public void testGetFollowersByIdUrl() {
                  urlHelper.getFollowerUsersUrl("12345"));
   }
 
-  @Test
-  public void testSearchTweetsUrl() {
-    //https://api.twitter.com/1.1/tweets/search/30day/DevImproveMyTwitter.json
-    assertEquals("https://api.twitter.com/1.1/tweets/search/30day/dev.json",
-                 URLHelper.SEARCH_TWEET_30_DAYS_URL);
-  }
-
-  @Test
-  public void testSearchTweetsUrlFull() {
-    // /search/fullarchive/:label.json
-    assertEquals("https://api.twitter.com/1.1/tweets/search/fullarchive/dev.json",
-                 URLHelper.SEARCH_TWEET_FULL_ARCHIVE_URL);
-  }
-
   @Test
   public void testSearchTweetsUrlStandard() {
     https:
@@ -138,13 +124,6 @@ public void testUnfollowByUserId() {
                  urlHelper.getUnfollowUrl("12345"));
   }
 
-  @Test
-  public void testLiveEventUrl() {
-    //https://api.twitter.com/1.1/account_activity/all/:env_name/webhooks.json
-    assertEquals("https://api.twitter.com/1.1/account_activity/all/dev/webhooks.json",
-                 URLHelper.LIVE_EVENT_URL);
-  }
-
   @Test
   public void testLikeUrl() {
     String tweetId = "12345";
@@ -220,7 +199,29 @@ public void testRetrieveFilteredStreamRulesUrl() {
   }
 
   @Test
-  public void testRetrieveFilteredStreamUrl() {
+  public void testFilteredStreamUrl() {
     assertEquals("https://api.twitter.com/2/tweets/search/stream", urlHelper.getFilteredStreamUrl());
   }
+
+  @Test
+  public void testSampledStreamUrl() {
+    assertEquals("https://api.twitter.com/2/tweets/sample/stream", urlHelper.getSampledStreamUrl());
+  }
+
+  @Test
+  public void testSearchFullArchiveUrl() {
+    assertEquals("https://api.twitter.com/1.1/tweets/search/fullarchive/dev.json", urlHelper.getSearchTweetFullArchiveUrl("dev"));
+  }
+
+  @Test
+  public void testSearch30daysUrl() {
+    assertEquals("https://api.twitter.com/1.1/tweets/search/30day/dev.json", urlHelper.getSearchTweet30DaysUrl("dev"));
+  }
+
+  @Test
+  public void testLiveEventUrl() {
+    assertEquals("https://api.twitter.com/1.1/account_activity/all/dev/webhooks.json",
+                 urlHelper.getLiveEventUrl("dev"));
+  }
+
 }