From fd8a5d04a47cfc30eb4ffac2db1f031a41bf7a01 Mon Sep 17 00:00:00 2001
From: Kazuki Yamaguchi <k@rhe.jp>
Date: Thu, 30 Aug 2018 17:49:20 +0900
Subject: service.rb: fix tweet object structure

The Tweet object retrieved by the retired Streaming API and the Account
Activity follows "Compatibility with additional extended_tweet in
payload". However, the REST API still defaults to the "Compatibility"
mode which does not support >140 characters texts at all. Let's adjust
the response from the REST API to mimic "Compatibility with additional
extended_tweet in payload" mode.
---
 README.txt | 10 ++++++++--
 service.rb | 25 ++++++++++++++++++++++++-
 2 files changed, 32 insertions(+), 3 deletions(-)

diff --git a/README.txt b/README.txt
index 9daac40..12fdd37 100644
--- a/README.txt
+++ b/README.txt
@@ -133,7 +133,12 @@ twitter-event-stream opens two endpoints for a client:
      :\r\n\r\n
 
 
-twitter-event-stream uses "OAuth Echo"[3] to authenticate a client, meaning
+In both endpoints, the Tweet object structure BASICALLY follows the
+"Compatibility with additional extended_tweet in payload" mode[3]. Each Tweet
+object has `extended_tweet` object containing `full_text` key.
+
+
+twitter-event-stream uses "OAuth Echo"[4] to authenticate a client, meaning
 an application must provide the following HTTP headers:
 
  - `x-auth-service-provider`
@@ -147,7 +152,8 @@ an application must provide the following HTTP headers:
     normally send when calling the account/verify_credentials API.
 
 [2] https://developer.twitter.com/en/docs/basics/authentication/overview/oauth-echo.html
-[3] https://developer.twitter.com/en/docs/accounts-and-users/subscribe-account-activity/guides/account-activity-data-objects
+[3] https://developer.twitter.com/en/docs/tweets/tweet-updates.html
+[4] https://developer.twitter.com/en/docs/accounts-and-users/subscribe-account-activity/guides/account-activity-data-objects
 
 License
 -------
diff --git a/service.rb b/service.rb
index c997409..915241a 100644
--- a/service.rb
+++ b/service.rb
@@ -187,7 +187,11 @@ class Service
         last_max = nil
         while true
           t = Time.now
-          opts = { "count" => 200, "since_id" => last_max ? last_max - 1 : 1 }
+          opts = {
+            "tweet_mode" => "extended",
+            "count" => 200,
+            "since_id" => last_max ? last_max - 1 : 1
+          }
           ret = twitter_get("/1.1/statuses/home_timeline.json", opts)
 
           unless ret.empty?
@@ -200,6 +204,25 @@ class Service
               end
             end
 
+            # Fix Tweet object structure so it follows "Compatibility with
+            # additional extended_tweet in payload" mode.
+            # https://developer.twitter.com/en/docs/tweets/tweet-updates.html
+            ret.each { |tweet|
+              tweet["extended_tweet"] = {
+                "full_text" => tweet["full_text"],
+                "display_text_range" => tweet["display_text_range"],
+                "entities" => tweet["entities"],
+                "extended_entities" => tweet["extended_entities"],
+              }
+              tweet["text"] = tweet["full_text"]
+
+              # NOTE: full_text should be removed from tweet, and then
+              # truncated, entities, extended_entities, and display_text_range
+              # should be modified according to the length of full_text. But
+              # this is probably not worth doing as clients will anyway process
+              # extended_tweet only so it can support >140 characters tweets.
+            }
+
             unless ret.empty?
               emit("twitter_event_stream_home_timeline", ret)
               last_max = ret.first["id"]
-- 
cgit v1.2.3