collector: optimize (creating model class instance is so slow)

4 files changed, 55 insertions, 25 deletions

diff --git a/app/models/favorite.rb b/app/models/favorite.rb
index da8087c..701b8ca 100644
--- a/app/models/favorite.rb
+++ b/app/models/favorite.rb
@@ -7,12 +7,16 @@ class Favorite < ActiveRecord::Base
   #
   # @param [Array] array An array of Streaming API events.
   def self.create_bulk_from_json(array)
+    return if array.empty?
+
     objects = array.map do |json|
-      self.new(user_id: json[:source][:id],
-               tweet_id: json[:target_object][:id])
+      {
+        user_id: json[:source][:id],
+        tweet_id: json[:target_object][:id]
+      }
     end
 
-    self.import(objects, ignore: true)
+    self.import(objects.first.keys, objects.map(&:values), ignore: true)
   end
 
   # Unregisters favorite event in bulk from an array of Streaming API 'unfavorite' events.
diff --git a/app/models/retweet.rb b/app/models/retweet.rb
index 0648525..cefd901 100644
--- a/app/models/retweet.rb
+++ b/app/models/retweet.rb
@@ -7,13 +7,17 @@ class Retweet < ActiveRecord::Base
   #
   # @param [Array] array An array of Streaming API messages.
   def self.create_bulk_from_json(array)
+    return if array.empty?
+
     objects = array.map do |json|
-      self.new(id: json[:id],
-               user_id: json[:user][:id],
-               tweet_id: json[:retweeted_status][:id])
+      {
+        id: json[:id],
+        user_id: json[:user][:id],
+        tweet_id: json[:retweeted_status][:id]
+      }
     end
 
-    self.import(objects, ignore: true)
+    self.import(objects.first.keys, objects.map(&:values), ignore: true)
   end
 
   # Unregisters retweet events in bulk from array of Streaming API's delete events.
diff --git a/app/models/tweet.rb b/app/models/tweet.rb
index 009f868..9ae277e 100644
--- a/app/models/tweet.rb
+++ b/app/models/tweet.rb
@@ -30,22 +30,33 @@ class Tweet < ActiveRecord::Base
     # @param [Hash] json Data from Twitter API
     # @return [Tweet] The new instance.
     def build_from_json(json)
-      self.new(id: json[:id],
-               text: extract_entities(json),
-               source: json[:source],
-               tweeted_at: Time.parse(json[:created_at]),
-               user_id: json[:user][:id],
-               in_reply_to_id: json[:in_reply_to_status_id],
-               favorites_count: json[:favorite_count],
-               retweets_count: json[:retweet_count],
-               reactions_count: json[:favorite_count] + json[:retweet_count])
+      self.new(transform_from_json_into_hash(json))
+    end
+
+    def transform_from_json_into_hash(json)
+      {
+        id: json[:id],
+        text: extract_entities(json),
+        source: json[:source],
+        tweeted_at: Time.parse(json[:created_at]),
+        user_id: json[:user][:id],
+        in_reply_to_id: json[:in_reply_to_status_id],
+        favorites_count: json[:favorite_count],
+        retweets_count: json[:retweet_count],
+        reactions_count: json[:favorite_count] + json[:retweet_count]
+      }
     end
 
     # Builds instances of Tweet and save them. This method is supposed to be used from collector daemon.
     # @param [Array<Hash>] array Data from collector.
     def create_bulk_from_json(array)
-      objects = array.map {|json| build_from_json(json) }
-      self.import(objects, on_duplicate_key_update: [:favorites_count, :retweets_count, :reactions_count])
+      return if array.empty?
+
+      objects = array.map {|json| transform_from_json_into_hash(json) }
+      keys = objects.first.keys
+      self.import(keys, objects.map(&:values),
+                  on_duplicate_key_update: [:favorites_count, :retweets_count, :reactions_count],
+                  validate: false)
     end
 
     # Destroys Tweets from database. This method is supposed to be used from collector daemon.
diff --git a/app/models/user.rb b/app/models/user.rb
index e008619..66620c7 100644
--- a/app/models/user.rb
+++ b/app/models/user.rb
@@ -18,12 +18,18 @@ class User < ActiveRecord::Base
       key && where(key => value).order(updated_at: :desc).first || raise(ActiveRecord::RecordNotFound, "Couldn't find User with #{key}=#{value}")
     end
 
+    def transform_from_json_into_hash(json)
+      {
+        id: json[:id],
+        screen_name: json[:screen_name],
+        name: json[:name],
+        profile_image_url: json[:profile_image_url_https] || json[:profile_image_url],
+        protected: json[:protected]
+      }
+    end
+
     def build_from_json(json)
-      self.new(id: json[:id],
-               screen_name: json[:screen_name],
-               name: json[:name],
-               profile_image_url: json[:profile_image_url_https] || json[:profile_image_url],
-               protected: json[:protected])
+      self.new(transform_from_json_into_hash(json))
     end
 
     def create_or_update_from_json(json)
@@ -31,8 +37,13 @@ class User < ActiveRecord::Base
     end
 
     def create_or_update_bulk_from_json(array)
-      objects = array.map {|json| build_from_json(json) }
-      import(objects, on_duplicate_key_update: [:screen_name, :name, :profile_image_url, :protected])
+      return if array.empty?
+
+      objects = array.map {|json| transform_from_json_into_hash(json) }
+      keys = objects.first.keys
+      self.import(keys, objects.map(&:values),
+                  on_duplicate_key_update: [:screen_name, :name, :profile_image_url, :protected],
+                  validate: false)
     end
   end