Refactor scraper

db/scrape.rb

@@ -13,21 +13,27 @@ class Scraper
     @first = first
     @last = last
     @log = log
+    @no_new_posts = false
 
     authenticate!
   end
 
   def scrape
     (@first..@last).each_with_index do |page_number, page_index|
-      page = Fetcher.new(cookie: @cookie).page(page_number)
-      threads = Parser.new.threads(page)
+      page = fetcher.page(page_number)
+      threads = parser.threads(page)
 
       threads.each do |t|
         next if page_index > 0 && t[:is_sticky]
-        no_new_posts = scrape_thread(t)
+
+        scrape_thread(t)
+
         if no_new_posts
-          next if t[:is_sticky]
-          return
+          if t[:is_sticky]
+            next
+          else
+            return
+          end
         end
       end
     end
@@ -35,50 +41,58 @@ class Scraper
 
   private
 
-  def authenticate!
-    @cookie = login(ENV['VLV_USERNAME'], ENV['VLV_PASSWORD'])
-
-    raise "Error logging into VLV. Check your credentials." if @cookie.nil?
-  end
+  attr_reader :cookie, :fetcher, :parser, :no_new_posts
 
   def scrape_thread(t)
+    @no_new_posts = false
+
     log t[:title]
 
-    page = Fetcher.new(cookie: @cookie).thread(t)
+    page = fetcher.thread(t)
     first_post = page.at_css('.postinfo:first-child')
     return false if first_post.nil?
 
     thread = DB.from(:threads).first(remote_id: t[:remote_id])
     if thread.nil?
-      t[:created_at] = Parser.new.thread_created_at(first_post)
-      thread = VLV::Thread.create(t.delete_if { |k| k == :is_sticky })
       log '  Inserting thread'
+      t[:created_at] = parser.thread_created_at(first_post)
+      thread = VLV::Thread.create(t.delete_if { |k| k == :is_sticky })
     end
 
     scrape_posts(thread, page)
   end
 
   def scrape_posts(thread, page)
-    posts = Parser.new.posts(thread, page)
+    posts = parser.posts(thread, page)
     last_post = posts.last
-    unless DB.from(:posts).first(remote_id: last_post[:remote_id]).nil?
-      log '  No new posts'
-      return true
-    end
 
-    posts_count = posts.size
-    posts.each_with_index do |p, index|
-      msg = "  Inserting post #{index + 1}/#{posts_count}"
-      print msg if @log
-      if DB.from(:posts).first(remote_id: p[:remote_id]).nil?
-        VLV::Post.create(p)
+    if DB.from(:posts).first(remote_id: last_post[:remote_id]).nil?
+      db_posts = VLV::Post.where(remote_id: posts.map { |p| p[:remote_id] }).all
+
+      posts = posts.each_with_index.map do |post|
+        if db_posts.detect { |db_post| db_post.remote_id == post[:remote_id] }.nil?
+          post
+        end
       end
-      print "\b" * msg.size unless index == posts_count - 1 if @log
+
+      log "  Inserting #{posts.size} posts"
+      VLV::Post.multi_insert(posts.compact)
+    else
+      no_new_posts!
+      log '  No new posts'
     end
+  end
 
-    log ''
+  def no_new_posts!
+    @no_new_posts = true
+  end
+
+  def authenticate!
+    @cookie = login(ENV['VLV_USERNAME'], ENV['VLV_PASSWORD'])
+    raise "Error logging into VLV. Check your credentials." if @cookie.nil?
 
-    false
+    @fetcher = Fetcher.new(cookie: cookie)
+    @parser = Parser.new
   end
 
   def log(msg)