Only break on the first thread with no new posts if that thread is not
sticky. Also, ignore sticky threads unless this is the first page.

Rakefile

@@ -9,7 +9,7 @@ task 'migrate' do
 end
 
 task 'scrape' do
-  scrape
+  scrape(log: ENV['APP_ENV'] == 'development')
 end
 
 task 'build' do

db/scrape.rb

@@ -16,6 +16,7 @@ def fetch_page(page_number, cookie)
 end
 
 def fetch_thread(thread, cookie)
+  sleep(0.5)
   url =
     URI(
       "http://board.vivalavinyl.com/thread/view/#{thread[:remote_id]}&ajax=true"
@@ -65,25 +66,28 @@ def parse_threads(page)
     thread_link = row.at_css('.subject > a')
     next if thread_link.nil?
     thread[:remote_id] = thread_link['href'].split('/')[3]
-    thread[:title] = thread_link.text
+    thread[:title] = thread_link.text.strip
     creator = row.at_css('.memberlink').text.strip
     creator = creator[0..-2] if creator.match(/\+$/)
     thread[:creator] = creator
+    thread[:is_sticky] = !!thread[:title].match(/^Sticky:/)
     threads << thread
   end
   threads
 end
 
-def scrape(first: 0, last: 0)
+def scrape(first: 0, last: 0, log: false)
   cookie = login(ENV['VLV_USERNAME'], ENV['VLV_PASSWORD'])
   db = connect
 
-  (first..last).each do |page_number|
+  (first..last).each_with_index do |page_number, index|
     page = fetch_page(page_number, cookie)
     threads = parse_threads(page)
     threads.each do |t|
-      puts t[:title]
-      sleep(1)
+      next if index > 0 && t[:is_sticky]
+
+      puts t[:title] if log
+
       page = fetch_thread(t, cookie)
       first_post = page.at_css('.postinfo:first-child')
 
@@ -96,7 +100,7 @@ def scrape(first: 0, last: 0)
       thread = db.from(:threads).first(remote_id: t[:remote_id])
       is_new_thread = thread.nil?
       if is_new_thread
-        puts '  Inserting thread'
+        puts '  Inserting thread' if log
         id =
           db.from(:threads).insert(
             title: t[:title],
@@ -112,13 +116,14 @@ def scrape(first: 0, last: 0)
 
       last_post = posts.last
       unless db.from(:posts).first(remote_id: last_post[:remote_id]).nil?
+        next if t[:is_sticky]
         break
       end
 
       posts_count = posts.size
       posts.each_with_index do |p, index|
         msg = "  Inserting post #{index + 1}/#{posts_count}"
-        print msg
+        print msg if log
         if is_new_thread || db.from(:posts).first(remote_id: p[:remote_id]).nil?
           db.from(:posts).insert(
             body: p[:body],
@@ -128,10 +133,10 @@ def scrape(first: 0, last: 0)
             remote_id: p[:remote_id]
           )
         end
-        print "\b" * msg.size unless index == posts_count - 1
+        print "\b" * msg.size unless index == posts_count - 1 if log
       end
 
-      puts
+      puts if log
     end
   end
 end