Add option to log or not

db/scrape.rb

@@ -16,6 +16,7 @@ def fetch_page(page_number, cookie)
 end
 
 def fetch_thread(thread, cookie)
+  sleep(0.5)
   url =
     URI(
       "http://board.vivalavinyl.com/thread/view/#{thread[:remote_id]}&ajax=true"
@@ -74,7 +75,7 @@ def parse_threads(page)
   threads
 end
 
-def scrape(first: 0, last: 0)
+def scrape(first: 0, last: 0, log: false)
   cookie = login(ENV['VLV_USERNAME'], ENV['VLV_PASSWORD'])
   db = connect
 
@@ -82,9 +83,11 @@ def scrape(first: 0, last: 0)
     page = fetch_page(page_number, cookie)
     threads = parse_threads(page)
     threads.each do |t|
-      sleep(1)
       is_sticky = t[:title].match(/^Sticky:/)
       next if index > 0 && is_sticky
+
+      puts t[:title] if log
+
       page = fetch_thread(t, cookie)
       first_post = page.at_css('.postinfo:first-child')
 
@@ -97,7 +100,7 @@ def scrape(first: 0, last: 0)
       thread = db.from(:threads).first(remote_id: t[:remote_id])
       is_new_thread = thread.nil?
       if is_new_thread
-        puts '  Inserting thread'
+        puts '  Inserting thread' if log
         id =
           db.from(:threads).insert(
             title: t[:title],
@@ -120,7 +123,7 @@ def scrape(first: 0, last: 0)
       posts_count = posts.size
       posts.each_with_index do |p, index|
         msg = "  Inserting post #{index + 1}/#{posts_count}"
-        print msg
+        print msg if log
         if is_new_thread || db.from(:posts).first(remote_id: p[:remote_id]).nil?
           db.from(:posts).insert(
             body: p[:body],
@@ -130,10 +133,10 @@ def scrape(first: 0, last: 0)
             remote_id: p[:remote_id]
           )
         end
-        print "\b" * msg.size unless index == posts_count - 1
+        print "\b" * msg.size unless index == posts_count - 1 if log
       end
 
-      puts
+      puts if log
     end
   end
 end