2 Commits

Author SHA1 Message Date
  Dylan Baker ad763890e5 Break as soon as there's a thread with no new posts 4 years ago
  Dylan Baker e0c560a446 Add sleep 4 years ago
1 changed files with 2 additions and 3 deletions
  1. 2
    3
      db/scrape.rb

+ 2
- 3
db/scrape.rb View File

83
     threads = parse_threads(page)
83
     threads = parse_threads(page)
84
     threads.each do |t|
84
     threads.each do |t|
85
       puts t[:title]
85
       puts t[:title]
86
-
86
+      sleep(1)
87
       page = fetch_thread(t, cookie)
87
       page = fetch_thread(t, cookie)
88
       first_post = page.at_css('.postinfo:first-child')
88
       first_post = page.at_css('.postinfo:first-child')
89
 
89
 
112
 
112
 
113
       last_post = posts.last
113
       last_post = posts.last
114
       unless db.from(:posts).first(remote_id: last_post[:remote_id]).nil?
114
       unless db.from(:posts).first(remote_id: last_post[:remote_id]).nil?
115
-        puts '  Up to date, skipping'
116
-        next
115
+        break
117
       end
116
       end
118
 
117
 
119
       posts_count = posts.size
118
       posts_count = posts.size

Loading…
Cancel
Save