2 Commits

Author SHA1 Message Date
  Dylan Baker ad763890e5 Break as soon as there's a thread with no new posts 4 years ago
  Dylan Baker e0c560a446 Add sleep 4 years ago
1 changed files with 2 additions and 3 deletions
  1. 2
    3
      db/scrape.rb

+ 2
- 3
db/scrape.rb View File

@@ -83,7 +83,7 @@ def scrape(first: 0, last: 0)
83 83
     threads = parse_threads(page)
84 84
     threads.each do |t|
85 85
       puts t[:title]
86
-
86
+      sleep(1)
87 87
       page = fetch_thread(t, cookie)
88 88
       first_post = page.at_css('.postinfo:first-child')
89 89
 
@@ -112,8 +112,7 @@ def scrape(first: 0, last: 0)
112 112
 
113 113
       last_post = posts.last
114 114
       unless db.from(:posts).first(remote_id: last_post[:remote_id]).nil?
115
-        puts '  Up to date, skipping'
116
-        next
115
+        break
117 116
       end
118 117
 
119 118
       posts_count = posts.size

Loading…
Cancel
Save