Browse Source

Fix breaking/sticky logic

Only break on the first thread with no new posts if that thread is not
sticky. Also, ignore sticky threads unless this is the first page.
master
Dylan Baker 4 years ago
parent
commit
210c1c5667
1 changed files with 4 additions and 2 deletions
  1. 4
    2
      db/scrape.rb

+ 4
- 2
db/scrape.rb View File

@@ -78,12 +78,13 @@ def scrape(first: 0, last: 0)
78 78
   cookie = login(ENV['VLV_USERNAME'], ENV['VLV_PASSWORD'])
79 79
   db = connect
80 80
 
81
-  (first..last).each do |page_number|
81
+  (first..last).each_with_index do |page_number, index|
82 82
     page = fetch_page(page_number, cookie)
83 83
     threads = parse_threads(page)
84 84
     threads.each do |t|
85
-      puts t[:title]
86 85
       sleep(1)
86
+      is_sticky = t[:title].match(/^Sticky:/)
87
+      next if index > 0 && is_sticky
87 88
       page = fetch_thread(t, cookie)
88 89
       first_post = page.at_css('.postinfo:first-child')
89 90
 
@@ -112,6 +113,7 @@ def scrape(first: 0, last: 0)
112 113
 
113 114
       last_post = posts.last
114 115
       unless db.from(:posts).first(remote_id: last_post[:remote_id]).nil?
116
+        next if is_sticky
115 117
         break
116 118
       end
117 119
 

Loading…
Cancel
Save