|
@@ -78,12 +78,13 @@ def scrape(first: 0, last: 0)
|
78
|
78
|
cookie = login(ENV['VLV_USERNAME'], ENV['VLV_PASSWORD'])
|
79
|
79
|
db = connect
|
80
|
80
|
|
81
|
|
- (first..last).each do |page_number|
|
|
81
|
+ (first..last).each_with_index do |page_number, index|
|
82
|
82
|
page = fetch_page(page_number, cookie)
|
83
|
83
|
threads = parse_threads(page)
|
84
|
84
|
threads.each do |t|
|
85
|
|
- puts t[:title]
|
86
|
85
|
sleep(1)
|
|
86
|
+ is_sticky = t[:title].match(/^Sticky:/)
|
|
87
|
+ next if index > 0 && is_sticky
|
87
|
88
|
page = fetch_thread(t, cookie)
|
88
|
89
|
first_post = page.at_css('.postinfo:first-child')
|
89
|
90
|
|
|
@@ -112,6 +113,7 @@ def scrape(first: 0, last: 0)
|
112
|
113
|
|
113
|
114
|
last_post = posts.last
|
114
|
115
|
unless db.from(:posts).first(remote_id: last_post[:remote_id]).nil?
|
|
116
|
+ next if is_sticky
|
115
|
117
|
break
|
116
|
118
|
end
|
117
|
119
|
|