|
@@ -66,10 +66,11 @@ def parse_threads(page)
|
66
|
66
|
thread_link = row.at_css('.subject > a')
|
67
|
67
|
next if thread_link.nil?
|
68
|
68
|
thread[:remote_id] = thread_link['href'].split('/')[3]
|
69
|
|
- thread[:title] = thread_link.text
|
|
69
|
+ thread[:title] = thread_link.text.strip
|
70
|
70
|
creator = row.at_css('.memberlink').text.strip
|
71
|
71
|
creator = creator[0..-2] if creator.match(/\+$/)
|
72
|
72
|
thread[:creator] = creator
|
|
73
|
+ thread[:is_sticky] = !!thread[:title].match(/^Sticky:/)
|
73
|
74
|
threads << thread
|
74
|
75
|
end
|
75
|
76
|
threads
|
|
@@ -83,8 +84,7 @@ def scrape(first: 0, last: 0, log: false)
|
83
|
84
|
page = fetch_page(page_number, cookie)
|
84
|
85
|
threads = parse_threads(page)
|
85
|
86
|
threads.each do |t|
|
86
|
|
- is_sticky = t[:title].match(/^Sticky:/)
|
87
|
|
- next if index > 0 && is_sticky
|
|
87
|
+ next if index > 0 && t[:is_sticky]
|
88
|
88
|
|
89
|
89
|
puts t[:title] if log
|
90
|
90
|
|
|
@@ -116,7 +116,7 @@ def scrape(first: 0, last: 0, log: false)
|
116
|
116
|
|
117
|
117
|
last_post = posts.last
|
118
|
118
|
unless db.from(:posts).first(remote_id: last_post[:remote_id]).nil?
|
119
|
|
- next if is_sticky
|
|
119
|
+ next if t[:is_sticky]
|
120
|
120
|
break
|
121
|
121
|
end
|
122
|
122
|
|