Browse Source

Parse stickiness in parse_threads

master
Dylan Baker 4 years ago
parent
commit
e46a9fa449
1 changed files with 4 additions and 4 deletions
  1. 4
    4
      db/scrape.rb

+ 4
- 4
db/scrape.rb View File

66
     thread_link = row.at_css('.subject > a')
66
     thread_link = row.at_css('.subject > a')
67
     next if thread_link.nil?
67
     next if thread_link.nil?
68
     thread[:remote_id] = thread_link['href'].split('/')[3]
68
     thread[:remote_id] = thread_link['href'].split('/')[3]
69
-    thread[:title] = thread_link.text
69
+    thread[:title] = thread_link.text.strip
70
     creator = row.at_css('.memberlink').text.strip
70
     creator = row.at_css('.memberlink').text.strip
71
     creator = creator[0..-2] if creator.match(/\+$/)
71
     creator = creator[0..-2] if creator.match(/\+$/)
72
     thread[:creator] = creator
72
     thread[:creator] = creator
73
+    thread[:is_sticky] = !!thread[:title].match(/^Sticky:/)
73
     threads << thread
74
     threads << thread
74
   end
75
   end
75
   threads
76
   threads
83
     page = fetch_page(page_number, cookie)
84
     page = fetch_page(page_number, cookie)
84
     threads = parse_threads(page)
85
     threads = parse_threads(page)
85
     threads.each do |t|
86
     threads.each do |t|
86
-      is_sticky = t[:title].match(/^Sticky:/)
87
-      next if index > 0 && is_sticky
87
+      next if index > 0 && t[:is_sticky]
88
 
88
 
89
       puts t[:title] if log
89
       puts t[:title] if log
90
 
90
 
116
 
116
 
117
       last_post = posts.last
117
       last_post = posts.last
118
       unless db.from(:posts).first(remote_id: last_post[:remote_id]).nil?
118
       unless db.from(:posts).first(remote_id: last_post[:remote_id]).nil?
119
-        next if is_sticky
119
+        next if t[:is_sticky]
120
         break
120
         break
121
       end
121
       end
122
 
122
 

Loading…
Cancel
Save