parse-postcodes.rb

#!/usr/bin/env ruby

$:.unshift "#{File.dirname(__FILE__)}/lib"

require 'mechanize_proxy'
require 'configuration'
require 'people'

conf = Configuration.new

agent = MechanizeProxy.new
agent.cache_subdirectory = "parse-postcodes"

puts "Reading Australia post office data..."
data = CSV.readlines("data/pc-full_20080529.csv")
# Ignore header
data.shift

valid_postcodes = data.map {|row| row.first}.uniq.sort

def extract_divisions_from_page(page)
  postcodes = []
  page.search('table').first.search('> tr').each do |row_tag|
    td_tag = row_tag.search('> td')[3]
    if td_tag
      postcode = td_tag.search('a').inner_text
      if postcode.nil?
        puts "Nil postcode in division #{division}"
      end
      postcodes << postcode
    end
  end
  postcodes
end

def other_pages?(page)
  table_tag = page.search('table')[1]
  !table_tag.search('> tr > td > a').map {|e| e.inner_text}.empty?
end

file = File.open("data/postcodes.csv", "w")

file.puts("Postcode,Electoral division name")
file.puts(",")

valid_postcodes.each do |postcode|
  page = agent.get("http://apps.aec.gov.au/esearch/LocalitySearchResults.aspx?filter=#{postcode}&filterby=Postcode")
  
  divisions = extract_divisions_from_page(page)
  
  if other_pages?(page)
    puts "WARNING: Multiple pages of data for postcode #{postcode}"
    file.puts("*** Double check data for postcode #{postcode} by hand ***")
  end
  
  if divisions.empty?
    puts "No divisions for postcode #{postcode}"
  else
    divisions.uniq.sort.each do |division|
      file.puts "#{postcode},#{division}"
    end
  end
end