module Beaker::DSL::Helpers::WebHelpers

def fetch_http_dir(url, dst_dir)
  logger.notify "fetch_http_dir (url: #{url}, dst_dir #{dst_dir})"
  if url[-1, 1] !~ /\//
    url += '/'
  end
  url = URI.parse(url)
  chunks = url.path.split('/')
  dst = File.join(dst_dir, chunks.last)
  #determine directory structure to cut
  #only want to keep the last directory, thus cut total number of dirs - 2 (hostname + last dir name)
  cut = chunks.length - 2
  wget_command = "wget -nv -P #{dst_dir} --reject \"index.html*\",\"*.gif\" --cut-dirs=#{cut} -np -nH --no-check-certificate -r #{url}"
  logger.notify "Fetching remote directory: #{url}"
  logger.notify "  and saving to #{dst}"
  logger.notify "  using command: #{wget_command}"
  #in ruby 1.9+ we can upgrade this to popen3 to gain access to the subprocess pid
  result = `#{wget_command} 2>&1`
  result.each_line do |line|
    logger.debug(line)
  end
  if $?.to_i != 0
    raise "Failed to fetch_remote_dir '#{url}' (exit code #{$?}"
  end
  dst
end