#!/bin/bash
#---------------------------------------------------------------------
##
## @Synopsis Functions that download and verify urls.
##
##
## This file contains functions for downloading and verifying urls.
## It does this by extracting information from a url with url handlers
## specific to that url type. The url handler also determines a download
## handler to actually download the url.
##
## FIXME
## For example, the request to download
## the following url is made through the generic F<url_download>
## function:
##
##     http://machinename.com/path/to/file.tar.bz2
##
## The F<url_download> function parses the url prefix (in this
## case, http) and passes the url to the http download handler
## (F<url_http_download>).  A similar approach is used for url 
## verification.
##
## This file provides an infrastructure that makes it relatively easy
## to add new url handlers.  In order to add new handlers, all that
## has to be done is add a new file to the sorcerer library directory
## with the new url handler functions defined in the file.  This new
## file will automatically be discovered and used by the sorcerer 
## scripts.
##
## The following section describes how to add new url handlers in
## a little more detail.
##
## <p>WRITING NEW URL HANDLERS</p>
##
## This section describes the steps needed to write new url handlers.
##
## <p>Decide on the Url Format</p>
##
## Urls must be of the form <prefix>://<address>.  The prefix should
## be something unique.  Only the prefix is used by this script,
## the address is not parsed or used, simply passed to the appropriate
## url handler.
##
## <p>Create a File to Hold the New Url Handling Functions</p>
##
## In the SGL library directory (i.e., the directory pointed to by the
## SGL_LIBRARY variable), create a new file called url_<prefix>.  For
## example, if your new url prefix is I<xyz>, you should create a new
## file called F<url_xyz>.  The file should be executable.
##
## <p>Implement Url Handlers</p>
##
## The next step is to write the actual functions that
## will handle url requests and put them in the new file you just
## created.  The functions that must be implemented are:
##
##    url_<URL_PREFIX>_bucketize <url>
##    url_<URL_PREFIX>_crack <url>
##    url_<URL_PREFIX>_expand <url>
##    url_<URL_PREFIX>_hostname <url>
##    url_<URL_PREFIX>_is_valid <url>
##    url_<URL_PREFIX>_netselect <url>
##    url_<URL_PREFIX>_verify <url>
##
## The easiest way to figure out what to do is to look at
## one of the existing files (e.g., url_http handles http requests).
##
## <p>Handling Multiple Url Types in a Single File</p>
##
## It's perfectly valid for a file to handle mutlple types of urls.
## The F<url_http> file actually handles ftp, http, and https urls.
## Take a look at the file to see how it's done.
##
##
## @Copyright Copyright 2002 by the Source Mage Team
##
#---------------------------------------------------------------------

#---------------------------------------------------------------------
# Load library files (url_*) that contain url handlers
#
# (2002/09/29) added if so it onlt loads the stuff once
#---------------------------------------------------------------------
if ! [[ $URL_HANDLER_FILES ]] ; then
  URL_HANDLER_FILES=`ls $SGL_LIBRARY_MODULES/url_handlers/url_*[^~]`
  for  url_handler_file  in  $URL_HANDLER_FILES;  do
    [  -x $url_handler_file  ]                               && 
    URL_PREFIX=`echo $url_handler_file | sed "s/.*\/url_//"` && 
    URL_TYPES[${#URL_TYPES[@]}]=$URL_PREFIX                  &&
    . $url_handler_file
  done
fi

#---------------------------------------------------------------------
## @param see url_download
##
## This is simply a wrapper around url_download which provides the
## additional functionality of expanding and ranking urls.
## Processes like summon should use this, more defined processes
## like scribe and sorcery update will probably directly call url_download.
##
#---------------------------------------------------------------------
function url_download_expand_sort() {
  $STD_DEBUG
  local target=$1
  local url_list=$2
  local hints=$3
  local udl_target=$4
  local udl_type=$5

  local url expanded_urls sorted_urls

  debug "liburl" "$FUNCNAME -- $@"
  # you can't expect me to download something without urls...
  [ -z "$url_list" ] && return 255

  # ensure list is newline seperated
  url_list="$(echo $url_list|tr ' ' '\n')"

  # expand urls
  url_expand_urls expanded_urls "$url_list"
  if ! [[ "$expanded_urls" ]] ; then
    message "No expanded urls! If you get this it is a sorcery bug"
    return 1
  fi
  # limit the number of expanded urls, only the linux kernel reaches this
  # limit and doing netselect on that many urls is absurd
  expanded_urls=$(echo "$expanded_urls"|head -n 50)
  debug "liburl" "expanded urls $expanded_urls"
  # sort urls 
  url_sort_urls sorted_urls "$expanded_urls"
  if ! [[ "$sorted_urls" ]] ; then
    message "No sorted urls! If you get this it is a sorcery bug"
    return 1
  fi
  # shorten the list to a manageable amount
  sorted_urls=$(echo "$sorted_urls"|head -n 10)
  debug "liburl" "sorted urls $sorted_urls"
  url_download "$1" "$sorted_urls" "$3" "$4" "$5"
}

#---------------------------------------------------------------------
## @param target	Expected target file or tree, this is only a
##                      suggestion, and the download handler may ignore it.
## @param url_list	List of urls to get the target from
## @param hints		Hints, these help the function determine what
##			type of thing it is downloading or other tunables.
## @param udl_target	Name of variable in which to store the name of the
##			result directory or file
## @param udl_type	Name of the variable in which to store the type of
##			thing downloaded
##
## @return 0 if file could be downloaded
## @return 1 otherwise
## Downloads the specified url.  Returns true if file could be 
## downloaded, false otherwise.
##
##
#---------------------------------------------------------------------
function url_download() {
  $STD_DEBUG
  local target=$1
  local url_list=$2
  local hints=$3
  local udl_target=$4
  local udl_type=$5

  local url

  debug "liburl" "$FUNCNAME -- $@"

  # you can't expect me to download something without urls...
  [ -z "$url_list" ] && return 255

  # sanity check urls
  local valid_urls
  url_get_valid_urls valid_urls "$url_list"
  if ! [[ "$valid_urls" ]] ; then
    message "No valid urls!"
    return 1
  fi

  hash_reset dl_buckets
  local bucket
  # bucketize
  for url in $valid_urls; do
    bucket=$(url_bucketize "$url")
    if [[ $bucket ]] ; then
      hash_append dl_buckets "$bucket" "$url"
    else
      message "Failed to find a download handler for $url, this is likely to be a sorcery bug"
    fi
  done
  
  # sort buckets, someday a better algorithm may be used, however
  # this should be sufficient for now.
  buckets=$(hash_get_table_fields dl_buckets|sort)

  # iterate through buckets
  for bucket in $buckets; do
    dl_get_bucket "$bucket" "$target" "$(hash_get dl_buckets $bucket)" \
                  "$hints" "$udl_target" "$udl_type"
    rc=$?
    if [[ $rc == 0 ]] ; then
      debug "liburl" "url_download -- downloaded $url $udl_r_target $udl_r_type"
      return 0
    fi
  done
  return 1
}


#---------------------------------------------------------------------
## @param urllist
#---------------------------------------------------------------------
function url_get_valid_urls() {
  $STD_DEBUG
  local upvar="$1"
  local url_list="$2"
  local tmp_list=$(
    for url in $url_list; do
      if url_is_valid "$url"; then
        echo "$url"
      fi
    done
  )
  eval "$upvar=\"\$tmp_list\""
}


#---------------------------------------------------------------------
## @param urllist
##
## Frontend to expand urls into a nice list.
#---------------------------------------------------------------------
function url_expand_urls() {
  $STD_DEBUG
  local upvar="$1"
  local url_list="$2"
  local tmp_list
  # put the original urls in the front of the list
  # awkuniq will remove duplicates thereafter
  tmp_list="$({ echo "$url_list" ; for url in $url_list; do
    # this should expand to something or return itself
    url_expand "$url"
  done ; } |awkuniq)"
  eval "$upvar=\"\$tmp_list\""
}

#---------------------------------------------------------------------
## @param urllist
#---------------------------------------------------------------------
function url_sort_urls() {
  $STD_DEBUG
  local upvar="$1"
  local url_list="$2"
  if [[ $NET_SELECT == on ]]  &&  spell_installed "netselect" ;  then
    lock_resources "network" "network"
    message -n "${CHECK_COLOR}Looking for the fastest mirror site...${DEFAULT_COLOR} "
    dl_connect
    url_list="$(url_rank $url_list 2> /dev/null)"
    dl_disconnect
    unlock_resources "network" "network"
    message " done."
  fi
  eval "$upvar=\"\$url_list\""
}

#---------------------------------------------------------------------
## @param urllist
## @Stdout new list
## Ranks the urls in order of speed from fastest to slowest using netselect
## Makes use of url_<url_type>_hostname functions.
##
## If multiple urls from the same hostname are passed in, their ordering
## is preserved, although that group of urls may move as a whole in
## the list.
#---------------------------------------------------------------------
function url_rank() {

  local urlList sortedList urlCounter
  local finalList url_speed
  local tmp_url tmp_prefix tmp_hostname tmp_list

  debug "liburl" "unsorted list: $*"

  urlList="$@"

  # Even if theres one url it might have multiple A records and we'll
  # want netselect to find the fastest one

  # The take home message is that a url can be /any/ random string of text,
  # and a url is officially defined by its handler. 

  # So we are having the handler give us the netselected output
  # from there we are sticking things into an array and letting
  # bash handle the sorting for us...
  function url_rank_divide_by_prefix() {
    local tmp_url=$1 tmp_prefix
    tmp_prefix=$(url_get_prefix $tmp_url)                 &&
    hash_append url_div_hash $tmp_prefix $tmp_url
  }
  iterate url_rank_divide_by_prefix " $IFS" $urlList

  for tmp_prefix in $(hash_get_table_fields url_div_hash) ; do
    tmp_list=$(hash_get url_div_hash $tmp_prefix)
    debug liburl "urls are $tmp_list"
    # the awk command turns netselect output like this:
    # 69 url1
    # 234 url2
    # into the following bash code, which is then executed
    # url_speed[69]="${url_speed[69]} url1"
    # url_speed[234]="${url_speed[234]} url2"
    eval $(url_netselect $tmp_list|
      awk '{ if ($2 > 0 && $2 < 65536 ) {
               printf "url_speed[%s]=\"${url_speed[%s]} %s\";",$1,$1,$2
             }
           }')
  done
  
  # since we put things in url_speed indexed by speed, the * should
  # expand back in sorted order

  sortedList=$(echo ${url_speed[@]})
  debug "liburl" "Ordered list pre-sanity check is $sortedList"

  # if all sites are ICMP Unreachable, return the unsorted list instead of null.
  if [[ -z "$sortedList" ]] ; then
    debug "liburl" "Failed to expand list"
    echo "$urlList"
    return
  fi

  # try really hard not to lose urls by appending missing ones to the end
  # of the list, please tell me if theres a faster way to do this
  function url_rank_iterator2() {
    echo $sortedList|grep -q "$1" || sortedList="$sortedList $1"
  }
  iterate url_rank_iterator2 " $IFS" $urlList

  # just in case something failed along the way just return what we had
  if [[ -z "$sortedList" ]] ; then
    debug "liburl" "Ordering failed somewhere, giving back original input"
    echo "$urlList"
  else
    debug "liburl" "Ordered URLs: $sortedList"
    echo "$sortedList"
  fi
}

#---------------------------------------------------------------------
## @Type Private
## @param url 
## @Stdout url prefix
## @return 0 valid url
## @return 1 otherwise
## Takes a url and echos the url prefix.  Returns
## true if a valid url could be found, returns false otherwise.
##
## This is the only place parsing of a url should take place outside of
## a url_handler. Doing so elsewhere is bugworthy.
##
#---------------------------------------------------------------------
function url_get_prefix()  {
  $STD_DEBUG
  local URL=$1
  local  URL_PREFIX=${URL/:\/\/*}
  [  -n  "$URL_PREFIX"  ]  &&
  [  "$URL_PREFIX" != "$URL" ]  &&
  echo    $URL_PREFIX
}


#---------------------------------------------------------------------
## @Type Private
## @param url 
## @param url prefix
## @stdout url sans prefix
#---------------------------------------------------------------------
function url_strip_prefix() {
  $STD_DEBUG
  echo $1 | sed "s!^$2://!!"
}

#---------------------------------------------------------------------
## url handler api functions. Use these to access a url specific
## functionality/data.
#---------------------------------------------------------------------

#---------------------------------------------------------------------
## @param url 
## @stdout dl handler
##
## Get the download handler for this url
#---------------------------------------------------------------------
function url_bucketize()  {
  $STD_DEBUG
  url_generic_apifunc bucketize $1
}

#---------------------------------------------------------------------
## @param url 
##
## Parse the url somehow. The results are url specific right now,
## this is usually only called by dl handlers who know what url types
## they can handle, and thus, understand the return value.
#---------------------------------------------------------------------
function url_crack()  {
  $STD_DEBUG
  url_generic_apifunc crack $1
}

#---------------------------------------------------------------------
## @param url(s)
## @stdout urls
##
## Attempt to get more similar urls to the given one based on the
## sorcery mirrors files. Most url types simply expand to the input.
#---------------------------------------------------------------------
function url_expand()  {
  $STD_DEBUG
  url_generic_apifunc expand "$@"
}

#---------------------------------------------------------------------
## @param url 
##
## Verify the url, this usually means going out to the internet and
## somehow determining if the link is good.
#---------------------------------------------------------------------
function url_verify()  {
  $STD_DEBUG
  url_generic_apifunc verify $1
}

#---------------------------------------------------------------------
## @param url 
## @Stdout url hostname
## @return 0 valid url
## @return 1 otherwise
## Takes a url and echos the url hostname. Returns
## true if a hostname could be found, returns false otherwise.
##
#---------------------------------------------------------------------
function url_hostname()  {
  $STD_DEBUG
  url_generic_apifunc hostname $1
}

#---------------------------------------------------------------------
## @param url 
## @Stdout url netslect output
## @return 0 valid url
## @return 1 otherwise
## Prints the netselect output from the url handlers attempt at
## running netselect.
##
#---------------------------------------------------------------------
function url_netselect()  {
  $STD_DEBUG
  url_generic_apifunc netselect "$@"
}

#---------------------------------------------------------------------
## @param url
## @return 0 valid url
## @return 1 otherwise
## Returns true if the given url is a valid url understood by the url
## library, returns false otherwise.
##
#---------------------------------------------------------------------
function url_is_valid()  {
  $STD_DEBUG
  url_generic_apifunc is_valid $1
}


#---------------------------------------------------------------------
## @param function name
## @param url(s)
##
## This implements the common code for simple url handler inheritence.
## The above url api functions call this which then calls the handler
## specific function, if it exists, or the default version. This
## allows url handlers to only override functions as necessary.
##
## If multiple urls are given, the prefix is assumed of all of them
## is assumed to be the same.
##
#---------------------------------------------------------------------
function url_generic_apifunc() {
  local tmp_func=$1
  shift
  local tmp_url=$1 tmp_prefix
  tmp_prefix=$(url_get_prefix $1)
  if misc_is_function url_${tmp_prefix}_${tmp_func}; then
    url_${tmp_prefix}_${tmp_func} "$@"
  else
    url_default_${tmp_func} "$@"
  fi
}

#---------------------------------------------------------------------
## This software is free software; you can redistribute it and/or modify
## it under the terms of the GNU General Public License as published by
## the Free Software Foundation; either version 2 of the License, or
## (at your option) any later version.
##
## This software is distributed in the hope that it will be useful,
## but WITHOUT ANY WARRANTY; without even the implied warranty of
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
## GNU General Public License for more details.
##
## You should have received a copy of the GNU General Public License
## along with this software; if not, write to the Free Software
## Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
#---------------------------------------------------------------------
