#! /bin/sh
##################################################################
#
# Advanced Package Tool (APT) repository generation program 
# for rpm packages 
# Original idea by Marcel Ritter 
# Written by Marcel Ritter and Richard Bos 
# Copyright (C) 2001,2002,2003 M Ritter and R Bos 
# aptate is a contamination of apt and update
#
# File:           $RCSfile: aptate.in,v $
# Revision:       $Revision: 1.181 $
# Last change:    $Date: 2003/03/06 21:55:03 $
# Last change by: $Author: rbos $
#
# Send suggestions to: apt4rpm-devel@lists.sourceforge.net
# Homepage: http://apt4rpm.sourceforge.net
#
# This program is free software; you can redistribute it and/or 
# modify it under the terms of the GNU General Public License 
# as published by the Free Software Foundation; either version 2 
# of the License, or (at your option) any later version. 
#
# This program is distributed in the hope that it will be useful, 
# but WITHOUT ANY WARRANTY; without even the implied warranty of 
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the 
# GNU General Public License for more details. 
#
# For a copy of the GNU General Public License, visit 
# http://www.gnu.org or write to the Free Software Foundation, Inc., 
# 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA. 
#
##################################################################

APTATE_CONFIG=${APTATE_CONFIG-/usr/lib/apt4rpm/aptate-config}

#########################
###### FUNCTIONS ########
#########################

function filesize()
{
  ls -ld "$1" | awk '{printf("%d", $5)}'
}

function apt_exit_on_trap()
###########################
{
  # reset trap, to prevent the exit function to be called multiple times
  trap > /dev/null

  echo -e "\n$PRGNAME interrupted, cleaning up temporary files..." >&2

  apt_exit $ERR_TRAP
}

function apt_exit()
###################
{
  # disable the trap for EXIT, preventing that it will call itself on this exit
  trap EXIT;

  displ_bad_rpms
  clean_up_rep_tmpdir $LOCK_TMPDIR
  clean_up_tmpdir
  clean_up_distid_lock

  echo "EXITVALUE: $1" >&6
  exit $1
}

function clean_up_rep_tmpdir()
##############################
{
  local DIR
  DIR=$1

  if [ -n "$DIR" ] && [ -d $DIR ]; then

    # Define a value for VERBOSE in case it is not set
    if [ ${VERBOSE:-0} -le ${DEBUG_LEVEL:-5} ]; then
      rm -rf $DIR
    else
      ( echo
        echo "Temporary directory \"$DIR\""
        echo "not removed for debugging purposes."
        echo
        echo "  The presence of this directory will prevent other aptate"
        echo "  instances to run.  Remove this directory as soon as possible"
        echo "  or move it to another location."
        echo
      ) >&2
    fi
  fi
}

function clean_up_distid_lock()
###############################
{
  if [ -n "$LOCK_DISTID" ]; then
    rm $LOCK_DISTID
  fi
}

function displ_bad_rpms()
#########################
{
  if [ -n "$TMPDIR" -a -s $TMPDIR/bad.rpms ]; then

    ( echo
      echo -e "Note: the packages below could not be queried successfully \c"
      echo "(rpm -qp <pkg>)"

      sed 's+^+  +' $TMPDIR/bad.rpms
      echo

      case $BAD_RPMS_MODE in

      rename)
        echo -n "  (They have been marked as bad rpm packages, by "
        echo "suffixing the files"
        echo "  with \".bad\")"
        ;;

      report-only)
        echo -n "  Take corrective action to prevent that this problem is "
        echo "reported over and"
        echo "  over again."
        echo
        echo "  Possible actions to be taken:"
        echo -n "  Download the rpms once more or in case you use e.g. rsync "
        echo "it's sufficient"
        echo -n "  to remove the bad rpms.  In case the packages are corrupted "
        echo "at the source,"
        echo "  you need to notify the package provider(s)."
        ;;

      esac

      echo

    ) >&2

    rm $TMPDIR/bad.rpms
  fi
}

function clean_up_tmpdir()
##########################
{
  if [ -n "$TMPDIR" ] && [ -d $TMPDIR ]; then
    if [ ${VERBOSE:-0} -le ${DEBUG_LEVEL:-5} ]; then
      rm -rf $TMPDIR
     else
      ( echo
        echo -n "Temporary directory \"$TMPDIR\" not "
        echo "removed for debugging purposes." >&2
      ) >&2
    fi
  fi
}

function hourglass()
####################
{
  local INDEX
  declare -i INDEX=0

  local CLOCKFILE=$1
  :> $CLOCKFILE

  # Create a spinning wheel
  # SWC: Spinning Wheel Character
  local SWC0="|"
  local SWC1="/"
  local SWC2="-"
  local SWC3="\\"

  while [ -f $CLOCKFILE ]
  do
    eval printf "%s\$ESC[D" \$SWC$INDEX >&4
    INDEX=$INDEX+1
    sleep 1
    [ $INDEX = 4 ] && INDEX=0
  done
}

function mirror()
#################
{
  local ID=$1
  local COMP=$2
  local FURL=$3
  local LOG_DIR=$4

  local RET
  local CLOCKFILE=$TMPDIR/mirror::clockrunning
  local WARG

  printf "${SCP}FTP access ... " >&4

  hourglass $CLOCKFILE &
  RET=$!

  if [ ! -d $LOG_DIR ]; then
    mkdir -p $LOG_DIR
  fi

  local LOG_FILE=$LOG_DIR/$COMP
  WARG="$(awk -F";" -v recid=$ID '{if (NR == recid) print $7}' $MIRRORTMP)"
  ( echo -ne "wget -P $UPDATEDIR --mirror --dont-remove-listing "
    echo -ne "-w $WGET_WAIT -o $LOG_FILE -t $WGET_TRIES "
    echo -ne "$WARG ftp://$FURL"
    echo
  ) >&6
  wget -P $UPDATEDIR --mirror --dont-remove-listing -w $WGET_WAIT \
    -o $LOG_FILE -t $WGET_TRIES $WARG ftp://${FURL##ftp://}

  rm -f $CLOCKFILE
  wait $RET

  printf "$RCP$CLEOL" >&4
  return 0
}

function check_rpm()
####################
{
  local FILENAME=$1
  local SEC_PKG=""

  # SuSE specific security setup
  if [ "$DIST" = "SuSE" ]; then
    local INFONAME=`dirname $FILENAME`
    INFONAME="${INFONAME}/"`rpm -q --queryformat '%{NAME}-%{VERSION}-%{RELEASE}.%{ARCH}_en.info' -p $FILENAME`

    if [ -f "$INFONAME" ] && grep -q -i "Security:[[:space:]]*Yes" "$INFONAME"; then
      SEC_PKG="SEC_OK"
    fi
  fi

  echo "$SEC_PKG"
}

function add_sourceslist_examples()
###################################
{
  local PREFIX
  local POSTFIX
  local LOCALKEY

  if [ -n "$1" ]; then
    LOCALKEY="[$1]"
  fi

  for POSTFIX in HTTP FTP FILE; do
    eval PREFIX=\$PREFIX_$POSTFIX
    if [ -n "$PREFIX" ]; then

      # The code below makes it possible that a user provides the url
      # with or without protocol specifier.  The following 2 constructions
      # will provide the same result: http://www.okna98.rm/apt and
      # www.okna98.rm/apt
      case $POSTFIX in
        FILE) PREFIX=$(echo file:${PREFIX##file:}) ;;
         FTP) PREFIX=$(echo  ftp://${PREFIX##ftp://}) ;;
        HTTP) PREFIX=$(echo http://${PREFIX##http://}) ;;
      esac

      ( echo "#"
        echo "# Repository created by: $PRGNAME (version $PRGVRSN)"
        echo -n "# At: "
        date
        echo "# More info about $PRGNAME at: http://apt4rpm.sourceforge.net"
        echo "#"
        echo "rpm     $LOCALKEY $PREFIX $ARCHIVE $COMP_LIST"
        echo "rpm-src $LOCALKEY $PREFIX $ARCHIVE $COMP_LIST"

      ) > $EXAMPLEDIR/sources.list.$POSTFIX
    fi
  done
}

function build_cache()
######################
{
  local BASENAME=$1
  local SRCDIR=$2
  local ID=$3
  local REBUILD_CACHE=$4

  local CACHEFILE=$CACHEDIR/$BASENAME
  local CHANGES=$TMPDIR/build_cache::changes
  local CURCACHE=$TMPDIR/build_cache::current_cache
  local NEWCACHE=$TMPDIR/build_cache::cache.$BASENAME
  local NEWPKGS=$TMPDIR/build_cache::pkgs
  local TMPFILE=$TMPDIR/build_cache::tmp

  local FILENAME
  local ACCEPT
  local REJECT

  printf " %-16s  -> " $SOURCE_NAME >&3
  printf "${SCP}Processing... " >&4

  # Change to the directory, as otherwise rpms won't be found in case the
  # top directory is actually a linked directory.  The find argument "-follow"
  # is not used by default to prevent that find moves out the find path.
  # An example of such a situation is a directory that is linked root (/).
  # If the linked subdirectories are located in find path they will be found
  # anyway.
  # Change the directory within "(....)" so the directory is changed in 
  # another shell.  After leaving the "(....)" we're still in the same
  # location.
  # Future code could be extended to report a warning in case linked
  # directories or files are encountered.  The check can be made with e.g.
  # $ find . -type l -xtype f -printf "%d %l %h\n" (check for linked files)
  # $ find . -type l -xtype d -printf "%d %l %h\n" (check ofr linked dirs)
  #
  ( cd $SRCDIR
    find . $FOLLOW -name "*.[sr]pm" -type f > $TMPFILE 2>/dev/null
  )

  ACCEPT=$(grep "^$ID;" $MIRRORTMP | cut -d";" -f8)
  REJECT=$(grep "^$ID;" $MIRRORTMP | cut -d";" -f9)

  # Add "--" to the grep arguments to prevent an error reported by grep, in case
  # the regular expression starts with a "-".  Define fallback values for the
  # ACCEPT or REJECT regular expressions in case they are not defined, 
  # respectively: "^" and "^$".
  sed s+^\.+$SRCDIR+ $TMPFILE | tr -s "/" | grep -E -- "${ACCEPT:-^}" | 
    grep -vE -- "${REJECT:-^$}" | sort > $NEWPKGS
  rm $TMPFILE

  if [ ! -f $CACHEFILE ] || [ "$REBUILD_CACHE" = "yes" ]; then

    sed 's+^+add;+' $NEWPKGS > $CHANGES

    # make sure to have a new cache (under any circumstances).
    touch $NEWCACHE

  else
    # Use the cache to prevent time consuming rpm queries to be run each
    # time aptate is run.
    typeset -i NOF
    NOF=$(awk -F\; '{print NF}' $CACHEFILE | sort -u)
    if [ $NOF -eq 9 ]; then
      local -i NUM=1 
      local -i COUNT=$(cat $CACHEFILE | wc -l)
      cat $CACHEFILE | while read LINE
      do
        printf "${RCP}Converting: %6d/%d " $NUM $COUNT >&4
        FILE=$(echo $LINE | cut -d\; -f2)
        FILESIZE=$(filesize $FILE 2>/dev/null)
        echo "$LINE;${FILESIZE:-0}"
        NUM=$NUM+1
      done > $CURCACHE
      printf $RCP$CLEOL >&4
    else
      cp $CACHEFILE $CURCACHE
    fi

    # Audit the cache: it might be that directories have disappeared since
    # aptate run for the last time
    CACHEDIRS=$(grep -v ";$SRCDIR/" $CURCACHE |
      awk -F";" '{ print $2 }' | while read DIR rest
      do
        dirname $DIR
      done | sort -u
    )

    for DIR in $CACHEDIRS
    do
      if [ ! -d $DIR ]; then
        grep -v ";$DIR" $CURCACHE > $TMPFILE
        mv $TMPFILE $CURCACHE
      fi
    done

    # Audit for a file that appears in cache multiple times.
    # This can happen if a package failed the gpg check initially, but
    # passes later on as it has been signed...
    local FILES
    FILES=$(awk -F";" '{ print $2 }' $CURCACHE | sort | uniq -c |
      grep -v "[[:space:]]1[[:space:]]$SRCDIR" | awk '{ print $2 }')

    local FILE
    for FILE in $FILES; do
      grep -v $FILE $CURCACHE > $TMPFILE
      mv $TMPFILE $CURCACHE
    done

    # The grep for SRCDIR is needed, because the rpm repository may contain,
    # for whatever reason, additional directories at the same level as
    # SRCDIR.  The rpms in the additional directories are unwanted.
    awk -F";" '{ print $2 }' $CURCACHE | grep ^$SRCDIR | sort > $TMPFILE

    # Determine the differences (adds and removes) between the cached rpms
    # and the rpms on disk.
    comm -13 $TMPFILE $NEWPKGS | sed 's+^+add;+' >  $CHANGES
    comm -23 $TMPFILE $NEWPKGS | sed 's+^+rem;+' >> $CHANGES
    rm -f $TMPFILE

    mv $CURCACHE $NEWCACHE
  fi

  if [ -s $CHANGES ]; then

    local -i COUNT
    local -i NUM
    local DATE=$(date '+%y%m%d')
    COUNT=$(cat $CHANGES | wc -l)
    NUM=1

    awk 'BEGIN { FS=";" }{ printf("%s %s\n", $1, $2)}' $CHANGES | (
    while read ACTION FILENAME; do

      # progress indicator
      printf "${RCP}Check: %11d/%d " $NUM $COUNT >&4

      if [ "$ACTION" == "rem" ]; then
        # package must be removed from the cache
        grep -v "$FILENAME" $NEWCACHE > $TMPFILE
        mv $TMPFILE $NEWCACHE
      else
        # a candidate package to be added to the cache
        local RPM_ARCH
        local RPM_CACHE_TYPE
        local RPM_CACHE_NAME
        local RPM_NVR
        local RPM_PATCHRPM=""
        local RPM_TYPE
        local RPM_OUT=""

        local RPM_QUALIFIER="RPM_QRYRES RPM_TYPE=%{SOURCERPM}"
        RPM_QUALIFIER="$RPM_QUALIFIER RPM_NAME=%{NAME}"
        RPM_QUALIFIER="$RPM_QUALIFIER RPM_NV=%{VERSION}-%{RELEASE}"
        RPM_QUALIFIER="$RPM_QUALIFIER RPM_ARCH=%{ARCH}\n"
        RPM_OUT=$(rpm -qp -requires --queryformat "$RPM_QUALIFIER" $FILENAME 2>/dev/null)

        if [ $? -eq 0 ]; then
          RPM_OUT=$(echo "$RPM_OUT" | 
           sed 's+rpmlib(PatchRPMs).*$+RPM_QRYRES RPM_PATCHRPM=yes+' |
           grep ^RPM_QRYRES | cut -d" " -f2-
          )
          eval $RPM_OUT

          if [ "$RPM_TYPE" = "none" ]; then

            # source rpm
            RPM_CACHE_TYPE=src

            case "$FILENAME" in
              *.nosrc.rpm) RPM_ARCH=nosrc ;;
              *)           RPM_ARCH=src ;;
            esac

          else
            if [ "$RPM_PATCHRPM" = "yes" ]; then
              RPM_CACHE_TYPE=pat
            else
              RPM_CACHE_TYPE=bin
            fi
          fi

          local SEC=$(check_rpm $FILENAME)

          #
          # check packages for MD5 sum/GPG signature
          # Remark: do not change the "echo" order of the strings MD5_.. and
          # GPG_..  Further on in the program regular expressions expect the
          # strings to be in this order.
          #
          local MD5_GPG
          local OUTPUT
          local RET
          OUTPUT=$(rpm --checksig $FILENAME 2>&1)
          RET=$?
        
          if [ $RET -eq 0 ]; then
            if echo "$OUTPUT" | grep -i -q gpg; then
              MD5_GPG="MD5_OK;GPG_OK"
            else
              MD5_GPG="MD5_OK;GPG_FAILED"
            fi
          else
            # rpm's exit value is 1 in case the gpg check failed.  This does not
            # necessarely mean that the MD5 check failed.
            if [ $RET -eq 1 ]; then
              if echo "$OUTPUT" | grep -i -q "md5.*ok"; then
                MD5_GPG="MD5_OK;GPG_FAILED"
              else
                MD5_GPG="MD5_FAILED;GPG_FAILED"
              fi
            else
              MD5_GPG="MD5_FAILED;GPG_FAILED"
            fi
          fi

          FILESIZE=$(filesize $FILENAME)

          ( echo -n "$RPM_CACHE_TYPE;$FILENAME;$RPM_NAME;$RPM_NV;$RPM_ARCH;"
            echo "$SEC;$MD5_GPG;$DATE;$FILESIZE"
          ) >> $NEWCACHE
        else

          case ${BAD_RPMS_MODE} in

          rename)
            # Mark the package as bad rpm, to prevent that this bad rpm is 
            # being processed unnecessarly each time aptate runs.
            mv $TOPDIR/$FILENAME $TOPDIR/$FILENAME.bad
            echo $TOPDIR/$FILENAME.bad >> $TMPDIR/bad.rpms
            ;;

          report-only)
            # Do nothing with the rpm. It's now up to the apt repository
            # maintainer to take action.
            echo $TOPDIR/$FILENAME >> $TMPDIR/bad.rpms
            ;;

          *)
            echo "Assert: unknown BAD_RPM_MODE mode" >&2
            ;;
          esac

        fi
      fi

      NUM=$NUM+1

    done )

    local -i ADD_CNT
    local -i REM_CNT
    ADD_CNT=$(grep -c "^add;" $CHANGES)
    REM_CNT=$(grep -c "^rem;" $CHANGES)

    # Output stream 4 takes care of the escape characters
    printf "$RCP$CLEOL" >&4

    # Use -- in the printf statement as the arrow "->" is otherwise
    # interpreted by printf as being an argument.
    ( printf "${RCP}RPMs added:   %4d\n" $ADD_CNT
      printf "                   "
      printf -- "-> RPMs removed: %4d\n" $REM_CNT
    ) >&3
  else
    printf "$RCP$CLEOL" >&4
    printf "No change\n" >&3
  fi

  # clean up
  rm -f $CHANGES $NEWPKGS
}

function make_links()
#####################
{
  local PATCH_TYPE=$1 # rpm patch type   (regular, security rpms)
  local COMPONENT_NAME=$2
  local ID=$3
  local REBUILD_LINK_USER=$4

  local BASENAME
  local NEW_CACHE
  local NEW_TMP_CACHE
  local OLD_CACHE
  local OLD_TMP_CACHE
  local PKG_TYPE
  local PKG_TYPES
  local SEARCHSTRING

  # Go for the PKG_TYPE = patch loop first. As the apt component containing
  # the (much smaller) patch rpms will then be listed before the component
  # containing the regular rpms in the sources.list example....
  if [ "$MK_PATCH_RPM_COMP" = "yes" ]; then
    PKG_TYPES="patch bug"
  else
   PKG_TYPES="bug"
  fi

  for PKG_TYPE in $PKG_TYPES
  do
    # Assign BASENAME here, as the name is being altered if PKG_TYPE = patch
    # to $BASENAME-patchrpm
    BASENAME=$COMPONENT_NAME

    case $PATCH_TYPE in

    regular)

      OLD_CACHE=$CACHEDIR/$BASENAME
      NEW_CACHE=$TMPDIR/build_cache::cache.$BASENAME
      ;;

    security)
      #
      # The virtual apt component security, is composed of all security 
      # packages available.
      #
      local FILE
      local SRC_NAME
      local OLD_CACHES
      local NEW_CACHES

      # prevent that old information is being used, while processing
      # the "for" loop
      unset OLD_CACHES
      unset NEW_CACHES
      for SRC_NAME in $SOURCE_NAMES
      do
        # Examine the old cache files, created in a previous aptate session
        FILE=$CACHEDIR/$SRC_NAME

        # The "if" is required, because it may happen that the
        # cache file is not present anymore or has not been created
        if [ -s $FILE ]; then
          OLD_CACHES="$OLD_CACHES $FILE"
        fi

        # Examine the new cache files, created in the current session
        FILE=$TMPDIR/build_cache::cache.$SRC_NAME

        if [ -s $FILE ]; then
          NEW_CACHES="$NEW_CACHES $FILE"
        fi
      done

      OLD_CACHE=$TMPDIR/make_links::cache-sec.old
      if [ -n "$OLD_CACHES" ]; then
        grep -h ";SEC_OK" $OLD_CACHES | sort > $OLD_CACHE
      else
        > $OLD_CACHE
      fi

      NEW_CACHE=$TMPDIR/make_links::cache-sec.new
      if [ -n "$NEW_CACHES" ]; then

        grep -h ";SEC_OK" $NEW_CACHES | sort > $NEW_CACHE
      else
        > $NEW_CACHE
      fi

    esac

    if [ "$PKG_TYPE" = "patch" ]; then
      BASENAME=${BASENAME}-${PATCH_RPM_STR}
    fi

    # The cachefile contains information about all rpm types: source, binary
    # and patch rpms.  While an apt component holds only a combination of
    # source|binary or source|patch rpms.  At the lines below a temporarely
    # cache is created to obtain a source|binary only or source|patch only
    # cache.
    local VALID_CACHE
    if [ -r $NEW_CACHE ]; then
      if [ "$PKG_TYPE" = "bug" ]; then
        # search for "^bin|^src" rpm packages
        SEARCHSTR="^[bs][ir][nc]"

        VALID_CACHE=yes
      else
        # patch rpm, search for "^pat|^src" rpm packages
        SEARCHSTR="^[ps][ar][tc]"

        # Are there any patch rpms?  If not, there is no need to create the
        # apt component at all
        grep -q ^pat $NEW_CACHE
        if [ $? -eq 0 ]; then
          VALID_CACHE=yes
        else
          VALID_CACHE=no
        fi
      fi
    else
     VALID_CACHE=no
    fi

    if [ "$VALID_CACHE" = "yes" ]; then

      printf " %-16s  -> " $BASENAME >&3
      printf $SCP >&4

      # create the source|binary only or source|patch only cache from the
      # old cache
      OLD_TMP_CACHE=$TMPDIR/make_links::cache-tmp
      if [ -f $OLD_CACHE ]; then
        grep "$SEARCHSTR" $OLD_CACHE > $OLD_TMP_CACHE
      fi

      # create for the source|binary only or source|patch only cache from the
      # new cache
      NEW_TMP_CACHE=$TMPDIR/make_links::cache-new
      if [ -f $NEW_CACHE ]; then
        # use the cache file created by function build_cache
        grep "$SEARCHSTR" $NEW_CACHE > $NEW_TMP_CACHE
      fi

      # Create the repository structure
      mckdir $ARCHIVE_TMP_BASE_DIR
      mckdir $ARCHIVE_TMP_ROOT/RPMS.$BASENAME
      mckdir $ARCHIVE_TMP_SRPM_ROOT/SRPMS.$BASENAME

      local CACHEDIFF
      if [ -r $OLD_TMP_CACHE ]; then
        CACHEDIFF=$(diff -q $OLD_TMP_CACHE $NEW_TMP_CACHE)
      else
        CACHEDIFF="new cache"
      fi

      if [ -n "$CACHEDIFF" ]; then

        CACHE_CHANGED_COMP_QUEUE="$CACHE_CHANGED_COMP_QUEUE $BASENAME"

        if [ "$TARGET" = "server" ]; then
          ACTION_CHANGED_COMP_QUEUE="$ACTION_CHANGED_COMP_QUEUE $BASENAME"
        fi

      else
        # The cache did not change.  It may never the less be required to
        # (re)build the apt repository.
        if [ "$TARGET" = "server" ]; then

          # Did the rpm repository change since the last run?
          # The size from a *.bz2 file is at least 14 bytes (just bzip2 a 
          # 0 byte file => it will become 14 bytes.
          # The file release.$BASENAME is always bigger than 0 byte, so if it
          # is 0 bytes something must have happened to the file....
          if [ -r $OLD_TMP_CACHE ] &&
             [ -s $ARCHIVE_BASE_DIR/release.$BASENAME ] &&
             [ -r $ARCHIVE_BASE_DIR/srclist.$BASENAME ] &&
             [ -r $ARCHIVE_BASE_DIR/pkglist.$BASENAME ] &&
             [ -s $ARCHIVE_BASE_DIR/pkglist.$BASENAME.bz2 ]
          then

            # Determine whether the cache has changed in a previous run
            if [ -f $SHARE_DIST_DIR/cache.state ]; then
              TIME1=$(grep "^$BASENAME;" $SHARE_DIST_DIR/cache.state |
                cut -d";" -f2)
            fi

            if [ -f $SHARE_DIST_DIR/server.cache.state ]; then
              TIME2=$(grep "^$BASENAME;" $SHARE_DIST_DIR/server.cache.state |
                cut -d";" -f2)
            fi

            # The cache may have changed in the previous run, if the cache
            # changed the apt repository must be recreated to reflect the
            # the changes....
            if [ ${TIME2:-0} -lt ${TIME1:-0} ]; then
              CACHEDIFF="due to a previous run for a different target"
              ACTION_CHANGED_COMP_QUEUE="$ACTION_CHANGED_COMP_QUEUE $BASENAME"
            else

              # User requested component rebuild?
              echo $REBUILD_LINK_USER | grep -Ewq "$BASENAME|all"
              if [ $? -eq 0 ]; then
                CACHEDIFF="user requested component rebuild"
              fi
            fi
          else
            CACHEDIFF="rebuild component"
          fi
        else
          # target is disc => always rebuild the links
          CACHEDIFF="apt repository on disc"
        fi
      fi

      if [ -n "$CACHEDIFF" ]; then

        if [ "$SIGNED_PKGS_ONLY" = "yes" ]; then
          SEARCHSTR=";GPG_OK"
        else
          SEARCHSTR=";MD5_OK"
        fi

        local -i NUM
        local -i COUNT
        local RPM_NAME
        local RPM_NV
        local RPM_ARCH
        COUNT=$(grep -c "$SEARCHSTR" $NEW_TMP_CACHE)
        NUM=1
        grep "$SEARCHSTR" $NEW_TMP_CACHE | tr ";" " " |
          while read RPMTYPE FILENAME RPM_NAME RPM_NV RPM_ARCH FLAGS; do

            printf "${RCP}Links: %11d/%d " $NUM $COUNT >&4

            DESTNAME="${RPM_NAME}-${RPM_NV}.${RPM_ARCH}.rpm"

            case $RPMTYPE in
            bin|pat)
              #
              # Make sure the link does not yet exist and create a unique one.
              # This is needed in case there are several packages with
              # the same internal name/version/release. This would cause
              # the latest link to be overwritten (others will be lost)
              # - this is not needed for SRPMs (and wouldn't work anyway)
              #
              # Relative links must be used as the links needs to remain
              # working, if they are exported (with e.g. NFS).
              # In this case the links are exported as they are - but all the
              # directories passed by following that link need to be exported
              # too! This can also happen when using FTP (FTP servers often
              # run in an chroot() environment).
              #
              while test -L $ARCHIVE_TMP_ROOT/RPMS.$BASENAME/$DESTNAME; do
                DESTNAME="_$DESTNAME"
              done
              ln -s $REL_ARCHIVE/$FILENAME $ARCHIVE_TMP_ROOT/RPMS.$BASENAME/$DESTNAME
              ;;

            src)
              DESTNAME=$ARCHIVE_TMP_SRPM_ROOT/SRPMS.$BASENAME/$DESTNAME
              test -L $DESTNAME || ln -s $REL_SRPM_ARCHIVE/$FILENAME $DESTNAME
            esac

            # progress indicator
            NUM=$NUM+1
          done

        CHANGED_COMP_LIST="$(add_component $BASENAME "$CHANGED_COMP_LIST")"

      else
        # The rpm repository did not change
        printf "Copying component... " >&4

        for FILE in pkglist.$BASENAME pkglist.$BASENAME.bz2 \
          srclist.$BASENAME srclist.$BASENAME.bz2
        do
          if [ -r $ARCHIVE_BASE_DIR/$FILE ]; then
           # make sure that file characterisc are being preserved (cp -p)
           cp -p $ARCHIVE_BASE_DIR/$FILE $ARCHIVE_TMP_BASE_DIR
          fi
        done

        cp -R $ARCHIVE_ROOT/RPMS.$BASENAME $ARCHIVE_TMP_ROOT
        cp -R $ARCHIVE_SRPM_ROOT/SRPMS.$BASENAME $ARCHIVE_TMP_SRPM_ROOT

        UNCHANGED_COMP_LIST="$(add_component $BASENAME "$UNCHANGED_COMP_LIST")"
      fi

      # Create a list with most recent binary rpms provided by the component
      local LIST_MOST_RECENT=$(grep "^$ID;" $MIRRORTMP | cut -d";" -f10)
      if [ "$LIST_MOST_RECENT" = "yes" ]; then

        local REBUILD_MRLIST
        if [ -n "$CACHEDIFF" -o "$OPT_REBUILD_MRLIST" = "all" ]; then
          REBUILD_MRLIST=yes
        else
          echo $OPT_REBUILD_MRLIST | grep -wq $SOURCE_NAME
          if [ $? = 0 ]; then
            REBUILD_MRLIST=yes
          else
            REBUILD_MRLIST=no
          fi
        fi

        if [ "$REBUILD_MRLIST" = "yes" ]; then

          # clear rest of line
          printf "${RCP}Most recent rpm list..." >&4

          if [ "$PKG_TYPE" = "bug" ]; then
            SEARCHSTR="bin"
          else
            SEARCHSTR="pat"
          fi

          /usr/lib/apt4rpm/compare_rpm_version --rpmtype $SEARCHSTR $NEW_CACHE |
            bzip2 > $ARCHIVE_TMP_BASE_DIR/mrlist.$BASENAME.bz2

        else
          if [ -r $ARCHIVE_BASE_DIR/mrlist.$BASENAME.bz2 ]; then
            cp -p $ARCHIVE_BASE_DIR/mrlist.$BASENAME.bz2 $ARCHIVE_TMP_BASE_DIR
          fi
        fi
      fi # create most recent list?

      # count created links, and files with correct MD5 sum/GPG signature
      local -i GPGNUM_SRC=$(grep -c "^src.*;GPG_OK" $NEW_TMP_CACHE)
      local -i MD5NUM_SRC=$(grep -c "^src.*;MD5_OK" $NEW_TMP_CACHE)
      local -i GPGNUM_BIN=$(grep -c "^bin.*;GPG_OK" $NEW_TMP_CACHE)
      local -i MD5NUM_BIN=$(grep -c "^bin.*;MD5_OK" $NEW_TMP_CACHE)
      local -i GPGNUM_PAT=$(grep -c "^pat.*;GPG_OK" $NEW_TMP_CACHE)
      local -i MD5NUM_PAT=$(grep -c "^pat.*;MD5_OK" $NEW_TMP_CACHE)

      # print some statistics
      printf "$RCP$CLEOL" >&4

      if [ "$PKG_TYPE" = "patch" ]; then
        printf "Pat: MD5/GPG: %4d/%d\n" $MD5NUM_PAT $GPGNUM_PAT >&3
      else
        printf "Bin: MD5/GPG: %4d/%d\n" $MD5NUM_BIN $GPGNUM_BIN >&3
      fi
      ( printf "                  "
        printf " -> Src: MD5/GPG: %4d/%d\n" $MD5NUM_SRC $GPGNUM_SRC
      ) >&3
    fi # valid cache?

    # clean up
    if [ "$PATCH_TYPE" = "security" ]; then
      rm -f $NEW_CACHE $OLD_CACHE
    fi

    rm -f $OLD_TMP_CACHE $NEW_TMP_CACHE
  done
}

function cntlines ()
###################
{
  #
  # $1 = search string
  # $2 = filename
  #
  # Usage (3 different possibilities):
  #        cntlines SEARCHSTRING FILENAME
  #        echo STRING | cntlines
  #        echo STRING | cntlines SEARCHSTRING
  #
  declare -i NUM
  NUM=$(grep -c "${1:-^}" ${2:-})
  echo $NUM
}

function mckdir()
#################
{
  #
  # mckdir: make check dir
  #
  local DIR2CREATE=$1

  mkdir -p $DIR2CREATE 2>/dev/null
  if [ $? -ne 0 ]; then
    echo "Failed to create directory: $DIR2CREATE" >&2
    apt_exit $ERR_DIR
  fi
}

function chkvar ()
##################
{
  local VNAME=$1
  local STR="$3"

  if [ -n "$STR" ]; then

    local INVEST_STR

    # Allowed non alphanumeric characters
    # The quotes around $STR are required to be able to detect an "*"
    INVEST_STR=$(echo "$STR" | tr -d "[\-_.]")

    if [ "$2" = "dir" ]; then
      INVEST_STR=$(echo "$INVEST_STR" | tr -d "/")
    fi

    echo "$INVEST_STR" | grep -q [[:punct:]]

    if [ $? -eq 0 ];  then
      ( echo "error: variable $VNAME contains invalid characters."
        echo -n " Allowed characters are: alphanumeric -_./ (\"/\" for "
        echo "directories only)."
        echo " Current value of $VNAME is: $STR"
        echo " Check configuration file: $CONFIG_FOUND"
      ) >&2
      apt_exit $ERR_CONFIGURATION
    fi
  else
    ( echo "error: variable $VNAME is not set."
      echo " Check configuration file: $CONFIG_FOUND"
    ) >&2
    apt_exit $ERR_CONFIGURATION
  fi
}

function add_component()
########################
{
  local COMP_NAME=$1
  local COMP_LIST="$2"
  declare -i PKG_CNT

  PKG_CNT=$(find $ARCHIVE_TMP_ROOT/RPMS.$COMP_NAME -type l | wc -l)
  if [ $PKG_CNT -ge 1 ]; then
    # void duplicate entries
    echo "$COMP_LIST" | grep -q "[[:space:]]$COMP_NAME[[:space:]]"
    if [ $? -ne 0 ]; then

      ( echo "Archive: stable"
        echo "Component: $1"
        echo "Origin: $DIST"
        echo "Label: $DIST"
        echo "Version: $VERSION"
        echo "Architecture: $ARCH"
        echo -n "Timestamp: "
        date
      ) > $ARCHIVE_TMP_BASE_DIR/release.$COMP_NAME

      if [ -z "$COMP_LIST" ]; then
        echo "$COMP_NAME"
      else
        echo "$COMP_LIST $COMP_NAME"
      fi
    fi
  else
    if [ -n "$COMP_LIST" ]; then
      echo "$COMP_LIST"
    fi
  fi
}

function var_debug_info()
#########################
{
  local VAR

  for VAR in $@
  do
    eval echo "DEBUG: $VAR=\$$VAR"
  done | sort
}

function audit_component()
##########################
{
  local TYPE=$1
  local COMP=$2
  local DIR=$3

  # genpkglist/gensrclist is in trouble, let's see if aptate can find the
  # the guilty one(2)....
  ( echo 
    echo "  Could not create the apt $TYPE database for component \"$COMP\""
    echo "  successfully.  This might be due to an inconsistent component cache."
    echo
    echo "  Auditing links in \"$DIR\""
  ) >&2

  local REPLACE_STR
  if [ "$TYPE" = "pkg" ]; then
    REPLACE_STR=$REL_ARCHIVE
  else
    REPLACE_STR=$REL_SRPM_ARCHIVE
  fi

  local FILES
  cd $DIR
  FILES=$(find . -follow 2>&1 >/dev/null | cut -d: -f2)

  local FILE
  for PKG in $FILES; do

    FILE=$(ls -l $(basename $PKG) | awk '{print $NF}' |
      sed s+$REPLACE_STR/++)
    echo -n "  error: file with incorrect linkage.  "
    grep -q ${FILE:-^$} $TMPDIR/build_cache::cache.$COMP
    if [ $? -eq 0 ]; then
      echo "File found in cache."
    else
      echo "File not found in cache."
    fi
    echo "    File = $FILE"
  done

  ( echo 
    echo -n "  Rerunning $PRGNAME once more might fix it.  If rerunning "
    echo "$PRGNAME does not"
    echo "  resolve the problem run $PRGNAME with argument \"--rebuild-comp $COMP\"."
  ) >&2
}

function bzip2_checked()
########################
{
  local FILE=$1

  bzip2 -c $FILE > $FILE.bz2

  if [ $? -ne 0 ]; then
    ( echo
      echo "error: bzip2 did not finish successfully."
      echo " Correct the error (full filesystem?) and rerun $PRGNAME."
    ) >&2
    apt_exit $ERR_BZIP2
  fi 
}

function cache_adm()
####################
{
  local TYPE=$1; shift
  local DATE=$1; shift
  local COMPONENTS=$@

  local COMPONENT
  local STATE_FILE
  local TMP

  TMP=$TMPDIR/main::cache.state

  if [ "$TYPE" = "common" ]; then
    STATE_FILE=$SHARE_DIST_DIR/cache.state
  else
    STATE_FILE=$SHARE_DIST_DIR/$TYPE.cache.state
  fi

  for COMPONENT in $COMPONENTS
  do
    if [ -r $STATE_FILE ]; then
      grep -v "^$COMPONENT;" $STATE_FILE > $TMP
    fi

    ( [ -r $TMP ] && cat $TMP
      echo "$COMPONENT;$DATE"

    ) > $STATE_FILE

  done
  rm -f $TMP
}

function usage()
#################
{
  ( echo "$SHORTHELP"
    echo "See $PRGNAME --help for extended help."
  ) >&2
  exit 0
}

function help ()
################
{
  cat << EOF >&1
Usage: $PRGNAME [options]
Options:
   -h           print short help and exit.
  --help        print this help and exit.
   -V, --version
   -q <q|qqqqqq>, --quiet   <q|qqqqqq>
   -v <v|vvvvvv>, --verbose <v|vvvvvv>

  --dist-id <distribution id,distribution id-2,...>

  --bad-rpms-mode <report-only|rename>
  --check-config
  --config-file <config file>
  --debug
  --force
  --rebuild-comp   <all|component-name,component-name>
  --rebuild-links  <all|component-name,component-name>
  --rebuild-mrlist <all|component-name,component-name>
  --test
  --topdir <repository directory>
  --update-rpms, --no-update-rpms

Report bugs to <apt4rpm-users@lists.sourceforge.net>.
EOF
}

##############################
########## CONFIG ############
##############################

# These variables must be readonly, to prevent them to be
# overwritten by the processing of the config files
# Have a restricted path variable as this application will be run by root
# though this is not required.
unset    PATH
export   PATH
readonly PATH=/bin:/usr/bin
readonly PRGNAME=$(basename $0)
readonly PRGVRSN=0.64.2

# Define the exit (error) codes
readonly ERR_LOCKED=1
readonly ERR_CONFIGURATION=2
readonly ERR_NO_COMPONENTS=3
readonly ERR_DIR=4
readonly ERR_ARGUMENT=5
readonly ERR_SYSTEM=6
readonly ERR_REPOSITORY=7
readonly ERR_BZIP2=8
readonly ERR_ASSERT=9
readonly ERR_TRAP=129

# default settings
# most of the default values are being determined by aptate's dtd
ARCH=""
DIST=""
KEY_NAME=""
KEY_EMAIL=""
PREFIX_FTP=""
PREFIX_FTP=""
PREFIX_HTTP=""
TOPDIR=""
VERSION=""

readonly SHORTHELP="Usage: $PRGNAME [-h] [-V] [-q <q|qqqqqq>] [-v <v|vvvvvv>]
  [--dist-id <dist-id,dist-id>] [--config-file <config file>] [--debug]
  [--check-config] [--test] [--bad-rpms-mode <report-only|rename>]
  [--(no-)update-rpms] [--topdir <repository directory>] [--force]
  [--rebuild-comp   <all|component-name,component-name]
  [--rebuild-links  <all|component-name,component-name]
  [--rebuild-mrlist <all|component-name,component-name]
  [--help] [--version] [--quiet <q|qqqqqq>] [--verbose <v|vvvvvv>]"

# Command line argument parsing, the allowed arguments are
# alphabetically listed, keep it this way please.
LOPT="action:,bad-rpms-mode:,config-file:,check-config,debug,dist-id:,force"
LOPT="${LOPT},help,no-update-rpms,quiet:,rebuild-comp:,rebuild-links:"
LOPT="${LOPT},rebuild-mrlist:,test,topdir:,update-rpms,verbose:,version"

# Note that we use `"$@"' to let each command-line parameter expand to a
# separate word. The quotes around `$@' are essential!
# We need TEMP as the `eval set --' would nuke the return value of getopt.
TEMP=$(getopt -options=q:hv:V --long $LOPT -n $PRGNAME -- "$@")

if [ $? -ne 0 ]; then
  echo "Terminating..." >&2
  exit $ERR_ARGUMENT
fi

# Note the quotes around `$TEMP': they are essential!
eval set -- "$TEMP"

while true
do
  case $1 in
  --action)              OPT_ACTION=$2; shift;;
  --bad-rpms-mode)       OPT_BAD_RPMS_MODE=$2; shift;;
  --check-config)        OPT_CHECKCONFIG=yes;;
  --config-file)         OPT_CONFIGFILE=$2; shift;;
  --debug)               OPT_DEBUG=yes;;
  --dist-id)             OPT_DISTID="$OPT_DISTID $2"; shift;;
  --force)               OPT_FORCE=yes;;
  --help)                help; exit;;
   -h)                   usage; exit;;
  --no-update-rpms)      OPT_UPDATE_RPMS=no;;
  --quiet|-q)            OPT_QUIET=$2; shift;;
  --rebuild-comp)        OPT_REBUILD_COMP=$2; shift;;
  --rebuild-links)       OPT_REBUILD_LINK=$2; shift;;
  --rebuild-mrlist)      OPT_REBUILD_MRLIST=$2; shift;;
  --test)                OPT_TEST=yes;;
  --topdir)              OPT_TOPDIR=$2; shift;;
  --update-rpms)         OPT_UPDATE_RPMS=yes;;
  --version|-V)          echo "$PRGNAME version: $PRGVRSN"; exit;;
  --verbose|-v)          OPT_VERBOSE=$2; shift;;
  --)                    shift; break;;
   *)                    echo "unknow argument \"$1\""; exit 1;;
  esac
  shift
done

#
# Define output streams.  Define them here as they are used in the
# function apt_exit, which can be called shortly from this point.
#
exec 3>/dev/null # progress markers
exec 4>/dev/null # progress markers containing ANSI characters
exec 5>/dev/null # Read 'config file' message
exec 6>/dev/null # debug output

#
# Configuration file selection: look for configuration files specified by the
# CONFIG_FILES variable.  As soon as a configuration file is found, stop
# looking for any other.  One could argue that another approach that allows
# configuration values to be overwritten as nicer, but that won't work with
# xml formatted configuration files.  The reason is that the default values
# for the variables are determined by the DTD, which is applied to all
# the config files.
#
# At last the command line argument values are being applied
#
unset CONFIG_FOUND
CONFIG_FILES="$OPT_CONFIGFILE $HOME/.aptate/aptate.conf /etc/apt/aptate.conf"

for CONFIGFILE in $CONFIG_FILES; do
  if [ -f $CONFIGFILE ]; then
    CONFIG_FOUND=$CONFIGFILE
    break
  fi
done

if [ -z "$CONFIG_FOUND" ]; then
  ( echo "error: no configuration file found."
    echo " (Searchlist: $CONFIG_FILES)"
    if [ -z "$OPT_CONFIGFILE" ]; then
      echo " Consider using the --config-file argument"
    fi
  ) >&2

  apt_exit $ERR_CONFIGURATION
fi

XMLOUT=$(xmllint --valid --noout $CONFIG_FOUND 2>&1)

if [ -z "$XMLOUT" ]; then
  if [ "$OPT_CHECKCONFIG" = "yes" ]; then
    echo "The configuration file is correct"
    apt_exit 0
  fi
else
  ( echo "The configuration file is incorrect, the errors are:"
    echo
    echo "$XMLOUT"
  ) >&2

  apt_exit $ERR_CONFIGURATION
fi

typeset -i VERBOSE
CONFIG=$(${APTATE_CONFIG} --action=config $CONFIG_FOUND)
case $? in
  0) eval $CONFIG
     ;;
  1) echo "error: incorrect config file \"$CONFIG_FOUND\"" >&2
     apt_exit $ERR_CONFIGURATION
     ;;
  *) echo "error: incorrect perl configuration" >&2
     apt_exit $ERR_SYSTEM
     ;;
esac

if [ -n "$OPT_QUIET" ] && [ -n "$OPT_VERBOSE" ]; then
  echo "error: --quiet and --verbose can't be used simulteanously" >&2
  apt_exit $ERR_ARGUMENT
fi

readonly DEBUG_LEVEL=5
if [ -n "$OPT_DEBUG" ]; then
  # Debug mode: show everything we have...
  VERBOSE=$DEBUG_LEVEL
fi

declare -i NUM
NUM=$(echo "$OPT_VERBOSE" | tr -dc "v" | wc -c)
VERBOSE=$VERBOSE+$NUM

NUM=$(echo "$OPT_QUIET"   | tr -dc "q" | wc -c)
VERBOSE=$VERBOSE-$NUM

#
# define which output streams are being written used
# default VERBOSE level is 3:
#  - no debug output and no "Read config file" message
#
if [ $VERBOSE -ge 1 ]; then
  # allow error message (exec 2>&2)

  if [ $VERBOSE -ge 2 ]; then
    # display progress messages
    exec 3>&2

    if [ "$VERBOSE" -ge 3 ]; then
      # display progress counters
      exec 4>&2

      if [ "$VERBOSE" -ge 4 ]; then
        # display the "Read config file" message
        exec 5>&2

        if [ "$VERBOSE" -ge $DEBUG_LEVEL ]; then
          # display debug output (echo of variables values)
          exec 6>&2

            # when collecting debug output the progress markers are unwanted in
            # the debug output stream which is standard error or &2
            exec 4>&1

            echo "DEBUG: verbose level = $VERBOSE" >&6

          if [ "$VERBOSE" -gt $DEBUG_LEVEL ]; then
            ( echo "DEBUG: _warning_ temporarly directories will not be removed"
              echo -n "DEBUG: this may prevent subsequent $PRGNAME instances "
              echo "to run."
            ) >&6
          else
            if [ -z "$OPT_VERBOSE" ]; then
              ( echo "DEBUG: it's possible to obtain more information by"
                echo "DEBUG: increasing the verbose level (-vv)."
              ) >&6
            fi
          fi
        fi
      fi
    fi
  fi
else
  # surpress error messages
  exec 2>/dev/null
fi

#
# Show the parsed config file.  Do it only here and not earlier as only
# at this location the quiet level has been determined and set
#
echo "Using configuration file: \"$CONFIG_FOUND\"" >&5

VARS="OPT_ACTION OPT_BAD_RPMS_MODE OPT_CONFIGFILE OPT_DEBUG OPT_DISTID"
VARS="$VARS OPT_FORCE OPT_UPDATE_RPMS OPT_QUIET OPT_REBUILD_COMP OPT_VERBOSE"
VARS="$VARS OPT_REBUILD_LINK OPT_REBUILD_MRLIST OPT_TEST OPT_TOPDIR"

( echo "DEBUG: === arguments ==="
  var_debug_info $VARS
) >&6

( echo "DEBUG: === global configuration information ==="
  echo "$CONFIG" | sort | sed 's+^+DEBUG: +'
) >&6

if [ "$FLAT" = "no" -a "$OPT_TEST" = "yes" ]; then
  echo "error: --test is available for flat apt repositories only" >&2
  apt_exit $ERR_ARGUMENT
fi

[ -n "$OPT_TOPDIR"           ] && TOPDIR=$OPT_TOPDIR
[ -n "$OPT_UPDATE_RPMS"      ] && UPDATE_RPMS=$OPT_UPDATE_RPMS

# check the variable TOPDIR for meta characters, as directories
# are created with it (the variable may neither be empty).
# The quotes around TOPDIR are requires to detect an "*".
chkvar TOPDIR dir "$TOPDIR"
TOPDIR=$(echo "$TOPDIR" | tr -s "/" | sed s+/$++)
readonly TOPDIR
readonly REP_DIR=$TOPDIR/apt

# Check the top level directories
if [ -d "$REP_DIR" ]; then
  if [ ! -w "$REP_DIR" ]; then
     ( echo "error: shared apt directory is not writable"
       echo " Run $PRGNAME as different user or change the directory permissions."
       echo " Current value: $REP_DIR"
     ) >&2
     apt_exit $ERR_DIR
  fi
else
  if [ -d "$TOPDIR" ]; then
    if [ -w "$TOPDIR" ]; then
      mckdir $REP_DIR
    else
       ( echo "error: top level directory is not writable."
         echo " This is needed to create the directory $REP_DIR."
         echo -n " Run $PRGNAME as different user, change the directory "
         echo "permissions or "
         echo " create the directory with the right permissions yourself."
         echo " Current value: $TOPDIR"
       ) >&2
       apt_exit $ERR_DIR
    fi
  else
    ( echo "error: incorrect apt/rpm repository directory."
      echo " current value: $TOPDIR"
    ) >&2
    apt_exit $ERR_CONFIGURATION
  fi
fi

# Aptate's share directory.  It must be made relative to TOPDIR, to
# allow relative linking
if [ -z "$SHAREDIR" ]; then
  REL_SHAREDIR="apt/share"
else
  chkvar SHAREDIR dir "$SHAREDIR"
  SHAREDIR=$(echo "$SHAREDIR" | tr -s "/" | sed -e s+^/++ -e s+/$++)
  REL_SHAREDIR=$SHAREDIR
fi

# Aptate's share directory, but now absolute (which is the preferred
# variable to be used)
SHAREDIR=$TOPDIR/$REL_SHAREDIR
readonly SHAREDIR

if [ ! -d $SHAREDIR ]; then
  mckdir $SHAREDIR
fi

if [ "$SIGN" = "yes" ] || [ "$SIGNED_PKGS_ONLY" = "yes" ]; then

  # check for gpg
  GPG_INSTALLED=no

  # Is gpg installed?
  # The redirect ">& /dev/null" means that all output streams are being
  # redirected to /dev/null
  if type gpg >& /dev/null; then
    # Is gpg configured?
    gpg --list-keys ${KEY_EMAIL:-joe@us.er} >& /dev/null
    if [ $? -eq 0 ]; then
      GPG_INSTALLED=yes
    else
      ( echo "error: GPG not configured."
        echo " No GPG key for user \"$KEY_EMAIL\"."
      ) >&2
    fi
  else
    echo "error: GPG not installed." >&2
  fi

  if [ "$GPG_INSTALLED" = "no" ]; then
    # provide additional information
    if [ "$SIGN" = "yes" ]; then
      echo "This is required for a gpg signed apt repository." >&2
    fi

    if [ "$SIGNED_PKGS_ONLY" = "yes" ]; then
      echo "This is required to accept gpg signed rpm packages." >&2
    fi

    apt_exit $ERR_SYSTEM
  fi
fi

# create database with extended (bloated) information?
if [ "$BLOAT" = "yes" ]; then
  BLOAT=--bloat
else
  unset BLOAT
fi

# should we put SRPMS besides RPMS?
if [ "$FLAT" = "yes" ]; then
  GENSRCLIST_ARG_FLAT=--flat
  SRPMDIR=""
else
  unset GENSRCLIST_ARG_FLAT
  SRPMDIR="/.."
fi

# should we follow symbolic links when searching
# for SRPMS/RPMS? Caution - this may be dangerous!
if [ "$FOLLOW" = "yes" ]; then
  FOLLOW=-follow
else
  unset FOLLOW
fi

case $OPT_BAD_RPMS_MODE in

  report-only)
    BAD_RPMS_MODE="report-only"
    ;;

  rename)
    BAD_RPMS_MODE="rename"
    ;;
  "")
    # BAD_RPMS_MODE=$BAD_RPMS_MODE
    # BAD_RPMS_MODE is assigned the default value, which has been set
    # while parsing the configuration file
    ;;
  *)
    ( echo "error: incorrect value \"$OPT_BAD_RPMS_MODE\" specified for --bad-rpm-mode"
      echo " valid values are: report-only and rename."
    ) >&2

    apt_exit $ERR_ARGUMENT
    ;;
esac

case $OPT_ACTION in

  # server|cd|dvd)
  server)
    TARGET=$OPT_ACTION
    MK_REP=yes
    ;;

  mirror)
    TARGET=$OPT_ACTION
    MK_REP=no
    ;;

  "")
    TARGET=server
    MK_REP=yes
    ;;

  *)
    ( echo "error: incorrect value specified for argument --action"
      # FIXME:
      # echo "  supported values are: server, cd, dvd and mirror (default: server)"
      echo " supported values are: server and mirror (default: server)"
      echo " current value: $OPT_ACTION"
    ) >&2

    apt_exit $ERR_ARGUMENT
esac

# escape sequences
readonly ESC=$(echo -en "\033")
# ESC: save cursor position
readonly SCP="$ESC"7
# ESC: restore cursor position
readonly RCP="$ESC"8
# ESC: clear from cursor to eol
readonly CLEOL="$ESC\1330K"

VARS="TOPDIR SIGN KEY_NAME KEY_EMAIL OLD_HASHFILE BAD_RPMS_MODE SHAREDIR"
VARS="$VARS BLOAT FLAT FOLLOW UPDATE_RPMS MK_SEC_COMP SIGNED_PKGS_ONLY"
VARS="$VARS WGET_WAIT WGET_TRIES VERBOSE PREFIX_FTP PREFIX_HTTP PREFIX_FILE"
VARS="$VARS MK_PATCH_RPM_COMP TARGET MK_REP"

( echo "DEBUG: === processed argument options ==="
  var_debug_info $VARS
) >&6

trap 'apt_exit_on_trap' HUP INT QUIT TERM EXIT

TMPDIR=$(mktemp -d /tmp/aptate.XXXXXX)
if [ $? -ne 0 ] ; then
  echo "error: temp directory creation failed" >&2
  apt_exit $ERR_DIR
fi

if test -n "$OPT_DISTID"; then
  for DISTRO in $(echo $OPT_DISTID | tr "," " ")
  do
    DIST=$(${APTATE_CONFIG} --action=query \
       "--xpath=attribute::id='$DISTRO'" ${CONFIG_FOUND})
    DISTROS="$DISTROS $DIST"
  done
  # remove the spaces at the beginning of line, this is needed for the 
  # if [ -z .. ] statement below
  DISTROS=$(echo $DISTROS | sed s/^[[:blank:]]*//)
else
  DISTROS=$(${APTATE_CONFIG} --action=query ${CONFIG_FOUND})
fi

if [ -z "$DISTROS" ]; then
  ( echo "error: no valid distribution found"
    echo -n " Choose from the following available distributions: $DISTROS"
    ${APTATE_CONFIG} --action=query ${CONFIG_FOUND}
  ) >&2

  if [ -n "$OPT_DISTID" ]; then
    apt_exit $ERR_ARGUMENT
  else
    apt_exit $ERR_CONFIGURATION
  fi
else
  echo "Distribution(s) to be processed: $DISTROS" >&3
fi

for DISTID in $DISTROS; do

  echo "Processing: $DISTID" >&3
  cd $TOPDIR

  # Aptate's relative and absolute share directory, but now including
  # the distribution
  REL_SHARE_DIST_DIR=$REL_SHAREDIR/$DISTID
  SHARE_DIST_DIR=$TOPDIR/$REL_SHARE_DIST_DIR

  # This is the first lock check, it checks whether another aptate instance
  # is running for the same DISTID.  The 2nd lock check, determines if another
  # aptate instance is already using the temp repository directory.
  # Checking whether the temp repository directory exits alone is not
  # sufficient, as aptate can be run with different (--)actions for the same
  # distid.  The different actions are using different tmp directories
  # (due to different apt repository structures).  However, the same caches
  # are being used.
  if [ -f $SHARE_DIST_DIR/lock ]; then

    printf "warning: other $PRGNAME instance active, $SCP" >&4

    typeset -i NUM=9
    while [ -f $SHARE_DIST_DIR/lock -a $NUM -gt 0 ]
    do
      printf "${RCP}waiting %d minutes... " $NUM >&4
      sleep 60
      NUM=$NUM-1
    done
    printf "\n"

    if [ $NUM -eq 0 ]; then
      ( echo "error: another $PRGNAME instance active."
        echo " check lock file: $SHARE_DIST_DIR/lock"
      ) >&2

      apt_exit $ERR_LOCKED
    fi
  fi

  # Do not define the variable LOCK_DISTID before, as the function 
  # clean_up_distid_lock assumes that it must remove the lock when the
  # variable LOCK_DISTID is set.  In case it is not set the lock belongs to
  # another aptate instance....
  if [ ! -d $SHARE_DIST_DIR ]; then
    mckdir $SHARE_DIST_DIR
  fi
  LOCK_DISTID=$SHARE_DIST_DIR/lock
  echo $$ > $LOCK_DISTID

  # In the update directory, the mirrored packages will be stored
  UPDATEDIR=$REL_SHARE_DIST_DIR/update

  # The log directory holds the logfiles that are being created during the
  # mirroring of the packages
  UPDATE_LOG_DIR=$SHARE_DIST_DIR/log

  # convert the xml formatted distribution part of the config file to something
  # bash can easily process
  MIRRORTMP=$TMPDIR/main::mirror_$DISTID

  ${APTATE_CONFIG} --action=mirror --updatedir=${UPDATEDIR} \
    --dist=$DISTID $CONFIG_FOUND > $MIRRORTMP
  if [ $? -ne 0 ]; then
    echo "error: incorrect configuration file \"$CONFIG_FOUND\"" >&2
    apt_exit $ERR_CONFIGURATION
  fi

  ( echo "DEBUG: === component information ==="
    cat $MIRRORTMP
  ) >&6

            IDS=$(cut -d";" -f1 $MIRRORTMP)
   SOURCE_NAMES=$(cut -d";" -f2 $MIRRORTMP)
        METHODS=$(cut -d";" -f3 $MIRRORTMP)
  UPD_RPM_COMPS=$(cut -d";" -f4 $MIRRORTMP)
    SOURCE_URLS=$(cut -d";" -f5 $MIRRORTMP)
    SEARCH_DIRS=$(cut -d";" -f6 $MIRRORTMP)

  declare -i NAMESNUM
  declare -i URLSNUM
  NAMESNUM=$(echo $SOURCE_NAMES | wc -w)
   URLSNUM=$(echo $SOURCE_URLS  | wc -w)
  if [ $NAMESNUM -eq 0 ]; then
    echo "No uri's found, check mirrorlist file: \"$CONFIG_FOUND\"" >&2
    apt_exit $ERR_CONFIGURATION
  else
    if [ $NAMESNUM -ne $URLSNUM ]; then
      ( echo -n "The number of names ($NAMESNUM) and uri's ($URLSNUM) "
        echo "are not balanced"
        echo "check the distribution part of the configuration file: \"$CONFIG_FOUND\""
        echo "Available names:"
        echo "${SOURCE_NAMES}" | sed 's+^+  +'
        echo
        echo "Available uri's:"
        echo "${SOURCE_URLS}"  | sed 's+^+  +'

      ) >&2
      apt_exit $ERR_CONFIGURATION
    fi
  fi

  if $(echo $SOURCE_NAMES | grep -qw security);then
    echo "error: \"security\" is a reserved $PRGNAME repository name" >&2
    apt_exit $ERR_CONFIGURATION
  fi

  if [ "$TARGET" = "mirror" ]; then

    # Is the mirror action in line with the update rpms setting?
    # It is, of course, possible to overrule the update rpms setting, but it
    # is better to let the user do this. 
    if [ "$UPDATE_RPMS" = "no" ]; then

      ( echo -n "error: use --update-rpms to override the configuration "
        echo "setting: update-rpms=no"
        echo " Check configuration file: $CONFIG_FOUND"
      ) >&2

      apt_exit $ERR_ARGUMENT
    fi
  fi

  if [ "$UPDATE_RPMS" = "yes" ]; then

    echo "Entering mirror stage" >&4

    NUMS=$(awk -F\; '{
      # find the components that are configured to be updated.
      # $3 = method
      # $4 = update allowed
      if ( $4 == "yes" && ( $3 == "ftp" || $3 == "script" ))  print $1
    }' $MIRRORTMP)

    if [ -z "$NUMS" ]; then
      echo "There are no components to be mirrored" >&3
    else
      for NUM in $NUMS; do
        METHOD=$(echo $METHODS | awk "{ print \$$NUM }" )
        SOURCE_NAME=$(echo $SOURCE_NAMES | awk "{ print \$$NUM }" )  

        if [ "$METHOD" = "ftp" -o "$METHOD" = "script" ]; then
          printf " %-20s Upd: " $SOURCE_NAME >&3

          case $METHOD in
            ftp)
              SOURCE_URL=$(echo $SOURCE_URLS | awk "{ print \$$NUM }" )  

              # FIXME: remove the echo
              mirror $NUM $SOURCE_NAME $SOURCE_URL $UPDATE_LOG_DIR
              RET=$?
              ;;

            script)
              # FIXME: remove the echo
              /usr/lib/apt4rpm/methods/script --distribution=$DISTID \
                --component=$SOURCE_NAME $CONFIG_FOUND >&6
              RET=$?
              ;;

            *)
              # programmer error
              echo "error: unknow method" >&2
              apt_exit $ERR_ASSERT
          esac

          if [ $RET -eq 0 ]; then
            printf "OK  \n" >&3
          else
            printf "Err \n" >&3
          fi
        fi
      done
    fi
  fi

  if [ "$MK_REP" = "yes" ]; then
    CONFIG=$(${APTATE_CONFIG} --action=dist --dist=$DISTID ${CONFIG_FOUND})
    if [ $? -ne 0 ]; then
      echo "error: incorrect configuration file \"$CONFIG_FOUND\"" >&2
      apt_exit $ERR_CONFIGURATION
    fi
    eval $CONFIG

    DISTATTRS=`${APTATE_CONFIG} --action=distattrs --dist=$DISTID $CONFIG_FOUND`
    eval $DISTATTRS

    if [ "$TARGET" != "server" ]; then
      # It does not make sense, to provide patch rpms together with the 
      # regular rpms on a cd/dvd disc set.
      MK_PATCH_RPM_COMP=no

      # The same is valid for rpms that have been marked as security rpms.
      # These rpms will be provided by the regular (other) apt components.
      MK_SEC_COMP=no
    fi

    VARS="TOPDIR SIGN KEY_NAME KEY_EMAIL OLD_HASHFILE BAD_RPMS_MODE SHAREDIR"
    VARS="$VARS BLOAT FLAT FOLLOW UPDATE_RPMS MK_SEC_COMP SIGNED_PKGS_ONLY"
    VARS="$VARS WGET_WAIT WGET_TRIES VERBOSE PREFIX_FTP PREFIX_HTTP PREFIX_FILE"
    VARS="$VARS MK_PATCH_RPM_COMP VERSION ARCH DIST ARCH_STRUCTURE LANGUAGE"
    VARS="$VARS ARCHIVE TARGET MK_REP"

    ( echo "DEBUG: === distribution dependend variables ==="
      var_debug_info $VARS
    ) >&6

    CACHEDIR=$SHARE_DIST_DIR/cache
    if [ ! -d $CACHEDIR ]; then
      mckdir $CACHEDIR
    fi

    case $ARCH_STRUCTURE in
      1)
         VARS="DIST VERSION ARCH"
         ARCHIVE="$DIST/$VERSION-$ARCH"
         ;;
      2)
         VARS="DIST VERSION ARCH"
         ARCHIVE="$DIST/$VERSION/$ARCH"
         ;;
      3)
         if [ -z "$LANGUAGE" ]; then
           ( echo "error: <language> must be populated in case archive type is 3."
             echo " Check config file: $CONFIG_FOUND"
           ) >&2
           apt_exit $ERR_CONFIGURATION
         fi

         VARS="DIST VERSION ARCH LANGUAGE"
         ARCHIVE="$DIST/$VERSION/$LANGUAGE/$ARCH"
         ;;
      4)
         if [ -n "$ARCHIVE" ]; then
           # ARCHIVE has been provided in the config file
           chkvar ARCHIVE dir "$ARCHIVE"

           VARS="DIST VERSION ARCH"
         else
           ( echo "error: <archive> must be populated in case archive type is 4."
             echo " Check config file: $CONFIG_FOUND"
           ) >&2
           apt_exit $ERR_CONFIGURATION
         fi

         ;;
    esac

    # check the required variables (in VARS) on meta characters, as directories
    # and/or files are created with them (the variables may neither be empty)
    for VAR in $VARS
    do
      # due to the eval the \" and \$ are needed, if not there an "*"
      # will not be reported as invalid character.
      eval chkvar $VAR var "\"\$$VAR"\"
    done

    BASE_ARCHIVE=$(echo $ARCHIVE | tr -s "/" | sed -e s+^/++ -e s+/$++)

    SLASH_COUNT=${BASE_ARCHIVE//[^\/]/}
    SLASH_COUNT=${#SLASH_COUNT}

    if [ "$SLASH_COUNT" = "0" ]; then
      ( echo "error: archive declaration \"$ARCHIVE\" incorrect."
        echo " Insufficient directories, at least 2 directories are needed."
        echo " Check config file: $CONFIG_FOUND"
      ) >&2
      apt_exit $ERR_CONFIGURATION
    fi

    echo "Archive structure: $ARCH_STRUCTURE => $ARCHIVE" >&3
    echo "Entering cache build stage" >&3

    unset PROCESSED_COMP
    for ID in $IDS; do
      SOURCE_NAME=$(echo $SOURCE_NAMES  | awk "{ print \$$ID }" )

      # Has the component been processed before?
      echo $PROCESSED_COMP | grep -qw $SOURCE_NAME
      if [ $? -ne 0 ]; then

        SOURCE_DIR=$(echo $SEARCH_DIRS | awk "{ print \$$ID }" )
        if [ -d "$SOURCE_DIR" ]; then

          echo $OPT_REBUILD_COMP | grep -Ewq "$SOURCE_NAME|all"
          if [ $? = 0 ]; then
            REBUILD_CACHE=yes
          else
            REBUILD_CACHE=no
          fi

          build_cache $SOURCE_NAME $SOURCE_DIR $ID $REBUILD_CACHE
          PROCESSED_COMP="$PROCESSED_COMP $SOURCE_NAME"
        else

          ( echo -n "error: not existing rpm repository directory "
            echo "for component: $SOURCE_NAME."
            echo " Expected directory: \"$TOPDIR/$SOURCE_DIR\"."
          ) >&2

          METHOD=$(echo $METHODS | awk "{ print \$$ID }" )
          if [ $METHOD = "ftp" -o $METHOD = "script" ]; then

            UPDATE_RPMS_COMP=$(echo $UPD_RPM_COMPS | awk "{ print \$$ID }" )
            if [ "$UPDATE_RPMS" = "yes" ]; then
              if [ "$UPDATE_RPMS_COMP" = "yes" ]; then

                UPDATE_LOG_FILE=$UPDATE_LOG_DIR/$SOURCE_NAME
                echo "  Maybe update failed? (see $UPDATE_LOG_FILE)" >&2
              else
                ( echo -n "  Update mode not activated in config file (set "
                  echo "update-rpms=yes) for"
                  echo "  this component ($SOURCE_NAME)."
                ) >&2
              fi
            else
              if [ "$UPDATE_RPMS_COMP" = "yes" ]; then
                echo "  Update mode not activated (use --update-rpms) " >&2
              else
                ( echo -n "  Update mode not activated (use --update-rpms to "
                  echo "start with)."
                ) >&2
              fi
            fi # update mode activated ($UPDATE_RPMS = "yes")
          fi # $METHOD = "ftp" -o $METHOD = "script"
        fi # Does $SOURCE_DIR exist?
      fi # Should the component be processed?
    done

    case $TARGET in
     cd)
       echo "Entering package list per disc stage" >&3
       for TYPE in bin src; do
         /usr/lib/apt4rpm/compare_rpm_version \
           --format path --rpmtype $TYPE $TMPDIR/build_cache::cache.extra
       done | sed s/^/extra\;/ > $TMPDIR/cd
       wc -l $TMPDIR/build_cache::cache.extra
       wc -l $TMPDIR/cd
       BEARERS="1 2"
       ARCHIVE_TMP_PREFIX=tmp_$$
       ;;

     dvd)
       BEARERS="1 2 3"
       ARCHIVE_TMP_PREFIX=tmp_$$
       ;;

     server)
       BEARERS=1
       # tmp_server_$$ may work as well
       ARCHIVE_TMP_PREFIX=tmp_$(echo $BASE_ARCHIVE | cut -d"/" -f1)
       ;;

     *)
       # Programmer error
       echo "error: unknown TARGET" >&2
       apt_exit $ERR_ASSERT
       ;;
    esac

    BASE_ARCHIVE_TMP_DIR=$REP_DIR/$ARCHIVE_TMP_PREFIX

    # Check for other running aptate instances, using the same temp directory 
    if [ -d $BASE_ARCHIVE_TMP_DIR ]; then

      printf "warning: other $PRGNAME instance active, $SCP" >&4

      typeset -i NUM=9
      while [ -d $BASE_ARCHIVE_TMP_DIR -a $NUM -gt 0 ]
      do
        printf "${RCP}waiting %d minutes... " $NUM >&4
        sleep 60
        NUM=$NUM-1
      done
      printf "\n"

      if [ $NUM -eq 0 ]; then
        ( echo "error: another $PRGNAME instance active."
          echo " check directory: $BASE_ARCHIVE_TMP_DIR"
        ) >&2

        apt_exit $ERR_LOCKED
      fi
    fi

    # Do not define the variable LOCK_TMPDIR before, as the function
    # clean_up_rep_tmpdir assumes that it must remove the lock when the
    # variable LOCK_TMPDIR is set.
    # An example where the lock should not be removed is when the lock is
    # is owned by another aptate instance and aptate is interrupted (with
    # e.g. ^C).
    LOCK_TMPDIR=$BASE_ARCHIVE_TMP_DIR
    mckdir $BASE_ARCHIVE_TMP_DIR

    echo "Entering apt repository creation stage" >&3

    for BEARER in $BEARERS
    do
      case $TARGET in
        cd)
          printf "Creating repository for: CD-%s\n" $BEARER >&3
          ARCHIVE="cd/$DISTID/$BEARER/$BASE_ARCHIVE"
          ;;

        dvd)
          printf "Creating repository for: DVD-%s\n" $BEARER >&3
          ARCHIVE="dvd/$DISTID/$BEARER/$BASE_ARCHIVE"
          ;;

        server)
          ARCHIVE=$BASE_ARCHIVE
          ;;
      esac

      ARCHIVE_ROOT=$REP_DIR/$ARCHIVE
      ARCHIVE_BASE_DIR=$ARCHIVE_ROOT/base

      typeset -i COUNT
      typeset -i MAX

      # Take the directory/ies leading to the archive and the component
      # directory into account.
      SLASH_COUNT=${ARCHIVE//[^\/]/}
      SLASH_COUNT=${#SLASH_COUNT}

      if [ $TARGET = "server" ]; then
        # ..../apt/tmp_$DIST/....<component>
        MAX=$SLASH_COUNT+2
      else
        # In case the target is cd or dvd there are 3 directories to take
        # into account apt/tmp_$$ and <component>
        # ..../apt/tmp_$$/cd/....<component>
        MAX=$SLASH_COUNT+3
      fi

      COUNT=0
      REL_ARCHIVE=$(while [ $COUNT -le $MAX ]
        do
          if [ $COUNT -eq 0 ]; then
            printf ".."
          else
            printf "/.."
          fi
          COUNT=$COUNT+1
        done
      )

      if [ "$FLAT" = "yes" ]; then
        SRPM_ARCHIVE=$ARCHIVE
        REL_SRPM_ARCHIVE=$REL_ARCHIVE
      else
        SRPM_ARCHIVE=${ARCHIVE%/*} # leave off last directory
        REL_SRPM_ARCHIVE=${REL_ARCHIVE%/*}
      fi

      ARCHIVE_SRPM_ROOT=$REP_DIR/$SRPM_ARCHIVE

      if [ $TARGET = "server" ]; then
        # In this case the temp directory and the archive overlap.
        # For example: ARCHIVE = mydistro/1234/xyz, the temp directory must
        # become ..../tmp_mydistro/1234/xyz.
        # The variable BASE_ARCHIVE_TMP_DIR already ends with ..../tmp_mydistro/
        # This requires that the first part of the variable $(SRPM_)ARCHIVE must
        # be removed.
        ARCHIVE_TMP_ROOT=$BASE_ARCHIVE_TMP_DIR/$( echo $ARCHIVE |
          cut -d"/" -f2-)

        echo $SRPM_ARCHIVE | grep -q "/"
        if [ $? -eq 0 ]; then
          ARCHIVE_TMP_SRPM_ROOT=$BASE_ARCHIVE_TMP_DIR/$(echo $SRPM_ARCHIVE |
            cut -d"/" -f2-)
        else
          ARCHIVE_TMP_SRPM_ROOT=$BASE_ARCHIVE_TMP_DIR
        fi
      else
        ARCHIVE_TMP_ROOT=$BASE_ARCHIVE_TMP_DIR/$ARCHIVE
        ARCHIVE_TMP_SRPM_ROOT=$BASE_ARCHIVE_TMP_DIR/$SRPM_ARCHIVE
      fi

      ARCHIVE_TMP_BASE_DIR=$ARCHIVE_TMP_ROOT/base

      VARS="ARCHIVE_ROOT ARCHIVE_TMP_ROOT ARCHIVE_BASE_DIR ARCHIVE_TMP_BASE_DIR"
      VARS="$VARS SRPM_ARCHIVE ARCHIVE_SRPM_ROOT ARCHIVE_TMP_SRPM_ROOT UPDATEDIR"
      VARS="$VARS REL_SHARE_DIST_DIR SHARE_DIST_DIR UPDATE_LOG_DIR MIRRORTMP"
      VARS="$VARS CACHEDIR"
      var_debug_info $VARS >&6

      # FIXME: what about the SRPMS.* directory in case a non flat directory
      # structure is being used?  Should ARCHIVE_TMP_SRPM_ROOT be used instead?
      # clean up
      if [ -d $ARCHIVE_TMP_ROOT ]; then
        rm -r $ARCHIVE_TMP_ROOT 2>/dev/null
        if [ $? -ne 0 ]; then
          echo "Failed to remove directory $ARCHIVE_TMP_ROOT" >&2
          apt_exit $ERR_DIR
        fi
      fi

      if [ "$TARGET" = "server" ]; then
        # Look if the srpm apt repository is located at the expected location.
        # If not a change of repository structure has been requested by
        # changing the <opt> attribute "flat" or it is the initial run for this
        # repository.
        ls -d $ARCHIVE_SRPM_ROOT/SRPMS.* >/dev/null 2>&1
        if [ $? -eq 0 ]; then
          REPOSITORY_CHANGE=""
        else
          if [ "$FLAT" = "yes" ]; then
            ls -d $ARCHIVE_SRPM_ROOT/../SRPMS.* >/dev/null 2>&1
            if [ $? -eq 0 ]; then
              if [ "$OPT_FORCE" = "yes" ]; then
                REPOSITORY_CHANGE=toflat
                OPT_REBUILD_LINK=all
              else
                ( echo "Use the --force option to have the repository characteristic"
                  echo "changed from noflat to flat"
                ) >&2
                apt_exit $ERR_CONFIGURATION
              fi
            else
              # initial run
              REPOSITORY_CHANGE=""
            fi
          else
            ls -d $ARCHIVE_ROOT/SRPMS.* >/dev/null 2>&1
            if [ $? -eq 0 ]; then
              if [ "$OPT_FORCE" = "yes" ]; then
                REPOSITORY_CHANGE=tonoflat
                OPT_REBUILD_LINK=all
              else
                ( echo "Use the --force option to have the repository characteristic"
                  echo "changed from flat to noflat"
                ) >&2
                apt_exit $ERR_CONFIGURATION
              fi
            else
              # initial run
              REPOSITORY_CHANGE=""
            fi
          fi
        fi

        if [ -n "$REPOSITORY_CHANGE" ]; then
          echo -n "Note: the repository characteristic will be changed from " >&2
          case $REPOSITORY_CHANGE in
            toflat)   echo "noflat to flat" >&2 ;;
            tonoflat) echo "flat to noflat" >&2 ;;
          esac
        fi
      fi # if [ $TARGET = server ]

      declare CHANGED_COMP_LIST=""
      declare UNCHANGED_COMP_LIST=""
      declare PROCESSED_COMP=""

      for NUM in $IDS; do

        SOURCE_NAME=$(echo $SOURCE_NAMES  | awk "{ print \$$NUM }" )

        # Has the component been processed before?
        echo $PROCESSED_COMP | grep -qw $SOURCE_NAME
        if [ $? -ne 0 ]; then

          make_links regular $SOURCE_NAME $NUM $OPT_REBUILD_COMP,$OPT_REBUILD_LINK
          PROCESSED_COMP="$PROCESSED_COMP $SOURCE_NAME"
        fi
      done

      # Create the virtual component security
      if [ "$MK_SEC_COMP" = "yes" ]; then
        make_links security security secid $OPT_REBUILD_COMP,$OPT_REBUILD_LINK
      fi

      ( echo "DEBUG: changed components:   $CHANGED_COMP_LIST"
        echo "DEBUG: Unchanged components: $UNCHANGED_COMP_LIST"
      ) >&6

      if [ "$CHANGED_COMP_LIST" ]; then

        #
        # The code below is similar to the code provided by genbasedir (from the
        # apt-rpm project).  Genbasedir is not used, as a lot of the functionality
        # in genbasedir is present in aptate as well.  To obtain more flexibility
        # and efficiency use our own genbasedir code.
        #
        echo -e "Entering apt database building stage" >&3

        > $TMPDIR/srcidx
        GENLISTARG="$BLOAT"

        if [ "$VERBOSE" -ge 3 ]; then
          GENLISTARG="$GENLISTARG --progress"
        fi

        for COMP in $CHANGED_COMP_LIST; do
          printf " %-16s  -> Bin: " $COMP >&3

          # Take preventive action to prevent genpkglist to crash.  This can happen
          # in case the source rpm (source of the link) is being removed.  Which
          # can happen e.g. due to an asynchronous running rsync process.
          # At busy servers, processing big component caches can consume a lot of
          # time.  This can cause the cache to be out sync while aptate is not even
          # finished.  The preventive action does not clean up the cache, this will
          # be taken care of during the next run of aptate.
          ( cd $ARCHIVE_TMP_ROOT/RPMS.$COMP
            find . -follow 2>&1 >/dev/null | cut -d: -f2 | xargs rm -f
          )

          ( cd $ARCHIVE_TMP_BASE_DIR
            genpkglist $GENLISTARG --index $TMPDIR/srcidx.$COMP $ARCHIVE_TMP_ROOT $COMP
          )

          # Check the exit value of genpkglist
          if [ $? != 0 ]; then

            echo "error: genpkglist did not finish successfully." >&2
            audit_component pkg $COMP $ARCHIVE_TMP_ROOT/RPMS.$COMP
            apt_exit $ERR_REPOSITORY
          fi

          FILE=$ARCHIVE_TMP_BASE_DIR/pkglist.$COMP
          if [ -r $FILE ]; then
            bzip2_checked $FILE
          fi

          cat $TMPDIR/srcidx.$COMP >> $TMPDIR/srcidx
          echo >&3
        done

        GENLISTARG="$GENSRCLIST_ARG_FLAT"

        if [ "$VERBOSE" -ge 3 ]; then
          GENLISTARG="$GENLISTARG --progress"
        fi

        for COMP in $CHANGED_COMP_LIST; do

          printf " %-16s  -> Src: " $COMP >&3

          # See comment above for genpkglist
          ( cd $ARCHIVE_TMP_SRPM_ROOT/SRPMS.$COMP
            find . -follow 2>&1 >/dev/null | cut -d: -f2 | xargs rm -f
          )

          ( cd $ARCHIVE_TMP_BASE_DIR
            gensrclist $GENLISTARG $ARCHIVE_TMP_ROOT${SRPMDIR} $COMP $TMPDIR/srcidx
          )

          # Check the exit value of gensrclist
          if [ $? != 0 ]; then

            echo "error: gensrclist did not finish successfully." >&2
            audit_component src $COMP $ARCHIVE_TMP_SRPM_ROOT/SRPMS.$COMP
            apt_exit $ERR_REPOSITORY
          fi

          FILE=$ARCHIVE_TMP_BASE_DIR/srclist.$COMP
          if [ -r $FILE ]; then
            bzip2_checked $FILE
          fi

          echo >&3
        done
      fi

      COMP_LIST="$CHANGED_COMP_LIST $UNCHANGED_COMP_LIST"
      printf "Creating global release file: " >&3
      printf $SCP >&4

      TMP_HASHFILE=$TMPDIR/main::hashfile
      MAX=$(echo $COMP_LIST | wc -w)
      COUNT=1

      echo "MD5SUM:" > $TMP_HASHFILE
      for COMP in $COMP_LIST; do
        printf "$RCP%02d/%02d " $COUNT $MAX >&4
        for FILE2 in pkglist.$COMP pkglist.$COMP.bz2 \
                     srclist.$COMP srclist.$COMP.bz2 release.$COMP ; do
          FILE=$ARCHIVE_TMP_BASE_DIR/$FILE2
          if [ -f $FILE ]; then
            MD5SUM=$(md5sum $FILE | cut -d' ' -f1)
            FILESIZE=$(filesize $FILE)
            echo " $MD5SUM $FILESIZE $ARCHIVE/base/$FILE2" >> $TMP_HASHFILE
          fi
        done
        COUNT=$COUNT+1
      done

      #
      # Global release file
      #
      ( echo "Origin: $DIST"
        echo "Label: $DIST"
        echo "Suite: unknown"
        echo "Codename: unknown"
        echo "Architecture: $ARCH"
        echo "Components: $COMP_LIST" | tr -s " "
        echo "Description: not available"
        sed -e s+$ARCHIVE/++ -e s+^MD5SUM:+MD5Sum:+ $TMP_HASHFILE

      ) > $ARCHIVE_TMP_BASE_DIR/release

      if [ "$OLD_HASHFILE" = "yes" ]; then
        mv $TMP_HASHFILE $ARCHIVE_TMP_BASE_DIR/hashfile
      else
        # Clean up.  The TMP_HASHFILE is not needed after this point
        rm -f $TMP_HASHFILE
      fi

      printf $RCP$CLEOL >&4
      printf "done\n" >&3

      #
      # End of code that is similar to the code provided by genbasedir
      #

      # Provide a list with all rpms and and by which component the rpms
      # are provided
      TMPFILE=$TMPDIR/allpkgs
      rm -f $TMPFILE
      for COMPONENT in $COMP_LIST
      do
        if [ -r $TMPDIR/build_cache::cache.$COMPONENT ]; then
          awk -F\; -v component=$COMPONENT '{
            printf("%s;%s;%s;%s\n", $3, $4, $5, component )
          }' $TMPDIR/build_cache::cache.$COMPONENT >> $TMPFILE
        fi
      done
      if [ -r $TMPFILE ]; then
        sort -u $TMPFILE | bzip2 -c > $ARCHIVE_TMP_BASE_DIR/contlist.bz2
      fi

      # EXAMPLEDIR is needed by the function add_sourceslist_examples
      if [ "$TARGET" = "server" ]; then
        EXAMPLEDIR=$ARCHIVE_TMP_ROOT/examples
        if [ ! -d $EXAMPLEDIR ]; then
          mckdir $EXAMPLEDIR
        fi
      fi

      # PREFIX_FILE defines the prefix for the file url in the sources.list example.
      # If not defined make an assumption.
      if [ -z "$PREFIX_FILE" ]; then
        PREFIX_FILE=$TOPDIR/apt
      fi

      if [ "$SIGN" != "yes" ]; then
        if [ "$TARGET" = "server" ]; then
          # create examples sources.list.
          add_sourceslist_examples
        fi
      else
        # SIGN = yes

        echo "Signing the apt repository..." >&3
        NAME=$(gpg --fingerprint $KEY_EMAIL | head -n 1 | cut -d' '  -f5-)
        FINGERPRINT=$(gpg --fingerprint $KEY_EMAIL | head -n 2 | tail -n 1 | \
                                       cut -d' '  -f9-  | sed -e s+" "++g)

        (
         echo "simple-key \"$KEY_NAME\""
         echo "{"
         echo "   Fingerprint \"$FINGERPRINT\";"
         echo "   Name \"$NAME\";"
         echo "}"
        ) > $EXAMPLEDIR/vendors.list

        eval gpg --sign -armour --detach-sign --yes --quiet --default-key $KEY_EMAIL \
          $ARCHIVE_TMP_BASE_DIR/release
        cat $ARCHIVE_TMP_BASE_DIR/release.asc >> $ARCHIVE_TMP_BASE_DIR/release
        rm -f $ARCHIVE_TMP_BASE_DIR/release.asc

        if [ "$OLD_HASHFILE" = "yes" ]; then
          # create the hashfile for apt version 0.3.19
          gpg --armour --export --default-key $KEY_EMAIL > $EXAMPLEDIR/$KEY_NAME.pub

          gpg -armour -qs --yes --default-key $KEY_EMAIL $ARCHIVE_TMP_BASE_DIR/hashfile
          mv -f $ARCHIVE_TMP_BASE_DIR/hashfile.asc $ARCHIVE_TMP_BASE_DIR/hashfile.gpg
        fi

        if [ "$TARGET" = "server" ]; then
          # create examples sources.list.
          add_sourceslist_examples
        fi
      fi

    done # for BEARER in BEARERS

    # Clean up.  All real components have been processed, the temporarely
    # mirror file is not needed anylonger
    rm -f $MIRRORTMP

    ### audit code ###
    # Provide code to find the root cause why sometimes files have
    # a unexpected length of 0 byte
    DISPLAY_AUDIT=""
    AUDIT_FILE=$TMPDIR/audit

    cd $ARCHIVE_TMP_BASE_DIR
    FILES=$(find . -size 0 -name "*.bz2")

    ( echo "Contents of tmp base dir: $ARCHIVE_TMP_BASE_DIR"
      ls -l
      echo
      echo "Free diskspace:"
      df . /tmp
      echo

    ) > $AUDIT_FILE
     
    cd $TOPDIR

    if [ -n "$FILES" ]; then
      DISPLAY_AUDIT=yes
    fi
    ### audit code ###

    #
    # If OPT_TEST is "yes" the new database will not be automatically updated
    # you'll have to do this by hand, if everything looks fine to you
    #
    if [ "$OPT_TEST" = "yes" ]; then

      ARCHIVE_TST_ROOT=$REP_DIR/test_$ARCHIVE
      ARCHIVE_INSTALL_DIR=${ARCHIVE_TST_ROOT%/*} # leave off last directory

      rm -rf $ARCHIVE_INSTALL_DIR
      mckdir $ARCHIVE_INSTALL_DIR

      if [ "$FLAT" = "yes" ]
      then
        mv $ARCHIVE_TMP_ROOT $ARCHIVE_INSTALL_DIR
      else
        for DIR in $ARCHIVE_TMP_SRPM_ROOT/*; do
          mv $DIR $ARCHIVE_INSTALL_DIR/${DIR##*/} # ${DIR##*/} = obtain the basename
        done
      fi

      CACHE_INSTALL_DIR=$ARCHIVE_TST_ROOT/cache
      mckdir $CACHE_INSTALL_DIR

      ( echo "A test version of the APT repository has been stored at:"
        echo "$ARCHIVE_INSTALL_DIR"
      ) >&3
    else

      ARCHIVE_INSTALL_DIR=${ARCHIVE_ROOT%/*} # retrieve the dirname
      CACHE_INSTALL_DIR=$CACHEDIR

      if [ -d $ARCHIVE_ROOT ]; then

        ARCHIVE_OLD_ROOT=$REP_DIR/$ARCHIVE.old

        # Create all required directories (a/b/c/...).  The "-p" mkdir argument
        # takes care that no error message is reported in case the install
        # directory already exists.
        mckdir $ARCHIVE_OLD_ROOT

        # Remove the data from the previous run
        rm   -rf $ARCHIVE_OLD_ROOT
        mv $ARCHIVE_ROOT $ARCHIVE_OLD_ROOT
        chmod go-r $ARCHIVE_OLD_ROOT
      else
        mckdir $ARCHIVE_INSTALL_DIR
      fi

      echo -n "Installing apt repository at: " >&3

      if [ "$FLAT" = "yes" ]
      then
        echo "$ARCHIVE_ROOT" >&3
        mv $ARCHIVE_TMP_ROOT $ARCHIVE_INSTALL_DIR

        MV_EXIT_VAL=$?
        ### audit code ###
        ( echo "mv exit value: $MV_EXIT_VAL"
          echo
          echo "Free diskspace:"
          df . /tmp
          echo

        ) >> $AUDIT_FILE

        if [ $MV_EXIT_VAL -ne 0 ]; then
          DISPLAY_AUDIT=yes
        fi
        ### audit code ###

        # Make sure that the SRPMS directories are removed in case the
        # previous repository was a noflat repository
        rm -rf $ARCHIVE_INSTALL_DIR/SRPMS.*
      else
        echo "$ARCHIVE_INSTALL_DIR" >&3

        for DIR in ${ARCHIVE_TMP_ROOT%/*}/*; do
          DEST=${DIR##*/} # retrieve the basename
          if [ -d $ARCHIVE_INSTALL_DIR/$DEST ]; then
            rm -rf $ARCHIVE_INSTALL_DIR/$DEST
          fi
          mv $DIR $ARCHIVE_INSTALL_DIR/$DEST
        done
      fi

      ### audit code ###
      cd $ARCHIVE_BASE_DIR
      FILES=$(find . -size 0 -name "*.bz2")

      ( echo "Contents of base dir: $ARCHIVE_BASE_DIR"
        ls -l
        echo

      ) >> $AUDIT_FILE

      cd $TOPDIR

      if [ -n "$FILES" ]; then
        DISPLAY_AUDIT=yes
      fi
    fi

    ### audit code ###
    if [ "$DISPLAY_AUDIT" = "yes" ] ;then
      ( echo "error: unexpected 0 byte files detected in $ARCHIVE_BASE_DIR"
        echo
        echo -n "The information below should help to find the root cause of "
        echo "this problem."
        echo
        cat $AUDIT_FILE
      ) >&2
    fi
    rm -f $AUDIT_FILE
    ### audit code ###

    # Save the caches
    for COMPONENT in $COMP_LIST
    do
      FILE=$TMPDIR/build_cache::cache.$COMPONENT
      if [ -r $FILE ]; then
        mv $FILE $CACHE_INSTALL_DIR/$COMPONENT
      fi
    done

    TIME=$(date '+%s')
    if [ -n "$CACHE_CHANGED_COMP_QUEUE" ]; then
      cache_adm common $TIME $CACHE_CHANGED_COMP_QUEUE
      unset CACHE_CHANGED_COMP_QUEUE
    fi

    if [ -n "$ACTION_CHANGED_COMP_QUEUE" ]; then
      cache_adm $TARGET $TIME $ACTION_CHANGED_COMP_QUEUE
      unset ACTION_CHANGED_COMP_QUEUE
    fi

    clean_up_rep_tmpdir "$LOCK_TMPDIR"
    unset LOCK_TMPDIR

  fi # if [ $MK_REP = yes ]

  clean_up_distid_lock
  unset LOCK_DISTID

done # for DISTID in $DISTROS

clean_up_tmpdir

# Prevent an exit trap call while exiting aptate
trap EXIT

echo "$PRGNAME finished successfully" >&3

