#!/bin/bash

# Source separate config file if given
CONFIG_FILE="$1"
[[ -r "$CONFIG_FILE" ]] && . "$CONFIG_FILE"

# Sync source
UPSTREAM=${UPSTREAM:-""}
UPSTREAM_DIR=${UPSTREAM_DIR:-""}

# Sync destination
LOCAL_DIR=${LOCAL_DIR:-""}

#declare -A DIST_COMPONENTs

# Optional fetch configuration
FETCH_I18N=${FETCH_I18N:-"yes"}             # Fetch translations
FETCH_SOURCES=${FETCH_SOURCES:-"no"}        # Fetch packages sources
FETCH_CONTENTS=${FETCH_CONTENTS:-"no"}      # Fetch 'Contents' file for distro
FETCH_INSTALLER=${FETCH_INSTALLER:="no"}    # Fetch separate 'debian-installer'
FETCH_DIFF=${FETCH_DIFF:-"no"}              # Fetch diffs of 'Packages' 
FETCH_INDICES=${FETCH_INDICES:-"yes"}       # Fetch indices 

# Misc
ARCH_ALL_IS_MANDATORY=${ARCH_ALL_IS_MANDATORY:-"no"}

#------------------------------------------------------------------------------#
POSSIBLE_COMPRESSIONS=( gz bz2 xz lzma )

BINROOT=$(dirname `readlink -f "$0"`)

. $BINROOT/util/msgs.sh
. $BINROOT/util/rsync.sh
. $BINROOT/util/dpkg.sh
. $BINROOT/util/checksum.sh

[[ -n "$UPSTREAM" ]] || fatal "UPSTREAM is not defined in config"
[[ -n "$UPSTREAM_DIR" ]] || fatal "UPSTREAM_DIR is not defined in config"
[[ -n "$LOCAL_DIR" ]] || fatal "LOCAL_DIR is not defined in config"
[[ -n "${ARCHs[@]}" ]] || fatal "ARCHs is not defined in config"
[[ -n "${DISTs[@]}" ]] || fatal "DISTs is not defined in config"

#------------------------------------------------------------------------------#
# Checks if 'value' contained within 'array'
# USAGE: contains 'what' 'where'
# $1 -- value to find in array
# $2 -- array to search
contains()
{
    local e
    for e in "${@:2}"; do [[ "$e" = "$1" ]] && return 0; done
    return 1
}

on_SIGINT()
{
    fatal "Got user interrupt, aborting"
    exit 1
}

#------------------------------------------------------------------------------#
# MAIN()
#------------------------------------------------------------------------------#
# Trap user abort
trap "on_SIGINT" INT

info "Started $0 $*"
debug "Upstream source is: $UPSTREAM::$UPSTREAM_DIR"
debug "Local dir is: $LOCAL_DIR"
debug "Architectures to sync: ${ARCHs[@]}"
debug "Dists to sync: ${DISTs[@]}"
debug "FETCH_I18N:              $FETCH_I18N           "
debug "FETCH_SOURCES:           $FETCH_SOURCES        "
debug "FETCH_CONTENTS:          $FETCH_CONTENTS       "
debug "FETCH_INSTALLER:         $FETCH_INSTALLER      "
debug "FETCH_DIFF:              $FETCH_DIFF           "
debug "FETCH_INDICES:           $FETCH_INDICES        "
debug "ARCH_ALL_IS_MANDATORY:   $ARCH_ALL_IS_MANDATORY"
debug "POSSIBLE_COMPRESSIONS:   ${POSSIBLE_COMPRESSIONS[@]}"
debug "BINROOT:                 $BINROOT              "
debug "PARTIAL_UPSTREAM:        $PARTIAL_UPSTREAM     "
debug "PARTIAL_UPSTREAM_PATH:   $PARTIAL_UPSTREAM_PATH"

# Create dirs
mkdir -p $LOCAL_DIR/dists
mkdir -p $LOCAL_DIR/pool

# Array of Packages files, that contains package descriptions
packages_pool_files=()
sources_pool_files=()

if rsync_file_exists "."; then
    info "Upstream mirror $UPSTREAM supports rsync protocol"
else
    fatal "Upstream mirror $UPSTREAM does not support rsync protocol, aborting"
fi

debug_job_start "Checking if upstream mirror update is in progress..."
while rsync_file_exists "Archive-Update-in-Progress*"; do
    info "'Archive-Update-in-Progress' file found on upstream mirror. Sleeping for 20 seconds"
    sleep 20
done ; debug_job_ok

################################################################################
# Stage 1
# Download metainformation files
################################################################################
for dist in "${DISTs[@]}"; do
    info "Fetching dist '$dist' lists"
    ############################################################################
    # Phase 1: Check if we have aliased distro and create necessary symlinks
    # aliases is specified after '@'-sign in dist name, separated by commas
    # For example: 'wheezy@testing,somealias' means dist 'wheezy' with symlinks
    # 'testing' and 'somealias' pointing to it
    ############################################################################
    # TODO: get aliases from Release suite
    if [ -n ${dist#*@} ]; then
        normal_dist="${dist%%@*}"
        for dist_alias in `echo ${dist#*@} | tr ',' ' '`; do
            if [[ "$dist_alias" == "$normal_dist" ]]; then
                continue
            fi

            if [ ! -L $LOCAL_DIR/dists/$dist_alias ]; then
                debug "Creating dist alias '$dist_alias' -> '$normal_dist'"
                ln -s "$normal_dist" "$LOCAL_DIR/dists/$dist_alias" || \
                    error "Error creating alias for $normal_dist ($dist_alias)" 
            fi
        done
        dist="$normal_dist"
        unset normal_dist
    fi

    ############################################################################
    # Phase 2: Create distribution dir
    ############################################################################
    mkdir -p $LOCAL_DIR/dists/$dist/

    ############################################################################
    # Phase 3: Fetch Release files
    # InRelease uses new scheme of inline Release signing
    # Old scheme implies separate 'Release' and 'Release.gpg' files
    ############################################################################
    debug "Fetching Release files"
    for rel_file in InRelease Release Release.gpg; do
        fetch "/dists/$dist/$rel_file" "$LOCAL_DIR/dists/$dist/"
    done
    release_file="$LOCAL_DIR/dists/$dist/InRelease"

    # Check InRelease file
    if [ -f "$release_file" ]; then
        inrl_valid=$(date -d "`grep Valid-Until /srv/mirror/debian/debian_bg/dists/wheezy-updates/InRelease  | awk '{$1=""; print $0}'`" +%s)
        now=$(date +%s)
        if [[ $(( $now - $inrl_valid )) -gt -86400 ]]; then
            info "InRelease file will expire before the next update, removing it..."
            rm -f "$release_file"
            release_file="$LOCAL_DIR/dists/$dist/Release"
        fi
    else
        release_file="$LOCAL_DIR/dists/$dist/Release"
    fi
    [ -f "$release_file" ] || fatal "Unable to find release file for dist $dist"
    debug "Got Release file '$release_file'"

    ############################################################################
    # Phase 4: check release signature
    ############################################################################
    if [[ "$release_file" =~ ".*InRelease$" ]]; then
        gpg --verify "$release_file" || \
            fatal "Failed to check signature for $release_file"
    elif [[ "$release_file" =~ ".*Release" ]]; then
        gpg --verify "${release_file}.gpg" "${release_file}" || \
            fatal "Failed to check signature for $release_file"
    fi

    ############################################################################
    # Phase 5: Determine which components and arches to download
    # Case A: If we have user specified component list, and hasn't found any 
    #         in distro description, then blindly use user given values
    # Case B: If we have no user specified component list, try to get them from
    #         repository Release file, if it fails - bail out
    # Case C: If we have both, calculate intersection of them
    ############################################################################
    debug "Calculating arches/components to fetch from dist"
    components=`get_dist_components $release_file "${DIST_COMPONENTs[$dist]}"`
    arches=`get_dist_architectures $release_file ${ARCHs[@]}`

    # Phase 6: Fork components into binary_components
    # That allows us to add special sub-components specific to binary components
    # such as 'debian-installer'
    binary_components="$components"

    ############################################################################
    # Phase 7: Check if we must fetch 'debian-installer' sub-components and add 
    # them to the binary_components list if needed
    ############################################################################
    if [[ "$FETCH_INSTALLER" = "yes" ]]; then
        for component in $components; do
            if rsync_file_exists "dists/$dist/$component/debian-installer"; then
                debug "Adding debian-installer '$component/debian-installer'"
                binary_components="$binary_components $component/debian-installer"
            else
                error "Not found debian-installer at '$component/debian-installer'"
            fi
        done
    fi

    ############################################################################
    # Phase 8: Fetch binary components 'Packages' indexes and diffs
    ############################################################################
    debug "Will fetch binary components: $binary_components"
    for component in $binary_components; do
        info "Fetching component '$component' binary package lists"
        # Component path
        comp_path="dists/$dist/$component"

        # Create component dir
        mkdir -p "$LOCAL_DIR/$comp_path"

        # First, fetch binary packages lists
        for arch in $arches; do
            arch_path="$comp_path/binary-$arch"

            # Check if remote the dir exists
            if ! rsync_file_exists "$arch_path"; then
                # Missing 'all' architecture in a non critical error 
                if [[ "$arch" = "all" ]] && [[ "$ARCH_ALL_IS_MANDATORY" != "yes" ]]; then
                    debug "Missing 'all' architecture in $dist/$component"
                    continue
                fi
                fatal "Arch '$arch' in '$dist/$component' doesn't exist"
            fi

            # Prepare component dir
            mkdir -p "$LOCAL_DIR/$arch_path"
            to_fetch=()

            # List of files that we want to dl
            to_fetch+=( "$arch_path/Release" )
            to_fetch+=( "$arch_path/Packages" )
            for ext in ${POSSIBLE_COMPRESSIONS[@]}; do
                to_fetch+=( "$arch_path/Packages.$ext" )
            done

            # Check if we want a Packages.diff files Index too
            if [[ "$FETCH_DIFF" = "yes" ]] && \
                rsync_file_exists "$arch_path/Packages.diff/Index"; then
                to_fetch+=( `rsync_ls "$arch_path/Packages.diff/*"` )
            fi

            # Download files in our wishlist and get names of actually
            # downloaded files
            fetched_files=`fetch_all "$LOCAL_DIR" ${to_fetch[@]}`

            # Verify all fetched files
            for file in ${fetched_files[@]}; do
                # Skip checking of diffs, they are mentioned in Index file
                # Validate only Index file
                if [[ "`dirname $file`" = "$LOCAL_DIR/$arch_path/Packages.diff" ]]; then
                    [[ "`basename $file`" != "Index" ]] && continue
                fi

                # Check file by Release file's checksum
                debug_job_start "Checking file $file"
                pkg_file_valid "$release_file" "${file#$LOCAL_DIR/dists/$dist/}" || \
                    fatal "Checksum check failed for $file"
                debug_job_ok
            done

            # Make sure that we have at least one valid packages list
            packages_file=`guess_filename "$LOCAL_DIR/$arch_path/Packages"`
            if [[ -z "$packages_file" ]]; then
                fatal "Failed to find Packages file at $arch_path"
            fi

            # Check integrity of .diffs if we got them
            diff_index=`guess_filename "$LOCAL_DIR/$arch_path/Packages.diff/Index"`
            if [[ "$FETCH_DIFF" = "yes" ]] && [[ -n "$diff_index" ]]; then
                diffs=`cat $diff_index | awk '/SHA1-Patches:/,0' | tail -n +2 | awk '{print $3}'`
                for diff in $diffs; do
                    debug_job_start "Checking file $LOCAL_DIR/$arch_path/Packages.diff/$diff"
                    diff_exp_sha1=`cat $diff_index | awk '/SHA1-Patches:/,0' | grep "$diff" | awk '{print $1}'`
                    diff_real_sha1=`read_file "$LOCAL_DIR/$arch_path/Packages.diff/$diff" | sha1sum | awk '{print $1}'`
                    if [[ "$diff_exp_sha1" != "$diff_real_sha1" ]]; then
                        debug_job_err
                        error "Checksum failed on file $arch_path/Packages.diff/$diff, removing all diffs"
                        rm -rf "$LOCAL_DIR/$arch_path/Packages.diff"
                        break
                    fi
                    debug_job_ok
                done
            fi

            # Parse package file and add packages from it to dl list
            packages_pool_files+=( "$packages_file" )
        done
    done

    ############################################################################
    # Phase 9: Fetch additional stuff for components, i18n, sources, 'Contents'
    ############################################################################
    for component in $components; do
        comp_path="dists/$dist/$component"
        mkdir -p "$LOCAL_DIR/$comp_path"

        # Second, the i18n packages
        info "Fetching section '$component' i18n"
        if [[ "$FETCH_I18N" = "yes" ]]; then
            mkdir -p "$LOCAL_DIR/$comp_path/i18n/"
            to_fetch=()
            to_fetch+=( "$comp_path/i18n/Index" )
            for i18n in ${I18Ns[@]}; do
                to_fetch+=( "$comp_path/i18n/Translation-$i18n" )
                # Translation files may have diffs too
                to_fetch+=( "$comp_path/i18n/Translation-$i18n.diff/*" )
                for ext in ${POSSIBLE_COMPRESSIONS[@]}; do
                    to_fetch+=( "$comp_path/i18n/Translation-$i18n.$ext" )
                done

            # Download files in our wishlist and get names of actually
            # downloaded files
            fetched_files=`fetch_all "$LOCAL_DIR" ${to_fetch[@]}`

            # Verify
            for file in ${fetched_files[@]}; do
                # Skip checking of diffs, except it's Index file
                if [[ "`dirname $file`" = "$LOCAL_DIR/$comp_path/i18n/Translation-$i18n.diff" ]]; then
                    [[ "`basename $file`" != "Index" ]] && continue
                fi

                debug_job_start "Checking file $file"
                pkg_file_valid "$release_file" "${file#$LOCAL_DIR/dists/$dist/}" || \
                    fatal "Checksum check failed for $file"
                debug_job_ok
            done

            # Check integrity of .diffs if we got them
            diff_index=`guess_filename "$LOCAL_DIR/$comp_path/i18n/Translation-$i18n.diff/Index"`
            if [[ -n "$diff_index" ]]; then
                diffs=`cat $diff_index | awk '/SHA1-Patches:/,0' | tail -n +2 | awk '{print $3}'`
                for diff in $diffs; do
                    debug_job_start "Checking file $LOCAL_DIR/$comp_path/i18n/Translation-$i18n.diff/$diff"
                    diff_exp_sha1=`cat $diff_index | awk '/SHA1-Patches:/,0' | grep "$diff" | awk '{print $1}'`
                    diff_real_sha1=`read_file "$LOCAL_DIR/$comp_path/i18n/Translation-$i18n.diff/$diff" | sha1sum | awk '{print $1}'`
                    if [[ "$diff_exp_sha1" != "$diff_real_sha1" ]]; then
                        debug_job_err
                        fatal "Checksum failed on file $comp_path/i18n/Translation-$i18n.diff/$diff"
                    fi
                    debug_job_ok
                done
            fi
            done
        fi

        # Third is the Sources
        if [[ "$FETCH_SOURCES" = "yes" ]]; then
            info "Fetching component '$component' source package lists"
            mkdir -p "$LOCAL_DIR/$comp_path/source/"
            to_fetch=()
            to_fetch+=( "$comp_path/source/Release" )
            to_fetch+=( "$comp_path/source/Sources" )
            for ext in ${POSSIBLE_COMPRESSIONS[@]}; do
                to_fetch+=( "$comp_path/source/Sources.$ext" )
            done

            # Download files in our wishlist and get names of actually
            # downloaded files
            fetched_files=`fetch_all "$LOCAL_DIR" ${to_fetch[@]}`

            # Verify
            for file in ${fetched_files[@]}; do
                pkg_file_valid "$release_file" "${file#$LOCAL_DIR/dists/$dist/}" || \
                    fatal "Checksum check failed for $file"
            done

            sources_file=`guess_filename "$LOCAL_DIR/$comp_path/source/Sources"`
            if [[ -z "$sources_file" ]]; then
                fatal "Failed to find Sources file at $LOCAL_DIR/$comp_path/source"
            fi


            # Parse sources file and add packages from it to dl list
            sources_pool_files+=( "$sources_file" )
        fi

        # Fetch the component contents packs
        if [[ "$FETCH_CONTENTS" = "yes" ]]; then
            info "Fetching component '$component' content lists"
            to_fetch=()
            for arch in $arches; do
                to_fetch+=( "$comp_path/Contents-$arch" )
                for ext in ${POSSIBLE_COMPRESSIONS[@]}; do
                    to_fetch+=( "$comp_path/Contents-$arch.$ext" )
                done
            done
            # Download files in our wishlist and get names of actually
            # downloaded files
            fetched_files=`fetch_all "$LOCAL_DIR" ${to_fetch[@]}`

            # Verify
            for file in ${fetched_files[@]}; do
                pkg_file_valid "$release_file" "${file#$LOCAL_DIR/dists/$dist/}" || \
                    fatal "Checksum check failed for $file"
            done

            # If our component is "main", make link in the root of distribution
            if [[ "$component" = "main" ]]; then
                for arch in $arches; do
                    if [[ -e "$LOCAL_DIR/dists/$dist/$component/Contents-$arch.gz" ]]; then
                        debug "Creating link to main/Contents-$arch.gz at $LOCAL_DIR/dists/$dist"
                        ln -sf main/Contents-$arch.gz $LOCAL_DIR/dists/$dist/Contents-$arch.gz
                    else
                        debug "Deleting link to main/Contents-$arch.gz at $LOCAL_DIR/dists/$dist"
                        rm -f "$LOCAL_DIR/dists/$dist/Contents-$arch.gz"
                    fi
                done
            fi
        fi
    done
done

    # Get the indices
    if [[ "$FETCH_INDICES" = "yes" ]]; then
        info "Fetching indices"
        mkdir -p "$LOCAL_DIR/indices/"
        for dist in "${DISTs[@]}"; do
            fetch "/indices/override.$dist.*" "$LOCAL_DIR/indices/"
        done
    fi

################################################################################
# Stage 2
# Download pool of packages
################################################################################

info "Parsing package and sources files:"
info "${packages_pool_files[@]}"
info "${sources_pool_files[@]}"

files_to_dl_list=`mktemp --suffix="-deb-mirror"`
# File that contains md5sums of deb pkgs
deb_md5=`mktemp --suffix="-deb-mirror-md5"`
rsync_log=`mktemp --suffix="-deb-mirror-rslog"`
sort_temp=`mktemp --suffix="-deb-mirror-sort"`

$BINROOT/util/parsePackages.py ${packages_pool_files[@]} > "$files_to_dl_list" \
                                                        2> "$deb_md5" && \
$BINROOT/util/parseSources.py ${sources_pool_files[@]}  >> "$files_to_dl_list" || \
    fatal "Unable to create list of packages to fetch"

sort -u "$files_to_dl_list" > "$sort_temp" ; mv "$sort_temp" "$files_to_dl_list"
sort -u -k 3,3 "$deb_md5" > "$sort_temp" ; mv "$sort_temp" "$deb_md5"

# If partial mirroring is enabled, get the list of partial mirror packages
if [[ "$PARTIAL_UPSTREAM" = "1" ]]; then

info "Resolving dependencies for partial mirror"

# Detect kernel version of debian-installer
fuel_config=$BINROOT/config/config.mk
export UBUNTU_RELEASE=`awk -F '[:=?]' '/^UBUNTU_RELEASE\>/ {print $NF}' $fuel_config`
export UBUNTU_NETBOOT_FLAVOR=`awk -F '[:=?]' '/^UBUNTU_NETBOOT_FLAVOR\>/ {print $NF}' $fuel_config`
export UBUNTU_KERNEL_FLAVOR=`awk -F '[:=?]' '/^UBUNTU_KERNEL_FLAVOR\>/ {print $NF}' $fuel_config`
export UBUNTU_ARCH=`awk -F '[:=?]' '/^UBUNTU_ARCH\>/ {print $NF}' $fuel_config`

INITRD_DIR="/dists/${UBUNTU_RELEASE}-updates/main/installer-${UBUNTU_ARCH}/current/images/${UBUNTU_NETBOOT_FLAVOR}/ubuntu-installer/${UBUNTU_ARCH}/"
mkdir -p "$LOCAL_DIR/$INITRD_DIR"
fetch "/$INITRD_DIR/initrd.gz" "$LOCAL_DIR/$INITRD_DIR"
export UBUNTU_INSTALLER_KERNEL_VERSION=`zcat "$LOCAL_DIR/$INITRD_DIR/initrd.gz" | cpio --list 'lib/modules/*/kernel' 2>/dev/null | cut -d"/" -f 3`
debug "Detected debian-installer kernel version: "$UBUNTU_INSTALLER_KERNEL_VERSION

# Generate list of MOS dependencies
export apt_altstate=`mktemp -d --suffix="-apt-altstate"`
export BINROOT

if [[ "$DOCKER_MODE" = "true" ]]; then
        ( docker ps -a | grep fuel-createmirror ) && docker rm -f fuel-createmirror 
        # docker pull ubuntu:latest
        docker -D run -d --name=fuel-createmirror --net=host -a stdout -a stderr -t \
               -e UBUNTU_RELEASE=$UBUNTU_RELEASE -e UBUNTU_NETBOOT_FLAVOR=$UBUNTU_NETBOOT_FLAVOR \
               -e UBUNTU_INSTALLER_KERNEL_VERSION=$UBUNTU_INSTALLER_KERNEL_VERSION -e UBUNTU_KERNEL_FLAVOR=$UBUNTU_KERNEL_FLAVOR \
               -e RSYNC_PROXY=$RSYNC_PROXY -e http_proxy=$http_proxy \
               -e UBUNTU_ARCH=$UBUNTU_ARCH -e BINROOT=$BINROOT \
               -e apt_altstate=$apt_altstate -v $BINROOT:$BINROOT:rw -v $apt_altstate:$apt_altstate:rw ubuntu:latest
        dockerctl shell fuel-createmirror $BINROOT/util/partial_ubuntu.sh || fatal "Cannot calculate list of dependencies"
        # cleanup ubuntu container
        docker rm -f fuel-createmirror
else
        $BINROOT/util/partial_ubuntu.sh || fatal "Cannot calculate list of dependencies"
fi

# Create download lists for deb and udeb
awk 'FNR==NR {arr[$0];next} $3 in arr' $apt_altstate/deb "$deb_md5" > $apt_altstate/deb_md5

grep "\.udeb$" "$files_to_dl_list" | egrep -v "generic|virtual" > $apt_altstate/udeb_nonkernel
grep "\.udeb$" "$files_to_dl_list" | egrep "generic|virtual" | grep $UBUNTU_INSTALLER_KERNEL_VERSION > $apt_altstate/udeb_kernel
cat $apt_altstate/udeb_nonkernel $apt_altstate/udeb_kernel | sort -u > $apt_altstate/udeb
awk 'FNR==NR {arr[$0];next} $3 in arr' $apt_altstate/udeb "$deb_md5" > $apt_altstate/udeb_md5

cat $apt_altstate/netboot.list $apt_altstate/udeb $apt_altstate/deb > "$files_to_dl_list"
cat $apt_altstate/netboot_md5.list $apt_altstate/udeb_md5 $apt_altstate/deb_md5 > "$deb_md5"

rm -rf "$apt_altstate"

fi # "$PARTIAL_UPSTREAM" = "1"

info "Downloading pool files"
rsync --verbose --out-format="%i %n" --stats \
    --recursive --perms --copy-links --times --hard-links --sparse --safe-links \
    --exclude=".tmp/" --exclude=".temp/" --exclude=".~tmp~/" \
    --files-from="$files_to_dl_list" \
    --bwlimit=5192 \
    "${UPSTREAM}::${UPSTREAM_DIR}/" "$LOCAL_DIR" | tee "$rsync_log"

#    --files-from="$files_to_dl_list" \--block-size=8192 
#--max-delete=40000 --delay-updates --delete --delete-after \
# Check if rsync was ok
if [[ $? != 0 ]]; then
    rm "$files_to_dl_list"
    fatal "Failed to sync all package files, see log for details"
    #error "Failed to sync all package files, see log for details"
else
    info "Primary sync successfully completed"
fi

# Let's check new file MD5sums
fresh_files=`egrep "^>f......... .*" "$rsync_log" | awk '{print $2}'`

for fresh_file in $fresh_files; do 
    check_file "$deb_md5" "$LOCAL_DIR" "$fresh_file"
    if [[ $? != 0 ]]; then 
        rm "$deb_md5"
        rm "$rsync_log"
        fatal "MD5sum check failed for file $LOCAL_DIR/$fresh_file"
    fi
done
rm "$deb_md5"
rm "$rsync_log"

# Now iterate through all downloaded files and check if any of them are symlink
# download neccessary files if needed
# Yeah, some times section can contain a metainfo for symlink to file in 
# diffirent section that is no longer exists in there, so it will be wiped as
# unused
wayback="`pwd`"
cd "$LOCAL_DIR/"

pool_current_files=`mktemp --suffix d-m_got`
pool_required_files=`mktemp --suffix d-m_req`

# Create lists of files that we got and that we need
find pool -type f -or -type l | sort -u > $pool_current_files

cat $files_to_dl_list | grep "^pool" | sort -u > $pool_required_files
cd "$wayback"

info "Cleaning up pool files"
# Clean obsolete files
obsolete_files=`comm -3 -2 "$pool_current_files" "$pool_required_files"`
for file in $obsolete_files; do
    debug_job_start "Deleting '$LOCAL_DIR/$file'"
    rm "$LOCAL_DIR/$file" && debug_job_ok || debug_job_err
done

info "Doublechecking that required pool files exists"
missing_files=`comm -3 -1 "$pool_current_files" "$pool_required_files"`

if [[ -n "$missing_files" ]]; then
    error "Some files are missing after sync!!!:"
    error "$missing_files"
    fatal "Aborting due to missing files"
fi

rm "$files_to_dl_list"
rm "$pool_required_files"
rm "$pool_current_files"

# Timestamp
echo "Updated at: `date`" > $LOCAL_DIR/.lastupdate

# If partial mirroring is enabled, get the list of partial mirror packages
if [[ "$PARTIAL_UPSTREAM" = "1" ]]; then

# Prepare directory structure for partial repository
info "Generating partial mirror"
mkdir -p ${PARTIAL_UPSTREAM_PATH}/pool/debian-installer
mkdir -p ${PARTIAL_UPSTREAM_PATH}/pool/main
mkdir -p ${PARTIAL_UPSTREAM_PATH}/indices
mkdir -p ${PARTIAL_UPSTREAM_PATH}/dists/${UBUNTU_RELEASE}/main/binary-amd64
mkdir -p ${PARTIAL_UPSTREAM_PATH}/dists/${UBUNTU_RELEASE}/main/debian-installer/binary-amd64
mkdir -p ${PARTIAL_UPSTREAM_PATH}/${INITRD_DIR}
temp_dir=`mktemp -d --suffix="-reposync"`
find $LOCAL_DIR/pool/ -name *.deb -type f -exec cp -vuni '{}' ${temp_dir} ";"
rsync -a --delete ${temp_dir}/ ${PARTIAL_UPSTREAM_PATH}/pool/main
rm -f ${temp_dir}/*
find ${LOCAL_DIR}/pool/ -name *.udeb -type f -exec cp -vuni '{}' ${temp_dir} ";"
rsync -a --delete ${temp_dir}/ ${PARTIAL_UPSTREAM_PATH}/pool/debian-installer
rm -rf ${temp_dir}
rsync -a --delete ${LOCAL_DIR}/${INITRD_DIR}/ ${PARTIAL_UPSTREAM_PATH}/${INITRD_DIR}
find ${PARTIAL_UPSTREAM_PATH} -type d -print0 | xargs -0 chmod 755

# Generate "indices" folder
cat $LOCAL_DIR/indices/*extra* | sort -u > ${PARTIAL_UPSTREAM_PATH}/indices/override.${UBUNTU_RELEASE}.extra.main
cat $LOCAL_DIR/indices/*.debian-installer | sort -u > ${PARTIAL_UPSTREAM_PATH}/indices/override.${UBUNTU_RELEASE}.main.debian-installer
pushd $LOCAL_DIR/indices/
ls --ignore="*extra*" --ignore="*src" --ignore="*debian-installer" --quoting-style=shell | xargs cat | sort -u > ${PARTIAL_UPSTREAM_PATH}/indices/override.${UBUNTU_RELEASE}.main
popd

# Generate Release file
cat <<EOF > ${PARTIAL_UPSTREAM_PATH}/dists/${UBUNTU_RELEASE}/Release
Architectures: amd64
Codename: ${UBUNTU_RELEASE}
Components: main
Date: `date`
Description: Ubuntu ${UBUNTU_RELEASE} partial mirror
Label: Ubuntu
Origin: Ubuntu
Suite: ${UBUNTU_RELEASE}
EOF

# Build partial mirror
info "Generating metadata for partial mirror"
info "Applying fix for upstream dpkg-scanpackages"
patch -N /usr/bin/dpkg-scanpackages < $BINROOT/util/dpkg.patch
export BINROOT
$BINROOT/util/regenerate_ubuntu_repo ${PARTIAL_UPSTREAM_PATH} ${UBUNTU_RELEASE} || fatal "Failed to generate partial mirror"
rm -rf $LOCAL_DIR
fi # "$PARTIAL_UPSTREAM" = "1"
info "Done"