#!/bin/bash # Check command line arguments and set options # Run before loading configuration, to allow branch overriding flag_INCREMENTAL="n" flag_VERBOSE_LOG="n" flag_SWITCH_ONLY="n" flag_SKIP_TAGS="n" OVERRIDE_DEFAULT_REPO_BRANCH="" for ((idx=1; idx<=$#; idx++)); do arg="${!idx}" if [ "$arg" = "-i" ]; then # continue from last updated module (Incremental) flag_INCREMENTAL="y" elif [ "$arg" = "-v" ]; then # display and log git command output (Verbose) flag_VERBOSE_LOG="y" elif [ "$arg" = "-ub" ]; then # branches to update (Update Branches) ((idx++)) OVERRIDE_UPDATE_BRANCHES="${!idx}" [[ -z "$OVERRIDE_DEFAULT_REPO_BRANCH" ]] && OVERRIDE_DEFAULT_REPO_BRANCH="${!idx}" elif [ "$arg" = "-db" ]; then # default branch after update (Default Branch) ((idx++)) if [[ "$flag_SWITCH_ONLY" != 'y' ]]; then # '-db' is only used if no '-so' argument is specified. If '-so ' # is given, '-db is ignored OVERRIDE_DEFAULT_REPO_BRANCH="${!idx}" fi elif [ "$arg" = "-so" ]; then # switch branch only (Switch Only) flag_SWITCH_ONLY="y" && ((idx++)) if ! [[ "${!idx}" =~ ^- ]]; then OVERRIDE_DEFAULT_REPO_BRANCH="${!idx}" else ((idx--)) fi fi done # Load common code export SCRIPT_DIR=$(dirname $(readlink -f "$0")) . ${SCRIPT_DIR}/internals/_build_common.sh init_common UPDATE_LOCK_FILENAME="/var/lock/TDE_update_repo_lock" # Lock file for incremental update declare -A RESULT_STRINGS=( [OK]="[ OK ]" [FAIL]="[ FAIL ]" [UPDATE]="[ UPDATE ]" [SKIP]="[ SKIP ]" [INV-OP]="[ INV-OP ]" ) declare -A COLOR_STRINGS=( [OK]="${CNone}" [FAIL]="${CLightRed}" [UPDATE]="${CLightGreen}" [SKIP]="${CBrown}" [INV-OP]="${CLightPurple}" ) #---------------------------- # Update a given module from the upstream repo # Parameters: # $1 - module folder # $2 - operation type # $3 - branch to update function _do_update() { local MOD_PATH=$1 local OP_TYPE=$2 local BRANCH=$3 local RESULT="" local LINE_CTRL_SUFFIX="" case "$OP_TYPE" in "fetch") if [[ -z `grep "^$BRANCH - $MOD_PATH$" "$UPDATE_LOCK_FILENAME"` ]]; then cd "$MOD_PATH" &>/dev/null eval git fetch --all --prune $GIT_NO_RECURSE_SUBMODULES $OPT_VERBOSE_LOG if [ $? -eq 0 ]; then RESULT="OK" else RESULT="FAIL" fi echo "$BRANCH - $MOD_PATH" >> "$UPDATE_LOCK_FILENAME" else RESULT="SKIP" fi ;; "update") cd "$MOD_PATH" &>/dev/null if [ $? -eq 0 ]; then # Clean up any possible uncommitted changes if [[ ! -z "`git status --porcelain $GIT_IGNORE_SUBMODULES`" ]]; then git reset --hard HEAD &>/dev/null git clean -dxff &>/dev/null fi # Make sure the local branch exists if [[ -z `git branch | grep -E "\b$BRANCH\b"` ]]; then eval git checkout -b \"$BRANCH\" \"origin/$BRANCH\" $OPT_VERBOSE_LOG else eval git checkout \"$BRANCH\" $OPT_VERBOSE_LOG fi # Make sure the local branch is a tracking branch if [[ -z `git config branch."$BRANCH".remote` ]]; then git branch -u "origin/$BRANCH" &>/dev/null #$ git reset --hard "origin/$BRANCH" &>/dev/null fi # Update eval git reset --hard HEAD $OPT_VERBOSE_LOG eval git clean -dxff $OPT_VERBOSE_LOG if [[ $(git rev-parse HEAD) != $(git rev-parse "origin/$BRANCH") ]]; then eval git rebase $OPT_VERBOSE_LOG if [[ `git rev-parse HEAD` == `git rev-parse "origin/$BRANCH"` ]]; then RESULT="UPDATE" else RESULT="FAIL" fi else RESULT="OK" fi else RESULT="FAIL" fi ;; "switch-to") cd "$MOD_PATH" &>/dev/null eval git checkout \"$BRANCH\" $OPT_VERBOSE_LOG eval git reset --hard HEAD $OPT_VERBOSE_LOG eval git clean -dxff $OPT_VERBOSE_LOG if [[ ! -z `git branch -v | grep -E "^\*\s+$BRANCH"` ]]; then RESULT="OK" else RESULT="FAIL" fi ;; *) RESULT="INV-OP" ;; esac if [ "$RESULT" != "OK" -o "$flag_VERBOSE_LOG" = "y" ]; then LINE_CTRL_SUFFIX="\n" fi echo -ne "\033[2K\r${COLOR_STRINGS[$RESULT]}${RESULT_STRINGS[$RESULT]} $MOD_PATH${CNone}$LINE_CTRL_SUFFIX" echo "${RESULT_STRINGS[$RESULT]} $MOD_PATH" >> "$LOG_UPDATE_REPO_FILENAME" } #---------------------------- # Update a given module from the upstream repo # Parameters: # $1 - module folder and name # $2 - operation type # $3 - module url (for "add_repo") # branch name (for other operations) # $4 - parent module path (only for check_module operation) # $5 - module string (only for check_module operation) function _do_worktree_update() { local MOD_FULLPATH=$1 local MOD_NAME=`echo ${MOD_FULLPATH} | sed "s|^.*/\([^/]\+\)$|\1|"` local OP_TYPE=$2 local MOD_REPO_PATH="${REPOS_DIR}/${MOD_NAME}.git" local RESULT="" local LINE_CTRL_SUFFIX="" local LOCAL_BRANCHES local REMOTE_BRANCHES case "$OP_TYPE" in "add_repo") if [ -z "`grep \"^${MOD_REPO_PATH} - ADD REPO$\" \"${UPDATE_LOCK_FILENAME}\"`" ]; then RESULT="OK" if [ ! -d "${MOD_REPO_PATH}" ]; then RESULT="UPDATE" local MOD_URLNAME="$3" [ -n "${MOD_URLNAME}" ] || MOD_URLNAME="${MOD_NAME}" eval git clone --bare --config "remote.origin.fetch=+refs/heads/*:refs/remotes/origin/*" \ "${ORIGIN_PATH}/${MOD_URLNAME}.git" "${MOD_REPO_PATH}" $OPT_VERBOSE_LOG || RESULT="FAIL" fi if [ "$RESULT" != "OK" -o "$flag_VERBOSE_LOG" = "y" ]; then LINE_CTRL_SUFFIX="\n" fi echo -ne "\033[2K\r${COLOR_STRINGS[$RESULT]}${RESULT_STRINGS[$RESULT]} ${MOD_NAME} - repo added${CNone}$LINE_CTRL_SUFFIX" echo "${RESULT_STRINGS[$RESULT]} ${MOD_NAME} - repo added" >> "$LOG_UPDATE_REPO_FILENAME" # echo "${MOD_REPO_PATH} - ADD REPO" >> "${UPDATE_LOCK_FILENAME}" fi ;; "fetch") if [[ -z `grep "^${MOD_REPO_PATH} - FETCH$" "${UPDATE_LOCK_FILENAME}"` ]]; then cd "${MOD_REPO_PATH}" &>/dev/null RESULT="FAIL" eval git fetch --all --prune $GIT_NO_RECURSE_SUBMODULES $OPT_VERBOSE_LOG && eval git worktree prune $OPT_VERBOSE_LOG && RESULT="OK" if [ "$RESULT" != "OK" -o "$flag_VERBOSE_LOG" = "y" ]; then LINE_CTRL_SUFFIX="\n" fi echo -ne "\033[2K\r${COLOR_STRINGS[$RESULT]}${RESULT_STRINGS[$RESULT]} ${MOD_NAME} - fetched${CNone}$LINE_CTRL_SUFFIX" echo "${RESULT_STRINGS[$RESULT]} ${MOD_NAME} - fetched" >> "$LOG_UPDATE_REPO_FILENAME" # echo "${MOD_REPO_PATH} - FETCH" >> "${UPDATE_LOCK_FILENAME}" fi ;; "check_worktree") # Check/add worktrees LOCAL_BRANCHES=$3 cd "${MOD_REPO_PATH}" &>/dev/null REMOTE_BRANCHES=(`git branch --remote | sed "s|origin/||g"`) for LCL_BR in $LOCAL_BRANCHES; do CURR_BR=`echo "$LCL_BR" | sed -e "s|^[[:space:]]*||" -e "s|[[:space:]]*$||"` local WORK_PATH="${WORKTREES_DIR}/${CURR_BR}/${MOD_FULLPATH}" for REM_BR in "${REMOTE_BRANCHES[@]}"; do if [ "${REM_BR}" = "${CURR_BR}" -a -z "`grep \"^${MOD_FULLPATH} - ${CURR_BR} - ADD WORKTREE$\" \"${UPDATE_LOCK_FILENAME}\"`" ]; then cd "${MOD_REPO_PATH}" &>/dev/null if [[ -z `git worktree list | grep "\[${CURR_BR}\]"` ]]; then RESULT="FAIL" eval git worktree add "${WORK_PATH}" "origin/${CURR_BR}" $OPT_VERBOSE_LOG && cd "${WORK_PATH}" &>/dev/null && eval git checkout ${CURR_BR} $OPT_VERBOSE_LOG && eval git branch --set-upstream-to=origin/${CURR_BR} $OPT_VERBOSE_LOG && RESULT="UPDATE" LINE_CTRL_SUFFIX="" [ "$RESULT" != "OK" -o "$flag_VERBOSE_LOG" = "y" ] && LINE_CTRL_SUFFIX="\n" echo -ne "\033[2K\r${COLOR_STRINGS[$RESULT]}${RESULT_STRINGS[$RESULT]} ${MOD_NAME} - branch ${CURR_BR} - worktree added${CNone}$LINE_CTRL_SUFFIX" echo "${RESULT_STRINGS[$RESULT]} ${MOD_NAME} - branch ${CURR_BR} - worktree added" >> "$LOG_UPDATE_REPO_FILENAME" fi # echo "${MOD_FULLPATH} - ${CURR_BR} - ADD WORKTREE" >> "${UPDATE_LOCK_FILENAME}" break fi done done ;; "check_module") # Make sure submodules are initialized and up to date if necessary if [ -z "`grep \"^${MOD_FULLPATH} - ${CURR_BR} - CHECK MODULE$\" \"${UPDATE_LOCK_FILENAME}\"`" ]; then local PARENT_PATH=$4 local SUBMOD_STR=$5 if [ -n "$PARENT_PATH" -a -n "$SUBMOD_STR" ]; then # Only do this on a real submodule RESULT="OK" cd "$PARENT_PATH" &>/dev/null git config submodule.$SUBMOD_STR.url "$MOD_REPO_PATH" &>/dev/null if [[ -n "`git submodule status -- ${SUBMOD_STR} | grep "^-.*"`" ]]; then RESULT="UPDATE" eval git submodule init -- ${SUBMOD_STR} $OPT_VERBOSE_LOG || RESULT="FAIL" fi if [ "$RESULT" != "FAIL" -a ! -e "${SUBMOD_STR}/.git" ]; then RESULT="UPDATE" eval git submodule update -- ${SUBMOD_STR} $OPT_VERBOSE_LOG || RESULT="FAIL" fi LINE_CTRL_SUFFIX="" [ "$RESULT" != "OK" -o "$flag_VERBOSE_LOG" = "y" ] && LINE_CTRL_SUFFIX="\n" echo -ne "\033[2K\r${COLOR_STRINGS[$RESULT]}${RESULT_STRINGS[$RESULT]} ${MOD_NAME} - module checked${CNone}$LINE_CTRL_SUFFIX" echo "${RESULT_STRINGS[$RESULT]} ${MOD_NAME} - module checked" >> "$LOG_UPDATE_REPO_FILENAME" fi # echo "${MOD_FULLPATH} - ${CURR_BR} - CHECK MODULE" >> "${UPDATE_LOCK_FILENAME}" fi ;; "update") LOCAL_BRANCHES=$3 REMOTE_BRANCHES=(`git branch --remote | sed "s|origin/||g"`) for LCL_BR in $LOCAL_BRANCHES; do CURR_BR=`echo "$LCL_BR" | sed -e "s|^[[:space:]]*||" -e "s|[[:space:]]*$||"` for REM_BR in "${REMOTE_BRANCHES[@]}"; do if [ "${REM_BR}" = "${CURR_BR}" -a -z "`grep \"^${MOD_FULLPATH} - ${CURR_BR} - WORKTREE UPDATED$\" \"${UPDATE_LOCK_FILENAME}\"`" ]; then local WORK_PATH="${WORKTREES_DIR}/${CURR_BR}/${MOD_FULLPATH}" RESULT="FAIL" if [ -d ${WORK_PATH} ]; then cd "${WORK_PATH}" &>/dev/null && # Clean up any possible uncommitted changes if [[ ! -z "`git status --porcelain $GIT_IGNORE_SUBMODULES`" ]]; then git reset --hard HEAD &>/dev/null git clean -dxff &>/dev/null fi # Make sure the local branch is a tracking branch if [[ -z `git config branch."${CURR_BR}".remote` ]]; then git branch -u "origin/${CURR_BR}" &>/dev/null #$ git reset --hard "origin/${CURR_BR}" &>/dev/null fi # If this is a clone of a local repository, make sure to do a fetch # operation first, otherwise this clone won't see the latest changes if [ "`git config --get remote.origin.url`" = "${MOD_REPO_PATH}" ]; then eval git fetch --all --prune $GIT_NO_RECURSE_SUBMODULES $OPT_VERBOSE_LOG eval git worktree prune $OPT_VERBOSE_LOG fi # Update eval git reset --hard HEAD $OPT_VERBOSE_LOG eval git clean -dxff $OPT_VERBOSE_LOG eval git checkout "${CURR_BR}" $OPT_VERBOSE_LOG if [[ $(git rev-parse HEAD) != $(git rev-parse "origin/${CURR_BR}") ]]; then eval git rebase $OPT_VERBOSE_LOG if [[ `git rev-parse HEAD` == `git rev-parse "origin/${CURR_BR}"` ]]; then RESULT="UPDATE" fi else RESULT="OK" fi fi LINE_CTRL_SUFFIX="" [ "$RESULT" != "OK" -o "$flag_VERBOSE_LOG" = "y" ] && LINE_CTRL_SUFFIX="\n" echo -ne "\033[2K\r${COLOR_STRINGS[$RESULT]}${RESULT_STRINGS[$RESULT]} ${MOD_NAME} - branch ${CURR_BR} - worktree updated${CNone}$LINE_CTRL_SUFFIX" echo "${RESULT_STRINGS[$RESULT]} ${MOD_NAME} - branch ${CURR_BR} - worktree updated" >> "$LOG_UPDATE_REPO_FILENAME" echo "${MOD_FULLPATH} - ${CURR_BR} - WORKTREE UPDATED" >> "${UPDATE_LOCK_FILENAME}" break fi done done ;; *) RESULT="INV-OP" ;; esac } #---------------------------- # Update a given module and all submodules from the upstream repo # Parameters: # $1 - module folder # $2 - operation type # $3 - branch to update function _update_module() { local MOD_PATH=$1 local OP_TYPE=$2 local BRANCH=$3 # Current module _do_update "$@" # Submodules local SUBMOD_LIST="$MOD_PATH/.gitmodules" if [[ -e "$SUBMOD_LIST" ]]; then sed -n "s|^\[submodule \"\([^\"]*\)\"\]$|\1|p" <$SUBMOD_LIST |\ while read -r SUBMOD_STRING; do cd "$MOD_PATH" &>/dev/null if [[ -z "`git config --get submodule.$SUBMOD_STRING.url`" ]]; then eval git submodule init -- \"$SUBMOD_STRING\" $OPT_VERBOSE_LOG fi if [[ ! -e "$MOD_PATH/$SUBMOD_STRING/.git" ]]; then eval git submodule update -- \"$SUBMOD_STRING\" $OPT_VERBOSE_LOG fi _update_module "$MOD_PATH/$SUBMOD_STRING" "$OP_TYPE" "$BRANCH" done fi } #---------------------------- # Update a given module and all submodules from the upstream repo # Parameters: # $1 - module folder and name (excluding base repo part) # $2 - list of branches to update # $3 - parent module path # $4 - module substring relative to parent function _update_worktree_module() { local MOD_FULLPATH=$1 local MOD_NAME=`echo ${MOD_FULLPATH} | sed "s|^.*/\([^/]\+\)$|\1|"` if [ -n "${MOD_FULLPATH}" -a -n "${MOD_NAME}" ]; then local BRANCH_LIST=$2 [ -n "${BRANCH_LIST}" ] || BRANCH_LIST="${UPDATE_BRANCHES}" # Current module _do_worktree_update "${MOD_FULLPATH}" "fetch" _do_worktree_update "${MOD_FULLPATH}" "check_worktree" "${BRANCH_LIST[@]}" _do_worktree_update "${MOD_FULLPATH}" "check_module" "dummy arg" "$3" "$4" _do_worktree_update "${MOD_FULLPATH}" "update" "${BRANCH_LIST[@]}" # Submodules for LCL_BR in ${BRANCH_LIST}; do local WORK_PATH="${WORKTREES_DIR}/${LCL_BR}/${MOD_FULLPATH}" if [ -d ${WORK_PATH} ]; then local SUBMOD_LIST="${WORK_PATH}/.gitmodules" if [ -e "${SUBMOD_LIST}" ]; then sed -n "s|^\[submodule \"\([^\"]*\)\"\]$|\1|p" <${SUBMOD_LIST} |\ while read -r SUBMOD_STRING; do cd "${WORK_PATH}" &>/dev/null SUBMOD_NAME=`echo ${SUBMOD_STRING} | sed "s|.*/\([^/]\+\)$|\1|"` SUBMOD_URLNAME=`git config --file .gitmodules --get submodule.${SUBMOD_STRING}.url | \ sed "s|.*/\([^/]\+\)$|\1|"` _do_worktree_update "${MOD_FULLPATH}/${SUBMOD_STRING}" "add_repo" "${SUBMOD_URLNAME}" _update_worktree_module "${MOD_FULLPATH}/${SUBMOD_STRING}" "${LCL_BR}" "${WORK_PATH}" "${SUBMOD_STRING}" done fi fi done else echo -ne "\033[2K\r${COLOR_STRINGS["FAIL"]}${RESULT_STRINGS["FAIL"]} Invalid module $1 passed to _update_worktree_module()${CNone}\n" echo "${RESULT_STRINGS["FAIL"]} Invalid module $1 passed to _update_worktree_module()" >> "$LOG_UPDATE_REPO_FILENAME" fi } #---------------------------- if [ "$flag_INCREMENTAL" = "y" ]; then [ ! -f "$UPDATE_LOCK_FILENAME" ] && flag_INCREMENTAL="n" else [ -f "$UPDATE_LOCK_FILENAME" ] && rm "$UPDATE_LOCK_FILENAME" fi OPT_VERBOSE_LOG="&>/dev/null" if [[ "$flag_VERBOSE_LOG" = "y" ]]; then OPT_VERBOSE_LOG=" |& tee -a \"$LOG_UPDATE_REPO_FILENAME\"" fi #---------------------------- # Check git abilities GIT_IGNORE_SUBMODULES="" if [[ -n "`git status --help 2>/dev/null|grep -- '--ignore-submodules'`" ]]; then GIT_IGNORE_SUBMODULES="--ignore-submodules" fi GIT_NO_RECURSE_SUBMODULES="" if [[ -n "`git pull --help |grep -- '--\[no-\]recurse-submodules'`" ]]; then GIT_NO_RECURSE_SUBMODULES="--no-recurse-submodules" fi # Start update if [ "$flag_INCREMENTAL" != "y" ]; then echo "TDE repositories update started" > "$UPDATE_LOCK_FILENAME" fi if [ "${USE_GIT_WORKTREES}" != "y" ]; then # Not using git worktree _LAST_BRANCH="" if [[ "$flag_SWITCH_ONLY" != "y" ]]; then # Update extra dependency repository if [[ "$USE_PREBUILD_EXTRA_DEPS" != "y" ]]; then echo_and_tee "${CLightCyan}---------------------------------------${CNone}" "$LOG_UPDATE_REPO_FILENAME" "y" echo_and_tee "${CLightCyan} Fetching extra dependencies ${CNone}" "$LOG_UPDATE_REPO_FILENAME" echo_and_tee "${CLightCyan}---------------------------------------${CNone}" "$LOG_UPDATE_REPO_FILENAME" # Fetch TDE extra dependency repository _update_module "$REPO_EXTRA_DEPENDENCIES" "fetch" _update_module "$REPO_EXTRA_DEPENDENCIES" "update" "master" echo -e "\033[2K" echo "" >> "$LOG_UPDATE_REPO_FILENAME" fi # Fetch all remotes first echo_and_tee "${CLightCyan}-------------------------------${CNone}" "$LOG_UPDATE_REPO_FILENAME" "y" echo_and_tee "${CLightCyan} Fetching repos ${CNone}" "$LOG_UPDATE_REPO_FILENAME" echo_and_tee "${CLightCyan}-------------------------------${CNone}" "$LOG_UPDATE_REPO_FILENAME" # Fetch TDE main repository _update_module "$REPO_TDE" "fetch" echo -e "\033[2K" echo "" >> "$LOG_UPDATE_REPO_FILENAME" # Branch update for branch in "${BRANCHES[@]}"; do _LAST_BRANCH="$branch" echo_and_tee "${CLightCyan}-------------------------------${CNone}" "$LOG_UPDATE_REPO_FILENAME" echo_and_tee "${CLightCyan} Updating branch ${CYellow}$branch ${CNone}" "$LOG_UPDATE_REPO_FILENAME" echo_and_tee "${CLightCyan}-------------------------------${CNone}" "$LOG_UPDATE_REPO_FILENAME" # Update TDE main repository _update_module "$REPO_TDE" "update" "$branch" echo -e "\033[2K" echo "" >> "$LOG_UPDATE_REPO_FILENAME" done fi # Switch to specified branch if necessary if [[ "$DEFAULT_REPO_BRANCH" != "$_LAST_BRANCH" ]]; then echo_and_tee "${CLightCyan}-------------------------------${CNone}" "$LOG_UPDATE_REPO_FILENAME" "$flag_SWITCH_ONLY" echo_and_tee "${CLightCyan} Switching to branch ${CYellow}$DEFAULT_REPO_BRANCH ${CNone}" "$LOG_UPDATE_REPO_FILENAME" echo_and_tee "${CLightCyan}-------------------------------${CNone}" "$LOG_UPDATE_REPO_FILENAME" # Switch TDE main repository _update_module "$REPO_TDE" "switch-to" "$DEFAULT_REPO_BRANCH" fi else # Using git worktree # Get TDE origin url cd "${REPOS_DIR}/tde.git" &>/dev/null ORIGIN_PATH=`git config --get remote.origin.url | sed "s|\(.*\)/tde.git$|\1|"` if [ ! -z "{ORIGIN_PATH}" ]; then # Update extra dependency repository if [[ "$USE_PREBUILD_EXTRA_DEPS" != "y" ]]; then echo_and_tee "${CLightCyan}-------------------------------${CNone}" "$LOG_UPDATE_REPO_FILENAME" "y" echo_and_tee "${CLightCyan} Updating extra dependencies ${CNone}" "$LOG_UPDATE_REPO_FILENAME" echo_and_tee "${CLightCyan}-------------------------------${CNone}" "$LOG_UPDATE_REPO_FILENAME" "y" _update_worktree_module "edeps" echo -e "\033[2K" echo "" >> "$LOG_UPDATE_REPO_FILENAME" fi echo_and_tee "${CLightCyan}-------------------------------${CNone}" "$LOG_UPDATE_REPO_FILENAME" "y" echo_and_tee "${CLightCyan} Updating TDE repos ${CNone}" "$LOG_UPDATE_REPO_FILENAME" echo_and_tee "${CLightCyan}-------------------------------${CNone}" "$LOG_UPDATE_REPO_FILENAME" # Main repo _update_worktree_module "tde" fi fi echo -e "\033[2K" echo "" >> "$LOG_UPDATE_REPO_FILENAME" # Update completed [ -f "$UPDATE_LOCK_FILENAME" ] && rm "$UPDATE_LOCK_FILENAME" cd $SCRIPT_DIR