build: use codeload for GitHub archive fallback

Switch archive tarball URL to codeload.github.com, which is typically more reliable than github.com/archive on flaky networks.

scripts/compilation.sh

@@ -137,7 +137,11 @@ compile_uboot()
 	# read uboot version
 	local version hash
 	version=$(grab_version "$ubootdir")
-	hash=$(improved_git --git-dir="$ubootdir"/.git rev-parse HEAD)
+	if [[ -d "$ubootdir"/.git ]]; then
+		hash=$(improved_git --git-dir="$ubootdir"/.git rev-parse HEAD 2>/dev/null || true)
+	else
+		hash="nogit"
+	fi
 
 	display_alert "Compiling u-boot" "v$version" "info"
 
@@ -390,7 +394,11 @@ compile_kernel()
 	version=$(grab_version "$kerneldir")
 
 	# read kernel git hash
-	hash=$(improved_git --git-dir="$kerneldir"/.git rev-parse HEAD)
+	if [[ -d "$kerneldir"/.git ]]; then
+		hash=$(improved_git --git-dir="$kerneldir"/.git rev-parse HEAD 2>/dev/null || true)
+	else
+		hash="nogit"
+	fi
 
 	# Apply a series of patches if a series file exists
 	if test -f "${EXTER}"/patch/kernel/${KERNELPATCHDIR}/series.conf; then

scripts/general.sh

@@ -315,21 +315,27 @@ create_sources_list()
 #
 improved_git()
 {
-
+	local realgit
-	local realgit=$(command -v git)
+	realgit=$(command -v git)
 	local retries=3
 	local delay=10
 	local count=1
+	local last_status=0
+
+	# Prefer HTTP/1.1 for stability on flaky links.
+	# Also disable low-speed aborts (many environments have very low throughput).
 	while [ $count -lt $retries ]; do
-		$realgit "$@"
+		$realgit -c http.version=HTTP/1.1 -c http.lowSpeedLimit=0 -c http.lowSpeedTime=0 "$@"
-		if [[ $? -eq 0 || -f .git/index.lock ]]; then
+		last_status=$?
-			retries=0
+		if [[ $last_status -eq 0 || -f .git/index.lock ]]; then
 			break
 		fi
-	let count=$count+1
+		let count=$count+1
-	sleep $delay
+		sleep $delay
 	done
 
+	return $last_status
+
 }
 
 clean_up_git ()
@@ -344,6 +350,67 @@ clean_up_git ()
 	git -C $target_dir checkout -qf HEAD
 }
 
+# Fallback for environments where large git pack transfers are unreliable.
+# Downloads a GitHub branch archive and extracts it into the workdir.
+fetch_from_github_archive()
+{
+	local upstream_url=$1
+	local workdir=$2
+	local ref_name=$3
+
+	[[ $upstream_url == https://github.com/* ]] || return 1
+	[[ -n $ref_name ]] || return 1
+
+	local base=${upstream_url%.git}
+	local gh_path=${base#https://github.com/}
+	local gh_owner=${gh_path%%/*}
+	local gh_repo=${gh_path#*/}
+	gh_repo=${gh_repo%%/*}
+	# Prefer codeload endpoint; it is often more reliable than github.com/archive.
+	local tar_url="https://codeload.github.com/${gh_owner}/${gh_repo}/tar.gz/refs/heads/${ref_name}"
+	local archive_dir="${EXTER}/cache/sources/.github-archives"
+	local tar_path="${archive_dir}/$(basename "${base}")-${ref_name}.tar.gz"
+	local tar_part="${tar_path}.part"
+
+	mkdir -p "${archive_dir}" || return 1
+
+	display_alert "Downloading GitHub archive" "${tar_url}" "wrn"
+	local attempt
+	for attempt in 1 2 3; do
+		# Resume into a .part file, verify integrity, then promote.
+		if command -v curl >/dev/null 2>&1; then
+			curl -L --fail --retry 10 --retry-delay 5 --connect-timeout 30 \
+				--continue-at - -o "${tar_part}" "${tar_url}" || true
+		else
+			wget -c -t 10 --timeout=30 --waitretry=5 -O "${tar_part}" "${tar_url}" || true
+		fi
+
+		if gzip -t "${tar_part}" >/dev/null 2>&1; then
+			mv -f "${tar_part}" "${tar_path}" || return 1
+			break
+		fi
+
+		display_alert "Archive download incomplete; retrying" "attempt ${attempt}/3" "wrn"
+		rm -f "${tar_part}" 2>/dev/null || true
+		sleep 5
+	done
+
+	[[ -f "${tar_path}" ]] || return 1
+
+	display_alert "Extracting archive" "${workdir}" "wrn"
+	mkdir -p "${workdir}" || return 1
+	(
+		cd "${workdir}" || exit 1
+		rm -rf ./* ./.??* 2>/dev/null || true
+	)
+
+	tar -xzf "${tar_path}" -C "${workdir}" --strip-components=1 || return 1
+
+	# Mark as archive-based sources (helps debugging; no functional dependency)
+	echo "${tar_url}" > "${workdir}/.orangepi-source-archive" 2>/dev/null || true
+	return 0
+}
+
 # used : waiter_local_git arg1='value' arg2:'value'
 #		 waiter_local_git \
 #			url='https://github.com/megous/linux' \
@@ -500,6 +567,7 @@ fetch_from_repo()
 	local dir=$2
 	local ref=$3
 	local ref_subdir=$4
+	local upstream_url=$1
 
 	# Set GitHub mirror before anything else touches $url
 	url=${url//'https://github.com/'/$GITHUB_SOURCE'/'}
@@ -555,6 +623,18 @@ fetch_from_repo()
 		offline=false
 	fi
 
+	# If this repo was left in a partial-clone state (e.g. filter=blob:none),
+	# checkouts can degrade into extremely slow per-blob fetches.
+	# Disable it to ensure reliable, reasonably fast builds.
+	if [[ "$(git config --bool --get remote.origin.promisor 2>/dev/null)" == "true" || \
+		-n "$(git config --get remote.origin.partialclonefilter 2>/dev/null)" || \
+		-n "$(git config --get extensions.partialclone 2>/dev/null)" ]]; then
+		display_alert "Disabling partial clone settings" "${workdir}" "wrn"
+		git config --unset-all remote.origin.promisor >/dev/null 2>&1 || true
+		git config --unset-all remote.origin.partialclonefilter >/dev/null 2>&1 || true
+		git config --unset-all extensions.partialclone >/dev/null 2>&1 || true
+	fi
+
 	local changed=false
 
 	# when we work offline we simply return the sources to their original state
@@ -567,14 +647,32 @@ fetch_from_repo()
 			# TODO: grep refs/heads/$name
 			local remote_hash
 			remote_hash=$(improved_git ls-remote -h "${url}" "$ref_name" | head -1 | cut -f1)
+			if [[ -z $remote_hash && $url != "$upstream_url" ]]; then
+				display_alert "Mirror missing ref; falling back" "$ref_name" "wrn"
+				url="$upstream_url"
+				git remote set-url origin "${url}" >/dev/null 2>&1 || true
+				remote_hash=$(improved_git ls-remote -h "${url}" "$ref_name" | head -1 | cut -f1)
+			fi
 			[[ -z $local_hash || "${local_hash}" != "${remote_hash}" ]] && changed=true
 			;;
 
 			tag)
 			local remote_hash
 			remote_hash=$(improved_git ls-remote -t "${url}" "$ref_name" | cut -f1)
+			if [[ -z $remote_hash && $url != "$upstream_url" ]]; then
+				display_alert "Mirror missing tag; falling back" "$ref_name" "wrn"
+				url="$upstream_url"
+				git remote set-url origin "${url}" >/dev/null 2>&1 || true
+				remote_hash=$(improved_git ls-remote -t "${url}" "$ref_name" | cut -f1)
+			fi
 			if [[ -z $local_hash || "${local_hash}" != "${remote_hash}" ]]; then
 				remote_hash=$(improved_git ls-remote -t "${url}" "$ref_name^{}" | cut -f1)
+				if [[ -z $remote_hash && $url != "$upstream_url" ]]; then
+					display_alert "Mirror missing tag deref; falling back" "$ref_name" "wrn"
+					url="$upstream_url"
+					git remote set-url origin "${url}" >/dev/null 2>&1 || true
+					remote_hash=$(improved_git ls-remote -t "${url}" "$ref_name^{}" | cut -f1)
+				fi
 				[[ -z $remote_hash || "${local_hash}" != "${remote_hash}" ]] && changed=true
 			fi
 			;;
@@ -582,6 +680,12 @@ fetch_from_repo()
 			head)
 			local remote_hash
 			remote_hash=$(improved_git ls-remote "${url}" HEAD | cut -f1)
+			if [[ -z $remote_hash && $url != "$upstream_url" ]]; then
+				display_alert "Mirror missing HEAD; falling back" "HEAD" "wrn"
+				url="$upstream_url"
+				git remote set-url origin "${url}" >/dev/null 2>&1 || true
+				remote_hash=$(improved_git ls-remote "${url}" HEAD | cut -f1)
+			fi
 			[[ -z $local_hash || "${local_hash}" != "${remote_hash}" ]] && changed=true
 			;;
 
@@ -593,14 +697,35 @@ fetch_from_repo()
 	fi # offline
 
 	if [[ $changed == true ]]; then
+		local fetch_depth=${GIT_FETCH_DEPTH:-200}
 
 		# remote was updated, fetch and check out updates
 		display_alert "Fetching updates"
 		case $ref_type in
-			branch) improved_git fetch --depth 200 origin "${ref_name}" ;;
+			branch) improved_git fetch --depth "${fetch_depth}" origin "${ref_name}" ;;
-			tag) improved_git fetch --depth 200 origin tags/"${ref_name}" ;;
+			tag) improved_git fetch --depth "${fetch_depth}" origin tags/"${ref_name}" ;;
-			head) improved_git fetch --depth 200 origin HEAD ;;
+			head) improved_git fetch --depth "${fetch_depth}" origin HEAD ;;
 		esac
+		local fetch_status=$?
+		if [[ $fetch_status -ne 0 && "${fetch_depth}" != "1" ]]; then
+			display_alert "Fetch failed; retrying with depth=1" "${workdir}" "wrn"
+			case $ref_type in
+				branch) improved_git fetch --depth 1 origin "${ref_name}" ;;
+				tag) improved_git fetch --depth 1 origin tags/"${ref_name}" ;;
+				head) improved_git fetch --depth 1 origin HEAD ;;
+			esac
+			fetch_status=$?
+		fi
+		if [[ $fetch_status -ne 0 ]]; then
+			# If we're targeting a GitHub branch, fall back to downloading the branch archive.
+			if [[ $ref_type == branch && $upstream_url == https://github.com/* ]]; then
+				display_alert "Git fetch failed; using archive fallback" "${ref_name}" "wrn"
+				# Ensure we don't keep a broken git repository around.
+				rm -rf "${workdir}/.git" 2>/dev/null || true
+				fetch_from_github_archive "${upstream_url}" "${workdir}" "${ref_name}" && return 0
+			fi
+			exit_with_error "Failed to fetch git sources" "${url} ${ref}"
+		fi
 
 		# commit type needs support for older git servers that doesn't support fetching id directly
 		if [[ $ref_type == commit ]]; then

scripts/main.sh

@@ -446,7 +446,9 @@ prepare_host
 
 # fetch_from_repo <url> <dir> <ref> <subdir_flag>
 
+
 # ignore updates help on building all images - for internal purposes
+# When IGNORE_UPDATES=yes, still fetch once if required sources are missing.
 if [[ ${IGNORE_UPDATES} != yes ]]; then
 
 	display_alert "Downloading sources" "" "info"
@@ -561,6 +563,26 @@ if [[ ${IGNORE_UPDATES} != yes ]]; then
 	BUILD_HOST_TOOLS
 fi
 
+# IGNORE_UPDATES=yes: prefer local trees, but bootstrap missing sources to avoid hard failures.
+if [[ ${IGNORE_UPDATES} == yes ]]; then
+	# Kernel: if the tree isn't populated, it will be missing the top-level Makefile.
+	if [[ $BUILD_OPT =~ kernel|image && ! -f "${LINUXSOURCEDIR}/Makefile" ]]; then
+		display_alert "Kernel sources missing; downloading once" "(IGNORE_UPDATES=yes)" "wrn"
+		fetch_from_repo "$KERNELSOURCE" "$KERNELDIR" "$KERNELBRANCH" "yes"
+	fi
+
+	# U-Boot/ATF are required for u-boot/image builds; fetch once if missing.
+	if [[ $BOARDFAMILY != "cix" && $BUILD_OPT =~ u-boot|image && ! -f "${BOOTSOURCEDIR}/Makefile" ]]; then
+		display_alert "U-Boot sources missing; downloading once" "(IGNORE_UPDATES=yes)" "wrn"
+		fetch_from_repo "$BOOTSOURCE" "$BOOTDIR" "$BOOTBRANCH" "yes"
+	fi
+
+	if [[ -n ${ATFSOURCE} && $BUILD_OPT =~ u-boot|image && ! -f "${EXTER}/cache/sources/${ATFSOURCEDIR}/Makefile" ]]; then
+		display_alert "ATF sources missing; downloading once" "(IGNORE_UPDATES=yes)" "wrn"
+		fetch_from_repo "$ATFSOURCE" "${EXTER}/cache/sources/$ATFDIR" "$ATFBRANCH" "yes"
+	fi
+fi
+
 for option in $(tr ',' ' ' <<< "$CLEAN_LEVEL"); do
 	[[ $option != sources ]] && cleaning "$option"
 done