Skip to content

Commit

Permalink
[WiP] OTA download for the partitions missing in the factory image
Browse files Browse the repository at this point in the history
TODO: Import the below in a better way
cyxx/extract_android_ota_payload#8
  • Loading branch information
chirayudesai committed Sep 16, 2019
1 parent 02a51ac commit 0012552
Show file tree
Hide file tree
Showing 9 changed files with 1,069 additions and 12 deletions.
63 changes: 59 additions & 4 deletions execute-all.sh
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@

set -e # fail on unhandled error
set -u # fail on undefined variable
#set -x # debug
set -x # debug

readonly SCRIPTS_ROOT="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
readonly TMP_WORK_DIR=$(mktemp -d "${TMPDIR:-/tmp}"/android_prepare_vendor.XXXXXX) || exit 1
Expand All @@ -25,6 +25,9 @@ readonly DOWNLOAD_SCRIPT="$SCRIPTS_ROOT/scripts/download-nexus-image.sh"
# Helper script to extract system & vendor images data
readonly EXTRACT_SCRIPT="$SCRIPTS_ROOT/scripts/extract-factory-images.sh"

# Helper script to extract ota data
readonly OTA_SCRIPT="$SCRIPTS_ROOT/scripts/extract-ota.sh"

# Helper script to generate "proprietary-blobs.txt" file
readonly GEN_BLOBS_LIST_SCRIPT="$SCRIPTS_ROOT/scripts/gen-prop-blobs-list.sh"

Expand Down Expand Up @@ -316,7 +319,7 @@ trap "abort 1" SIGINT SIGTERM
export PATH="$PATH:$LC_BIN"

# Global variables
DEVICE=""
DEVICE=""DEVICE
BUILDID=""
OUTPUT_DIR=""
INPUT_IMG=""
Expand All @@ -341,6 +344,7 @@ USE_FUSEEXT2=false
FORCE_VIMG=false
JAVA_FOUND=false
TIMESTAMP=""
OTA=false

# Compatibility
check_bash_version
Expand Down Expand Up @@ -502,17 +506,59 @@ if [[ "$INPUT_IMG" == "" ]]; then
echo "[-] Images download failed"
abort 1
}
factoryImgArchive="$(find "$OUT_BASE" -iname "*$DEV_ALIAS*$BUILDID*.tgz" -or \
-iname "*$DEV_ALIAS*$BUILDID*.zip" | head -1)"
factoryImgArchive="$(find "$OUT_BASE" -iname "*$DEV_ALIAS*$BUILDID-factory*.tgz" -or \
-iname "*$DEV_ALIAS*$BUILDID-factory*.zip" | head -1)"
else
factoryImgArchive="$INPUT_IMG"
fi

readonly OTA_IMGS_LIST="$(jqIncRawArrayTop "ota-partitions" "$CONFIG_FILE")"
if [[ "$OTA_IMGS_LIST" != "" ]]; then
OTA=true
fi

if [ "$OTA" = true ]; then
OtaArchive=""
if [[ "$INPUT_IMG" == "" ]]; then

# Factory image alias for devices with naming incompatibilities with AOSP
if [[ "$DEVICE" == "flounder" && "$DEV_ALIAS" == "" ]]; then
echo "[-] Building for flounder requires setting the device alias option - 'volantis' or 'volantisg'"
abort 1
fi
if [[ "$DEV_ALIAS" == "" ]]; then
DEV_ALIAS="$DEVICE"
fi

__extraArgs=""
if [ $AUTO_TOS_ACCEPT = true ]; then
__extraArgs="--yes"
fi

$DOWNLOAD_SCRIPT --device "$DEVICE" --alias "$DEV_ALIAS" \
--buildID "$BUILDID" --output "$OUT_BASE" $__extraArgs --ota || {
echo "[-] OTA download failed"
abort 1
}
OtaArchive="$(find "$OUT_BASE" -iname "*$DEV_ALIAS*ota-$BUILDID*.tgz" -or \
-iname "*$DEV_ALIAS*ota-$BUILDID*.zip" | head -1)"
else
OtaArchive="$INPUT_IMG"
fi
fi

if [[ "$factoryImgArchive" == "" ]]; then
echo "[-] Failed to locate factory image archive"
abort 1
fi

if [ "$OTA" = true ]; then
if [[ "$OtaArchive" == "" ]]; then
echo "[-] Failed to locate OTA archive"
abort 1
fi
fi

# Clear old data if present & extract data from factory images
if [ -d "$FACTORY_IMGS_DATA" ]; then
# Previous run might have been with --keep which keeps the mount-points. Check
Expand Down Expand Up @@ -544,6 +590,15 @@ $EXTRACT_SCRIPT "${EXTRACT_SCRIPT_ARGS[@]}" --conf-file "$CONFIG_FILE" || {
abort 1
}

if [ "$OTA" = true ]; then
OTA_SCRIPT_ARGS=(--input "$OtaArchive" --output "$FACTORY_IMGS_DATA")

$OTA_SCRIPT "${OTA_SCRIPT_ARGS[@]}" --conf-file "$CONFIG_FILE" || {
echo "[-] OTA data extract failed"
abort 1
}
fi

# system.img contents are different between Nexus & Pixel
SYSTEM_ROOT="$FACTORY_IMGS_DATA/system"
if [[ -d "$FACTORY_IMGS_DATA/system/system" && -f "$FACTORY_IMGS_DATA/system/system/build.prop" ]]; then
Expand Down
3 changes: 2 additions & 1 deletion scripts/constants.sh
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,8 @@ declare -ra SUPPORTED_DEVICES=(

# URLs to download factory images from
readonly NID_URL="https://google.com"
readonly GURL="https://developers.google.com/android/nexus/images"
readonly GURL="https://developers.google.com/android/images"
readonly GURL2="https://developers.google.com/android/ota"
readonly TOSURL="https://developers.google.com/profile/acknowledgeNotification"

# oatdump dependencies URLs as compiled from AOSP matching API levels
Expand Down
35 changes: 28 additions & 7 deletions scripts/download-nexus-image.sh
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ cat <<_EOF
-b|--buildID : BuildID string (e.g. MMB29P)
-o|--output : Path to save images archived
-y|--yes : Default accept Google ToS
--ota : Download OTA instead of factory image
_EOF
abort 1
}
Expand Down Expand Up @@ -82,6 +83,7 @@ DEV_ALIAS=""
BUILDID=""
OUTPUT_DIR=""
AUTO_TOS_ACCEPT=false
OTA=false

while [[ $# -gt 0 ]]
do
Expand All @@ -106,6 +108,9 @@ do
-y|--yes)
AUTO_TOS_ACCEPT=true
;;
--ota)
OTA=true
;;
*)
echo "[-] Invalid argument '$1'"
usage
Expand Down Expand Up @@ -148,22 +153,38 @@ done

# Accept news ToS page
accept_tos
xsrf_token=$(curl -L -b "$COOKIE_FILE" --silent "$GURL" | \
if [ "$OTA" = true ]; then
URL="$GURL2"
else
URL="$GURL"
fi
xsrf_token=$(curl -L -b "$COOKIE_FILE" --silent "$URL" | \
grep -io "<meta name=\"xsrf_token\" content=\".*\" />" | \
cut -d '"' -f4)
response=$(curl -b "$COOKIE_FILE" -X POST -d "notification_id=wall-nexus-image-tos" \
-H "X_XSRFToken: $xsrf_token" --write-out "%{http_code}" --output /dev/null \
--silent "$TOSURL")
if [ "$OTA" = true ]; then
response=$(curl -b "$COOKIE_FILE" -X POST -d "notification_id=wall-nexus-ota-tos" \
-H "X_XSRFToken: $xsrf_token" --write-out "%{http_code}" --output /dev/null \
--silent "$TOSURL")
else
response=$(curl -b "$COOKIE_FILE" -X POST -d "notification_id=wall-nexus-image-tos" \
-H "X_XSRFToken: $xsrf_token" --write-out "%{http_code}" --output /dev/null \
--silent "$TOSURL")
fi
if [[ "$response" != "200" ]]; then
echo "[-] Nexus ToS accept request failed"
abort 1
fi

# Then retrieve the index page
url=$(curl -L -b "$COOKIE_FILE" --silent -H "X_XSRFToken: $xsrf_token" "$GURL" | \
grep -i "<a href=.*$DEV_ALIAS-$BUILDID-" | cut -d '"' -f2)
if [ "$OTA" = true ]; then
url=$(curl -L -b "$COOKIE_FILE" --silent -H "X_XSRFToken: $xsrf_token" "$URL" | \
grep -i "<a href=.*$DEV_ALIAS-ota-$BUILDID-" | cut -d '"' -f2)
else
url=$(curl -L -b "$COOKIE_FILE" --silent -H "X_XSRFToken: $xsrf_token" "$URL" | \
grep -i "<a href=.*$DEV_ALIAS-$BUILDID-" | cut -d '"' -f2)
fi
if [ "$url" == "" ]; then
echo "[-] Image URL not found"
echo "[-] URL not found"
abort 1
fi

Expand Down
152 changes: 152 additions & 0 deletions scripts/extract-ota.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,152 @@
#!/usr/bin/env bash
#
# Extract images from ota zip
#

set -e # fail on unhandled error
set -u # fail on undefined variable
#set -x # debug

readonly SCRIPTS_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
readonly CONSTS_SCRIPT="$SCRIPTS_DIR/constants.sh"
readonly COMMON_SCRIPT="$SCRIPTS_DIR/common.sh"
readonly EXTRACT_PAYLOAD_SCRIPT="$SCRIPTS_DIR/extract_android_ota_payload/extract_android_ota_payload.py"
readonly TMP_WORK_DIR=$(mktemp -d "${TMPDIR:-/tmp}"/android_img_extract.XXXXXX) || exit 1
declare -a SYS_TOOLS=("tar" "find" "unzip" "uname" "du" "stat" "tr" "cut")

abort() {
# If debug keep work dir for bugs investigation
if [[ "$-" == *x* ]]; then
echo "[*] Workspace available at '$TMP_WORK_DIR' - delete manually when done"
else
rm -rf "$TMP_WORK_DIR"
fi
exit "$1"
}

usage() {
cat <<_EOF
Usage: $(basename "$0") [options]
OPTIONS:
-i|--input : Archive with ota as downloaded from
Google Nexus ota website
-o|--output : Path to save contents extracted from ota
--conf-file : Device configuration file
_EOF
abort 1
}

extract_archive() {
local in_archive="$1"
local out_dir="$2"
local archiveFile

echo "[*] Extracting '$in_archive'"

archiveFile="$(basename "$in_archive")"
local f_ext="${archiveFile##*.}"
if [[ "$f_ext" == "tar" || "$f_ext" == "tar.gz" || "$f_ext" == "tgz" ]]; then
tar -xf "$in_archive" -C "$out_dir" || { echo "[-] tar extract failed"; abort 1; }
elif [[ "$f_ext" == "zip" ]]; then
unzip -qq "$in_archive" -d "$out_dir" || { echo "[-] zip extract failed"; abort 1; }
else
echo "[-] Unknown archive format '$f_ext'"
abort 1
fi
}

extract_payload() {
local otaFile="$1"
local out_dir="$2"
python $EXTRACT_PAYLOAD_SCRIPT $out_dir/payload.bin $out_dir $otaFile
}

trap "abort 1" SIGINT SIGTERM
. "$CONSTS_SCRIPT"
. "$COMMON_SCRIPT"

INPUT_ARCHIVE=""
OUTPUT_DIR=""
CONFIG_FILE=""

# Compatibility
HOST_OS=$(uname)
if [[ "$HOST_OS" != "Linux" && "$HOST_OS" != "Darwin" ]]; then
echo "[-] '$HOST_OS' OS is not supported"
abort 1
fi

while [[ $# -gt 0 ]]
do
arg="$1"
case $arg in
-o|--output)
OUTPUT_DIR=$(echo "$2" | sed 's:/*$::')
shift
;;
-i|--input)
INPUT_ARCHIVE=$2
shift
;;
--conf-file)
CONFIG_FILE="$2"
shift
;;
*)
echo "[-] Invalid argument '$1'"
usage
;;
esac
shift
done

# Check that system tools exist
for i in "${SYS_TOOLS[@]}"
do
if ! command_exists "$i"; then
echo "[-] '$i' command not found"
abort 1
fi
done

# Input args check
check_dir "$OUTPUT_DIR" "Output"
check_file "$INPUT_ARCHIVE" "Input archive"
check_file "$CONFIG_FILE" "Device Config File"

# Fetch required values from config
readonly VENDOR="$(jqRawStrTop "vendor" "$CONFIG_FILE")"
readonly OTA_IMGS_LIST="$(jqIncRawArrayTop "ota-partitions" "$CONFIG_FILE")"
if [[ "$OTA_IMGS_LIST" != "" ]]; then
readarray -t OTA_IMGS < <(echo "$OTA_IMGS_LIST")
fi

RADIO_DATA_OUT="$OUTPUT_DIR/radio"
if [ -d "$RADIO_DATA_OUT" ]; then
rm -rf "${RADIO_DATA_OUT:?}"/*
fi
mkdir -p "$RADIO_DATA_OUT"

archiveName="$(basename "$INPUT_ARCHIVE")"
fileExt="${archiveName##*.}"
archName="$(basename "$archiveName" ".$fileExt")"
extractDir="$TMP_WORK_DIR/$archName"
mkdir -p "$extractDir"

# Extract archive
extract_archive "$INPUT_ARCHIVE" "$extractDir"

# For Pixel devices with AB partitions layout, copy additional images required for OTA
if [[ "$VENDOR" == "google" && "$OTA_IMGS_LIST" != "" ]]; then
for img in "${OTA_IMGS[@]}"
do
extract_payload "$img.img" "$extractDir"
if [ ! -f "$extractDir/$img.img" ]; then
echo "[-] Failed to locate '$img.img' in ota zip"
abort 1
fi
mv "$extractDir/$img.img" "$RADIO_DATA_OUT/"
done
fi

abort 0
1 change: 1 addition & 0 deletions scripts/extract_android_ota_payload/.gitignore
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
*.pyc
34 changes: 34 additions & 0 deletions scripts/extract_android_ota_payload/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
# extract_android_ota_payload.py

Extract Android firmware images from an OTA payload.bin file.

With the introduction of the A/B system update, the OTA file format changed.
This tool allows to extract and decompress the firmware images packed using the 'brillo' toolset.

Incremental firmware images are not supported (source_copy, source_bsdiff operations).

## Usage

```
$ extract_android_ota_payload.py <payload.bin> [target_dir] [partition.img]
<payload.bin> : file extracted from the OTA zip file or the OTA zip file
<target_dir> : output directory for the extracted file
<partition_img> : name of partition to be extracted
```

## Example

```
$ python extract_android_ota_payload.py marlin-ota-opm4.171019.021.d1-fd6998a5.zip /tmp/
Extracting 'boot.img'
Extracting 'system.img'
Extracting 'vendor.img'
...
Extracting 'modem.img'
```

## Dependencies

```
python-protobuf
```
Loading

0 comments on commit 0012552

Please sign in to comment.