scripts: call devserver/payload generators from chroot & delete moved scripts

BUG=chromium-os:5246

TEST=below, During all tests, the directory platform/dev was manually removed
1) Ran cros_au_test_harness.py and saw it succeed
2) Ran generate_au_zip.py, compared the generated .zip with one before this
patch - they were identical
3) Ran cros_image_to_target.py and saw it succeed

Change-Id: Iab2620245a45442b79ee8369f3af1a3990f4644e

Review URL: http://codereview.chromium.org/6576023
This commit is contained in:
Zdenek Behan 2011-03-01 22:57:33 +01:00
parent 87dd2be709
commit 99f7a5397d
8 changed files with 15 additions and 389 deletions

View File

@ -737,7 +737,7 @@ class DevServerWrapper(threading.Thread):
RunCommand(['sudo', 'pkill', '-f', 'devserver.py'], error_ok=True,
print_cmd=False)
RunCommand(['sudo',
'./start_devserver',
'start_devserver',
'--archive_dir=./static',
'--client_prefix=ChromeOSUpdateEngine',
'--production',
@ -844,7 +844,7 @@ def _PregenerateUpdates(parser, options):
'--nogit_config',
'--',
'sudo',
'./start_devserver',
'start_devserver',
'--pregenerate_update',
'--exit',
]
@ -979,7 +979,7 @@ def CleanPreviousWork(options):
Info('Cleaning up previous work.')
# Wipe devserver cache.
RunCommandCaptureOutput(
['sudo', './start_devserver', '--clear_cache', '--exit', ],
['sudo', 'start_devserver', '--clear_cache', '--exit', ],
enter_chroot=True, print_cmd=False, combine_stdout_stderr=True)
# Clean previous vm images if they exist.

View File

@ -206,7 +206,7 @@ class CrosEnv(object):
self.Info('Using cached update image %s' % dst)
return True
if not self.cmd.Run(self.CrosUtilsPath('cros_generate_update_payload'),
if not self.cmd.Run(self.ChrootPath('/usr/bin/cros_generate_update_payload'),
'--image=%s' % src, '--output=%s' % dst,
'--patch_kernel'):
self.Error('generate_payload failed')
@ -221,8 +221,8 @@ class CrosEnv(object):
self.Info('Using cached stateful %s' % dst_file)
return True
return self.cmd.Run(self.CrosUtilsPath(
'cros_generate_stateful_update_payload'),
return self.cmd.Run(
self.ChrootPath('/usr/bin/cros_generate_stateful_update_payload'),
'--image=%s' % src, '--output=%s' % dst_dir)
def GetSize(self, filename):
@ -343,7 +343,7 @@ class CrosEnv(object):
return False
self.Info('Update complete - running update script on client')
self.ssh_cmd.Copy(self.CrosUtilsPath('../platform/dev/stateful_update'),
self.ssh_cmd.Copy(self.ChrootPath('/usr/bin/stateful_update'),
'/tmp')
if not self.ssh_cmd.Run('/tmp/stateful_update', url_base,
remote_tunnel=(port, port)):

View File

@ -1 +0,0 @@
cros_generate_stateful_update_payload.py

View File

@ -1,95 +0,0 @@
#!/usr/bin/python2.6
# Copyright (c) 2010 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""This module is responsible for generate a stateful update payload."""
import logging
import optparse
import os
import subprocess
import tempfile
STATEFUL_FILE = 'stateful.tgz'
def GenerateStatefulPayload(image_path, output_directory, logger):
"""Generates a stateful update payload given a full path to an image.
Args:
image_path: Full path to the image.
output_directory: Path to the directory to leave the resulting output.
logger: logging instance.
"""
logger.info('Generating stateful update file.')
from_dir = os.path.dirname(image_path)
image = os.path.basename(image_path)
output_gz = os.path.join(output_directory, STATEFUL_FILE)
crosutils_dir = os.path.dirname(__file__)
# Temporary directories for this function.
rootfs_dir = tempfile.mkdtemp(suffix='rootfs', prefix='tmp')
stateful_dir = tempfile.mkdtemp(suffix='stateful', prefix='tmp')
# Mount the image to pull out the important directories.
try:
# Only need stateful partition, but this saves us having to manage our
# own loopback device.
subprocess.check_call(['%s/mount_gpt_image.sh' % crosutils_dir,
'--from=%s' % from_dir,
'--image=%s' % image,
'--read_only',
'--rootfs_mountpt=%s' % rootfs_dir,
'--stateful_mountpt=%s' % stateful_dir,
])
logger.info('Tarring up /usr/local and /var!')
subprocess.check_call(['sudo',
'tar',
'-czf',
output_gz,
'--directory=%s' % stateful_dir,
'--hard-dereference',
'--transform=s,^dev_image,dev_image_new,',
'--transform=s,^var,var_new,',
'dev_image',
'var',
])
except:
logger.error('Failed to create stateful update file')
raise
finally:
# Unmount best effort regardless.
subprocess.call(['%s/mount_gpt_image.sh' % crosutils_dir,
'--unmount',
'--rootfs_mountpt=%s' % rootfs_dir,
'--stateful_mountpt=%s' % stateful_dir,
])
# Clean up our directories.
os.rmdir(rootfs_dir)
os.rmdir(stateful_dir)
logger.info('Successfully generated %s' % output_gz)
def main():
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(os.path.basename(__file__))
parser = optparse.OptionParser()
parser.add_option('-i', '--image_path',
help='The image to generate the stateful update for.')
parser.add_option('-o', '--output_dir',
help='The path to the directory to output the update file.')
options, unused_args = parser.parse_args()
if not options.image_path:
parser.error('Missing image for stateful payload generator')
if not options.output_dir:
parser.error('Missing output directory for the payload generator')
GenerateStatefulPayload(os.path.abspath(options.image_path),
options.output_dir, logger)
if __name__ == '__main__':
main()

View File

@ -1,271 +0,0 @@
#!/bin/bash
# Copyright (c) 2010 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
# Script to generate a Chromium OS update for use by the update engine.
# If a source .bin is specified, the update is assumed to be a delta update.
# --- BEGIN COMMON.SH BOILERPLATE ---
# Load common CrOS utilities. Inside the chroot this file is installed in
# /usr/lib/crosutils. Outside the chroot we find it relative to the script's
# location.
find_common_sh() {
local thisdir="$(dirname "$(readlink -f "$0")")"
local common_paths=(/usr/lib/crosutils "${thisdir}")
local path
SCRIPT_ROOT=
for path in "${common_paths[@]}"; do
if [ -r "${path}/common.sh" ]; then
SCRIPT_ROOT=${path}
break
fi
done
# HACK(zbehan): We have to fake GCLIENT_ROOT in case we're running inside
# au_zip enviroment. GCLIENT_ROOT detection became fatal...
[ "${SCRIPT_ROOT}" == "${thisdir}" ] && \
export GCLIENT_ROOT="."
}
find_common_sh
. "${SCRIPT_ROOT}/common.sh" || (echo "Unable to load common.sh" && exit 1)
# --- END COMMON.SH BOILERPLATE ---
# Load functions and constants for chromeos-install
# NOTE: Needs to be called from outside the chroot.
. "/usr/lib/installer/chromeos-common.sh" &> /dev/null || \
. "${SRC_ROOT}/platform/installer/chromeos-common.sh" &> /dev/null || \
. "./chromeos-common.sh" || \
die "Unable to load /usr/lib/installer/chromeos-common.sh"
SRC_MNT=""
DST_MNT=""
SRC_KERNEL=""
SRC_ROOT=""
DST_KERNEL=""
DST_ROOT=""
STATE_MNT=""
STATE_LOOP_DEV=""
# Pass an arg to not exit 1 at the end
cleanup() {
set +e
if [ -n "$SRC_MNT" ]; then
sudo umount -d "$SRC_MNT"
[ -d "$SRC_MNT" ] && rmdir "$SRC_MNT"
SRC_MNT=""
fi
if [ -n "$DST_MNT" ]; then
sudo umount -d "$DST_MNT"
[ -d "$DST_MNT" ] && rmdir "$DST_MNT"
DST_MNT=""
fi
if [ -n "$STATE_MNT" ]; then
sudo umount "$STATE_MNT"
[ -d "$STATE_MNT" ] && rmdir "$STATE_MNT"
STATE_MNT=""
fi
if [ -n "$STATE_LOOP_DEV" ]; then
sudo losetup -d "$STATE_LOOP_DEV"
STATE_LOOP_DEV=""
fi
rm -f "$SRC_KERNEL"
rm -f "$SRC_ROOT"
rm -f "$DST_KERNEL"
rm -f "$DST_ROOT"
[ -n "$1" ] || exit 1
}
extract_partition_to_temp_file() {
local filename="$1"
local partition="$2"
local temp_file="$3"
if [ -z "$temp_file" ]; then
temp_file=$(mktemp /tmp/cros_generate_update_payload.XXXXXX)
echo "$temp_file"
fi
local offset=$(partoffset "${filename}" ${partition}) # 512-byte sectors
local length=$(partsize "${filename}" ${partition}) # 512-byte sectors
local bs=512
local sectors_per_two_mib=$((2 * 1024 * 1024 / 512))
if [ $(( $offset % $sectors_per_two_mib )) -eq 0 -a \
$(( $length % $sectors_per_two_mib )) -eq 0 ]; then
bs=$((2 * 1024 * 1024))
offset=$(($offset / $sectors_per_two_mib))
length=$(($length / $sectors_per_two_mib))
else
warn "partition offset or length not at 2MiB boundary"
fi
dd if="$filename" of="$temp_file" bs=$bs count="$length" \
skip="$offset" 2>/dev/null
}
patch_kernel() {
local IMAGE="$1"
local KERN_FILE="$2"
echo "Patching kernel" $KERN_FILE
echo " into" $IMAGE
STATE_LOOP_DEV=$(sudo losetup -f)
[ -n "$STATE_LOOP_DEV" ] || die "no free loop device"
local offset=$(partoffset "${IMAGE}" 1)
offset=$(($offset * 512))
sudo losetup -o "$offset" "$STATE_LOOP_DEV" "$IMAGE"
STATE_MNT=$(mktemp -d /tmp/state.XXXXXX)
sudo mount --read-only "$STATE_LOOP_DEV" "$STATE_MNT"
dd if="$STATE_MNT"/vmlinuz_hd.vblock of="$KERN_FILE" conv=notrunc 2>/dev/null
sudo umount "$STATE_MNT"
STATE_MNT=""
sudo losetup -d "$STATE_LOOP_DEV"
STATE_LOOP_DEV=""
}
extract_kern_root() {
local bin_file="$1"
local kern_out="$2"
local root_out="$3"
if [ -z "$kern_out" ]; then
die "missing kernel output filename"
fi
if [ -z "$root_out" ]; then
die "missing root output filename"
fi
extract_partition_to_temp_file "$bin_file" 2 "$kern_out"
if [ "$FLAGS_patch_kernel" -eq "$FLAGS_TRUE" ]; then
patch_kernel "$bin_file" "$kern_out"
fi
extract_partition_to_temp_file "$bin_file" 3 "$root_out"
}
DEFINE_string image "" "The image that should be sent to clients."
DEFINE_string src_image "" "Optional: a source image. If specified, this makes\
a delta update."
DEFINE_boolean old_style "$FLAGS_TRUE" "Generate an old-style .gz full update."
DEFINE_string output "" "Output file"
DEFINE_boolean outside_chroot "$FLAGS_FALSE" "Running outside of chroot."
DEFINE_boolean patch_kernel "$FLAGS_FALSE" "Whether or not to patch the kernel \
with the patch from the stateful partition (default: false)"
DEFINE_string private_key "" "Path to private key in .pem format."
DEFINE_boolean extract "$FLAGS_FALSE" "If set, extract old/new kernel/rootfs \
to [old|new]_[kern|root].dat. Useful for debugging (default: false)"
DEFINE_boolean full_kernel "$FLAGS_FALSE" "Generate a full kernel update even \
if generating a delta update (default: false)"
# Parse command line
FLAGS "$@" || exit 1
eval set -- "${FLAGS_ARGV}"
set -e
if [ -n "$FLAGS_src_image" ] && \
[ "$FLAGS_outside_chroot" -eq "$FLAGS_FALSE" ]; then
# We need to be in the chroot for generating delta images.
# by specifying --outside_chroot you can choose not to assert
# this will allow us to run this script outside chroot.
# Running this script outside chroot requires copying delta_generator binary
# and also copying few shared libraries with it.
assert_inside_chroot
fi
locate_gpt
if [ "$FLAGS_extract" -eq "$FLAGS_TRUE" ]; then
if [ -n "$FLAGS_src_image" ]; then
extract_kern_root "$FLAGS_src_image" old_kern.dat old_root.dat
fi
if [ -n "$FLAGS_image" ]; then
extract_kern_root "$FLAGS_image" new_kern.dat new_root.dat
fi
echo Done extracting kernel/root
exit 0
fi
DELTA=$FLAGS_TRUE
[ -n "$FLAGS_output" ] || die \
"Error: you must specify an output filename with --output FILENAME"
if [ -z "$FLAGS_src_image" ]; then
DELTA=$FLAGS_FALSE
fi
if [ "$DELTA" -eq "$FLAGS_TRUE" -o "$FLAGS_old_style" -eq "$FLAGS_FALSE" ]; then
echo "Generating a delta update"
# Sanity check that the real generator exists:
GENERATOR="$(which delta_generator)"
[ -x "$GENERATOR" ] || die "can't find delta_generator"
trap cleanup INT TERM EXIT
if [ "$DELTA" -eq "$FLAGS_TRUE" ]; then
if [ "$FLAGS_full_kernel" -eq "$FLAGS_FALSE" ]; then
SRC_KERNEL=$(extract_partition_to_temp_file "$FLAGS_src_image" 2)
if [ "$FLAGS_patch_kernel" -eq "$FLAGS_TRUE" ]; then
patch_kernel "$FLAGS_src_image" "$SRC_KERNEL"
fi
echo md5sum of src kernel:
md5sum "$SRC_KERNEL"
else
echo "Generating a full kernel update."
fi
SRC_ROOT=$(extract_partition_to_temp_file "$FLAGS_src_image" 3)
echo md5sum of src root:
md5sum "$SRC_ROOT"
fi
DST_KERNEL=$(extract_partition_to_temp_file "$FLAGS_image" 2)
if [ "$FLAGS_patch_kernel" -eq "$FLAGS_TRUE" ]; then
patch_kernel "$FLAGS_image" "$DST_KERNEL"
fi
DST_ROOT=$(extract_partition_to_temp_file "$FLAGS_image" 3)
if [ "$DELTA" -eq "$FLAGS_TRUE" ]; then
SRC_MNT=$(mktemp -d /tmp/src_root.XXXXXX)
sudo mount -o loop,ro "$SRC_ROOT" "$SRC_MNT"
DST_MNT=$(mktemp -d /tmp/src_root.XXXXXX)
sudo mount -o loop,ro "$DST_ROOT" "$DST_MNT"
sudo LD_LIBRARY_PATH=${LD_LIBRARY_PATH} PATH=${PATH} "$GENERATOR" \
-new_dir "$DST_MNT" -new_image "$DST_ROOT" -new_kernel "$DST_KERNEL" \
-old_dir "$SRC_MNT" -old_image "$SRC_ROOT" -old_kernel "$SRC_KERNEL" \
-out_file "$FLAGS_output" -private_key "$FLAGS_private_key"
else
"$GENERATOR" \
-new_image "$DST_ROOT" -new_kernel "$DST_KERNEL" \
-out_file "$FLAGS_output" -private_key "$FLAGS_private_key"
fi
trap - INT TERM EXIT
cleanup noexit
if [ "$DELTA" -eq "$FLAGS_TRUE" ]; then
echo "Done generating delta."
else
echo "Done generating new style full update."
fi
else
echo "Generating old-style full update"
trap cleanup INT TERM EXIT
DST_KERNEL=$(extract_partition_to_temp_file "$FLAGS_image" 2)
if [ "$FLAGS_patch_kernel" -eq "$FLAGS_TRUE" ]; then
patch_kernel "$FLAGS_image" "$DST_KERNEL"
fi
DST_ROOT=$(extract_partition_to_temp_file "$FLAGS_image" 3)
GENERATOR="${SCRIPTS_DIR}/mk_memento_images.sh"
CROS_GENERATE_UPDATE_PAYLOAD_CALLED=1 "$GENERATOR" "$DST_KERNEL" "$DST_ROOT"
mv "$(dirname "$DST_KERNEL")/update.gz" "$FLAGS_output"
trap - INT TERM EXIT
cleanup noexit
echo "Done generating full update."
fi

View File

@ -106,7 +106,7 @@ def CopyRequiredFiles(dest_files_root):
'/usr/bin/bspatch', '/usr/bin/cgpt']
# statically linked files and scripts etc.,
static_files = ['~/trunk/src/scripts/common.sh',
'~/trunk/src/scripts/cros_generate_update_payload',
'/usr/bin/cros_generate_update_payload',
'~/trunk/src/scripts/chromeos-common.sh']
# We need directories to be copied recursively to a destination within tempdir
recurse_dirs = {'~/trunk/src/scripts/lib/shflags': 'lib/shflags'}

View File

@ -180,7 +180,7 @@ function start_dev_server {
--src_image=\"$(reinterpret_path_for_chroot ${FLAGS_src_image})\""
info "Starting devserver with flags ${devserver_flags}"
./enter_chroot.sh -- sudo sh -c "./start_devserver ${devserver_flags} \
./enter_chroot.sh -- sudo sh -c "start_devserver ${devserver_flags} \
--client_prefix=ChromeOSUpdateEngine \
--board=${FLAGS_board} \
--port=${FLAGS_devserver_port} > ${FLAGS_server_log} 2>&1" &
@ -222,8 +222,10 @@ function run_stateful_update {
info "Starting stateful update using URL ${stateful_url}"
# Copy over update script and run update.
local dev_dir="${SCRIPTS_DIR}/../platform/dev"
remote_cp_to "${dev_dir}/stateful_update" "/tmp"
local chroot_path="${SCRIPTS_DIR}/../../chroot"
local stateful_update_script="/usr/bin/stateful_update"
remote_cp_to "${chroot_path}/${stateful_update_script}" "/tmp"
remote_sh "/tmp/stateful_update ${stateful_update_args} ${stateful_url}"
}

View File

@ -1,5 +1,4 @@
#!/bin/bash
# Copyright (c) 2010 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
@ -32,13 +31,5 @@ if [[ "$1" != "--archive_dir" ]]; then
restart_in_chroot_if_needed "$@"
fi
# Temporary workaround: to start devserver for update engine, pass in args
# --client_prefix ChromeOSUpdateEngine
# Set PKG_INSTALL_MASK if it's not set already.
if [ -z "${PKG_INSTALL_MASK+x}" ]; then
export PKG_INSTALL_MASK="${DEFAULT_INSTALL_MASK}"
fi
echo PKG_INSTALL_MASK=$PKG_INSTALL_MASK
cd ${GCLIENT_ROOT}/src/platform/dev && python devserver.py "$@"
# This is the location of the script now.
exec $(dirname ${0})/../../chroot/usr/bin/start_devserver "$@"