tools/update_datasets.sh - github/openstack/turbo-hipster - Gitiles

 #!/bin/bash
 #
 # Copyright 2014 Rackspace Australia
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you may
 # not use this file except in compliance with the License. You may obtain
 # a copy of the License at
 #
 # http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
 # License for the specific language governing permissions and limitations
 # under the License.

 # A tool to update a given dataset to a given version. Used to keep datasets
 # somewhat fresh rather than over-exercising old migrations.


 # Usage: ./update_datasets.sh VENV_NAME WORKING_DIR GIT_PATH DB_USER DB_PASS DB_NAME SEED_DATA OUTPUT_DATA
 # apt-get install git virtualenvwrapper python-pip mysql-server python-lxml build-essential libffi-dev

 # $1 is the unique job id
 # $2 is the working dir path
 # $3 is the path to the git repo path
 # $4 is the nova db user
 # $5 is the nova db password
 # $6 is the nova db name
 # $7 is the path to the seed dataset to test against
 # $8 is the logging.conf for openstack
 # $9 is the pip cache dir

 UNIQUE_ID=$1
 WORKING_DIR_PATH=`realpath $2`
 GIT_REPO_PATH=`realpath $3`
 DB_USER=$4
 DB_PASS=$5
 DB_NAME=$6
 DATASET_SEED_SQL=`realpath $7`
 DATASET_OUTPUT_SQL=$8

 SCRIPT_DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )

 # We also support the following environment variables to tweak our behavour:
 #   NOCLEANUP: if set to anything, don't cleanup at the end of the run

 pip_requires() {
   pip install -q mysql-python
   pip install -q eventlet
   requires="tools/pip-requires"
   if [ ! -e $requires ]
   then
     requires="requirements.txt"
   fi
   echo "Install pip requirements from $requires"
   pip install -q -r $requires
   echo "Requirements installed"
 }

 db_sync() {
   # $1 is the test target (ie branch name)
   # $2 is an (optional) destination version number

   # Create a nova.conf file
   cat - > $WORKING_DIR_PATH/nova-$1.conf <<EOF
 [DEFAULT]
 sql_connection = mysql://$DB_USER:$DB_PASS@localhost/$DB_NAME?charset=utf8
 #log_config = $LOG_CONF_FILE
 EOF

   # Silently return git to a known good state (delete untracked files)
   git clean -xfdq

   echo "***** Start DB upgrade to state of $1 *****"
   echo "HEAD of branch under test is:"
   git log -n 1

   echo "Setting up the nova-manage entry point"
   python setup.py -q clean
   python setup.py -q develop
   python setup.py -q install

   # Log the migrations present
   echo "Migrations present:"
   ls $GIT_REPO_PATH/nova/db/sqlalchemy/migrate_repo/versions/*.py | sed 's/.*\///' | egrep "^[0-9]+_"

   # Flush innodb's caches
   echo "Restarting mysql"
   sudo service mysql stop
   sudo service mysql start

   start_version=`mysql -u $DB_USER --password=$DB_PASS $DB_NAME -e "select * from migrate_version \G" | grep version | sed 's/.*: //'`

   if [ "%$2%" == "%%" ]
   then
     end_version=`ls $GIT_REPO_PATH/nova/db/sqlalchemy/migrate_repo/versions/*.py | sed 's/.*\///' | egrep "^[0-9]+_" | tail -1 | cut -f 1 -d "_"`
   else
     end_version=$2
   fi

   echo "Test will migrate from $start_version to $end_version"
   if [ $end_version -lt $start_version ]
   then
     increment=-1
     end_version=$(( $end_version + 1 ))
   else
     increment=1
     start_version=$(( $start_version + 1))
   fi

   for i in `seq $start_version $increment $end_version`
   do
     set -x
     $SCRIPT_DIR/nova-manage-wrapper.sh $VENV_PATH --config-file $WORKING_DIR_PATH/nova-$1.conf --verbose db sync --version $i
     manage_exit=$?
     set +x

     echo "nova-manage returned exit code $manage_exit"
     if [ $manage_exit -gt 0 ]
     then
       echo "Aborting early"
       exit $manage_exit
     fi
   done

   echo "***** Finished DB upgrade to state of $1 *****"
 }

 stable_release_db_sync() {
   version=`mysql -u $DB_USER --password=$DB_PASS $DB_NAME -e "select * from migrate_version \G" | grep version | sed 's/.*: //'`

   # Some databases are from Folsom
   echo "Schema version is $version"
   if [ $version -lt "161" ]
   then
     echo "Database is from Folsom! Upgrade via Grizzly"
     git branch -D eol/grizzly || true
     git remote update
     git checkout -b eol/grizzly
     # Use tag
     git reset --hard grizzly-eol
     pip_requires
     db_sync "grizzly"
   fi

   version=`mysql -u $DB_USER --password=$DB_PASS $DB_NAME -e "select * from migrate_version \G" | grep version | sed 's/.*: //'`
   # Some databases are from Grizzly
   echo "Schema version is $version"
   if [ $version -lt "216" ]
   then
     echo "Database is from Grizzly! Upgrade via Havana"
     git branch -D eol/havana || true
     git remote update
     git checkout -b eol/havana
     # Use tag
     git reset --hard havana-eol
     pip_requires
     db_sync "havana"
   fi

   version=`mysql -u $DB_USER --password=$DB_PASS $DB_NAME -e "select * from migrate_version \G" | grep version | sed 's/.*: //'`
   # Some databases are from Havana
   echo "Schema version is $version"
   if [ $version -lt "234" ]
   then
     echo "Database is from Havana! Upgrade via Icehouse"
     git branch -D eol/icehouse || true
     git remote update
     git checkout -b eol/icehouse
     # Use tag
     git reset --hard icehouse-eol
     pip_requires
     db_sync "icehouse"
   fi

   version=`mysql -u $DB_USER --password=$DB_PASS $DB_NAME -e "select * from migrate_version \G" | grep version | sed 's/.*: //'`
   # Some databases are from Icehouse
   echo "Schema version is $version"
   if [ $version -lt "254" ]
   then
     echo "Database is from Icehouse! Upgrade via Juno"
     git branch -D stable/juno || true
     git remote update
     git checkout -b stable/juno
     git reset --hard remotes/origin/stable/juno
     pip_requires
     db_sync "juno"
   fi

   version=`mysql -u $DB_USER --password=$DB_PASS $DB_NAME -e "select * from migrate_version \G" | grep version | sed 's/.*: //'`
   # Some databases are from Juno
   echo "Schema version is $version"
   if [ $version -lt "280" ]
   then
     echo "Database is from Juno! Upgrade via Kilo"
     git branch -D stable/kilo || true
     git remote update
     git checkout -b stable/kilo
     git reset --hard remotes/origin/stable/kilo
     pip_requires
     db_sync "kilo"

     # TODO(jhesketh): This is a bit of a hack until we update our datasets to
     # have the flavour data migrated. We have to do this before upgrading from
     set -x
     $SCRIPT_DIR/nova-manage-wrapper.sh $VENV_PATH --config-file $WORKING_DIR_PATH/nova-kilo.conf --verbose db migrate_flavor_data --force
     set +x
   fi

   # TODO(jhesketh): Add in Liberty here once released

   # TODO(jhesketh): Make this more DRY and/or automatically match migration
   # numbers to releases.
 }

 echo "Test running on "`hostname`" as "`whoami`" ("`echo ~`", $HOME)"
 echo "To execute this script manually, run this:"
 echo "$0 $@"

 # Setup the environment
 set -x
 export PATH=/usr/lib/ccache:$PATH
 #export PIP_INDEX_URL="http://www.rcbops.com/pypi/mirror"
 export PIP_INDEX_URL="http://pypi.openstack.org/simple/"
 export PIP_EXTRA_INDEX_URL="https://pypi.python.org/simple/"
 which pip
 pip --version
 which virtualenv
 virtualenv --version
 which mkvirtualenv
 set +x

 # Restore database to known good state
 echo "Restoring test database $DB_NAME"
 set -x
 mysql -u $DB_USER --password=$DB_PASS -e "drop database $DB_NAME"
 mysql -u $DB_USER --password=$DB_PASS -e "create database $DB_NAME"
 mysql -u $DB_USER --password=$DB_PASS $DB_NAME < $DATASET_SEED_SQL
 set +x

 echo "Build test environment"
 cd $GIT_REPO_PATH

 echo "Setting up virtual env"
 source ~/.bashrc
 export WORKON_HOME=`pwd`/envs
 mkdir -p $WORKON_HOME
 VENV_PATH=$WORKON_HOME/$UNIQUE_ID
 rm -rf $VENV_PATH
 source /usr/local/bin/virtualenvwrapper.sh
 source /etc/bash_completion.d/virtualenvwrapper
 mkvirtualenv --no-site-packages $UNIQUE_ID
 #toggleglobalsitepackages
 export PYTHONPATH=$PYTHONPATH:$GIT_REPO_PATH

 if [ ! -e $VENV_PATH ]
 then
   echo "Error: making the virtual env failed"
   exit 1
 fi

 stable_release_db_sync

 # Determine the final schema version
 version=`mysql -u $DB_USER --password=$DB_PASS $DB_NAME -e "select * from migrate_version \G" | grep version | sed 's/.*: //'`
 echo "Final schema version is $version"

 if [ "%$NOCLEANUP%" == "%%" ]
 then
   # Cleanup virtual env
   echo "Cleaning up virtual env"
   deactivate
   rmvirtualenv $UNIQUE_ID
 fi

 cd $SCRIPT_DIR
 mysqldump -u $DB_USER --password=$DB_PASS $DB_NAME > $DATASET_OUTPUT_SQL
	#!/bin/bash
	#
	# Copyright 2014 Rackspace Australia
	#
	# Licensed under the Apache License, Version 2.0 (the "License"); you may
	# not use this file except in compliance with the License. You may obtain
	# a copy of the License at
	#
	# http://www.apache.org/licenses/LICENSE-2.0
	#
	# Unless required by applicable law or agreed to in writing, software
	# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
	# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
	# License for the specific language governing permissions and limitations
	# under the License.

	# A tool to update a given dataset to a given version. Used to keep datasets
	# somewhat fresh rather than over-exercising old migrations.


	# Usage: ./update_datasets.sh VENV_NAME WORKING_DIR GIT_PATH DB_USER DB_PASS DB_NAME SEED_DATA OUTPUT_DATA
	# apt-get install git virtualenvwrapper python-pip mysql-server python-lxml build-essential libffi-dev

	# $1 is the unique job id
	# $2 is the working dir path
	# $3 is the path to the git repo path
	# $4 is the nova db user
	# $5 is the nova db password
	# $6 is the nova db name
	# $7 is the path to the seed dataset to test against
	# $8 is the logging.conf for openstack
	# $9 is the pip cache dir

	UNIQUE_ID=$1
	WORKING_DIR_PATH=`realpath $2`
	GIT_REPO_PATH=`realpath $3`
	DB_USER=$4
	DB_PASS=$5
	DB_NAME=$6
	DATASET_SEED_SQL=`realpath $7`
	DATASET_OUTPUT_SQL=$8

	SCRIPT_DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )

	# We also support the following environment variables to tweak our behavour:
	# NOCLEANUP: if set to anything, don't cleanup at the end of the run

	pip_requires() {
	pip install -q mysql-python
	pip install -q eventlet
	requires="tools/pip-requires"
	if [ ! -e $requires ]
	then
	requires="requirements.txt"
	fi
	echo "Install pip requirements from $requires"
	pip install -q -r $requires
	echo "Requirements installed"
	}

	db_sync() {
	# $1 is the test target (ie branch name)
	# $2 is an (optional) destination version number

	# Create a nova.conf file
	cat - > $WORKING_DIR_PATH/nova-$1.conf <<EOF
	[DEFAULT]
	sql_connection = mysql://$DB_USER:$DB_PASS@localhost/$DB_NAME?charset=utf8
	#log_config = $LOG_CONF_FILE
	EOF

	# Silently return git to a known good state (delete untracked files)
	git clean -xfdq

	echo "*** Start DB upgrade to state of $1 ***"
	echo "HEAD of branch under test is:"
	git log -n 1

	echo "Setting up the nova-manage entry point"
	python setup.py -q clean
	python setup.py -q develop
	python setup.py -q install

	# Log the migrations present
	echo "Migrations present:"
	ls $GIT_REPO_PATH/nova/db/sqlalchemy/migrate_repo/versions/.py \| sed 's/.\///' \| egrep "^[0-9]+_"

	# Flush innodb's caches
	echo "Restarting mysql"
	sudo service mysql stop
	sudo service mysql start

	start_version=`mysql -u $DB_USER --password=$DB_PASS $DB_NAME -e "select * from migrate_version \G" \| grep version \| sed 's/.*: //'`

	if [ "%$2%" == "%%" ]
	then
	end_version=`ls $GIT_REPO_PATH/nova/db/sqlalchemy/migrate_repo/versions/.py \| sed 's/.\///' \| egrep "^[0-9]+_" \| tail -1 \| cut -f 1 -d "_"`
	else
	end_version=$2
	fi

	echo "Test will migrate from $start_version to $end_version"
	if [ $end_version -lt $start_version ]
	then
	increment=-1
	end_version=$(( $end_version + 1 ))
	else
	increment=1
	start_version=$(( $start_version + 1))
	fi

	for i in `seq $start_version $increment $end_version`
	do
	set -x
	$SCRIPT_DIR/nova-manage-wrapper.sh $VENV_PATH --config-file $WORKING_DIR_PATH/nova-$1.conf --verbose db sync --version $i
	manage_exit=$?
	set +x

	echo "nova-manage returned exit code $manage_exit"
	if [ $manage_exit -gt 0 ]
	then
	echo "Aborting early"
	exit $manage_exit
	fi
	done

	echo "*** Finished DB upgrade to state of $1 ***"
	}

	stable_release_db_sync() {
	version=`mysql -u $DB_USER --password=$DB_PASS $DB_NAME -e "select * from migrate_version \G" \| grep version \| sed 's/.*: //'`

	# Some databases are from Folsom
	echo "Schema version is $version"
	if [ $version -lt "161" ]
	then
	echo "Database is from Folsom! Upgrade via Grizzly"
	git branch -D eol/grizzly \|\| true
	git remote update
	git checkout -b eol/grizzly
	# Use tag
	git reset --hard grizzly-eol
	pip_requires
	db_sync "grizzly"
	fi

	version=`mysql -u $DB_USER --password=$DB_PASS $DB_NAME -e "select * from migrate_version \G" \| grep version \| sed 's/.*: //'`
	# Some databases are from Grizzly
	echo "Schema version is $version"
	if [ $version -lt "216" ]
	then
	echo "Database is from Grizzly! Upgrade via Havana"
	git branch -D eol/havana \|\| true
	git remote update
	git checkout -b eol/havana
	# Use tag
	git reset --hard havana-eol
	pip_requires
	db_sync "havana"
	fi

	version=`mysql -u $DB_USER --password=$DB_PASS $DB_NAME -e "select * from migrate_version \G" \| grep version \| sed 's/.*: //'`
	# Some databases are from Havana
	echo "Schema version is $version"
	if [ $version -lt "234" ]
	then
	echo "Database is from Havana! Upgrade via Icehouse"
	git branch -D eol/icehouse \|\| true
	git remote update
	git checkout -b eol/icehouse
	# Use tag
	git reset --hard icehouse-eol
	pip_requires
	db_sync "icehouse"
	fi

	version=`mysql -u $DB_USER --password=$DB_PASS $DB_NAME -e "select * from migrate_version \G" \| grep version \| sed 's/.*: //'`
	# Some databases are from Icehouse
	echo "Schema version is $version"
	if [ $version -lt "254" ]
	then
	echo "Database is from Icehouse! Upgrade via Juno"
	git branch -D stable/juno \|\| true
	git remote update
	git checkout -b stable/juno
	git reset --hard remotes/origin/stable/juno
	pip_requires
	db_sync "juno"
	fi

	version=`mysql -u $DB_USER --password=$DB_PASS $DB_NAME -e "select * from migrate_version \G" \| grep version \| sed 's/.*: //'`
	# Some databases are from Juno
	echo "Schema version is $version"
	if [ $version -lt "280" ]
	then
	echo "Database is from Juno! Upgrade via Kilo"
	git branch -D stable/kilo \|\| true
	git remote update
	git checkout -b stable/kilo
	git reset --hard remotes/origin/stable/kilo
	pip_requires
	db_sync "kilo"

	# TODO(jhesketh): This is a bit of a hack until we update our datasets to
	# have the flavour data migrated. We have to do this before upgrading from
	set -x
	$SCRIPT_DIR/nova-manage-wrapper.sh $VENV_PATH --config-file $WORKING_DIR_PATH/nova-kilo.conf --verbose db migrate_flavor_data --force
	set +x
	fi

	# TODO(jhesketh): Add in Liberty here once released

	# TODO(jhesketh): Make this more DRY and/or automatically match migration
	# numbers to releases.
	}

	echo "Test running on "`hostname`" as "`whoami`" ("`echo ~`", $HOME)"
	echo "To execute this script manually, run this:"
	echo "$0 $@"

	# Setup the environment
	set -x
	export PATH=/usr/lib/ccache:$PATH
	#export PIP_INDEX_URL="http://www.rcbops.com/pypi/mirror"
	export PIP_INDEX_URL="http://pypi.openstack.org/simple/"
	export PIP_EXTRA_INDEX_URL="https://pypi.python.org/simple/"
	which pip
	pip --version
	which virtualenv
	virtualenv --version
	which mkvirtualenv
	set +x

	# Restore database to known good state
	echo "Restoring test database $DB_NAME"
	set -x
	mysql -u $DB_USER --password=$DB_PASS -e "drop database $DB_NAME"
	mysql -u $DB_USER --password=$DB_PASS -e "create database $DB_NAME"
	mysql -u $DB_USER --password=$DB_PASS $DB_NAME < $DATASET_SEED_SQL
	set +x

	echo "Build test environment"
	cd $GIT_REPO_PATH

	echo "Setting up virtual env"
	source ~/.bashrc
	export WORKON_HOME=`pwd`/envs
	mkdir -p $WORKON_HOME
	VENV_PATH=$WORKON_HOME/$UNIQUE_ID
	rm -rf $VENV_PATH
	source /usr/local/bin/virtualenvwrapper.sh
	source /etc/bash_completion.d/virtualenvwrapper
	mkvirtualenv --no-site-packages $UNIQUE_ID
	#toggleglobalsitepackages
	export PYTHONPATH=$PYTHONPATH:$GIT_REPO_PATH

	if [ ! -e $VENV_PATH ]
	then
	echo "Error: making the virtual env failed"
	exit 1
	fi

	stable_release_db_sync

	# Determine the final schema version
	version=`mysql -u $DB_USER --password=$DB_PASS $DB_NAME -e "select * from migrate_version \G" \| grep version \| sed 's/.*: //'`
	echo "Final schema version is $version"

	if [ "%$NOCLEANUP%" == "%%" ]
	then
	# Cleanup virtual env
	echo "Cleaning up virtual env"
	deactivate
	rmvirtualenv $UNIQUE_ID
	fi

	cd $SCRIPT_DIR
	mysqldump -u $DB_USER --password=$DB_PASS $DB_NAME > $DATASET_OUTPUT_SQL