1.1 --- /dev/null	Thu Jan 01 00:00:00 1970 +0000
     1.2 +++ b/examples/teleios/runChain.sh	Fri Nov 09 02:14:41 2012 +0200
     1.3 @@ -0,0 +1,275 @@
     1.4 +#! /bin/bash
     1.5 +
     1.6 +#
     1.7 +# This Source Code Form is subject to the terms of the Mozilla Public
     1.8 +# License, v. 2.0. If a copy of the MPL was not distributed with this
     1.9 +# file, You can obtain one at http://mozilla.org/MPL/2.0/.
    1.10 +#
    1.11 +# Copyright (C) 2010, 2011, 2012, Pyravlos Team
    1.12 +#
    1.13 +# http://www.strabon.di.uoa.gr/
    1.14 +#
    1.15 +
    1.16 +LOC="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
    1.17 +
    1.18 +ENDPOINT="http://localhost:8080/NOA"
    1.19 +DB="NOA2012"
    1.20 +
    1.21 +HOTSPOTS_URL="http://localhost/MSG1"
    1.22 +ENDPOINT_SCRIPT=../../scripts/endpoint
    1.23 +
    1.24 +logFile="chain.log"
    1.25 +
    1.26 +function timer()
    1.27 +{
    1.28 +   if [[ $# -eq 0 ]]; then
    1.29 +       t=$(date '+%s%N')
    1.30 +       t=$((t/1000000))
    1.31 +       echo $t
    1.32 +   else
    1.33 +       local  stime=$1
    1.34 +       etime=$(date '+%s%N')
    1.35 +       etime=$((etime/1000000))
    1.36 +
    1.37 +       if [[ -z "$stime" ]]; then stime=$etime; fi
    1.38 +       dt=$((etime - stime)) #dt in milliseconds
    1.39 +       dM=$((dt%1000))
    1.40 +       Dt=$((dt/1000)) #delta t in seconds
    1.41 +       ds=$((Dt % 60))
    1.42 +       dm=$(((Dt / 60) % 60))
    1.43 +       dh=$((Dt / 3600))
    1.44 +       printf '%d:%02d:%02d.%03d' $dh $dm $ds $dM
    1.45 +   fi
    1.46 +}
    1.47 +
    1.48 +# find out the postgres service to use
    1.49 +postgres=$(ls -1 /etc/init.d/| grep postgres | head -1)
    1.50 +
    1.51 +tomcat=
    1.52 +function chooseTomcat()
    1.53 +{
    1.54 +	if test -s /etc/fedora-release ; then
    1.55 +		tomcat="tomcat"
    1.56 +	#elif test -s /etc/centos-release ; then
    1.57 +	#elif test -s /etc/yellowdog-release ; then
    1.58 +	#elif test -s /etc/redhat-release ; then
    1.59 +	#elif test -s /etc/SuSE-release ; then
    1.60 +	#elif test -s /etc/gentoo-release ; then
    1.61 +	elif test -s /etc/lsb-release ; then # Ubuntu
    1.62 +			tomcat=$(ls -1 /etc/init.d/| grep tomcat | head -1)
    1.63 +	elif test -s /etc/debian_version ; then
    1.64 +		tomcat="tomcat"
    1.65 +	fi
    1.66 +
    1.67 +	# check for service availability
    1.68 +	if ! test -e "/etc/init.d/${tomcat}"; then
    1.69 +		tomcat=
    1.70 +	fi
    1.71 +}
    1.72 +
    1.73 +insertMunicipalities=`cat ${LOC}/insertMunicipalities.rq`
    1.74 +deleteSeaHotspots=`cat ${LOC}/deleteSeaHotspots.rq` # | sed 's/\"/\\\"/g'`
    1.75 +refinePartialSeaHotspots=`cat ${LOC}/refinePartialSeaHotspots.rq` # | sed 's/\"/\\\"/g'`
    1.76 +invalidForFires=`cat ${LOC}/landUseInvalidForFires.rq`
    1.77 +refineTimePersistence=`cat ${LOC}/refineTimePersistence.rq` # | sed 's/\"/\\\"/g'`
    1.78 +discover=`cat ${LOC}/discover.rq`
    1.79 +#InsertMunicipalities =`cat ${LOC}/InsertMunicipalities.sparql` # | sed 's/\"/\\\"/g'`
    1.80 +
    1.81 +# Initialize (stop tomcat, restart postgres, drop/create database, start tomcat)
    1.82 +chooseTomcat
    1.83 +echo "stopping tomcat"
    1.84 +if test -z "${tomcat}"; then
    1.85 +	# work-around for babis (standalone tomcat, with start_tomcat.sh and stop_tomcat.sh scripts)
    1.86 +	stop_tomcat.sh
    1.87 +else
    1.88 +	sudo service ${tomcat} stop
    1.89 +fi
    1.90 +
    1.91 +sudo service ${postgres} restart
    1.92 +
    1.93 +# get the main version of postgres
    1.94 +POSTGRES_MAIN_VERSION=$(sudo service ${postgres} status | grep -o '.\..' | cut -b 1)
    1.95 +
    1.96 +echo "Dropping endpoint database";
    1.97 +dropdb -U postgres ${DB}
    1.98 +
    1.99 +echo "Creating endpoint database"
   1.100 +createdb -U postgres ${DB} 
   1.101 +
   1.102 +# load data
   1.103 +#curl -s http://dev.strabon.di.uoa.gr/rdf/Kallikratis-Coastline-Corine-dump-postgres-${POSTGRES_MAIN_VERSION}.tgz | tar xz -O | psql -d ${DB}
   1.104 +psql -U postgres -d ${DB} -f /opt/data/Additional-GIS-Data/Kallikratis-Coastline-ExcludeArea-dump.sql
   1.105 +psql -U postgres ${DB} -c 'VACUUM ANALYZE' 
   1.106 +
   1.107 +echo "starting tomcat"
   1.108 +if test -z "${tomcat}"; then
   1.109 +	# work-around for babis (standalone tomcat, with start_tomcat.sh and stop_tomcat.sh scripts)
   1.110 +	start_tomcat.sh
   1.111 +	sleep 2
   1.112 +else
   1.113 +	sudo service ${tomcat} start
   1.114 +fi
   1.115 +
   1.116 +echo "initializing database"
   1.117 +echo "Timestamp Store Municipalities DeleteInSea InvalidForFires RefineInCoast TimePersistence" > stderr.txt
   1.118 +
   1.119 +
   1.120 +#./scripts/endpoint query ${ENDPOINT} "SELECT (COUNT(*) AS ?C) WHERE {?s ?p ?o}"
   1.121 +#sudo -u postgres psql -d endpoint -c 'CREATE INDEX datetime_values_idx_value ON datetime_values USING btree(value)';
   1.122 +#sudo -u postgres psql -d endpoint -c 'VACUUM ANALYZE;';
   1.123 +
   1.124 +#for y in 2007 2008 2010 2011; do
   1.125 +for y in 2012; do
   1.126 +	# get hotpost URLS
   1.127 +	for hot in $(curl -s ${HOTSPOTS_URL}/${y}/ | grep -o '>HMSG.*\.nt' | colrm 1 1); do
   1.128 +		file="${HOTSPOTS_URL}/${y}/${hot}"
   1.129 +
   1.130 +		time_status=$(echo ${hot} | egrep -o '[[:digit:]]{6}_[[:digit:]]{4}')
   1.131 +		
   1.132 +		# get sensor
   1.133 +		SENSOR=$(echo ${hot} | grep -o 'MSG.')
   1.134 +
   1.135 +		# get satellite and set number of acquisitions per hour
   1.136 +		if test "${SENSOR}" = "MSG2"; then
   1.137 +			SAT="METEOSAT9"
   1.138 +
   1.139 +			N_ACQUISITIONS=3.0
   1.140 +		else
   1.141 +			SAT="METEOSAT8"
   1.142 +			SENSOR="MSG1_RSS"
   1.143 +
   1.144 +			N_ACQUISITIONS=7.0
   1.145 +		fi
   1.146 +
   1.147 +		# get time information for acquisition and construct timestamp
   1.148 +		year="20$(expr substr ${time_status} 1 2)"
   1.149 +		month=$(expr substr ${time_status} 3 2)
   1.150 +		day=$(expr substr ${time_status} 5 2)
   1.151 +		time2=$(expr substr ${time_status} 8 2)
   1.152 +		time2="${time2}:$(expr substr ${time_status} 10 2)"
   1.153 +
   1.154 +		# construct timestamp
   1.155 +		TIMESTAMP="${year}-${month}-${day}T${time2}:00"
   1.156 +
   1.157 +		# store file
   1.158 +		echo -n "storing " $file; echo; echo; 
   1.159 +		# ${countTime} ./strabon -db endpoint store $file
   1.160 +
   1.161 +		# print timestamp
   1.162 +		echo -n "${TIMESTAMP} " >> stderr.txt
   1.163 +
   1.164 +		tmr1=$(timer)
   1.165 +		${ENDPOINT_SCRIPT} store ${ENDPOINT} N-Triples -u ${file}
   1.166 +		tmr2=$(timer)
   1.167 +		printf '%s ' $((tmr2-tmr1)) >> stderr.txt
   1.168 +
   1.169 +		# sudo -u postgres psql -d endpoint -c 'VACUUM ANALYZE;';
   1.170 +
   1.171 +		echo;echo;echo;echo "File ${file} stored!" >> ${logFile}
   1.172 +
   1.173 +		# insertMunicipalities
   1.174 +		echo -n "inserting Municipalities " ;echo; echo; echo;
   1.175 +		# query=`echo "${insertMunicipalities}" `
   1.176 +		# ${countTime} ./strabon -db endpoint update "${query}"
   1.177 +
   1.178 +		tmr1=$(timer)
   1.179 +
   1.180 +		query=`echo "${insertMunicipalities}" | sed "s/TIMESTAMP/${TIMESTAMP}/g" | \
   1.181 +		sed "s/PROCESSING_CHAIN/DynamicThresholds/g" | \
   1.182 +		sed "s/SENSOR/${SENSOR}/g"`
   1.183 +
   1.184 +		${ENDPOINT_SCRIPT} update ${ENDPOINT} "${query}"
   1.185 +		
   1.186 +		tmr2=$(timer)
   1.187 +		printf '%s ' $((tmr2-tmr1)) >>stderr.txt
   1.188 +		echo;echo;echo;echo "File ${file} inserted Municipalities!"
   1.189 +
   1.190 +		# execute an explicit VACUUM ANALYZE when a query takes longer than it should
   1.191 +		duration=$((tmr2-tmr1))
   1.192 +		if test ${duration} -ge 30000; then
   1.193 +			psql -U postgres ${DB} -c 'VACUUM ANALYZE' 
   1.194 +			echo "Explicit VACUUM ANALYZE"
   1.195 +		fi
   1.196 +		
   1.197 +		# deleteSeaHotspots
   1.198 +		echo -n "Going to deleteSeaHotspots ${TIMESTAMP} " ;echo; echo; echo;
   1.199 +		query=`echo "${deleteSeaHotspots}" | sed "s/TIMESTAMP/${TIMESTAMP}/g" | \
   1.200 +		sed "s/PROCESSING_CHAIN/DynamicThresholds/g" | \
   1.201 +		sed "s/SENSOR/${SENSOR}/g"`
   1.202 +		# ${countTime} ./strabon -db endpoint update "${query}"
   1.203 +
   1.204 +		tmr1=$(timer)
   1.205 +		${ENDPOINT_SCRIPT} update ${ENDPOINT} "${query}"
   1.206 +
   1.207 +		tmr2=$(timer)
   1.208 +		printf '%s ' $((tmr2-tmr1)) >>stderr.txt
   1.209 +		echo;echo;echo;echo "File ${file} deleteSeaHotspots done!"
   1.210 +
   1.211 +		# invalidForFires
   1.212 +		echo -n "invalidForFires ${TIMESTAMP} "  ; echo; echo ; echo;
   1.213 +		query=`echo "${invalidForFires}" | sed "s/TIMESTAMP/${TIMESTAMP}/g" | \
   1.214 +		sed "s/PROCESSING_CHAIN/DynamicThresholds/g" | \
   1.215 +		sed "s/SENSOR/${SENSOR}/g"` 
   1.216 +
   1.217 +		# ${countTime} ./strabon -db endpoint update "${query}"
   1.218 +		tmr1=$(timer)
   1.219 +		${ENDPOINT_SCRIPT} update ${ENDPOINT} "${query}"
   1.220 +		tmr2=$(timer)
   1.221 +		printf '%s ' $((tmr2-tmr1)) >>stderr.txt
   1.222 +		echo "File ${file} invalidForFires done!"
   1.223 + 
   1.224 +		# refinePartialSeaHotspots
   1.225 +		echo -n "refinePartialSeaHotspots ${TIMESTAMP} "  ; echo; echo ; echo;
   1.226 +		query=`echo "${refinePartialSeaHotspots}" | sed "s/TIMESTAMP/${TIMESTAMP}/g" | \
   1.227 +		sed "s/PROCESSING_CHAIN/DynamicThresholds/g" | \
   1.228 +		sed "s/SENSOR/${SENSOR}/g" |\
   1.229 +		sed "s/SAT/${SAT}/g"`
   1.230 +		# ${countTime} ./strabon -db endpoint update "${query}"
   1.231 +		tmr1=$(timer)
   1.232 +		${ENDPOINT_SCRIPT} update ${ENDPOINT} "${query}"
   1.233 +		tmr2=$(timer)
   1.234 +		printf '%s ' $((tmr2-tmr1)) >>stderr.txt
   1.235 +
   1.236 +		echo "File ${file} refinePartialSeaHotspots done!"
   1.237 +		# echo "Continue?"
   1.238 +		# read a
   1.239 +
   1.240 +		# refineTimePersistence
   1.241 +		echo -n "Going to refineTimePersistence ${TIMESTAMP} ";echo;echo;echo; 
   1.242 +		min_acquisition_time=`date --date="${year}-${month}-${day} ${time2}:00 EEST -30 minutes" +%Y-%m-%dT%H:%M:00`
   1.243 +		query=`echo "${refineTimePersistence}" | sed "s/TIMESTAMP/${TIMESTAMP}/g" | \
   1.244 +		sed "s/PROCESSING_CHAIN/DynamicThresholds/g" | \
   1.245 +		sed "s/SENSOR/${SENSOR}/g" | \
   1.246 +		sed "s/ACQUISITIONS_IN_HALF_AN_HOUR/${N_ACQUISITIONS}/g" | \
   1.247 +		sed "s/MIN_ACQUISITION_TIME/${min_acquisition_time}/g" |\
   1.248 +		sed "s/SAT/${SAT}/g"`
   1.249 +
   1.250 +		#sudo -u postgres psql -d ${DB} -c 'VACUUM ANALYZE;';
   1.251 +
   1.252 +		tmr1=$(timer)
   1.253 +		${ENDPOINT_SCRIPT} update ${ENDPOINT} "${query}"
   1.254 +		 tmr2=$(timer)
   1.255 +		printf '%s \n' $((tmr2-tmr1)) >>stderr.txt
   1.256 +		echo;echo;echo;echo "File ${file} timePersistence done!"
   1.257 +		# echo "Continue?"
   1.258 +		# read a
   1.259 +
   1.260 +
   1.261 +		# discover
   1.262 +		echo -n "Going to discover ${TIMESTAMP} ";echo;echo;echo; 
   1.263 +		min_acquisition_time=`date --date="${year}-${month}-${day} 00:00 EEST" +%Y-%m-%dT%H:%M:00`
   1.264 +		max_acquisition_time=`date --date="${year}-${month}-${day} 23:59 EEST" +%Y-%m-%dT%H:%M:00`
   1.265 +		query=`echo "${discover}" | \
   1.266 +			sed "s/PROCESSING_CHAIN/DynamicThresholds/g" | \
   1.267 +			sed "s/SENSOR/${SENSOR}/g" | \
   1.268 +			sed "s/MIN_ACQUISITION_TIME/${min_acquisition_time}/g" |\
   1.269 +			sed "s/MAX_ACQUISITION_TIME/${max_acquisition_time}/g"`
   1.270 +			
   1.271 +		tmr1=$(timer)
   1.272 +		${ENDPOINT_SCRIPT} query ${ENDPOINT} "${query}"
   1.273 +		tmr2=$(timer)
   1.274 +		printf '%s \n' $((tmr2-tmr1)) >>discover.txt
   1.275 +		echo;echo;echo;echo "Discovered hotspots done!"
   1.276 +	done
   1.277 +done
   1.278 +