Strabon

changeset 457:3197adbd785e

Adding script that executes the refinement processing chain on NOA data hosted in papos
author Kostis Kyzirakos <kkyzir@di.uoa.gr>
date Thu Jul 19 15:36:41 2012 +0300 (2012-07-19)
parents da5b9a0d9659
children bca9421e4cfc
files scripts/v2.2/runNoaRefinementChain-papos.sh
line diff
     1.1 --- /dev/null	Thu Jan 01 00:00:00 1970 +0000
     1.2 +++ b/scripts/v2.2/runNoaRefinementChain-papos.sh	Thu Jul 19 15:36:41 2012 +0300
     1.3 @@ -0,0 +1,252 @@
     1.4 +#!/bin/bash
     1.5 +LOC="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
     1.6 +
     1.7 +############################ CONFIGURATION #############################
     1.8 +ENDPOINT="http://localhost:8080/endpoint"
     1.9 +DB="endpoint"
    1.10 +
    1.11 +HOTSPOTS_URL="http://papos.space.noa.gr/rdf"
    1.12 +URLDIR=(   "msg1_rss"              "msg2")
    1.13 +URLPREFIX=("HMSG1_RSS_IR_039_s7_"  "HMSG2_IR_039_s7_")
    1.14 +SENSOR=(   "MSG1_RSS"              "MSG2")
    1.15 +SATELITE=( "METEOSAT9"             "METEOSAT9")
    1.16 +SUFFIX=".hotspots.n3"
    1.17 +PROCESSING_CHAIN="DynamicThresholds"
    1.18 +ACQUISITIONS_IN_HALF_AN_HOUR="3.0"
    1.19 +
    1.20 +# log files
    1.21 +logFile="chain.log"
    1.22 +timings="chain-times.log"
    1.23 +timingsDiscover="discover.txt"
    1.24 +
    1.25 +# stSPARQL statements
    1.26 +insertMunicipalities=`cat ${LOC}/insertMunicipalities.sparql` 
    1.27 +deleteSeaHotspots=`cat ${LOC}/deleteSeaHotspots.sparql` 
    1.28 +invalidForFires=`cat ${LOC}/landUseInvalidForFires.sparql`
    1.29 +refinePartialSeaHotspots=`cat ${LOC}/refinePartialSeaHotspots.sparql`
    1.30 +refineTimePersistence=`cat ${LOC}/refineTimePersistence.sparql`
    1.31 +discover=`cat ${LOC}/discover.sparql`
    1.32 +# | sed 's/\"/\\\"/g'`
    1.33 +########################################################################
    1.34 +
    1.35 +function timer()
    1.36 +{
    1.37 +   if [[ $# -eq 0 ]]; then
    1.38 +       t=$(date '+%s%N')
    1.39 +       t=$((t/1000000))
    1.40 +       echo $t
    1.41 +   else
    1.42 +       local  stime=$1
    1.43 +       etime=$(date '+%s%N')
    1.44 +       etime=$((etime/1000000))
    1.45 +
    1.46 +       if [[ -z "$stime" ]]; then stime=$etime; fi
    1.47 +       dt=$((etime - stime)) #dt in milliseconds
    1.48 +       dM=$((dt%1000))
    1.49 +       Dt=$((dt/1000)) #delta t in seconds
    1.50 +       ds=$((Dt % 60))
    1.51 +       dm=$(((Dt / 60) % 60))
    1.52 +       dh=$((Dt / 3600))
    1.53 +       printf '%d:%02d:%02d.%03d' $dh $dm $ds $dM
    1.54 +   fi
    1.55 +}
    1.56 +
    1.57 +# find out the postgres service to use
    1.58 +postgres=$(ls -1 /etc/init.d/| grep postgres | head -1)
    1.59 +
    1.60 +tomcat=
    1.61 +function chooseTomcat()
    1.62 +{
    1.63 +	if test -s /etc/fedora-release ; then
    1.64 +		tomcat="tomcat"
    1.65 +	#elif test -s /etc/centos-release ; then
    1.66 +	#elif test -s /etc/yellowdog-release ; then
    1.67 +	#elif test -s /etc/redhat-release ; then
    1.68 +	#elif test -s /etc/SuSE-release ; then
    1.69 +	#elif test -s /etc/gentoo-release ; then
    1.70 +	elif test -s /etc/lsb-release ; then # Ubuntu
    1.71 +			if test -s /etc/init.d/tomcat6 ; then
    1.72 +				tomcat="tomcat6"
    1.73 +			elif test -s /etc/init.d/tomcat7 ; then
    1.74 +				tomcat="tomcat7"
    1.75 +			fi
    1.76 +	elif test -s /etc/debian_version ; then
    1.77 +			tomcat="tomcat"
    1.78 +	fi
    1.79 +
    1.80 +	# check for service availability
    1.81 +	if ! test -e "/etc/init.d/${tomcat}"; then
    1.82 +		tomcat=
    1.83 +	fi
    1.84 +}
    1.85 +
    1.86 +# Initialize (stop tomcat, restart postgres, drop/create database, start tomcat)
    1.87 +chooseTomcat
    1.88 +echo "stopping tomcat"
    1.89 +if test -z "${tomcat}"; then
    1.90 +	# work-around for babis (standalone tomcat, with start_tomcat.sh and stop_tomcat.sh scripts)
    1.91 +	stop_tomcat.sh
    1.92 +else
    1.93 +	sudo service ${tomcat} stop
    1.94 +fi
    1.95 +
    1.96 +sudo service ${postgres} restart
    1.97 +
    1.98 +# get the main version of postgres
    1.99 +POSTGRES_MAIN_VERSION=$(sudo service ${postgres} status | grep -o '.\..' | cut -b 1)
   1.100 +
   1.101 +echo "Dropping endpoint database";
   1.102 +dropdb -U postgres ${DB}
   1.103 +
   1.104 +echo "Creating endpoint database"
   1.105 +createdb -U postgres ${DB} 
   1.106 +
   1.107 +# load background data
   1.108 +echo "initializing database"
   1.109 +curl -s  http://dev.strabon.di.uoa.gr/rdf/Kallikratis-Coastline-Corine-dump-postgres-${POSTGRES_MAIN_VERSION}.tgz | tar xz -O | psql -U postgres -d ${DB}
   1.110 +psql ${DB} -U postgres -c 'VACUUM ANALYZE '
   1.111 +
   1.112 +echo "starting tomcat"
   1.113 +if test -z "${tomcat}"; then
   1.114 +	# work-around for babis (standalone tomcat, with start_tomcat.sh and stop_tomcat.sh scripts)
   1.115 +	start_tomcat.sh
   1.116 +else
   1.117 +	sudo service ${tomcat} start
   1.118 +fi
   1.119 +
   1.120 +# the chain :)
   1.121 +echo "Store Municipalities DeleteInSea InvalidForFires RefineInCoast TimePersistence" > ${timings}
   1.122 +for (( i = 0 ; i < ${#URLDIR[@]} ; i++ )) do
   1.123 +	dir=${URLDIR[$i]}
   1.124 +	prefix=${URLPREFIX[$i]}
   1.125 +	sensor=${SENSOR[$i]}
   1.126 +	satelite=${SATELITE[$i]}
   1.127 +	
   1.128 +	# get hotpost URLS
   1.129 +	for hot in $(curl -s ${HOTSPOTS_URL}/${dir}/ | grep -o ">${prefix}.*\.n3" | colrm 1 1); do
   1.130 +		echo $hot
   1.131 +		file="${HOTSPOTS_URL}/${dir}/${hot}"
   1.132 +		echo $file
   1.133 +
   1.134 +		# get time information for acquisition
   1.135 +		offset=$(( ${#prefix} + 1 ))
   1.136 +		year=$(expr substr ${hot} ${offset} 2)
   1.137 +		month=$(expr substr ${hot} $(( ${offset} + 2 )) 2)
   1.138 +		day=$(expr substr ${hot} $(( ${offset} + 4 )) 2)
   1.139 +		t1=$(expr substr ${hot} $(( ${offset} + 7 )) 2)
   1.140 +		t2=$(expr substr ${hot} $(( ${offset} + 9 )) 2)
   1.141 +		time2="${t1}:${t2}"
   1.142 +
   1.143 +		# store file
   1.144 +		echo -n "storing " $file; echo; echo; 
   1.145 +		# ${countTime} ./strabon -db endpoint store $file
   1.146 +
   1.147 +		tmr1=$(timer)
   1.148 +		../endpoint store ${ENDPOINT} N-Triples -u ${file}
   1.149 +		tmr2=$(timer)
   1.150 +		printf '%s ' $((tmr2-tmr1)) >>  ${timings}
   1.151 +
   1.152 +		# sudo -u postgres psql -d endpoint -c 'VACUUM ANALYZE;';
   1.153 +
   1.154 +		echo;echo;echo;echo "File ${file} stored!" >> ${logFile}
   1.155 +
   1.156 +		# insertMunicipalities
   1.157 +		echo -n "inserting Municipalities " ;echo; echo; echo;
   1.158 +		# query=`echo "${insertMunicipalities}" `
   1.159 +		# ${countTime} ./strabon -db endpoint update "${query}"
   1.160 +
   1.161 +		tmr1=$(timer)
   1.162 +
   1.163 +		query=`echo "${insertMunicipalities}" | sed "s/TIMESTAMP/20${year}-${month}-${day}T${time2}:00/g" | \
   1.164 +		sed "s/PROCESSING_CHAIN/${PROCESSING_CHAIN}/g" | \
   1.165 +		sed "s/SENSOR/${sensor}/g"`
   1.166 +
   1.167 +		../endpoint update ${ENDPOINT} "${query}"
   1.168 +		
   1.169 +		tmr2=$(timer)
   1.170 +printf '%s ' $((tmr2-tmr1)) >> ${timings}
   1.171 +		echo;echo;echo;echo "File ${file} inserted Municipalities!"
   1.172 +		
   1.173 +		# deleteSeaHotspots
   1.174 +		echo -n "Going to deleteSeaHotspots 20${year}-${month}-${day}T${time2}:00 " ;echo; echo; echo;
   1.175 +		query=`echo "${deleteSeaHotspots}" | sed "s/TIMESTAMP/20${year}-${month}-${day}T${time2}:00/g" | \
   1.176 +		sed "s/PROCESSING_CHAIN/${PROCESSING_CHAIN}/g" | \
   1.177 +		sed "s/SENSOR/${sensor}/g"`
   1.178 +		# ${countTime} ./strabon -db endpoint update "${query}"
   1.179 +
   1.180 +		tmr1=$(timer)
   1.181 +		../endpoint update ${ENDPOINT} "${query}"
   1.182 +
   1.183 +		tmr2=$(timer)
   1.184 +		printf '%s ' $((tmr2-tmr1)) >> ${timings}
   1.185 +		echo;echo;echo;echo "File ${file} deleteSeaHotspots done!"
   1.186 +
   1.187 +		# echo "Continue?"
   1.188 +		# read a
   1.189 +			# invalidForFires
   1.190 +		echo -n "invalidForFires 20${year}-${month}-${day}T${time2}:00 "  ; echo; echo ; echo;
   1.191 +		query=`echo "${invalidForFires}" | sed "s/TIMESTAMP/20${year}-${month}-${day}T${time2}:00/g" | \
   1.192 +		sed "s/PROCESSING_CHAIN/${PROCESSING_CHAIN}/g" | \
   1.193 +		sed "s/SENSOR/${sensor}/g" |\
   1.194 +		sed "s/SAT/${satelite}/g"`
   1.195 +		# ${countTime} ./strabon -db endpoint update "${query}"
   1.196 +		tmr1=$(timer)
   1.197 +		../endpoint update ${ENDPOINT} "${query}"
   1.198 +		tmr2=$(timer)
   1.199 +		printf '%s ' $((tmr2-tmr1)) >> ${timings}
   1.200 +		echo "File ${file} invalidForFires done!"
   1.201 + 
   1.202 +		# refinePartialSeaHotspots
   1.203 +		echo -n "refinePartialSeaHotspots 20${year}-${month}-${day}T${time2}:00 "  ; echo; echo ; echo;
   1.204 +		query=`echo "${refinePartialSeaHotspots}" | sed "s/TIMESTAMP/20${year}-${month}-${day}T${time2}:00/g" | \
   1.205 +		sed "s/PROCESSING_CHAIN/${PROCESSING_CHAIN}/g" | \
   1.206 +		sed "s/SENSOR/${sensor}/g" |\
   1.207 +		sed "s/SAT/${satelite}/g"`
   1.208 +		# ${countTime} ./strabon -db endpoint update "${query}"
   1.209 +		tmr1=$(timer)
   1.210 +		../endpoint update ${ENDPOINT} "${query}"
   1.211 +		tmr2=$(timer)
   1.212 +		printf '%s ' $((tmr2-tmr1)) >> ${timings}
   1.213 +
   1.214 +		echo "File ${file} refinePartialSeaHotspots done!"
   1.215 +		# echo "Continue?"
   1.216 +		# read a
   1.217 +
   1.218 +		# refineTimePersistence
   1.219 +		echo -n "Going to refineTimePersistence 20${year}-${month}-${day}T${time2}:00 ";echo;echo;echo; 
   1.220 +		min_acquisition_time=`date --date="20${year}-${month}-${day} ${time2}:00 EEST -30 minutes" +%Y-%m-%dT%H:%m:00`
   1.221 +		query=`echo "${refineTimePersistence}" | sed "s/TIMESTAMP/20${year}-${month}-${day}T${time2}:00/g" | \
   1.222 +		sed "s/PROCESSING_CHAIN/${PROCESSING_CHAIN}/g" | \
   1.223 +		sed "s/SENSOR/${sensor}/g" | \
   1.224 +		sed "s/ACQUISITIONS_IN_HALF_AN_HOUR/${ACQUISITIONS_IN_HALF_AN_HOUR}/g" | \
   1.225 +		sed "s/MIN_ACQUISITION_TIME/${min_acquisition_time}/g" |\
   1.226 +		sed "s/SAT/${satelite}/g"`
   1.227 +
   1.228 +		#sudo -u postgres psql -d ${DB} -c 'VACUUM ANALYZE;';
   1.229 +
   1.230 +		tmr1=$(timer)
   1.231 +		../endpoint update ${ENDPOINT} "${query}"
   1.232 +		 tmr2=$(timer)
   1.233 +		printf '%s \n' $((tmr2-tmr1)) >> ${timings}
   1.234 +		echo;echo;echo;echo "File ${file} timePersistence done!"
   1.235 +		# echo "Continue?"
   1.236 +		# read a
   1.237 +
   1.238 +
   1.239 +		# discover
   1.240 +		echo -n "Going to discover 20${year}-${month}-${day}T${time2}:00 ";echo;echo;echo; 
   1.241 +		min_acquisition_time=`date --date="20${year}-${month}-${day} 00:00 EEST" +%Y-%m-%dT%H:%m:00`
   1.242 +		max_acquisition_time=`date --date="20${year}-${month}-${day} 23:59 EEST" +%Y-%m-%dT%H:%m:00`
   1.243 +		query=`echo "${discover}" | \
   1.244 +			sed "s/PROCESSING_CHAIN/${PROCESSING_CHAIN}/g" | \
   1.245 +			sed "s/SENSOR/${sensor}/g" | \
   1.246 +			sed "s/MIN_ACQUISITION_TIME/${min_acquisition_time}/g" |\
   1.247 +			sed "s/MAX_ACQUISITION_TIME/${max_acquisition_time}/g"`
   1.248 +			
   1.249 +		tmr1=$(timer)
   1.250 +		../endpoint query ${ENDPOINT} "${query}"
   1.251 +		tmr2=$(timer)
   1.252 +		printf '%s \n' $((tmr2-tmr1)) >> ${timingsDiscover}
   1.253 +		echo;echo;echo;echo "Discovered hotspots done!"
   1.254 +	done
   1.255 +done