Strabon

changeset 827:bc8d029d2088 DiffDateTime

New version of runChain.sh
author George Garbis <ggarbis@di.uoa.gr>
date Fri Dec 21 16:29:08 2012 +0200 (2012-12-21)
parents 7ebfdbbce0d5
children 52396ec09aae
files examples/teleios/runChain.sh
line diff
     1.1 --- a/examples/teleios/runChain.sh	Fri Dec 21 16:27:59 2012 +0200
     1.2 +++ b/examples/teleios/runChain.sh	Fri Dec 21 16:29:08 2012 +0200
     1.3 @@ -8,6 +8,11 @@
     1.4  # 
     1.5  # http://www.strabon.di.uoa.gr/ 
     1.6  #
     1.7 +# Author: George Garbis <ggarbis@di.uoa.gr>
     1.8 +# Author: Charalampos (Babis) Nikolaou <charnik@di.uoa.gr>
     1.9 +# Author: Manos Karpathiotakis <mk@di.uoa.gr>
    1.10 +# Author: Konstantina Bereta <Konstantina.Bereta@di.uoa.gr>
    1.11 +#
    1.12  
    1.13  # Command name
    1.14  cmd="$(basename ${0})" 
    1.15 @@ -59,7 +64,7 @@
    1.16  	# find out the postgres service to use
    1.17  	postgres=$(ls -1 /etc/init.d/| grep postgres | head -1)
    1.18  
    1.19 -	echo "Postgres service received command: $1"
    1.20 +	echo "Service ${postgres} received command: $1"
    1.21  	sudo service ${postgres} $1
    1.22  }
    1.23  
    1.24 @@ -74,7 +79,14 @@
    1.25  	local options="$*"
    1.26  	case "${command}" in
    1.27  		create)
    1.28 -			echo "Creating database ${db}..."
    1.29 +            if test "${options}" = "spatial"; then
    1.30 +                options="-T template_postgis"
    1.31 +            elif test ! -z "${options}"; then
    1.32 +				echo "ERROR: only spatial is allowed for create option"
    1.33 +				echo "options: ${options}"
    1.34 +				exit -1
    1.35 +			fi
    1.36 +			echo "Creating database ${db}... with options ${options}"
    1.37  			createdb -U postgres ${db} ${options}	
    1.38  			;;
    1.39  		drop)
    1.40 @@ -88,13 +100,15 @@
    1.41  			;;
    1.42  		vacuum)
    1.43  			if test "${options}" = "analyze"; then
    1.44 -				psql -U postgres ${DB} -c 'VACUUM ANALYZE' 
    1.45 +				psql -U postgres ${db} -c 'VACUUM ANALYZE' 
    1.46 +                echo "VACUUM ANALYZE ${db}"
    1.47  			else
    1.48 -				psql -U postgres ${DB} -c 'VACUUM' 
    1.49 +				psql -U postgres ${db} -c 'VACUUM' 
    1.50 +                echo "VACUUM ${db}"
    1.51  			fi
    1.52  			;;
    1.53 -		run)
    1.54 -			if test ! [ -f "${options}" ]; then
    1.55 +		runscript)
    1.56 +			if test ! -f "${options}"; then
    1.57  				echo "ERROR: No dump file to run"
    1.58  				exit -1
    1.59  			fi
    1.60 @@ -145,13 +159,13 @@
    1.61  	local bgFile=$2
    1.62  
    1.63  	if test -f ${bgFile}; then
    1.64 -		handlePostgresDatabase run ${db} ${bgFile}	
    1.65 +		handlePostgresDatabase runscript ${db} ${bgFile}	
    1.66  	elif test "${bgFile:0:7}" = "http://"; then
    1.67  		#curl -s ${bgFile} | tar xz -O | psql -d ${DB}
    1.68  		wget ${bgFile} -O /tmp/bgFile$$.tar.gz
    1.69  		tar xz -O /tmp/bgFile$$.tar.gz
    1.70  		
    1.71 -		handlePostgresDatabase run ${db} /tmp/bgFile$$.tar.tz
    1.72 +		handlePostgresDatabase runscript ${db} /tmp/bgFile$$.tar.tz
    1.73  		rm /tmp/bgFiile$$.tar.gz
    1.74  	else
    1.75  		echo "Backgound file not foung"
    1.76 @@ -167,55 +181,59 @@
    1.77  function handleStrabonEndpoint(){
    1.78  	endpoint=$1
    1.79  	command=$2
    1.80 -	option=$3
    1.81 +	options=$3
    1.82  
    1.83  	endpointScript=${loc}/../../scripts/endpoint
    1.84  	case ${command} in
    1.85  		store)
    1.86  			url=${options}
    1.87  
    1.88 -			# print timestamp
    1.89 -			echo -n "${TIMESTAMP} " >> ${logFile}
    1.90 -
    1.91  			tmr1=$(timer)
    1.92  			${endpointScript} store ${endpoint} N-Triples -u ${url}
    1.93  			tmr2=$(timer)
    1.94 +
    1.95 +    		# execute an explicit VACUUM ANALYZE when a query takes longer than it should
    1.96 +    		duration=$((tmr2-tmr1))
    1.97 +    		if test ${duration} -ge 30000; then
    1.98 +                handlePostgresDatabase vacuum ${db} analyze
    1.99 +#    			psql -U postgres ${DB} -c 'VACUUM ANALYZE' 
   1.100 +#    			echo "Explicit VACUUM ANALYZE"
   1.101 +        	    tmr2=$(timer)
   1.102 +    		fi
   1.103  			printf '%s ' $((tmr2-tmr1)) >> ${logFile}
   1.104  			;;
   1.105  		query)
   1.106 -			query=${option}
   1.107 -
   1.108 +			query=${options}
   1.109  			tmr1=$(timer)
   1.110  			${endpointScript} query ${endpoint} "${query}"
   1.111  			tmr2=$(timer)
   1.112  			printf '%s ' $((tmr2-tmr1)) >> ${logFile} 
   1.113  			;;
   1.114  		update)
   1.115 -			update=${option}
   1.116 +			update=${options}
   1.117  			tmr1=$(timer)
   1.118  			${endpointScript} update ${endpoint} "${update}"
   1.119  			tmr2=$(timer)
   1.120 -			printf '%s ' $((tmr2-tmr1)) >> stderr.txt
   1.121 +			printf '%s ' $((tmr2-tmr1)) >> ${logFile}
   1.122  			;;
   1.123  		*)
   1.124 -			echo "ERROR: ..."
   1.125 +			echo "ERROR: Unknown endpoint command"
   1.126  			exit -1
   1.127  			;;
   1.128  	esac
   1.129  }
   1.130  
   1.131 -# $1: queryFile
   1.132 -function makeUpdate() {
   1.133 -	queryFile=$1
   1.134 -	instansiateScript=${loc}/../scripts/instantiate.sh
   1.135 -}
   1.136  # default values
   1.137  endpoint="http://pathway.di.uoa.gr:8080/endpoint"
   1.138  db="NOA2012"
   1.139 -hotspots_url="http://pathway.di.uoag.r/hotspots/2012"
   1.140 +hotspotsURL="http://pathway.di.uoa.gr/hotspots"
   1.141  bgFile="http://dev.strabon.di.uoa.gr/rdf/Kallikratis-Coastline-Corine-dump-postgres-9.tgz"
   1.142  logFile="runChain.log"
   1.143  
   1.144 +chain="DynamicThresholds"
   1.145 +persistence=10
   1.146 +repeatInPers=3
   1.147 +
   1.148  # read script options
   1.149  while test $# -gt 0 -a "X${1:0:1}" == "X-"; do
   1.150      case "${1}" in
   1.151 @@ -258,162 +276,109 @@
   1.152  
   1.153  echo "endpoint: ${endpoint}"
   1.154  echo "db: ${db}"
   1.155 -echo "hotspots: ${hotspots_url}"
   1.156 +echo "hotspots: ${hotspotsURL}"
   1.157  echo "background: ${bgFile}"
   1.158 -echo "logFile: ${logfile}"
   1.159 +echo "logFile: ${logFile}"
   1.160  
   1.161 -
   1.162 -# Get queries and updates
   1.163 -insertMunicipalities=`cat ${loc}/insertMunicipalities.rq`
   1.164 -deleteSeaHotspots=`cat ${loc}/deleteSeaHotspots.rq` # | sed 's/\"/\\\"/g'`
   1.165 -refinePartialSeaHotspots=`cat ${loc}/refinePartialSeaHotspots.rq` # | sed 's/\"/\\\"/g'`
   1.166 -invalidForFires=`cat ${loc}/landUseInvalidForFires.rq`
   1.167 -refineTimePersistence=`cat ${loc}/refineTimePersistence.rq` # | sed 's/\"/\\\"/g'`
   1.168 -discover=`cat ${loc}/discover.rq`
   1.169 -InsertMunicipalities=`cat ${loc}/InsertMunicipalities.sparql` # | sed 's/\"/\\\"/g'`
   1.170 -
   1.171 +instantiate=${loc}/instantiate.sh
   1.172  
   1.173  #Initialize (stop tomcat, restart postgres, drop/create database, start tomcat)
   1.174  handleTomcatService stop
   1.175  handlePostgresService restart
   1.176  
   1.177 -# get the main version of postgres
   1.178 -postgres_main_version=$(getPostgresMainVersion)
   1.179 -
   1.180  handlePostgresDatabase drop ${db}
   1.181  handlePostgresDatabase create ${db}
   1.182  
   1.183 -storeBackgroundData ${db} ${bgFile}
   1.184 +storeBackgroundData ${db} ${bgFile} # ~/Temp/Kallikratis-Coastline-Corine-postgres-9.sql
   1.185  
   1.186  handleTomcatService start
   1.187  
   1.188 -echo "Timestamp Store Municipalities DeleteInSea InvalidForFires RefineInCoast
   1.189 -TimePersistence" $ > stderr.txt
   1.190 +#${loc}/../../scripts/endpoint query ${endpoint} size 
   1.191 +#exit -1
   1.192 +echo "Timestamp Store Municipalities DeleteInSea InvalidForFires DeleteReflections RefineInCoast TimePersistence" > ${logFile}
   1.193  
   1.194 -years="2007 2008 2010 2011"
   1.195 -for year in "${years}"; do
   1.196 -for month in `seq 5 9`; do
   1.197 -for day in `seq 1 31`; do
   1.198 -for hour in `seq 0 23`; do
   1.199 -for minute in `seq 0 60 5`; do
   1.200 +years="2012" #"2007 2008 2010 2011"
   1.201 +for y in ${years}; do
   1.202 +    hotspots="`ls /var/www/hotspots/${y} | sort | grep -o 'HMSG.*\.nt'`"
   1.203 +	# get hotpost URLS
   1.204 +#	for hot in $(curl -s ${hotspotsURL}/${y}/ | grep -o '>HMSG.*\.nt' | colrm 1 1); do
   1.205 +	for hot in ${hotspots}; do
   1.206 +		file="${hotspotsURL}/${y}/${hot}"
   1.207  
   1.208 -		month=$(printf "%02d" ${month})
   1.209 -		day=$(printf "%02d" ${day})k
   1.210 -		hour=$(printf "%02d" ${hour})
   1.211 -		minute=$(printf "%02d" ${minute})k
   1.212 +		time_status=$(echo ${hot} | egrep -o '[[:digit:]]{6}_[[:digit:]]{4}')
   1.213 +		
   1.214 +		# get sensor
   1.215 +		sensor=$(echo ${hot} | grep -o 'MSG.')
   1.216 +
   1.217 +		# get satellite and set number of acquisitions per hour
   1.218 +		if test "${sensor}" = "MSG1"; then
   1.219 +			sensor="MSG1_RSS"
   1.220 +		fi
   1.221 +
   1.222 +		# get time information for acquisition and construct timestamp
   1.223 +		year="20$(expr substr ${time_status} 1 2)"
   1.224 +		month=$(expr substr ${time_status} 3 2)
   1.225 +		day=$(expr substr ${time_status} 5 2)
   1.226 +		time2=$(expr substr ${time_status} 8 2)
   1.227 +		time2="${time2}:$(expr substr ${time_status} 10 2)"
   1.228  
   1.229  		# construct timestamp
   1.230 -		timestamp="${year}-${month}-${day}T${hour}:${minute}:00"
   1.231 +		timestamp="${year}-${month}-${day}T${time2}:00"
   1.232 +		# print timestamp
   1.233 +		echo -n "${timestamp} " >> ${logFile}
   1.234 +        
   1.235 +		handleStrabonEndpoint ${endpoint} store ${file}
   1.236 +        echo "Store ${file}" ; # read t
   1.237  
   1.238 -		fileURL="HMSG1_RSS_039_s7_${year}${month}${day}_${hour}${minute}.hotspots.nt"
   1.239 -		handleStrabonEndpoint store ${fileURL} ${fileURL}
   1.240 +		# Insert Municipalities	
   1.241 +		update="`${instantiate} -t ${timestamp} -c ${chain} -s ${sensor} ${loc}/insertMunicipalities.rq`"
   1.242 +#        echo "Insert Municipalities: ${update}" ; read t
   1.243 +        handleStrabonEndpoint ${endpoint} update "${update}"
   1.244  
   1.245 -##		# sudo -u postgres psql -d endpoint -c 'VACUUM ANALYZE;';
   1.246  
   1.247 +		# Delete Sea Hotspots
   1.248 +		update="`${instantiate} -t ${timestamp} -c ${chain} -s ${sensor} ${loc}/deleteSeaHotspots.rq`"
   1.249 +#       echo "Delete Sea Hotspots: ${update}" ; read t
   1.250 +		handleStrabonEndpoint ${endpoint} update "${update}"
   1.251  
   1.252 -##		# insertMunicipalities
   1.253 -##		echo -n "inserting Municipalities " ;echo; echo; echo;
   1.254 -##		# query=`echo "${insertMunicipalities}" `
   1.255 -##		# ${countTime} ./strabon -db endpoint update "${query}"
   1.256 -##
   1.257 -##		tmr1=$(timer)
   1.258 -##
   1.259 -##		query=`echo "${insertMunicipalities}" | sed "s/TIMESTAMP/${TIMESTAMP}/g" | \
   1.260 -##		sed "s/PROCESSING_CHAIN/DynamicThresholds/g" | \
   1.261 -##		sed "s/SENSOR/${SENSOR}/g"`
   1.262 -##
   1.263 -##		${ENDPOINT_SCRIPT} update ${ENDPOINT} "${query}"
   1.264 -##		
   1.265 -##		tmr2=$(timer)
   1.266 -##		printf '%s ' $((tmr2-tmr1)) >>stderr.txt
   1.267 -##		echo;echo;echo;echo "File ${file} inserted Municipalities!"
   1.268 -##
   1.269 -##		# execute an explicit VACUUM ANALYZE when a query takes longer than it should
   1.270 -##		duration=$((tmr2-tmr1))
   1.271 -##		if test ${duration} -ge 30000; then
   1.272 -##			psql -U postgres ${DB} -c 'VACUUM ANALYZE' 
   1.273 -##			echo "Explicit VACUUM ANALYZE"
   1.274 -##		fi
   1.275 -##		
   1.276 -##		# deleteSeaHotspots
   1.277 -##		echo -n "Going to deleteSeaHotspots ${TIMESTAMP} " ;echo; echo; echo;
   1.278 -##		query=`echo "${deleteSeaHotspots}" | sed "s/TIMESTAMP/${TIMESTAMP}/g" | \
   1.279 -##		sed "s/PROCESSING_CHAIN/DynamicThresholds/g" | \
   1.280 -##		sed "s/SENSOR/${SENSOR}/g"`
   1.281 -##		# ${countTime} ./strabon -db endpoint update "${query}"
   1.282 -##
   1.283 -##		tmr1=$(timer)
   1.284 -##		${ENDPOINT_SCRIPT} update ${ENDPOINT} "${query}"
   1.285 -##
   1.286 -##		tmr2=$(timer)
   1.287 -##		printf '%s ' $((tmr2-tmr1)) >>stderr.txt
   1.288 -##		echo;echo;echo;echo "File ${file} deleteSeaHotspots done!"
   1.289 -##
   1.290 -##		# invalidForFires
   1.291 -##		echo -n "invalidForFires ${TIMESTAMP} "  ; echo; echo ; echo;
   1.292 -##		query=`echo "${invalidForFires}" | sed "s/TIMESTAMP/${TIMESTAMP}/g" | \
   1.293 -##		sed "s/PROCESSING_CHAIN/DynamicThresholds/g" | \
   1.294 -##		sed "s/SENSOR/${SENSOR}/g"` 
   1.295 -##
   1.296 -##		# ${countTime} ./strabon -db endpoint update "${query}"
   1.297 -##		tmr1=$(timer)
   1.298 -##		${ENDPOINT_SCRIPT} update ${ENDPOINT} "${query}"
   1.299 -##		tmr2=$(timer)
   1.300 -##		printf '%s ' $((tmr2-tmr1)) >>stderr.txt
   1.301 -##		echo "File ${file} invalidForFires done!"
   1.302 -## 
   1.303 -##		# refinePartialSeaHotspots
   1.304 -##		echo -n "refinePartialSeaHotspots ${TIMESTAMP} "  ; echo; echo ; echo;
   1.305 -##		query=`echo "${refinePartialSeaHotspots}" | sed "s/TIMESTAMP/${TIMESTAMP}/g" | \
   1.306 -##		sed "s/PROCESSING_CHAIN/DynamicThresholds/g" | \
   1.307 -##		sed "s/SENSOR/${SENSOR}/g" |\
   1.308 -##		sed "s/SAT/${SAT}/g"`
   1.309 -##		# ${countTime} ./strabon -db endpoint update "${query}"
   1.310 -##		tmr1=$(timer)
   1.311 -##		${ENDPOINT_SCRIPT} update ${ENDPOINT} "${query}"
   1.312 -##		tmr2=$(timer)
   1.313 -##		printf '%s ' $((tmr2-tmr1)) >>stderr.txt
   1.314 -##
   1.315 -##		echo "File ${file} refinePartialSeaHotspots done!"
   1.316 -##		# echo "Continue?"
   1.317 -##		# read a
   1.318 -##
   1.319 -##		# refineTimePersistence
   1.320 -##		echo -n "Going to refineTimePersistence ${TIMESTAMP} ";echo;echo;echo; 
   1.321 -##		min_acquisition_time=`date --date="${year}-${month}-${day} ${time2}:00 EEST -30 minutes" +%Y-%m-%dT%H:%M:00`
   1.322 -##		query=`echo "${refineTimePersistence}" | sed "s/TIMESTAMP/${TIMESTAMP}/g" | \
   1.323 -##		sed "s/PROCESSING_CHAIN/DynamicThresholds/g" | \
   1.324 -##		sed "s/SENSOR/${SENSOR}/g" | \
   1.325 -##		sed "s/ACQUISITIONS_IN_HALF_AN_HOUR/${N_ACQUISITIONS}/g" | \
   1.326 -##		sed "s/MIN_ACQUISITION_TIME/${min_acquisition_time}/g" |\
   1.327 -##		sed "s/SAT/${SAT}/g"`
   1.328 -##
   1.329 +		# Invalid For Fires
   1.330 +		update="`${instantiate} -t ${timestamp} -c ${chain} -s ${sensor} ${loc}/landUseInvalidForFires.rq`"
   1.331 +#       echo "Invalid For Fires: ${update}" ; read t
   1.332 +		handleStrabonEndpoint ${endpoint} update "${update}"
   1.333 + 
   1.334 +        # Delete Reflections
   1.335 +    	minTime=`date --date="${year}-${month}-${day} ${time2}:00 EEST -60 minutes" +%Y-%m-%dT%H:%M:00`
   1.336 +		update="`${instantiate} -t ${timestamp} -c ${chain} -s ${sensor} -m ${minTime} ${loc}/deleteReflections.rq`"
   1.337 +        echo "Delete Reflections: ${update}" ;
   1.338 +		handleStrabonEndpoint ${endpoint} update "${update}"
   1.339 +
   1.340 +		# Refine Partial Sea Hotspots
   1.341 +		update="`${instantiate} -t ${timestamp} -c ${chain} -s ${sensor} ${loc}/refinePartialSeaHotspots.rq`"
   1.342 +#       echo "Refine Partial Sea Hotspots: ${update}" ; read t
   1.343 +		handleStrabonEndpoint ${endpoint} update "${update}"
   1.344 +
   1.345 +		# Refine Time Persistence
   1.346 +		minTime=`date --date="${year}-${month}-${day} ${time2}:00 EEST -30 minutes" +%Y-%m-%dT%H:%M:00`
   1.347 +		update="`${instantiate} -t ${timestamp} -c ${chain} -s ${sensor} -m ${minTime} ${loc}/refineTimePersistence.rq`"
   1.348 +#       echo "Refine Time Persistence: ${update}" ; read t
   1.349 +		handleStrabonEndpoint ${endpoint} update "${update}"
   1.350 +
   1.351  ##		#sudo -u postgres psql -d ${DB} -c 'VACUUM ANALYZE;';
   1.352 -##
   1.353 -##		tmr1=$(timer)
   1.354 -##		${ENDPOINT_SCRIPT} update ${ENDPOINT} "${query}"
   1.355 -##		 tmr2=$(timer)
   1.356 -##		printf '%s \n' $((tmr2-tmr1)) >>stderr.txt
   1.357 -##		echo;echo;echo;echo "File ${file} timePersistence done!"
   1.358 -##		# echo "Continue?"
   1.359 -##		# read a
   1.360 -##
   1.361 -##
   1.362 -##		# discover
   1.363 -##		echo -n "Going to discover ${TIMESTAMP} ";echo;echo;echo; 
   1.364 -##		min_acquisition_time=`date --date="${year}-${month}-${day} 00:00 EEST" +%Y-%m-%dT%H:%M:00`
   1.365 -##		max_acquisition_time=`date --date="${year}-${month}-${day} 23:59 EEST" +%Y-%m-%dT%H:%M:00`
   1.366 -##		query=`echo "${discover}" | \
   1.367 -##			sed "s/PROCESSING_CHAIN/DynamicThresholds/g" | \
   1.368 -##			sed "s/SENSOR/${SENSOR}/g" | \
   1.369 -##			sed "s/MIN_ACQUISITION_TIME/${min_acquisition_time}/g" |\
   1.370 -##			sed "s/MAX_ACQUISITION_TIME/${max_acquisition_time}/g"`
   1.371 -##			
   1.372 -##		tmr1=$(timer)
   1.373 -##		${ENDPOINT_SCRIPT} query ${ENDPOINT} "${query}"
   1.374 -##		tmr2=$(timer)
   1.375 -##		printf '%s \n' $((tmr2-tmr1)) >>discover.txt
   1.376 -##		echo;echo;echo;echo "Discovered hotspots done!"
   1.377 -##	done
   1.378 -##done
   1.379 -##
   1.380 +        
   1.381 +		# Discover
   1.382 +		minTime=`date --date="${year}-${month}-${day} 00:00 EEST" +%Y-%m-%dT%H:%M:00`
   1.383 +		maxTime=`date --date="${year}-${month}-${day} 23:59 EEST" +%Y-%m-%dT%H:%M:00`
   1.384 +        query="`${instantiate} -c ${chain} -s ${sensor} -m ${minTime} -M ${maxTime} ${loc}/discover.rq`"
   1.385 +       echo "Discover: ${query}" ; read t
   1.386 +		handleStrabonEndpoint ${endpoint} query "${query}" | tee /home/ggarbis/discover.log
   1.387 +    
   1.388 +		# Discover Fires
   1.389 +		minTime=`date --date="${year}-${month}-${day} 00:00 EEST" +%Y-%m-%dT%H:%M:00`
   1.390 +		maxTime=`date --date="${year}-${month}-${day} 23:59 EEST" +%Y-%m-%dT%H:%M:00`
   1.391 +        query="`${instantiate} -c ${chain} -s ${sensor} -m ${minTime} -M ${maxTime} -p 10 -r 3 ${loc}/discoverFires.rq`"
   1.392 +        echo "Discover Fires: ${query}" ; #read t
   1.393 +		handleStrabonEndpoint ${endpoint} query "${query}" #| tee /home/ggarbis/discoverFires.log
   1.394 +#    
   1.395 +        echo >> ${logFile}    
   1.396 +	done
   1.397 +done