Strabon

changeset 439:4e38d658b9d6

made runNoaRefinementChain.sh more portable
author Babis Nikolaou <charnik@di.uoa.gr>
date Wed Jul 18 15:53:40 2012 +0300 (2012-07-18)
parents 32d7cfd5f785
children 7ed217b88b49
files scripts/v2.2/runNoaRefinementChain.sh
line diff
     1.1 --- a/scripts/v2.2/runNoaRefinementChain.sh	Wed Jul 18 14:36:50 2012 +0300
     1.2 +++ b/scripts/v2.2/runNoaRefinementChain.sh	Wed Jul 18 15:53:40 2012 +0300
     1.3 @@ -3,7 +3,7 @@
     1.4  
     1.5  ENDPOINT="http://localhost:8080/endpoint"
     1.6  DB="endpoint"
     1.7 -GRIDURL="http://kk.di.uoa.gr/grid_4.nt"
     1.8 +#GRIDURL="http://kk.di.uoa.gr/grid_4.nt"
     1.9  GRIDURL="http://jose.di.uoa.gr/rdf/coastline/grid_4.nt"
    1.10  #INIT="http://jose.di.uoa.gr/rdf/Kallikratis-Coastline.ntriples"
    1.11  INIT="../Kalli_coast.sql"
    1.12 @@ -47,6 +47,10 @@
    1.13     fi
    1.14  }
    1.15  
    1.16 +# find out the postgres service to use
    1.17 +postgres=$(ls -1 /etc/init.d/| grep postgres | head -1)
    1.18 +
    1.19 +tomcat=
    1.20  function chooseTomcat()
    1.21  {
    1.22  	if test -s /etc/fedora-release ; then
    1.23 @@ -61,6 +65,11 @@
    1.24  	elif test -s /etc/debian_version ; then
    1.25  			tomcat="tomcat"
    1.26  	fi
    1.27 +
    1.28 +	# check for service availability
    1.29 +	if ! test -e "/etc/init.d/${tomcat}"; then
    1.30 +		tomcat=
    1.31 +	fi
    1.32  }
    1.33  
    1.34  insertMunicipalities=`cat ${LOC}/InsertMunicipalities.sparql` 
    1.35 @@ -72,17 +81,40 @@
    1.36  #InsertMunicipalities =`cat ${LOC}/InsertMunicipalities.sparql` # | sed 's/\"/\\\"/g'`
    1.37  
    1.38  
    1.39 -# Initialize
    1.40 +# Initialize (stop tomcat, restart postgres, drop/create database, start tomcat)
    1.41  chooseTomcat
    1.42 -sudo service postgresql restart
    1.43 +echo "stopping tomcat"
    1.44 +if test -z "${tomcat}"; then
    1.45 +	# work-around for babis (standalone tomcat, with start_tomcat.sh and stop_tomcat.sh scripts)
    1.46 +	stop_tomcat.sh
    1.47 +else
    1.48 +	sudo service ${tomcat} stop
    1.49 +fi
    1.50 +
    1.51 +sudo service ${postgres} restart
    1.52 +
    1.53 +# get the main version of postgres
    1.54 +POSTGRES_MAIN_VERSION=$(sudo service ${postgres} status | grep -o '.\..' | cut -b 1)
    1.55 +
    1.56  echo "Dropping endpoint database";
    1.57 -sudo -u postgres dropdb ${DB}
    1.58 +dropdb -U postgres ${DB}
    1.59 +
    1.60  echo "Creating endpoint database"
    1.61 -sudo -u postgres createdb ${DB} 
    1.62 -sudo -u postgres sh -c "curl -s  http://dev.strabon.di.uoa.gr/rdf/Kallikratis-Coastline-Corine-dump.tgz|tar xz -O|psql -d ${DB}"
    1.63 -sudo -u postgres psql ${DB} -c 'VACUUM ANALYZE '
    1.64 -echo "restarting tomcat"
    1.65 -sudo service ${tomcat} restart
    1.66 +createdb -U postgres ${DB} 
    1.67 +
    1.68 +# load data
    1.69 +curl -s  http://dev.strabon.di.uoa.gr/rdf/Kallikratis-Coastline-Corine-dump-postgres-${POSTGRES_MAIN_VERSION}.tgz | tar xz -O | psql -U postgres -d ${DB}
    1.70 +psql ${DB} -U postgres -c 'VACUUM ANALYZE '
    1.71 +
    1.72 +echo "starting tomcat"
    1.73 +if test -z "${tomcat}"; then
    1.74 +	# work-around for babis (standalone tomcat, with start_tomcat.sh and stop_tomcat.sh scripts)
    1.75 +	start_tomcat.sh
    1.76 +else
    1.77 +	sudo service ${tomcat} start
    1.78 +fi
    1.79 +
    1.80 +exit
    1.81  
    1.82  echo "initializing database"
    1.83  echo "IM S D R TP" >stderr.txt
    1.84 @@ -98,139 +130,141 @@
    1.85  
    1.86  
    1.87  for y in 7 8 10 11 ;do
    1.88 -for mon in `seq 4 10`; do
    1.89 -for d in `seq 1 30`; do
    1.90 -for h in `seq 0 23 `; do
    1.91 -    for m in `seq 0 15 45`; do
    1.92 -            time=`printf "%02d%02d\n" $h $m`
    1.93 -            time2=`printf "%02d:%02d\n" $h $m`
    1.94 -	    	day=`printf "%02d" $d`
    1.95 -            month=`printf "%02d" $mon`
    1.96 -            year=`printf "%02d" $y`
    1.97 -            file=${dataDir}${year}/${name}${year}${month}${day}_${time}$suffix
    1.98 -#            file=${dataUrl}${name}_${time}$suffix
    1.99 +	for mon in `seq 4 10`; do
   1.100 +		for d in `seq 1 30`; do
   1.101 +			for h in `seq 0 23 `; do
   1.102 +				for m in `seq 0 15 45`; do
   1.103 +					time=`printf "%02d%02d\n" $h $m`
   1.104 +					time2=`printf "%02d:%02d\n" $h $m`
   1.105 +					day=`printf "%02d" $d`
   1.106 +					month=`printf "%02d" $mon`
   1.107 +					year=`printf "%02d" $y`
   1.108 +					file=${dataDir}${year}/${name}${year}${month}${day}_${time}$suffix
   1.109 +					# file=${dataUrl}${name}_${time}$suffix
   1.110  
   1.111 -   		 check=${dataDir}${year}/${name}${year}${month}${day}_${time}$suffix
   1.112 -		 wget -q --spider $check
   1.113 -   	
   1.114 -	      if [[   $? -ne 0 ]];
   1.115 -              then echo "FILE" $check "NOT EXISTS" ; continue
   1.116 -	      fi
   1.117 +					check=${dataDir}${year}/${name}${year}${month}${day}_${time}$suffix
   1.118 +					wget -q --spider $check
   1.119 +				
   1.120 +					if [[   $? -ne 0 ]];
   1.121 +					then echo "FILE" $check "NOT EXISTS" ; continue
   1.122 +					fi
   1.123  
   1.124  
   1.125 -            # store file
   1.126 -            echo -n "storing " $file; echo; echo; 
   1.127 -	  # echo "Hotspot : " $h:$m >> stderr.txt
   1.128 -#            ${countTime} ./strabon -db endpoint store $file
   1.129 +				   	# store file
   1.130 +					echo -n "storing " $file; echo; echo; 
   1.131 +					# echo "Hotspot : " $h:$m >> stderr.txt
   1.132 +					# ${countTime} ./strabon -db endpoint store $file
   1.133  
   1.134 - tmr1=$(timer)
   1.135 -            ../endpoint store ${ENDPOINT} N-Triples -u ${file}
   1.136 - tmr2=$(timer)
   1.137 -printf '%s ' $((tmr2-tmr1)) >>stderr.txt
   1.138 +					tmr1=$(timer)
   1.139 +					../endpoint store ${ENDPOINT} N-Triples -u ${file}
   1.140 +					tmr2=$(timer)
   1.141 +					printf '%s ' $((tmr2-tmr1)) >>stderr.txt
   1.142  
   1.143 -	   # sudo -u postgres psql -d endpoint -c 'VACUUM ANALYZE;';
   1.144 +					# sudo -u postgres psql -d endpoint -c 'VACUUM ANALYZE;';
   1.145  
   1.146 -            echo;echo;echo;echo "File ${file} stored!" >> ${logFile}
   1.147 +					echo;echo;echo;echo "File ${file} stored!" >> ${logFile}
   1.148  
   1.149 -            # insertMunicipalities
   1.150 -            echo -n "inserting Municipalities " ;echo; echo; echo;
   1.151 -           # query=`echo "${insertMunicipalities}" `
   1.152 -#            ${countTime} ./strabon -db endpoint update "${query}"
   1.153 +					# insertMunicipalities
   1.154 +					echo -n "inserting Municipalities " ;echo; echo; echo;
   1.155 +					# query=`echo "${insertMunicipalities}" `
   1.156 +					# ${countTime} ./strabon -db endpoint update "${query}"
   1.157  
   1.158 -tmr1=$(timer)
   1.159 +					tmr1=$(timer)
   1.160  
   1.161 -                 query=`echo "${insertMunicipalities}" | sed "s/TIMESTAMP/20${year}-${month}-${day}T${time2}:00/g" | \
   1.162 -                sed "s/PROCESSING_CHAIN/DynamicThresholds/g" | \
   1.163 -                sed "s/SENSOR/MSG2/g"`
   1.164 +					query=`echo "${insertMunicipalities}" | sed "s/TIMESTAMP/20${year}-${month}-${day}T${time2}:00/g" | \
   1.165 +					sed "s/PROCESSING_CHAIN/DynamicThresholds/g" | \
   1.166 +					sed "s/SENSOR/MSG2/g"`
   1.167  
   1.168 -  ../endpoint update ${ENDPOINT} "${query}"
   1.169 -                
   1.170 -tmr2=$(timer)
   1.171 +					../endpoint update ${ENDPOINT} "${query}"
   1.172 +					
   1.173 +					tmr2=$(timer)
   1.174  
   1.175 -            echo;echo;echo;echo "File ${file} inserted Municipalities!"
   1.176 -            
   1.177 -            # deleteSeaHotspots
   1.178 -            echo -n "Going to deleteSeaHotspots 20${year}-${month}-${day}T${time2}:00 " ;echo; echo; echo;
   1.179 -            query=`echo "${deleteSeaHotspots}" | sed "s/TIMESTAMP/20${year}-${month}-${day}T${time2}:00/g" | \
   1.180 -                sed "s/PROCESSING_CHAIN/DynamicThresholds/g" | \
   1.181 -                sed "s/SENSOR/MSG2/g"`
   1.182 -#            ${countTime} ./strabon -db endpoint update "${query}"
   1.183 +					echo;echo;echo;echo "File ${file} inserted Municipalities!"
   1.184 +					
   1.185 +					# deleteSeaHotspots
   1.186 +					echo -n "Going to deleteSeaHotspots 20${year}-${month}-${day}T${time2}:00 " ;echo; echo; echo;
   1.187 +					query=`echo "${deleteSeaHotspots}" | sed "s/TIMESTAMP/20${year}-${month}-${day}T${time2}:00/g" | \
   1.188 +					sed "s/PROCESSING_CHAIN/DynamicThresholds/g" | \
   1.189 +					sed "s/SENSOR/MSG2/g"`
   1.190 +					# ${countTime} ./strabon -db endpoint update "${query}"
   1.191  
   1.192 -tmr1=$(timer)
   1.193 -  ../endpoint update ${ENDPOINT} "${query}"
   1.194 +					tmr1=$(timer)
   1.195 +					../endpoint update ${ENDPOINT} "${query}"
   1.196  
   1.197 -tmr2=$(timer)
   1.198 -printf '%s ' $((tmr2-tmr1)) >>stderr.txt
   1.199 +					tmr2=$(timer)
   1.200 +					printf '%s ' $((tmr2-tmr1)) >>stderr.txt
   1.201 +					echo;echo;echo;echo "File ${file} deleteSeaHotspots done!"
   1.202  
   1.203 -            echo;echo;echo;echo "File ${file} deleteSeaHotspots done!"
   1.204 -#            echo "Continue?"
   1.205 -#            read a
   1.206 -            # invalidForFires
   1.207 -            echo -n "invalidForFires 20${year}-${month}-${day}T${time2}:00 "  ; echo; echo ; echo;
   1.208 -            query=`echo "${invalidForFires}" | sed "s/TIMESTAMP/20${year}-${month}-${day}T${time2}:00/g" | \
   1.209 -                sed "s/PROCESSING_CHAIN/DynamicThresholds/g" | \
   1.210 -                sed "s/SENSOR/MSG2/g" |\
   1.211 -		sed "s/SAT/METEOSAT9/g"`
   1.212 -#            ${countTime} ./strabon -db endpoint update "${query}"
   1.213 -tmr1=$(timer)
   1.214 -              ../endpoint update ${ENDPOINT} "${query}"
   1.215 -tmr2=$(timer)
   1.216 -printf '%s ' $((tmr2-tmr1)) >>stderr.txt
   1.217 - echo "File ${file} invalidForFires done!"
   1.218 - 
   1.219 -            # refinePartialSeaHotspots
   1.220 -            echo -n "refinePartialSeaHotspots 20${year}-${month}-${day}T${time2}:00 "  ; echo; echo ; echo;
   1.221 -            query=`echo "${refinePartialSeaHotspots}" | sed "s/TIMESTAMP/20${year}-${month}-${day}T${time2}:00/g" | \
   1.222 -                sed "s/PROCESSING_CHAIN/DynamicThresholds/g" | \
   1.223 -                sed "s/SENSOR/MSG2/g" |\
   1.224 -		sed "s/SAT/METEOSAT9/g"`
   1.225 -#            ${countTime} ./strabon -db endpoint update "${query}"
   1.226 -tmr1=$(timer)
   1.227 -              ../endpoint update ${ENDPOINT} "${query}"
   1.228 -tmr2=$(timer)
   1.229 -printf '%s ' $((tmr2-tmr1)) >>stderr.txt
   1.230 +					# echo "Continue?"
   1.231 +					# read a
   1.232 +						# invalidForFires
   1.233 +					echo -n "invalidForFires 20${year}-${month}-${day}T${time2}:00 "  ; echo; echo ; echo;
   1.234 +					query=`echo "${invalidForFires}" | sed "s/TIMESTAMP/20${year}-${month}-${day}T${time2}:00/g" | \
   1.235 +					sed "s/PROCESSING_CHAIN/DynamicThresholds/g" | \
   1.236 +					sed "s/SENSOR/MSG2/g" |\
   1.237 +					sed "s/SAT/METEOSAT9/g"`
   1.238 +					# ${countTime} ./strabon -db endpoint update "${query}"
   1.239 +					tmr1=$(timer)
   1.240 +					../endpoint update ${ENDPOINT} "${query}"
   1.241 +					tmr2=$(timer)
   1.242 +					printf '%s ' $((tmr2-tmr1)) >>stderr.txt
   1.243 +					echo "File ${file} invalidForFires done!"
   1.244 +			 
   1.245 +					# refinePartialSeaHotspots
   1.246 +					echo -n "refinePartialSeaHotspots 20${year}-${month}-${day}T${time2}:00 "  ; echo; echo ; echo;
   1.247 +					query=`echo "${refinePartialSeaHotspots}" | sed "s/TIMESTAMP/20${year}-${month}-${day}T${time2}:00/g" | \
   1.248 +					sed "s/PROCESSING_CHAIN/DynamicThresholds/g" | \
   1.249 +					sed "s/SENSOR/MSG2/g" |\
   1.250 +					sed "s/SAT/METEOSAT9/g"`
   1.251 +					# ${countTime} ./strabon -db endpoint update "${query}"
   1.252 +					tmr1=$(timer)
   1.253 +					../endpoint update ${ENDPOINT} "${query}"
   1.254 +					tmr2=$(timer)
   1.255 +					printf '%s ' $((tmr2-tmr1)) >>stderr.txt
   1.256  
   1.257 -            echo "File ${file} refinePartialSeaHotspots done!"
   1.258 -#            echo "Continue?"
   1.259 -#            read a
   1.260 +					echo "File ${file} refinePartialSeaHotspots done!"
   1.261 +					# echo "Continue?"
   1.262 +					# read a
   1.263  
   1.264 -            # refineTimePersistence
   1.265 -            echo -n "Going to refineTimePersistence 20${year}-${month}-${day}T${time2}:00 ";echo;echo;echo; 
   1.266 -            min_acquisition_time=`date --date="20${year}-${month}-${day} ${time2}:00 EEST -30 minutes" +%Y-%m-%dT%H:%m:00`
   1.267 -            query=`echo "${refineTimePersistence}" | sed "s/TIMESTAMP/20${year}-${month}-${day}T${time2}:00/g" | \
   1.268 -                sed "s/PROCESSING_CHAIN/DynamicThresholds/g" | \
   1.269 -                sed "s/SENSOR/MSG2/g" | \
   1.270 -                sed "s/ACQUISITIONS_IN_HALF_AN_HOUR/3.0/g" | \
   1.271 -                sed "s/MIN_ACQUISITION_TIME/${min_acquisition_time}/g" |\
   1.272 -		sed "s/SAT/METEOSAT9/g"`
   1.273 +					# refineTimePersistence
   1.274 +					echo -n "Going to refineTimePersistence 20${year}-${month}-${day}T${time2}:00 ";echo;echo;echo; 
   1.275 +					min_acquisition_time=`date --date="20${year}-${month}-${day} ${time2}:00 EEST -30 minutes" +%Y-%m-%dT%H:%m:00`
   1.276 +					query=`echo "${refineTimePersistence}" | sed "s/TIMESTAMP/20${year}-${month}-${day}T${time2}:00/g" | \
   1.277 +					sed "s/PROCESSING_CHAIN/DynamicThresholds/g" | \
   1.278 +					sed "s/SENSOR/MSG2/g" | \
   1.279 +					sed "s/ACQUISITIONS_IN_HALF_AN_HOUR/3.0/g" | \
   1.280 +					sed "s/MIN_ACQUISITION_TIME/${min_acquisition_time}/g" |\
   1.281 +					sed "s/SAT/METEOSAT9/g"`
   1.282  
   1.283 - #sudo -u postgres psql -d ${DB} -c 'VACUUM ANALYZE;';
   1.284 +					#sudo -u postgres psql -d ${DB} -c 'VACUUM ANALYZE;';
   1.285  
   1.286 -tmr1=$(timer)
   1.287 -              ../endpoint update ${ENDPOINT} "${query}"
   1.288 - tmr2=$(timer)
   1.289 -printf '%s \n' $((tmr2-tmr1)) >>stderr.txt
   1.290 -            echo;echo;echo;echo "File ${file} timePersistence done!"
   1.291 -#            echo "Continue?"
   1.292 -#            read a
   1.293 -    done #minutes
   1.294 -done #hours
   1.295 -            # discover
   1.296 -            echo -n "Going to discover 20${year}-${month}-${day}T${time2}:00 ";echo;echo;echo; 
   1.297 -            min_acquisition_time=`date --date="20${year}-${month}-${day} 00:00 EEST" +%Y-%m-%dT%H:%m:00`
   1.298 -            max_acquisition_time=`date --date="20${year}-${month}-${day} 23:59 EEST" +%Y-%m-%dT%H:%m:00`
   1.299 -            query=`echo "${discover}" | \
   1.300 -                sed "s/PROCESSING_CHAIN/DynamicThresholds/g" | \
   1.301 -                sed "s/SENSOR/MSG2/g" | \
   1.302 -                sed "s/MIN_ACQUISITION_TIME/${min_acquisition_time}/g" |\
   1.303 -                sed "s/MAX_ACQUISITION_TIME/${max_acquisition_time}/g"`
   1.304 -                
   1.305 -                tmr1=$(timer)
   1.306 -              ../endpoint query ${ENDPOINT} "${query}"
   1.307 - tmr2=$(timer)
   1.308 -printf '%s \n' $((tmr2-tmr1)) >>discover.txt
   1.309 -            echo;echo;echo;echo "Discovered hotspots done!"
   1.310 -            
   1.311 -done #days
   1.312 -done #months
   1.313 +					tmr1=$(timer)
   1.314 +					../endpoint update ${ENDPOINT} "${query}"
   1.315 +					 tmr2=$(timer)
   1.316 +					printf '%s \n' $((tmr2-tmr1)) >>stderr.txt
   1.317 +					echo;echo;echo;echo "File ${file} timePersistence done!"
   1.318 +					# echo "Continue?"
   1.319 +					# read a
   1.320 +				done #minutes
   1.321 +			done #hours
   1.322 +
   1.323 +			# discover
   1.324 +			echo -n "Going to discover 20${year}-${month}-${day}T${time2}:00 ";echo;echo;echo; 
   1.325 +			min_acquisition_time=`date --date="20${year}-${month}-${day} 00:00 EEST" +%Y-%m-%dT%H:%m:00`
   1.326 +			max_acquisition_time=`date --date="20${year}-${month}-${day} 23:59 EEST" +%Y-%m-%dT%H:%m:00`
   1.327 +			query=`echo "${discover}" | \
   1.328 +				sed "s/PROCESSING_CHAIN/DynamicThresholds/g" | \
   1.329 +				sed "s/SENSOR/MSG2/g" | \
   1.330 +				sed "s/MIN_ACQUISITION_TIME/${min_acquisition_time}/g" |\
   1.331 +				sed "s/MAX_ACQUISITION_TIME/${max_acquisition_time}/g"`
   1.332 +				
   1.333 +			tmr1=$(timer)
   1.334 +			../endpoint query ${ENDPOINT} "${query}"
   1.335 +			tmr2=$(timer)
   1.336 +			printf '%s \n' $((tmr2-tmr1)) >>discover.txt
   1.337 +			echo;echo;echo;echo "Discovered hotspots done!"
   1.338 +	    
   1.339 +		done #days
   1.340 +	done #months
   1.341  done #years
   1.342 +