Strabon
changeset 439:4e38d658b9d6
made runNoaRefinementChain.sh more portable
| author | Babis Nikolaou <charnik@di.uoa.gr> |
|---|---|
| date | Wed Jul 18 15:53:40 2012 +0300 (2012-07-18) |
| parents | 32d7cfd5f785 |
| children | 7ed217b88b49 |
| files | scripts/v2.2/runNoaRefinementChain.sh |
line diff
1.1 --- a/scripts/v2.2/runNoaRefinementChain.sh Wed Jul 18 14:36:50 2012 +0300 1.2 +++ b/scripts/v2.2/runNoaRefinementChain.sh Wed Jul 18 15:53:40 2012 +0300 1.3 @@ -3,7 +3,7 @@ 1.4 1.5 ENDPOINT="http://localhost:8080/endpoint" 1.6 DB="endpoint" 1.7 -GRIDURL="http://kk.di.uoa.gr/grid_4.nt" 1.8 +#GRIDURL="http://kk.di.uoa.gr/grid_4.nt" 1.9 GRIDURL="http://jose.di.uoa.gr/rdf/coastline/grid_4.nt" 1.10 #INIT="http://jose.di.uoa.gr/rdf/Kallikratis-Coastline.ntriples" 1.11 INIT="../Kalli_coast.sql" 1.12 @@ -47,6 +47,10 @@ 1.13 fi 1.14 } 1.15 1.16 +# find out the postgres service to use 1.17 +postgres=$(ls -1 /etc/init.d/| grep postgres | head -1) 1.18 + 1.19 +tomcat= 1.20 function chooseTomcat() 1.21 { 1.22 if test -s /etc/fedora-release ; then 1.23 @@ -61,6 +65,11 @@ 1.24 elif test -s /etc/debian_version ; then 1.25 tomcat="tomcat" 1.26 fi 1.27 + 1.28 + # check for service availability 1.29 + if ! test -e "/etc/init.d/${tomcat}"; then 1.30 + tomcat= 1.31 + fi 1.32 } 1.33 1.34 insertMunicipalities=`cat ${LOC}/InsertMunicipalities.sparql` 1.35 @@ -72,17 +81,40 @@ 1.36 #InsertMunicipalities =`cat ${LOC}/InsertMunicipalities.sparql` # | sed 's/\"/\\\"/g'` 1.37 1.38 1.39 -# Initialize 1.40 +# Initialize (stop tomcat, restart postgres, drop/create database, start tomcat) 1.41 chooseTomcat 1.42 -sudo service postgresql restart 1.43 +echo "stopping tomcat" 1.44 +if test -z "${tomcat}"; then 1.45 + # work-around for babis (standalone tomcat, with start_tomcat.sh and stop_tomcat.sh scripts) 1.46 + stop_tomcat.sh 1.47 +else 1.48 + sudo service ${tomcat} stop 1.49 +fi 1.50 + 1.51 +sudo service ${postgres} restart 1.52 + 1.53 +# get the main version of postgres 1.54 +POSTGRES_MAIN_VERSION=$(sudo service ${postgres} status | grep -o '.\..' | cut -b 1) 1.55 + 1.56 echo "Dropping endpoint database"; 1.57 -sudo -u postgres dropdb ${DB} 1.58 +dropdb -U postgres ${DB} 1.59 + 1.60 echo "Creating endpoint database" 1.61 -sudo -u postgres createdb ${DB} 1.62 -sudo -u postgres sh -c "curl -s http://dev.strabon.di.uoa.gr/rdf/Kallikratis-Coastline-Corine-dump.tgz|tar xz -O|psql -d ${DB}" 1.63 -sudo -u postgres psql ${DB} -c 'VACUUM ANALYZE ' 1.64 -echo "restarting tomcat" 1.65 -sudo service ${tomcat} restart 1.66 +createdb -U postgres ${DB} 1.67 + 1.68 +# load data 1.69 +curl -s http://dev.strabon.di.uoa.gr/rdf/Kallikratis-Coastline-Corine-dump-postgres-${POSTGRES_MAIN_VERSION}.tgz | tar xz -O | psql -U postgres -d ${DB} 1.70 +psql ${DB} -U postgres -c 'VACUUM ANALYZE ' 1.71 + 1.72 +echo "starting tomcat" 1.73 +if test -z "${tomcat}"; then 1.74 + # work-around for babis (standalone tomcat, with start_tomcat.sh and stop_tomcat.sh scripts) 1.75 + start_tomcat.sh 1.76 +else 1.77 + sudo service ${tomcat} start 1.78 +fi 1.79 + 1.80 +exit 1.81 1.82 echo "initializing database" 1.83 echo "IM S D R TP" >stderr.txt 1.84 @@ -98,139 +130,141 @@ 1.85 1.86 1.87 for y in 7 8 10 11 ;do 1.88 -for mon in `seq 4 10`; do 1.89 -for d in `seq 1 30`; do 1.90 -for h in `seq 0 23 `; do 1.91 - for m in `seq 0 15 45`; do 1.92 - time=`printf "%02d%02d\n" $h $m` 1.93 - time2=`printf "%02d:%02d\n" $h $m` 1.94 - day=`printf "%02d" $d` 1.95 - month=`printf "%02d" $mon` 1.96 - year=`printf "%02d" $y` 1.97 - file=${dataDir}${year}/${name}${year}${month}${day}_${time}$suffix 1.98 -# file=${dataUrl}${name}_${time}$suffix 1.99 + for mon in `seq 4 10`; do 1.100 + for d in `seq 1 30`; do 1.101 + for h in `seq 0 23 `; do 1.102 + for m in `seq 0 15 45`; do 1.103 + time=`printf "%02d%02d\n" $h $m` 1.104 + time2=`printf "%02d:%02d\n" $h $m` 1.105 + day=`printf "%02d" $d` 1.106 + month=`printf "%02d" $mon` 1.107 + year=`printf "%02d" $y` 1.108 + file=${dataDir}${year}/${name}${year}${month}${day}_${time}$suffix 1.109 + # file=${dataUrl}${name}_${time}$suffix 1.110 1.111 - check=${dataDir}${year}/${name}${year}${month}${day}_${time}$suffix 1.112 - wget -q --spider $check 1.113 - 1.114 - if [[ $? -ne 0 ]]; 1.115 - then echo "FILE" $check "NOT EXISTS" ; continue 1.116 - fi 1.117 + check=${dataDir}${year}/${name}${year}${month}${day}_${time}$suffix 1.118 + wget -q --spider $check 1.119 + 1.120 + if [[ $? -ne 0 ]]; 1.121 + then echo "FILE" $check "NOT EXISTS" ; continue 1.122 + fi 1.123 1.124 1.125 - # store file 1.126 - echo -n "storing " $file; echo; echo; 1.127 - # echo "Hotspot : " $h:$m >> stderr.txt 1.128 -# ${countTime} ./strabon -db endpoint store $file 1.129 + # store file 1.130 + echo -n "storing " $file; echo; echo; 1.131 + # echo "Hotspot : " $h:$m >> stderr.txt 1.132 + # ${countTime} ./strabon -db endpoint store $file 1.133 1.134 - tmr1=$(timer) 1.135 - ../endpoint store ${ENDPOINT} N-Triples -u ${file} 1.136 - tmr2=$(timer) 1.137 -printf '%s ' $((tmr2-tmr1)) >>stderr.txt 1.138 + tmr1=$(timer) 1.139 + ../endpoint store ${ENDPOINT} N-Triples -u ${file} 1.140 + tmr2=$(timer) 1.141 + printf '%s ' $((tmr2-tmr1)) >>stderr.txt 1.142 1.143 - # sudo -u postgres psql -d endpoint -c 'VACUUM ANALYZE;'; 1.144 + # sudo -u postgres psql -d endpoint -c 'VACUUM ANALYZE;'; 1.145 1.146 - echo;echo;echo;echo "File ${file} stored!" >> ${logFile} 1.147 + echo;echo;echo;echo "File ${file} stored!" >> ${logFile} 1.148 1.149 - # insertMunicipalities 1.150 - echo -n "inserting Municipalities " ;echo; echo; echo; 1.151 - # query=`echo "${insertMunicipalities}" ` 1.152 -# ${countTime} ./strabon -db endpoint update "${query}" 1.153 + # insertMunicipalities 1.154 + echo -n "inserting Municipalities " ;echo; echo; echo; 1.155 + # query=`echo "${insertMunicipalities}" ` 1.156 + # ${countTime} ./strabon -db endpoint update "${query}" 1.157 1.158 -tmr1=$(timer) 1.159 + tmr1=$(timer) 1.160 1.161 - query=`echo "${insertMunicipalities}" | sed "s/TIMESTAMP/20${year}-${month}-${day}T${time2}:00/g" | \ 1.162 - sed "s/PROCESSING_CHAIN/DynamicThresholds/g" | \ 1.163 - sed "s/SENSOR/MSG2/g"` 1.164 + query=`echo "${insertMunicipalities}" | sed "s/TIMESTAMP/20${year}-${month}-${day}T${time2}:00/g" | \ 1.165 + sed "s/PROCESSING_CHAIN/DynamicThresholds/g" | \ 1.166 + sed "s/SENSOR/MSG2/g"` 1.167 1.168 - ../endpoint update ${ENDPOINT} "${query}" 1.169 - 1.170 -tmr2=$(timer) 1.171 + ../endpoint update ${ENDPOINT} "${query}" 1.172 + 1.173 + tmr2=$(timer) 1.174 1.175 - echo;echo;echo;echo "File ${file} inserted Municipalities!" 1.176 - 1.177 - # deleteSeaHotspots 1.178 - echo -n "Going to deleteSeaHotspots 20${year}-${month}-${day}T${time2}:00 " ;echo; echo; echo; 1.179 - query=`echo "${deleteSeaHotspots}" | sed "s/TIMESTAMP/20${year}-${month}-${day}T${time2}:00/g" | \ 1.180 - sed "s/PROCESSING_CHAIN/DynamicThresholds/g" | \ 1.181 - sed "s/SENSOR/MSG2/g"` 1.182 -# ${countTime} ./strabon -db endpoint update "${query}" 1.183 + echo;echo;echo;echo "File ${file} inserted Municipalities!" 1.184 + 1.185 + # deleteSeaHotspots 1.186 + echo -n "Going to deleteSeaHotspots 20${year}-${month}-${day}T${time2}:00 " ;echo; echo; echo; 1.187 + query=`echo "${deleteSeaHotspots}" | sed "s/TIMESTAMP/20${year}-${month}-${day}T${time2}:00/g" | \ 1.188 + sed "s/PROCESSING_CHAIN/DynamicThresholds/g" | \ 1.189 + sed "s/SENSOR/MSG2/g"` 1.190 + # ${countTime} ./strabon -db endpoint update "${query}" 1.191 1.192 -tmr1=$(timer) 1.193 - ../endpoint update ${ENDPOINT} "${query}" 1.194 + tmr1=$(timer) 1.195 + ../endpoint update ${ENDPOINT} "${query}" 1.196 1.197 -tmr2=$(timer) 1.198 -printf '%s ' $((tmr2-tmr1)) >>stderr.txt 1.199 + tmr2=$(timer) 1.200 + printf '%s ' $((tmr2-tmr1)) >>stderr.txt 1.201 + echo;echo;echo;echo "File ${file} deleteSeaHotspots done!" 1.202 1.203 - echo;echo;echo;echo "File ${file} deleteSeaHotspots done!" 1.204 -# echo "Continue?" 1.205 -# read a 1.206 - # invalidForFires 1.207 - echo -n "invalidForFires 20${year}-${month}-${day}T${time2}:00 " ; echo; echo ; echo; 1.208 - query=`echo "${invalidForFires}" | sed "s/TIMESTAMP/20${year}-${month}-${day}T${time2}:00/g" | \ 1.209 - sed "s/PROCESSING_CHAIN/DynamicThresholds/g" | \ 1.210 - sed "s/SENSOR/MSG2/g" |\ 1.211 - sed "s/SAT/METEOSAT9/g"` 1.212 -# ${countTime} ./strabon -db endpoint update "${query}" 1.213 -tmr1=$(timer) 1.214 - ../endpoint update ${ENDPOINT} "${query}" 1.215 -tmr2=$(timer) 1.216 -printf '%s ' $((tmr2-tmr1)) >>stderr.txt 1.217 - echo "File ${file} invalidForFires done!" 1.218 - 1.219 - # refinePartialSeaHotspots 1.220 - echo -n "refinePartialSeaHotspots 20${year}-${month}-${day}T${time2}:00 " ; echo; echo ; echo; 1.221 - query=`echo "${refinePartialSeaHotspots}" | sed "s/TIMESTAMP/20${year}-${month}-${day}T${time2}:00/g" | \ 1.222 - sed "s/PROCESSING_CHAIN/DynamicThresholds/g" | \ 1.223 - sed "s/SENSOR/MSG2/g" |\ 1.224 - sed "s/SAT/METEOSAT9/g"` 1.225 -# ${countTime} ./strabon -db endpoint update "${query}" 1.226 -tmr1=$(timer) 1.227 - ../endpoint update ${ENDPOINT} "${query}" 1.228 -tmr2=$(timer) 1.229 -printf '%s ' $((tmr2-tmr1)) >>stderr.txt 1.230 + # echo "Continue?" 1.231 + # read a 1.232 + # invalidForFires 1.233 + echo -n "invalidForFires 20${year}-${month}-${day}T${time2}:00 " ; echo; echo ; echo; 1.234 + query=`echo "${invalidForFires}" | sed "s/TIMESTAMP/20${year}-${month}-${day}T${time2}:00/g" | \ 1.235 + sed "s/PROCESSING_CHAIN/DynamicThresholds/g" | \ 1.236 + sed "s/SENSOR/MSG2/g" |\ 1.237 + sed "s/SAT/METEOSAT9/g"` 1.238 + # ${countTime} ./strabon -db endpoint update "${query}" 1.239 + tmr1=$(timer) 1.240 + ../endpoint update ${ENDPOINT} "${query}" 1.241 + tmr2=$(timer) 1.242 + printf '%s ' $((tmr2-tmr1)) >>stderr.txt 1.243 + echo "File ${file} invalidForFires done!" 1.244 + 1.245 + # refinePartialSeaHotspots 1.246 + echo -n "refinePartialSeaHotspots 20${year}-${month}-${day}T${time2}:00 " ; echo; echo ; echo; 1.247 + query=`echo "${refinePartialSeaHotspots}" | sed "s/TIMESTAMP/20${year}-${month}-${day}T${time2}:00/g" | \ 1.248 + sed "s/PROCESSING_CHAIN/DynamicThresholds/g" | \ 1.249 + sed "s/SENSOR/MSG2/g" |\ 1.250 + sed "s/SAT/METEOSAT9/g"` 1.251 + # ${countTime} ./strabon -db endpoint update "${query}" 1.252 + tmr1=$(timer) 1.253 + ../endpoint update ${ENDPOINT} "${query}" 1.254 + tmr2=$(timer) 1.255 + printf '%s ' $((tmr2-tmr1)) >>stderr.txt 1.256 1.257 - echo "File ${file} refinePartialSeaHotspots done!" 1.258 -# echo "Continue?" 1.259 -# read a 1.260 + echo "File ${file} refinePartialSeaHotspots done!" 1.261 + # echo "Continue?" 1.262 + # read a 1.263 1.264 - # refineTimePersistence 1.265 - echo -n "Going to refineTimePersistence 20${year}-${month}-${day}T${time2}:00 ";echo;echo;echo; 1.266 - min_acquisition_time=`date --date="20${year}-${month}-${day} ${time2}:00 EEST -30 minutes" +%Y-%m-%dT%H:%m:00` 1.267 - query=`echo "${refineTimePersistence}" | sed "s/TIMESTAMP/20${year}-${month}-${day}T${time2}:00/g" | \ 1.268 - sed "s/PROCESSING_CHAIN/DynamicThresholds/g" | \ 1.269 - sed "s/SENSOR/MSG2/g" | \ 1.270 - sed "s/ACQUISITIONS_IN_HALF_AN_HOUR/3.0/g" | \ 1.271 - sed "s/MIN_ACQUISITION_TIME/${min_acquisition_time}/g" |\ 1.272 - sed "s/SAT/METEOSAT9/g"` 1.273 + # refineTimePersistence 1.274 + echo -n "Going to refineTimePersistence 20${year}-${month}-${day}T${time2}:00 ";echo;echo;echo; 1.275 + min_acquisition_time=`date --date="20${year}-${month}-${day} ${time2}:00 EEST -30 minutes" +%Y-%m-%dT%H:%m:00` 1.276 + query=`echo "${refineTimePersistence}" | sed "s/TIMESTAMP/20${year}-${month}-${day}T${time2}:00/g" | \ 1.277 + sed "s/PROCESSING_CHAIN/DynamicThresholds/g" | \ 1.278 + sed "s/SENSOR/MSG2/g" | \ 1.279 + sed "s/ACQUISITIONS_IN_HALF_AN_HOUR/3.0/g" | \ 1.280 + sed "s/MIN_ACQUISITION_TIME/${min_acquisition_time}/g" |\ 1.281 + sed "s/SAT/METEOSAT9/g"` 1.282 1.283 - #sudo -u postgres psql -d ${DB} -c 'VACUUM ANALYZE;'; 1.284 + #sudo -u postgres psql -d ${DB} -c 'VACUUM ANALYZE;'; 1.285 1.286 -tmr1=$(timer) 1.287 - ../endpoint update ${ENDPOINT} "${query}" 1.288 - tmr2=$(timer) 1.289 -printf '%s \n' $((tmr2-tmr1)) >>stderr.txt 1.290 - echo;echo;echo;echo "File ${file} timePersistence done!" 1.291 -# echo "Continue?" 1.292 -# read a 1.293 - done #minutes 1.294 -done #hours 1.295 - # discover 1.296 - echo -n "Going to discover 20${year}-${month}-${day}T${time2}:00 ";echo;echo;echo; 1.297 - min_acquisition_time=`date --date="20${year}-${month}-${day} 00:00 EEST" +%Y-%m-%dT%H:%m:00` 1.298 - max_acquisition_time=`date --date="20${year}-${month}-${day} 23:59 EEST" +%Y-%m-%dT%H:%m:00` 1.299 - query=`echo "${discover}" | \ 1.300 - sed "s/PROCESSING_CHAIN/DynamicThresholds/g" | \ 1.301 - sed "s/SENSOR/MSG2/g" | \ 1.302 - sed "s/MIN_ACQUISITION_TIME/${min_acquisition_time}/g" |\ 1.303 - sed "s/MAX_ACQUISITION_TIME/${max_acquisition_time}/g"` 1.304 - 1.305 - tmr1=$(timer) 1.306 - ../endpoint query ${ENDPOINT} "${query}" 1.307 - tmr2=$(timer) 1.308 -printf '%s \n' $((tmr2-tmr1)) >>discover.txt 1.309 - echo;echo;echo;echo "Discovered hotspots done!" 1.310 - 1.311 -done #days 1.312 -done #months 1.313 + tmr1=$(timer) 1.314 + ../endpoint update ${ENDPOINT} "${query}" 1.315 + tmr2=$(timer) 1.316 + printf '%s \n' $((tmr2-tmr1)) >>stderr.txt 1.317 + echo;echo;echo;echo "File ${file} timePersistence done!" 1.318 + # echo "Continue?" 1.319 + # read a 1.320 + done #minutes 1.321 + done #hours 1.322 + 1.323 + # discover 1.324 + echo -n "Going to discover 20${year}-${month}-${day}T${time2}:00 ";echo;echo;echo; 1.325 + min_acquisition_time=`date --date="20${year}-${month}-${day} 00:00 EEST" +%Y-%m-%dT%H:%m:00` 1.326 + max_acquisition_time=`date --date="20${year}-${month}-${day} 23:59 EEST" +%Y-%m-%dT%H:%m:00` 1.327 + query=`echo "${discover}" | \ 1.328 + sed "s/PROCESSING_CHAIN/DynamicThresholds/g" | \ 1.329 + sed "s/SENSOR/MSG2/g" | \ 1.330 + sed "s/MIN_ACQUISITION_TIME/${min_acquisition_time}/g" |\ 1.331 + sed "s/MAX_ACQUISITION_TIME/${max_acquisition_time}/g"` 1.332 + 1.333 + tmr1=$(timer) 1.334 + ../endpoint query ${ENDPOINT} "${query}" 1.335 + tmr2=$(timer) 1.336 + printf '%s \n' $((tmr2-tmr1)) >>discover.txt 1.337 + echo;echo;echo;echo "Discovered hotspots done!" 1.338 + 1.339 + done #days 1.340 + done #months 1.341 done #years 1.342 +