Strabon

view scripts/v2.2/runNoaRefinementChain.sh @ 455:206040860011

chain script fix
author Konstantina Mpereta <grad1110@di.uoa.gr>
date Wed Jul 18 21:06:36 2012 +0300 (2012-07-18)
parents 1caabaa1a742
children e81a36154bb9
line source
1 #!/bin/bash
2 LOC="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
4 ENDPOINT="http://localhost:8080/endpoint"
5 DB="endpoint"
6 #GRIDURL="http://kk.di.uoa.gr/grid_4.nt"
7 GRIDURL="http://jose.di.uoa.gr/rdf/coastline/grid_4.nt"
8 #INIT="http://jose.di.uoa.gr/rdf/Kallikratis-Coastline.ntriples"
9 INIT="../Kalli_coast.sql"
11 #CHECKDIR="/home/konstantina/allhot/"
12 #CHECKDIR="${HOME}/teleios/nkua/Hotspots/"
14 POSTGISTEMPLATE="postgistemplate"
15 #POSTGISTEMPLATE="template_postgis"
17 #dataDir="http://localhost/noa-teleios/out_triples/"
18 #dataDir="http://kk.di.uoa.gr/out_triples/"
19 #dataDir="http://godel.di.uoa.gr/allhot/"
20 dataDir="http://jose.di.uoa.gr/rdf/hotspots/20"
21 name="HMSG2_IR_039_s7_"
22 suffix=".hotspots.nt"
24 HOTSPOTS_URL="http://jose.di.uoa.gr/rdf/hotspots"
26 logFile="chain.log"
27 #countWTime="/usr/bin/time -p %e"
28 #echo > ${logFile}
30 function timer()
31 {
32 if [[ $# -eq 0 ]]; then
33 t=$(date '+%s%N')
34 t=$((t/1000000))
35 echo $t
36 else
37 local stime=$1
38 etime=$(date '+%s%N')
39 etime=$((etime/1000000))
41 if [[ -z "$stime" ]]; then stime=$etime; fi
42 dt=$((etime - stime)) #dt in milliseconds
43 dM=$((dt%1000))
44 Dt=$((dt/1000)) #delta t in seconds
45 ds=$((Dt % 60))
46 dm=$(((Dt / 60) % 60))
47 dh=$((Dt / 3600))
48 printf '%d:%02d:%02d.%03d' $dh $dm $ds $dM
49 fi
50 }
52 # find out the postgres service to use
53 postgres=$(ls -1 /etc/init.d/| grep postgres | head -1)
55 tomcat=
56 function chooseTomcat()
57 {
58 if test -s /etc/fedora-release ; then
59 tomcat="tomcat"
60 #elif test -s /etc/centos-release ; then
61 #elif test -s /etc/yellowdog-release ; then
62 #elif test -s /etc/redhat-release ; then
63 #elif test -s /etc/SuSE-release ; then
64 #elif test -s /etc/gentoo-release ; then
65 elif test -s /etc/lsb-release ; then # Ubuntu
66 tomcat="tomcat7"
67 elif test -s /etc/debian_version ; then
68 tomcat="tomcat"
69 fi
71 # check for service availability
72 if ! test -e "/etc/init.d/${tomcat}"; then
73 tomcat=
74 fi
75 }
77 insertMunicipalities=`cat ${LOC}/insertMunicipalities.sparql`
78 deleteSeaHotspots=`cat ${LOC}/deleteSeaHotspots.sparql` # | sed 's/\"/\\\"/g'`
79 refinePartialSeaHotspots=`cat ${LOC}/refinePartialSeaHotspots.sparql` # | sed 's/\"/\\\"/g'`
80 refineTimePersistence=`cat ${LOC}/refineTimePersistence.sparql` # | sed 's/\"/\\\"/g'`
81 invalidForFires=`cat ${LOC}/landUseInvalidForFires.sparql`
82 discover=`cat ${LOC}/discover.sparql`
83 #InsertMunicipalities =`cat ${LOC}/InsertMunicipalities.sparql` # | sed 's/\"/\\\"/g'`
85 # Initialize (stop tomcat, restart postgres, drop/create database, start tomcat)
86 chooseTomcat
87 echo "stopping tomcat"
88 if test -z "${tomcat}"; then
89 # work-around for babis (standalone tomcat, with start_tomcat.sh and stop_tomcat.sh scripts)
90 stop_tomcat.sh
91 else
92 sudo service ${tomcat} stop
93 fi
95 sudo service ${postgres} restart
97 # get the main version of postgres
98 POSTGRES_MAIN_VERSION=$(sudo service ${postgres} status | grep -o '.\..' | cut -b 1)
100 echo "Dropping endpoint database";
101 dropdb ${DB}
103 echo "Creating endpoint database"
104 createdb ${DB}
106 # load data
107 curl -s http://dev.strabon.di.uoa.gr/rdf/Kallikratis-Coastline-Corine-dump-postgres-${POSTGRES_MAIN_VERSION}.tgz | tar xz -O | psql -d ${DB}
108 psql ${DB} -c 'VACUUM ANALYZE '
110 echo "starting tomcat"
111 if test -z "${tomcat}"; then
112 # work-around for babis (standalone tomcat, with start_tomcat.sh and stop_tomcat.sh scripts)
113 start_tomcat.sh
114 else
115 sudo service ${tomcat} start
116 fi
118 echo "initializing database"
119 echo "S M D IF R TP" >stderr.txt
122 #./scripts/endpoint query ${ENDPOINT} "SELECT (COUNT(*) AS ?C) WHERE {?s ?p ?o}"
123 #sudo -u postgres psql -d endpoint -c 'CREATE INDEX datetime_values_idx_value ON datetime_values USING btree(value)';
124 #sudo -u postgres psql -d endpoint -c 'VACUUM ANALYZE;';
126 #echo "Continue?"
127 #read a
129 #for y in 2008; do
130 for y in 2007 2008 2010 2011 ;do
131 # get hotpost URLS
132 for hot in $(curl -s ${HOTSPOTS_URL}/${y}/ | grep -o '>HMSG2.*\.nt' | colrm 1 1); do
133 file="${HOTSPOTS_URL}/${y}/${hot}"
135 # get time information for acquisition
136 year=${y}
137 month=$(expr substr ${hot} 19 2)
138 day=$(expr substr ${hot} 21 2)
139 time2=$(expr substr ${hot} 24 2)
140 time2="${time2}:$(expr substr ${hot} 26 2)"
142 # store file
143 echo -n "storing " $file; echo; echo;
144 # ${countTime} ./strabon -db endpoint store $file
146 tmr1=$(timer)
147 ../endpoint store ${ENDPOINT} N-Triples -u ${file}
148 tmr2=$(timer)
149 printf '%s ' $((tmr2-tmr1)) >> stderr.txt
151 # sudo -u postgres psql -d endpoint -c 'VACUUM ANALYZE;';
153 echo;echo;echo;echo "File ${file} stored!" >> ${logFile}
155 # insertMunicipalities
156 echo -n "inserting Municipalities " ;echo; echo; echo;
157 # query=`echo "${insertMunicipalities}" `
158 # ${countTime} ./strabon -db endpoint update "${query}"
160 tmr1=$(timer)
162 query=`echo "${insertMunicipalities}" | sed "s/TIMESTAMP/${year}-${month}-${day}T${time2}:00/g" | \
163 sed "s/PROCESSING_CHAIN/DynamicThresholds/g" | \
164 sed "s/SENSOR/MSG2/g"`
166 ../endpoint update ${ENDPOINT} "${query}"
168 tmr2=$(timer)
169 printf '%s ' $((tmr2-tmr1)) >>stderr.txt
170 echo;echo;echo;echo "File ${file} inserted Municipalities!"
172 # deleteSeaHotspots
173 echo -n "Going to deleteSeaHotspots ${year}-${month}-${day}T${time2}:00 " ;echo; echo; echo;
174 query=`echo "${deleteSeaHotspots}" | sed "s/TIMESTAMP/${year}-${month}-${day}T${time2}:00/g" | \
175 sed "s/PROCESSING_CHAIN/DynamicThresholds/g" | \
176 sed "s/SENSOR/MSG2/g"`
177 # ${countTime} ./strabon -db endpoint update "${query}"
179 tmr1=$(timer)
180 ../endpoint update ${ENDPOINT} "${query}"
182 tmr2=$(timer)
183 printf '%s ' $((tmr2-tmr1)) >>stderr.txt
184 echo;echo;echo;echo "File ${file} deleteSeaHotspots done!"
186 # echo "Continue?"
187 # read a
188 # invalidForFires
189 echo -n "invalidForFires ${year}-${month}-${day}T${time2}:00 " ; echo; echo ; echo;
190 query=`echo "${invalidForFires}" | sed "s/TIMESTAMP/${year}-${month}-${day}T${time2}:00/g" | \
191 sed "s/PROCESSING_CHAIN/DynamicThresholds/g" | \
192 sed "s/SENSOR/MSG2/g" |\
193 sed "s/SAT/METEOSAT9/g"`
194 # ${countTime} ./strabon -db endpoint update "${query}"
195 tmr1=$(timer)
196 ../endpoint update ${ENDPOINT} "${query}"
197 tmr2=$(timer)
198 printf '%s ' $((tmr2-tmr1)) >>stderr.txt
199 echo "File ${file} invalidForFires done!"
201 # refinePartialSeaHotspots
202 echo -n "refinePartialSeaHotspots ${year}-${month}-${day}T${time2}:00 " ; echo; echo ; echo;
203 query=`echo "${refinePartialSeaHotspots}" | sed "s/TIMESTAMP/${year}-${month}-${day}T${time2}:00/g" | \
204 sed "s/PROCESSING_CHAIN/DynamicThresholds/g" | \
205 sed "s/SENSOR/MSG2/g" |\
206 sed "s/SAT/METEOSAT9/g"`
207 # ${countTime} ./strabon -db endpoint update "${query}"
208 tmr1=$(timer)
209 ../endpoint update ${ENDPOINT} "${query}"
210 tmr2=$(timer)
211 printf '%s ' $((tmr2-tmr1)) >>stderr.txt
213 echo "File ${file} refinePartialSeaHotspots done!"
214 # echo "Continue?"
215 # read a
217 # refineTimePersistence
218 echo -n "Going to refineTimePersistence ${year}-${month}-${day}T${time2}:00 ";echo;echo;echo;
219 min_acquisition_time=`date --date="${year}-${month}-${day} ${time2}:00 EEST -30 minutes" +%Y-%m-%dT%H:%m:00`
220 query=`echo "${refineTimePersistence}" | sed "s/TIMESTAMP/${year}-${month}-${day}T${time2}:00/g" | \
221 sed "s/PROCESSING_CHAIN/DynamicThresholds/g" | \
222 sed "s/SENSOR/MSG2/g" | \
223 sed "s/ACQUISITIONS_IN_HALF_AN_HOUR/3.0/g" | \
224 sed "s/MIN_ACQUISITION_TIME/${min_acquisition_time}/g" |\
225 sed "s/SAT/METEOSAT9/g"`
227 #sudo -u postgres psql -d ${DB} -c 'VACUUM ANALYZE;';
229 tmr1=$(timer)
230 ../endpoint update ${ENDPOINT} "${query}"
231 tmr2=$(timer)
232 printf '%s \n' $((tmr2-tmr1)) >>stderr.txt
233 echo;echo;echo;echo "File ${file} timePersistence done!"
234 # echo "Continue?"
235 # read a
238 # discover
239 echo -n "Going to discover ${year}-${month}-${day}T${time2}:00 ";echo;echo;echo;
240 min_acquisition_time=`date --date="${year}-${month}-${day} 00:00 EEST" +%Y-%m-%dT%H:%m:00`
241 max_acquisition_time=`date --date="${year}-${month}-${day} 23:59 EEST" +%Y-%m-%dT%H:%m:00`
242 query=`echo "${discover}" | \
243 sed "s/PROCESSING_CHAIN/DynamicThresholds/g" | \
244 sed "s/SENSOR/MSG2/g" | \
245 sed "s/MIN_ACQUISITION_TIME/${min_acquisition_time}/g" |\
246 sed "s/MAX_ACQUISITION_TIME/${max_acquisition_time}/g"`
248 tmr1=$(timer)
249 ../endpoint query ${ENDPOINT} "${query}"
250 tmr2=$(timer)
251 printf '%s \n' $((tmr2-tmr1)) >>discover.txt
252 echo;echo;echo;echo "Discovered hotspots done!"
254 done
255 done