Strabon
changeset 770:6aa5f315664a temporals
added class and scripts for experiments
author | Kostis Kyzirakos <kkyzir@di.uoa.gr> |
---|---|
date | Fri Dec 07 18:13:07 2012 +0200 (2012-12-07) |
parents | d715fd304e9e |
children | 069e5acada90 |
files | runtime/pom.xml runtime/src/main/java/eu/earthobservatory/runtime/generaldb/Strabon.java runtime/src/main/java/eu/earthobservatory/utils/ExperimentsReal.java runtime/src/main/resources/pg_restart_clear_caches.sh scripts/experiments/exp scripts/experiments/setenv |
line diff
1.1 --- a/runtime/pom.xml Sat Dec 01 21:21:42 2012 +0200 1.2 +++ b/runtime/pom.xml Fri Dec 07 18:13:07 2012 +0200 1.3 @@ -408,6 +408,27 @@ 1.4 </execution> 1.5 </executions> 1.6 </plugin> 1.7 + 1.8 + <!-- retain permissions --> 1.9 + <plugin> 1.10 + <groupId>org.apache.maven.plugins</groupId> 1.11 + <artifactId>maven-antrun-plugin</artifactId> 1.12 + <version>1.6</version> 1.13 + <executions> 1.14 + <execution> 1.15 + <id>process-classes</id> 1.16 + <phase>process-classes</phase> 1.17 + <configuration> 1.18 + <target> 1.19 + <chmod file="target/classes/pg_restart_clear_caches.sh" perm="777" /> 1.20 + </target> 1.21 + </configuration> 1.22 + <goals> 1.23 + <goal>run</goal> 1.24 + </goals> 1.25 + </execution> 1.26 + </executions> 1.27 + </plugin> 1.28 </plugins> 1.29 </build> 1.30 </project>
2.1 --- a/runtime/src/main/java/eu/earthobservatory/runtime/generaldb/Strabon.java Sat Dec 01 21:21:42 2012 +0200 2.2 +++ b/runtime/src/main/java/eu/earthobservatory/runtime/generaldb/Strabon.java Fri Dec 07 18:13:07 2012 +0200 2.3 @@ -225,7 +225,7 @@ 2.4 logger.info("[Strabon.query] Executing query: \n{}", queryString); 2.5 2.6 // check for null stream 2.7 - if (out == null) { 2.8 + if ((out == null) && (resultsFormat != org.openrdf.query.resultio.Format.EXP)) { 2.9 logger.error("[Strabon.query] Cannot write to null stream."); 2.10 2.11 return false; 2.12 @@ -255,13 +255,14 @@ 2.13 long t2 = System.nanoTime(); 2.14 2.15 while (result.hasNext()) { 2.16 + String r = result.toString(); 2.17 results++; 2.18 } 2.19 2.20 long t3 = System.nanoTime(); 2.21 2.22 - //return new long[]{t2-t1, t3-t2, t3-t1, results}; 2.23 - break; 2.24 + return new long[]{t2-t1, t3-t2, t3-t1, results}; 2.25 + //break; 2.26 2.27 default: 2.28 // get the writer for the specified format
3.1 --- /dev/null Thu Jan 01 00:00:00 1970 +0000 3.2 +++ b/runtime/src/main/java/eu/earthobservatory/utils/ExperimentsReal.java Fri Dec 07 18:13:07 2012 +0200 3.3 @@ -0,0 +1,225 @@ 3.4 +package eu.earthobservatory.utils; 3.5 + 3.6 +import java.io.BufferedWriter; 3.7 +import java.io.File; 3.8 +import java.io.FileWriter; 3.9 +import java.io.IOException; 3.10 +import java.net.URL; 3.11 +import java.sql.Connection; 3.12 +import java.sql.DriverManager; 3.13 +import java.sql.ResultSet; 3.14 +import java.sql.SQLException; 3.15 +import java.sql.Statement; 3.16 +import java.text.DecimalFormat; 3.17 +import java.util.SortedSet; 3.18 +import java.util.TreeSet; 3.19 + 3.20 +import org.apache.log4j.chainsaw.Main; 3.21 +import org.openrdf.query.MalformedQueryException; 3.22 +import org.openrdf.query.QueryEvaluationException; 3.23 +import org.openrdf.query.QueryLanguage; 3.24 +import org.openrdf.query.TupleQuery; 3.25 +import org.openrdf.query.TupleQueryResult; 3.26 +import org.openrdf.query.TupleQueryResultHandlerException; 3.27 +import org.openrdf.repository.RepositoryException; 3.28 + 3.29 + 3.30 +import eu.earthobservatory.runtime.monetdb.Strabon; 3.31 + 3.32 +public class ExperimentsReal { 3.33 + 3.34 + private static Strabon strabon = null; 3.35 + 3.36 + /** 3.37 + * @param args 3.38 + * @throws Exception 3.39 + */ 3.40 + public static void main(String[] args) throws Exception { 3.41 + 3.42 + if (args.length < 6) { 3.43 + System.err.println("Usage: eu.ist.semsorgrid4env.strabon.ExperimentReal <HOST> <PORT> <DATABASE> <USERNAME> <PASSWORD> <LOG_PATH>"); 3.44 + System.err.println(" where <HOST> is the postgis database host to connect to"); 3.45 + System.err.println(" <PORT> is the port to connect to on the database host"); 3.46 + System.err.println(" <DATABASE> is the spatially enabled postgis database that Strabon will use as a backend, "); 3.47 + System.err.println(" <USERNAME> is the username to use when connecting to the database "); 3.48 + System.err.println(" <PASSWORD> is the password to use when connecting to the database"); 3.49 + System.err.println(" <LOG_PATH> where to store the logs"); 3.50 + System.exit(0); 3.51 + } 3.52 + 3.53 + String host = args[0]; 3.54 + Integer port = new Integer(args[1]); 3.55 + String db = args[2]; 3.56 + String user = args[3]; 3.57 + String passwd = args[4]; 3.58 + String logPath = args[5]; 3.59 + 3.60 + String[] queries = { 3.61 + //Q1: members of Congress after Clinton. 3.62 + "select distinct ?name2 where { ?x1 <http://xmlns.com/foaf/0.1/name> \"William Clinton\" . ?x1 <http://www.rdfabout.com/rdf/schema/politico/hasRole> ?term1 ?t1 . ?x2 <http://xmlns.com/foaf/0.1/name> ?name2 . ?x2 <http://www.rdfabout.com/rdf/schema/politico/hasRole> ?term2 ?t2 . FILTER(<http://strdf.di.uoa.gr/ontology#afterPeriod>(?t2, ?t1)) } ", 3.63 + //Q2: members of Congress, that there isn't another member of Congress after them. 3.64 + "select distinct ?name where { ?x1 <http://www.rdfabout.com/rdf/schema/politico/hasRole> ?term1 ?t1 . ?x1 <http://xmlns.com/foaf/0.1/name> ?name . optional {?x2 <http://www.rdfabout.com/rdf/schema/politico/hasRole> ?term2 ?t2 . FILTER(<http://strdf.di.uoa.gr/ontology#afterPeriod>(?t2, ?t1)) . } FILTER(!bound(?x2)) } ", 3.65 + //Q3: members of Congress in [1-1-1863, 1-1-1864]. 3.66 + "select distinct ?name where { ?x1 <http://www.rdfabout.com/rdf/schema/politico/hasRole> ?term1 ?t1 . ?x1 <http://xmlns.com/foaf/0.1/name> ?name . FILTER(<http://strdf.di.uoa.gr/ontology#PeriodContains>(?t1, \"[1-1-1863 00:00:00, 1-1-1864 00:00:00]\"^^<http://strdf.di.uoa.gr/ontology#validTime>)) } " , 3.67 + }; 3.68 + 3.69 + int repetitions = 5; 3.70 + 3.71 + try { 3.72 + for (int qno = 0; qno < queries.length; qno++) { 3.73 + 3.74 + String queryString = queries[qno]; 3.75 + 3.76 + long coldtime = 0; 3.77 + long warmtime = 0; 3.78 + long results = 0; 3.79 + 3.80 + // cold runs 3.81 + long[][] coldruns = new long[repetitions][4]; 3.82 + for (int i = 0; i < repetitions; i++) { 3.83 + strabon = new Strabon(db, user, passwd, port, host, true); 3.84 + coldruns[i] = (long[])run("cold", queryString, i); 3.85 + strabon.close(); 3.86 + strabon = null; 3.87 + 3.88 + URL script = Main.class.getResource("/pg_restart_clear_caches.sh"); 3.89 + String restart_script = script.toString().substring(5); 3.90 + 3.91 + Process p = Runtime.getRuntime().exec(restart_script); 3.92 + p.waitFor(); 3.93 + System.gc(); 3.94 + Thread.sleep(5000); 3.95 + } 3.96 + 3.97 + //Strabon strabon = new Strabon(db, user, passwd, port, host, true, cachepath); 3.98 + strabon = new Strabon(db, user, passwd, port, host, true); 3.99 + 3.100 + // warm runs 3.101 + long[][] warmruns = new long[repetitions][4]; 3.102 + // warm up caches 3.103 + long[] onerun = (long[])run("warm", queryString, 0); 3.104 + results = onerun[3]; 3.105 + 3.106 + for (int i = 0; i < repetitions; i++) { 3.107 + warmruns[i] = (long[])run("warm", queryString, i); 3.108 + } 3.109 + 3.110 + strabon.close(); 3.111 + strabon = null; 3.112 + 3.113 + // sort results 3.114 + SortedSet<Long> coldtimes = new TreeSet<Long>(); 3.115 + for (int i = 0; i < repetitions; i++) { 3.116 + coldtimes.add(coldruns[i][2]); 3.117 + } 3.118 + 3.119 + SortedSet<Long> warmtimes = new TreeSet<Long>(); 3.120 + for (int i = 0; i < repetitions; i++) { 3.121 + warmtimes.add(warmruns[i][2]); 3.122 + } 3.123 + 3.124 + // calculate median 3.125 + int i = 0; 3.126 + for (Long long1 : coldtimes) { 3.127 + i++; 3.128 + if ((repetitions % 2 == 1) && ((repetitions/2+1) == i)) { 3.129 + coldtime = long1; 3.130 + } else if ((repetitions % 2 == 0) && ((repetitions/2) == i)) { 3.131 + coldtime = long1; 3.132 + } else if ((repetitions % 2 == 0) && ((repetitions/2+1) == i)) { 3.133 + coldtime = coldtime + long1; 3.134 + coldtime = coldtime / 2; 3.135 + } 3.136 + } 3.137 + 3.138 + i = 0; 3.139 + for (Long long1 : warmtimes) { 3.140 + i++; 3.141 + if ((repetitions % 2 == 1) && ((repetitions/2+1) == i)) { 3.142 + warmtime = long1; 3.143 + } else if ((repetitions % 2 == 0) && ((repetitions/2) == i)) { 3.144 + warmtime = long1; 3.145 + } else if ((repetitions % 2 == 0) && ((repetitions/2+1) == i)) { 3.146 + warmtime = warmtime + long1; 3.147 + warmtime = warmtime / 2; 3.148 + } 3.149 + } 3.150 + 3.151 + // print logs 3.152 + FileWriter fstream; 3.153 + BufferedWriter out; 3.154 + try { 3.155 + 3.156 + fstream = new FileWriter(logPath + "/warm_key_q" + (qno+1), 3.157 + true); 3.158 + out = new BufferedWriter(fstream); 3.159 + out.write("GARBAGE " + results + " " + warmtime + "\n"); 3.160 + out.close(); 3.161 + 3.162 + fstream = new FileWriter(logPath + "/warm_key_q" + (qno+1) 3.163 + + "_long", true); 3.164 + out = new BufferedWriter(fstream); 3.165 + for (int j = 0; j < repetitions; j++) { 3.166 + out.write("GARBAGE " + results + " " + warmruns[j][0] 3.167 + + " " + warmruns[j][1] + " " + warmruns[j][2] 3.168 + + "\n"); 3.169 + } 3.170 + out.close(); 3.171 + 3.172 + fstream = new FileWriter(logPath + "/cold_key_q" + (qno+1), 3.173 + true); 3.174 + out = new BufferedWriter(fstream); 3.175 + out.write("GARBAGE " + results + " " + coldtime + "\n"); 3.176 + out.close(); 3.177 + 3.178 + fstream = new FileWriter(logPath + "/cold_key_q" + (qno+1) 3.179 + + "_long", true); 3.180 + out = new BufferedWriter(fstream); 3.181 + for (int j = 0; j < repetitions; j++) { 3.182 + out.write("GARBAGE " + results + " " + coldruns[j][0] 3.183 + + " " + coldruns[j][1] + " " + coldruns[j][2] 3.184 + + "\n"); 3.185 + } 3.186 + out.close(); 3.187 + } catch (Exception e) { 3.188 + System.err.println("Error: " + e.getMessage()); 3.189 + } 3.190 + } 3.191 + 3.192 + } catch (MalformedQueryException e) { 3.193 + System.out.println("Please use a well-formed query"); 3.194 + e.printStackTrace(); 3.195 + } catch (QueryEvaluationException e) { 3.196 + e.printStackTrace(); 3.197 + } catch (TupleQueryResultHandlerException e) { 3.198 + e.printStackTrace(); 3.199 + } catch (IOException e) { 3.200 + e.printStackTrace(); 3.201 + } 3.202 + } 3.203 + 3.204 + private static long[] run(String cacheState, String query, int repetition) throws Exception { 3.205 + 3.206 + System.out.println("Executing query. Caches: " + cacheState + ". Repetition: " + repetition); 3.207 + 3.208 + long[] resp = (long[])strabon.query(query, org.openrdf.query.resultio.Format.EXP, null); 3.209 + 3.210 +// Statement st; 3.211 +// System.out.println("Evaluating query (naive)."); 3.212 +// long t1 = System.nanoTime(); 3.213 +// TupleQuery tupleQuery = repo.getConnection().prepareTupleQuery(QueryLanguage.SPARQL, query); 3.214 +// TupleQueryResult result = tupleQuery.evaluate(); 3.215 +// long t2 = System.nanoTime(); 3.216 +// int results = 0; 3.217 +// while(result.hasNext()) { 3.218 +// String rrrr = result.next().toString(); 3.219 +// results++; 3.220 +// } 3.221 +// long t3 = System.nanoTime(); 3.222 +// System.out.println("Evaluated query (naive). results="+results); 3.223 +// resp = new long[]{t2-t1, t3-t2, t3-t1,results}; 3.224 + 3.225 + return resp; 3.226 + 3.227 + } 3.228 +}
4.1 --- /dev/null Thu Jan 01 00:00:00 1970 +0000 4.2 +++ b/runtime/src/main/resources/pg_restart_clear_caches.sh Fri Dec 07 18:13:07 2012 +0200 4.3 @@ -0,0 +1,8 @@ 4.4 +#!/bin/bash 4.5 +DIR="$( cd -P "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" 4.6 + 4.7 +service postgresql stop 4.8 +sync && echo 3 > /proc/sys/vm/drop_caches; 4.9 +service postgresql start 4.10 +echo "clearing cache... `date`" 4.11 +sleep 1;
5.1 --- /dev/null Thu Jan 01 00:00:00 1970 +0000 5.2 +++ b/scripts/experiments/exp Fri Dec 07 18:13:07 2012 +0200 5.3 @@ -0,0 +1,20 @@ 5.4 +#!/bin/bash 5.5 +INITPATH="$( cd -P "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" 5.6 +source $INITPATH/setenv 5.7 + 5.8 +#DBNAME=$1 5.9 + 5.10 +cd $LOG_PATH; 5.11 +dirs=`ls -1d $DBNAME* | sort | tail -n 1`; 5.12 +maxdir=${dirs/$DBNAME-/''}; 5.13 +maxdir=`echo $maxdir | sed 's/^0//'`; 5.14 +dir=`printf "%02d" $((++maxdir))`; 5.15 + 5.16 +if [ -z $dir ] ; then dir="01"; fi 5.17 + 5.18 +mkdir $DBNAME-$dir; 5.19 + 5.20 +cd $STRABONEXP_PATH; 5.21 +CPATH=$(JARS=(*.jar); IFS=:; echo "${JARS[*]}") 5.22 +java -Xmx10G -cp $CPATH eu.earthobservatory.utils.ExperimentsReal $HOST $PORT $DBNAME $USERNAME $PASSWORD $LOG_PATH/$DBNAME-$dir 1> $LOG_PATH/$DBNAME-$dir/$DBNAME.out 2>$LOG_PATH/$DBNAME-$dir/$DBNAME.err 5.23 +cd $INITPATH;
6.1 --- /dev/null Thu Jan 01 00:00:00 1970 +0000 6.2 +++ b/scripts/experiments/setenv Fri Dec 07 18:13:07 2012 +0200 6.3 @@ -0,0 +1,7 @@ 6.4 +#!/bin/bash 6.5 +export DBTYPE=strabon #monet, postgres, rdf3x, strabon, naive 6.6 +export HOST='localhost' 6.7 +export PORT='5432' 6.8 +export USERNAME='strabon' 6.9 +export PASSWORD='strabon' 6.10 +export LOG_PATH="/home/strabon/experiments/logs-$DBTYPE"