Commit 6f842b24 authored by Stephan Herrnkind's avatar Stephan Herrnkind
Browse files

added tag jakarta/2014.318.01

parent 156d52ad
......@@ -41,11 +41,11 @@ EXEC_PROGRAM(
IF (SC_GLOBAL_PYTHON_WRAPPER)
FIND_PACKAGE(PythonLibs REQUIRED)
FIND_PACKAGE(PythonInterp REQUIRED)
#SET(PYTHON_LIBRARY_SUFFIX /python${PYTHON_VERSION}/site-packages)
SET(PYTHON_LIBRARY_SUFFIX /python)
SET(PYTHON_LIBRARY_PATH lib${PYTHON_LIBRARY_SUFFIX})
ENDIF (SC_GLOBAL_PYTHON_WRAPPER)
SET(PYTHON_LIBRARY_SUFFIX /python)
SET(PYTHON_LIBRARY_PATH lib${PYTHON_LIBRARY_SUFFIX})
INCLUDE(AddFileDependencies)
INCLUDE(CreateLibrary)
......
......@@ -62,7 +62,7 @@ copyright = u'2014, GFZ Potsdam, gempa GmbH'
# built documents.
#
# The short X.Y version.
version = '2014.255'
version = '2014.318'
# The full version, including alpha/beta/rc tags.
release = 'Jakarta'
......
......@@ -558,11 +558,11 @@ def _main(SSLpasswordDict, addr, request_format, data_format, label, resp_dict,
logs.warning(ln)
## Prepare to download
canJoin = True
canJoin = False
volumecount = 0
if req_type == "WAVEFORM" and req_args.get("format") != "MSEED":
canJoin = False
if req_type == "WAVEFORM" and req_args.get("format") == "MSEED":
canJoin = True
for req in req_ok:
for vol in req.status().volume:
......
......@@ -18,7 +18,7 @@ from seiscomp import sds
from seiscomp.arclink.manager import *
from seiscomp3 import Logging
VERSION = "1.1 (2013.008)"
VERSION = "1.1 (2014.316)"
class InvalidCommandError(Exception):
pass
......@@ -132,8 +132,8 @@ class RequestProxy(object):
return self.__user
def add(self, network, station, stream, loc_id, begin, end, constraints):
tw_tuple = (network, station, stream, loc_id, begin, end,
constraints, set())
tw_tuple = RequestLine(begin, end, network, station, stream, loc_id,
constraints)
if self.__rtype == "WAVEFORM" and (g_options.local or self.__sds.exists(begin, end, network, station, stream, _checkdot(loc_id))):
self.__tw_local.append(tw_tuple)
......@@ -900,7 +900,7 @@ def process_options():
parser.add_option("-O", "--organization", type="string", dest="organization",
help="organization name (default %default)")
parser.add_option("-d", "--dcid", type="string", dest="organization",
parser.add_option("-d", "--dcid", type="string", dest="dcid",
help="datacenter ID (default %default)")
parser.add_option("-S", "--max-sessions", type="int", dest="max_sessions",
......
#!/bin/sh
#
# Helper script to manually remove a day e.g. due to a bad report:
#
# Begun by Peter L. Evans, May 2014?
# <pevans@gfz-potsdam.de>
#
# ----------------------------------------------------------------------
set -u
progname=`basename $0`
function show_usage() {
echo "Usage: ${progname} {db} {DCID | id} {date}"
cat <<EOF
Remove a record from all tables for the report.
{db} - the SQLite3 database containing Arclink statistics summaries
{DCID} - string e.g. "GFZ"
{id} - numeric internal id used inside the database; see ArcStatsSource.
{date} - start day of the record to be remove, as YYYY-MM-DD
EOF
}
function examine() {
echo "Looking into DB for relevant records..."
echo "Don't remove this row in ArcStatsSource:"
sqlite3 ${db} <<EOF
SELECT * FROM ArcStatsSource where id='${src}'; /* Don't delete this! */
EOF
echo "Do remove all of these:"
sqlite3 ${db} <<EOF
SELECT * FROM ArcStatsClientIP where start_day = "${day}" and src='${src}';
SELECT * FROM ArcStatsMessages where start_day = "${day}" and src='${src}';
SELECT * FROM ArcStatsNetwork where start_day = "${day}" and src='${src}';
SELECT * FROM ArcStatsReport where start_day = "${day}";
SELECT * FROM ArcStatsRequest where start_day = "${day}" and src='${src}';
SELECT * FROM ArcStatsStation where start_day = "${day}" and src='${src}';
SELECT * FROM ArcStatsSummary where start_day = "${day}" and src='${src}';
SELECT * FROM ArcStatsUser where start_day = "${day}" and src='${src}';
SELECT * FROM ArcStatsUserIP where start_day = "${day}" and src='${src}';
SELECT * FROM ArcStatsVolume where start_day = "${day}" and src='${src}';
EOF
}
if [ $# -ne 3 ] ; then
show_usage
exit 2
fi
db=$1 # var/reqlogstats-2014.db
src=$2 # 4, RESIF
day=$3 # "2014-03-03"
if [ ! -r ${db} ] ; then
echo "Error, database file ${db} isn't readable" >&2
exit 1
fi
if [[ "$src" =~ ^[0-9]*$ ]] ; then
echo "Removing record for id=$src"
else
echo "Looking up source DCID..."
id=$(echo "SELECT id FROM ArcStatsSource WHERE dcid='$src';" | sqlite3 $db)
src=$id
fi
echo "Found source:"
echo "SELECT * FROM ArcStatsSource WHERE id='$src';" | sqlite3 $db
if [[ "${day}" =~ ^[1-9][0-9][0-9][0-9]-[0-1][0-9]-[0-3][0-9]$ ]] ; then
echo "Start day: $day"
else
echo "Error, not a proper day, use YYYY-MM-DD format." >&2
exit 1
fi
examine
# Then:
# sqlite> DELETE FROM ArcStatsSummary where start_day = "${day}" and src="${src}";
# and so on...
# Make sure the offending e-mail isn't in ~/eida_stats, otherwise
# it might just get reloaded. Then run reqlogstats.sh again.
#!/bin/bash
#
# Make graph of number of users per day or month over a whole year.
#
# Begun by Peter L. Evans, Oct 2014.
# Quick hack of make_year_graph.sh
#
# Input: reqlogstats-*.db SQLite database
# Parameters: network code [optional]
# Output: PNG plot - total
# PNG plot - break-out by source.
# text total
#
# ----------------------------------------------------------------------
set -u
progname=`basename $0`
today=`date +%F`
start_year=`date +%Y`
start_month=`date +%m`
img_dir='/srv/www/webdc/eida/data'
db_dir='/home/sysop/reqlogstats/var'
if [ ! -d ${img_dir} ] ; then
echo "${progname}: Images directory ${img_dir} does not exist. Bye."
exit 1
fi
if [ ! -d ${db_dir} ] ; then
echo "${progname}: SQLite DB directory ${db_dir} does not exist. Bye."
exit 1
fi
function show_usage() {
echo "Usage: ${progname} [--dcid {dcid} ] [ {month} [ {year} [ {db file} ]]]"
echo
echo "Create usage images from {db file} for the given date."
echo "If {dcid} is not given, all DCIDs are included."
echo "If {month} or {year} are not given, use today's date."
echo "If {db file} is not given, use a default."
}
dcid=
dcid_constr=""
table=
if [ $# -gt 0 ] ; then
first=$1
if [ "$first" == "--dcid" ] ; then
dcid=$2
dcid_constr="AND Y.dcid = '${dcid}'"
shift 2;
fi
fi
echo "Restricted to dcid=${dcid}; setting constraint: '${dcid_constr}'"
if [ $# -ge 1 ] ; then
start_month=$1 # Should be a two-digit number
fi
if [ $# -ge 2 ] ; then
start_year=$2 # Should be a four-digit number
fi
if [ $# -ge 3 ] ; then
dbfile=$3
else
dbfile="${db_dir}/reqlogstats-${start_year}.db"
fi
echo "Looking in ${dbfile} for ${start_year} month ${start_month}"
if [ ! -s ${dbfile} ] ; then
echo "Error: ${dbfile} not found. Bye"
exit 1
fi
# Normalise GiB,MiB etc to MiB ... Python, awk, or what?
cat > t1.py <<EOF
import sys
for x in sys.stdin.readlines():
line = x.strip()
if line.endswith('GiB'):
words = line.split();
val = words[-2];
print '\t'.join(words[0:-2]), '\t', float(val) * 1024.0, 'MiB'
else:
print line
EOF
# Rearrange to a table for histogram plotting, and summation by size:
cat > t2.py <<EOF
import sys
dcid_list = ['BGR', 'ETHZ', 'GFZ', 'INGV', 'IPGP', 'LMU', 'ODC', 'RESIF']
curday = None
row = {}
def flush_day(day, row):
s = sum(float(row[x]) for x in row.keys())
print "%s %10.1f" % (day, s),
for dcid in dcid_list:
if row.has_key(dcid):
print "%8.1f" % (float(row[dcid])),
else:
print "%8d" % 0,
print
print "# DAY ", " TOTAL", " ".join("%8s" % x for x in dcid_list)
for x in sys.stdin.readlines():
line = x.strip()
words = line.split()
day = words[0]
dcid = words[1]
val = words[2]
if day == curday:
row[dcid] = val
else:
# new day, flush and reload
if (curday != None):
flush_day(curday, row)
curday = day
row = {}
row[dcid] = val
if (curday != None):
flush_day(curday, row)
EOF
if [ -z "${dcid}" ] ; then
cmd="SELECT start_day, dcid, count(userID) FROM ArcStatsUser as X JOIN ArcStatsSource as Y WHERE X.src = Y.id ${dcid_constr} AND substr(X.start_day, 1, 4) = '${start_year}' GROUP BY start_day, dcid ORDER BY start_day, dcid;"
else
cmd="SELECT start_day, dcid, count(userID) FROM ArcStatsUser as X JOIN ArcStatsSource as Y WHERE X.src = Y.id ${dcid_constr} GROUP BY start_day, dcid ORDER BY start_day, dcid;"
fi
echo ${cmd}
echo ${cmd} \
| sqlite3 ${dbfile} | sed -e 's/|/ /g' \
| python t2.py > days3.dat
if [ $(wc -l days3.dat | awk '{print $1}') -le 1 ] ; then
echo "Nothing in db with '${dcid_constr}'."
exit 0
fi
head -1 days3.dat
tail -5 days3.dat
start_month_name=$(date +%B -d "$start_year-$start_month-01")
gnuplot <<EOF
set xdata time
set timefmt "%Y-%m-%d"
set xlabel 'Date in $start_year'
set xrange ['$start_year-01-01':]
set xtics 14*24*3600
set xtics format "%d\n%b"
set ylabel 'Distinct users'
#set logscale y
set yrange [0:210] # For GFZ in 2014: [0:130] is good.
set key top left
set grid x
set style data linespoints
set terminal svg font "arial,14" size 960,480 # even "giant" is not enough font.
set output 'out.svg'
plot 'days3.dat' using 1:2 title 'All EIDA nodes', \
'' using 1:5 title 'GFZ'
#set terminal dumb
#set output
#replot
EOF
if [ -z "${dcid}" ] ; then
out_dir="${img_dir}"
outfile="${out_dir}/total-users-${start_year}.svg"
else
echo "Sorry, can't do it yet. Bye."
exit 22
fi
if [ -s out.svg ] ; then
mkdir -p ${out_dir}
mv out.svg $outfile
echo "Wrote $outfile"
else
echo "No output!"
rm -f out.svg
exit 0
fi
# ----------------------------------------------------------------------
gnuplot <<EOF
set xlabel 'Day in $start_year'
set ylabel 'Distinct users'
set yrange [0:300] # Good for all 8 EIDA nodes stacked in 2014.
set key top left
set nogrid
set style data histograms
set style histogram rowstacked
set boxwidth 0.7 relative
set style fill solid 1.0 border 0
set terminal svg font "arial,14" size 960,480
set output 'out.svg'
# Default for ls 6 is dark blue, too close to pure blue for GFZ:
set style line 3 linecolor rgb "#00589C"
set style line 6 linecolor rgb "violet"
plot '<cut -c9- days3.dat' using 3:xtic(int(\$0) % 14 == 0?sprintf("%i", \$0):"") title 'BGR' ls 2, \
'' using 4 title 'ETHZ' ls 1, \
'' using 5 title 'GFZ' ls 3, \
'' using 6 title 'INGV' ls 4, \
'' using 7 title 'IPGP' ls 6, \
'' using 8 title 'LMU' ls 7, \
'' using 9 title 'ODC' ls 9, \
'' using 10 title 'RESIF' ls 8
#set terminal dumb
#set output
#replot
EOF
if [ -z "${dcid}" ] ; then
out_dir="${img_dir}"
outfile="${out_dir}/sources-users-${start_year}.svg"
txtfile="${out_dir}/total-users-${start_year}.txt"
else
echo "Sorry, can't do it yet. Bye."
exit 22
#out_dir="${img_dir}/${start_year}/${start_month}"
fi
if [ -s out.svg ] ; then
mkdir -p "${out_dir}"
mv out.svg $outfile
echo "Wrote $outfile"
else
rm -f out.svg
echo "No SVG output!"
fi
if [ -s days3.dat ] ; then
mv days3.dat $txtfile
else
echo "No text file output!"
fi
rm -f t1.py t2.py days3.dat
......@@ -64,7 +64,13 @@ def parse_table(table):
content = td.attrib["sorttable_customkey"]
else:
content = td.text
rowdata.append(str(content).strip())
try:
rowdata.append(content.strip().encode('ascii', 'replace'))
except UnicodeEncodeError as e:
print "While working on row", len(result), "of table with headers:", "|".join(headers)
print "ERROR working on content: {", content, "}"
raise e
#print '|'.join([td.tag, str(td.attrib), str(content).strip(), str(td.tail)])
result.append(tuple(rowdata))
return result
......
......@@ -142,7 +142,8 @@ $year = substr($date,0,4);
$month = substr($date,5,2);
$day = substr($date,8,2);
$db = new ReqLogSQ3($reqlogstats_db_dir . "/reqlogstats-$year.db");
$db_filename = $reqlogstats_db_dir . "/reqlogstats-$year.db";
$db = new ReqLogSQ3($db_filename);
$img_base_url = "../data";
$img1 = "$img_base_url/total-$year-$month.svg";
......
SC_BEGIN_PACKAGE(GUI)
OPTION(SC_GLOBAL_GUI "Build graphical user interfaces (requires Qt4)" OFF)
OPTION(SC_GLOBAL_GUI "Build graphical user interfaces (requires Qt4)" ON)
SET(CPACK_PACKAGE_EXECUTABLES "scrttv;scrttv" "scmv;scmv" "scolv;scolv" "scesv;scesv")
INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR}/libs)
......@@ -13,9 +13,19 @@ INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR}/../trunk/libs)
INCLUDE_DIRECTORIES(${CMAKE_CURRENT_BINARY_DIR}/../trunk/libs)
IF (SC_GLOBAL_GUI)
FIND_PACKAGE(Qt4 REQUIRED)
SET(Boost_DETAILED_FAILURE_MSG ON)
FIND_PACKAGE(Boost REQUIRED COMPONENTS program_options thread filesystem iostreams regex)
IF (WIN32)
ADD_DEFINITIONS("-DBOOST_ALL_NO_LIB")
ADD_DEFINITIONS("-DBOOST_LIB_DIAGNOSTIC")
ENDIF (WIN32)
LINK_DIRECTORIES(${Boost_LIBRARY_DIRS})
INCLUDE_DIRECTORIES(${Boost_INCLUDE_DIRS})
FIND_PACKAGE(Qt4 REQUIRED)
FIND_PACKAGE(OpenSSL REQUIRED)
SET(SC_LICENSE_LIB ${CMAKE_CURRENT_SOURCE_DIR}/license.cpp)
INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR})
INCLUDE_DIRECTORIES(${OPENSSL_INCLUDE_DIR})
......
......@@ -52,3 +52,9 @@ olv.arrivalTable.visibleColumns = Used, Status, Phase, Net, Sta, Loc/Cha, Res, D
# enable/disable advanced options (magnitude parameters) for artificial
# origin creations
olv.artificialOriginAdvanced = false
# If a locator does not populate the take off angle in its arrivals
# the first motion plot will not show picked polarities. This option
# defines whether to compute take off angles that are not present
# in the arrivals or not.
olv.computeMissingTakeOffAngles = true
......@@ -91,8 +91,12 @@
Sets the drawing of grid lines in the plot.
</description>
</parameter>
<parameter name="computeMissingTakeOffAngles" type="boolean" default="false">
<parameter name="computeMissingTakeOffAngles" type="boolean" default="true">
<description>
If a locator does not populate the take off angle in its arrivals
the first motion plot will not show picked polarities. This option
defines whether to compute take off angles that are not present
in the arrivals or not.
</description>
</parameter>
<parameter name="defaultAddStationsDistance" type="double" unit="deg" default="15">
......@@ -170,6 +174,12 @@
when in picking mode.
</description>
</parameter>
<parameter name="ignoreUnconfiguredStations" type="boolean" default="false">
<description>
If enabled the picker will not add stations that are not
configured with a detecStream.
</description>
</parameter>
<parameter name="loadAllComponents" type="boolean" default="true">
<description>
Loads all three components by default. If disabled then additional
......@@ -233,6 +243,14 @@
Minimum data time window length.
</description>
</parameter>
<parameter name="alignmentPosition" type="double" default="0.5">
<description>
Defines the relative position of the alignment when traces are
aligned at a phase. This value is bound
to interval [0:1] where 0 is the left side and 1 is right
side of the trace window.
</description>
</parameter>
<group name="uncertainties">
<parameter name="preferred" type="list:string">
<description>
......
......@@ -232,7 +232,7 @@ MainFrame::MainFrame(){
catch ( ... ) { locatorConfig.drawGridLines = true; }
try { locatorConfig.computeMissingTakeOffAngles = SCApp->configGetBool("olv.computeMissingTakeOffAngles"); }
catch ( ... ) { locatorConfig.computeMissingTakeOffAngles = false; }
catch ( ... ) { locatorConfig.computeMissingTakeOffAngles = true; }
try { locatorConfig.defaultEventRadius = SCApp->configGetDouble("olv.map.event.defaultRadius"); }
catch ( ... ) {}
......@@ -240,6 +240,9 @@ MainFrame::MainFrame(){
try { pickerConfig.showCrossHair = SCApp->configGetBool("picker.showCrossHairCursor"); }
catch ( ... ) { pickerConfig.showCrossHair = false; }
try { pickerConfig.ignoreUnconfiguredStations = SCApp->configGetBool("picker.ignoreUnconfiguredStations"); }
catch ( ... ) { pickerConfig.ignoreUnconfiguredStations = false; }
try { pickerConfig.loadAllComponents = SCApp->configGetBool("picker.loadAllComponents"); }
catch ( ... ) { pickerConfig.loadAllComponents = true; }
......@@ -279,6 +282,14 @@ MainFrame::MainFrame(){
try { pickerConfig.minimumTimeWindow = Core::TimeSpan(SCApp->configGetInt("picker.minimumTimeWindow")); }
catch ( ... ) { pickerConfig.minimumTimeWindow = Core::TimeSpan(1800); }
try { pickerConfig.alignmentPosition = SCApp->configGetDouble("picker.alignmentPosition"); }
catch ( ... ) { pickerConfig.alignmentPosition = 0.5; }
if ( pickerConfig.alignmentPosition < 0 )
pickerConfig.alignmentPosition = 0;
else if ( pickerConfig.alignmentPosition > 1 )
pickerConfig.alignmentPosition = 1;
try {
vector<string> uncertaintyProfiles =
SCApp->configGetStrings("picker.uncertainties.preferred");
......@@ -933,6 +944,7 @@ void MainFrame::configureAcquisition() {
SCApp->configSetBool("olv.hideStationsWithoutData", pc.hideStationsWithoutData);
SCApp->configSetBool("picker.showCrossHairCursor", pc.showCrossHair);
SCApp->configSetBool("picker.ignoreUnconfiguredStations", pc.ignoreUnconfiguredStations);
SCApp->configSetBool("picker.loadAllComponents", pc.loadAllComponents);
SCApp->configSetBool("picker.loadAllPicks", pc.loadAllPicks);
SCApp->configSetBool("picker.loadStrongMotion", pc.loadStrongMotionData);
......@@ -944,6 +956,7 @@ void MainFrame::configureAcquisition() {
SCApp->configSetInt("picker.preOffset", pc.preOffset.seconds());
SCApp->configSetInt("picker.postOffset", pc.postOffset.seconds());
SCApp->configSetInt("picker.minimumTimeWindow", pc.minimumTimeWindow.seconds());
SCApp->configSetDouble("picker.alignmentPosition", pc.alignmentPosition);
SCApp->configSetBool("picker.removeAutomaticPicksFromStationAfterManualReview", pc.removeAutomaticStationPicks);
SCApp->configSetBool("picker.removeAllAutomaticPicksAfterManualReview", pc.removeAutomaticPicks);
......
......@@ -19,7 +19,10 @@
#include <QString>
#include <QRectF>
#ifndef Q_MOC_RUN
#include <seiscomp3/client/application.h>
#endif
#include <seiscomp3/gui/core/maps.h>
#include <seiscomp3/gui/core/scheme.h>
#include <seiscomp3/gui/core/messagethread.h>
......
......@@ -48,7 +48,7 @@ QColor Gradient::colorAt(qreal position, bool discrete) const {
else if ( it.key() > position ) {
if ( last != end() ) {
if ( discrete ) {
return (last.key()>=0)?last.value().first: it.value().first;
return last.value().first;
}
else {
qreal v1 = last.key();
......
......@@ -263,7 +263,10 @@ void RecordStreamThread::run()
}
}
}