Commit f2b70095 authored by sarsonl's avatar sarsonl
Browse files

Do not return sections if epoch time not valid

When datetime query parameters were used to filter hydraulics,
The sections of the borehole would all be returned regardless
of the section epoch time. Now logic has been added to also
filter sections on epoch time if this does not overlap with
the hydraulic search criteria.

A test case was added to the populate_db_cases.py to test.
parent 42751187
......@@ -6,6 +6,8 @@
.. moduleauthor:: Laura Sarson <laura.sarson@sed.ethz.ch>
"""
import datetime
from sqlalchemy import or_
from sqlalchemy.orm.exc import NoResultFound
from hydws.db.orm import Borehole, BoreholeSection, HydraulicSample
......@@ -44,6 +46,10 @@ filter_hydraulics = [# capital.
('fluidph_value', 'ge', 'minfluidph'),
('fluidph_value', 'le', 'maxfluidph')]
filter_sections = [
('starttime', 'le', 'endtime'),
('endtime', 'ge', 'starttime')]
filter_boreholes = [
('latitude_value', 'ge', 'minlatitude'),
('latitude_value', 'le', 'maxlatitude'),
......@@ -51,6 +57,11 @@ filter_boreholes = [
('longitude_value', 'le', 'maxlongitude')]
#class InvalidOperator(ErrorWithTraceback):
# def __init__(self,*args,**kwargs):
# Exception.__init__(self,*args,**kwargs)
class DynamicQuery(object):
"""
......@@ -86,6 +97,7 @@ class DynamicQuery(object):
:rtype: dict
"""
#MultipleResultsFound from sqlalchemy.orm.exc
return self.query.one_or_none()
def paginate_query(self, limit, page=None, error_flag=False):
......@@ -124,6 +136,13 @@ class DynamicQuery(object):
# define a specific error here
raise Exception(f"Invalid operator: {op}")
def filter_section_epoch(self, column, attr, param_value):
# special case as have to deal with open epochs.
eq_attr = self.operator_attr(column, 'eq')
filt = getattr(column, attr)(param_value)
return filt
def filter_query(self, query_params, filter_level):
"""Update self.query with chained filters based
on query_params
......@@ -142,6 +161,10 @@ class DynamicQuery(object):
elif filter_level == "borehole":
orm_class = Borehole
filter_condition = filter_boreholes
elif filter_level == "section":
orm_class = BoreholeSection
#print(dir(Borehole._sections))
filter_condition = filter_sections
else:
raise Exception(f'filter level not handled: {filter_level}')
......@@ -168,6 +191,9 @@ class DynamicQuery(object):
filt = column.in_(param_value.split(","))
else:
attr = self.operator_attr(column, op)
filt = getattr(column, attr)(param_value)
if filter_level == "section":
filt = self.filter_section_epoch(column, attr, param_value)
else:
filt = getattr(column, attr)(param_value)
self.query = self.query.filter(filt)
......@@ -22,6 +22,7 @@
# Copyright (c) Sven Marti (ETH), Daniel Armbruster (ETH), Fabian Euchner (ETH)
#
# REVISION AND CHANGES
# 2019/05/13 Repurposed for HYDWS Laura Sarson
# 2018/06/05 V0.1 Sven Marti
# =============================================================================
"""
......
......@@ -199,12 +199,42 @@ bh3_section3 = orm.BoreholeSection(
holediameter_value=0.3,
casingdiameter_value=0.25, ) # This has been altered from bh3_section2
# Third borehole, sections but no hydraulics
bh4 = orm.Borehole(
publicid='smi:ch.ethz.sed/bh/11111111-e4a0-4692-bf29-33b5591eb7123',
depth_value=1000,
latitude_value=10.66320713,
latitude_uncertainty=0.5368853227,
longitude_value=10.66320713,
longitude_uncertainty=0.7947170871,
bedrockdepth_value=0)
bh4_section1 = orm.BoreholeSection(
publicid='smi:ch.ethz.sed/bh/section/'
'11111111-8d89-4f13-95e7-526ade73c123',
topclosed=False,
bottomclosed=False,
toplatitude_value=15.63484349,
toplatitude_uncertainty=0.0008854447,
toplongitude_value=-50.66323323,
toplongitude_uncertainty=0.7947170871,
topdepth_value=0,
bottomlatitude_value=50.66323327,
bottomlatitude_uncertainty=0.5368853227,
bottomlongitude_value=50.66323330,
bottomlongitude_uncertainty=0.7947170871,
bottomdepth_value=100,
holediameter_value=0.3,
casingdiameter_value=0.28, )
def insert_orm_values(db_url):
bh1_section1._hydraulics.append(sample1)
bh1_section1._hydraulics.append(sample2)
bh1_section1._hydraulics.append(sample3)
bh1._sections = [bh1_section1, bh1_section2]
bh4._sections = [bh4_section1]
bh3._sections = [bh3_section1, bh3_section2, bh3_section3]
......@@ -217,6 +247,7 @@ def insert_orm_values(db_url):
session.add(bh1)
session.add(bh2)
session.add(bh3)
session.add(bh4)
session.commit()
session.close()
......
......@@ -49,11 +49,11 @@ class FDSNWSDateTime(fields.DateTime):
<http://www.fdsn.org/webservices/FDSN-WS-Specifications-1.1.pdf>`_.
"""
SERIALIZATION_FUNCS = fields.DateTime.SERIALIZATION_FUNCS.copy()
#SERIALIZATION_FUNCS = fields.DateTime.SERIALIZATION_FUNCS.copy()
DESERIALIZATION_FUNCS = fields.DateTime.DESERIALIZATION_FUNCS.copy()
SERIALIZATION_FUNCS['fdsnws'] = fdsnws_isoformat
#SERIALIZATION_FUNCS['fdsnws'] = fdsnws_isoformat
DESERIALIZATION_FUNCS['fdsnws'] = from_fdsnws_datetime
......@@ -206,7 +206,7 @@ class BoreholeHydraulicSampleListResourceSchema(HydraulicsSchemaMixin,
raise ValidationError on hydraulic level query parameters.
"""
if data.get('level') in ('borehole', 'section'):
hydraulic_params = HydraulicsSchemaMixin().dump(data)
hydraulic_params = HydraulicsSchemaMixin(exclude=['starttime', 'endtime']).dump(data)
if hydraulic_params:
raise ValidationError(
'Hydraulic query parameters not allowed: {}'.\
......
......@@ -5,14 +5,15 @@
.. moduleauthor:: Laura Sarson <laura.sarson@sed.ethz.ch>
"""
import datetime as datetime
import logging
from sqlalchemy import literal
from sqlalchemy import literal, and_
from flask_restful import Api, Resource
from sqlalchemy.orm.exc import NoResultFound
from webargs.flaskparser import use_kwargs
from marshmallow import fields
from sqlalchemy.orm import subqueryload, eagerload, joinedload, contains_eager, lazyload
from sqlalchemy.orm import subqueryload, eagerload, joinedload, joinedload_all, contains_eager, lazyload, aliased, selectinload
from hydws import __version__
from hydws.db.orm import Borehole, BoreholeSection, HydraulicSample
......@@ -83,8 +84,7 @@ class BoreholeListResource(ResourceBase):
@use_kwargs(BoreholeListResourceSchema, locations=("query",))
@with_strict_args(BoreholeListResourceSchema, locations=("query",))
def get(self, **query_params):
params_schema = BoreholeListResourceSchema()
query_params = params_schema.dump(query_params)
self.logger.debug(
f"Received request: "
f"query_params={query_params}")
......@@ -129,16 +129,16 @@ class BoreholeHydraulicSampleListResource(ResourceBase):
locations=("query", ))
def get(self, borehole_id, **query_params):
borehole_id = decode_publicid(borehole_id)
params_schema = BoreholeHydraulicSampleListResourceSchema()
query_params = params_schema.dump(query_params)
map_return_levels = [{'hydraulic': 'section',}]
self.logger.debug(
f"Received request: borehole_id={borehole_id}, "
f"query_params={query_params}")
resp = self._process_request(db.session, borehole_id=borehole_id,
**query_params)
if not resp:
self._handle_nodata(query_params)
# TODO(damb): Serialize according to query_param format=JSON|XML
......@@ -155,25 +155,49 @@ class BoreholeHydraulicSampleListResource(ResourceBase):
raise ValueError(f"Invalid borehole identifier: {borehole_id!r}")
level = query_params.get('level')
order_by_levels = []
hyd_subquery = DynamicQuery(session.query(HydraulicSample.boreholesection_oid))
sec_subquery = DynamicQuery(session.query(BoreholeSection.borehole_oid))
if level in ['section', 'hydraulic']:
sec_subquery.filter_query(query_params, 'section')
if level == 'hydraulic':
hyd_subquery.filter_query(query_params,
'hydraulic')
hyd_query = hyd_subquery.query.subquery(name="hyd_query")
sec_query = sec_subquery.query.subquery(name="sec_query")
print(sec_query)
query = session.query(Borehole)
if level == 'section':
query = query.options(subqueryload(Borehole._sections)).join(BoreholeSection)
order_by_levels.append('section')
query = query.options(lazyload(Borehole._sections)).\
join(BoreholeSection, isouter=True).options(contains_eager("_sections"))
#TODO(sarsonl): Currently if querying at the hydraulic level, and no hydraulics
# exist, then no result is output. Not sure why as using left outer joins.
elif level == 'hydraulic':
query = query.options(lazyload(Borehole._sections).\
lazyload(BoreholeSection._hydraulics)).\
join(BoreholeSection).join(HydraulicSample)
join(sec_query, Borehole._oid==sec_query.c.borehole_oid , isouter=True).\
join(hyd_query, sec_query.c._oid==hyd_query.c.boreholesection_oid,
isouter=True).\
options(contains_eager("_sections").\
contains_eager("_hydraulics"))
query = query.filter(Borehole.publicid==borehole_id)
query = query.filter(Borehole.publicid==borehole_id)
dynamic_query = DynamicQuery(query)
dynamic_query.filter_query(query_params,
'hydraulic')
#if level in ['section', 'hydraulic']:
# dynamic_query.filter_query(query_params, 'section')
#if level == 'hydraulic':
# dynamic_query.filter_query(query_params,
# 'hydraulic')
print(dynamic_query.query)
if query_params.get('limit'):
paginate_obj = dynamic_query.paginate_query(
query_params.get('limit'), query_params.get('page'))
......@@ -196,8 +220,7 @@ class SectionHydraulicSampleListResource(ResourceBase):
def get(self, borehole_id, section_id, **query_params):
borehole_id = decode_publicid(borehole_id)
section_id = decode_publicid(section_id)
params_schema = SectionHydraulicSampleListResourceSchema()
query_params = params_schema.dump(query_params)
self.logger.debug(
f"Received request: borehole_id={borehole_id}, "
f"section_id={section_id}, "
......
......@@ -22,10 +22,12 @@ _name = 'hydws'
_description = 'REST webservice allowing access to hydraulic data.'
_authors = [
'Daniel Armbruster',
'Lukas Heiniger', ]
'Lukas Heiniger',
'Laura Sarson', ]
_authors_email = [
'daniel.armbruster@sed.ethz.ch',
'lukas.heiniger@sed.ethz.ch', ]
'lukas.heiniger@sed.ethz.ch',
'laura.sarson@sed.ethz.ch']
_install_requires = [
'Flask>=1.0.2',
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment