Commit e57f4947 authored by sarsonl's avatar sarsonl
Browse files

Add filter query class to centralize column filtering.

Implemented cleaner way of doing column comparison to input query
parameters. No functionality currently to do specialised filters which
use joins, but this use will be added if and when required.

Unit tests writen for new class.
parent 4871aab4
# Copyright (C) 2019, ETH Zurich - Swiss Seismological Service SED
Well related test facilities.
import unittest
from unittest.mock import MagicMock, patch
from hydws.server.v1 import routes
class MockQuery(object):
def __init__(self, query_text):
self.val = query_text
def filter(self, filt):
self.val += str(filt)
return self
class MockColumn(object):
def __init__(self):
self.mock_attr = 'mock_attr'
def mock_method(self, val):
return val
class MockClass(object):
mock_column = MockColumn()
def mock_method(self):
class MockClassUnderscore(object):
def mock_method_(self):
class MockClassDoubleUnderscore(object):
def __mock_method__(self):
class DynamicFilterTestCase(unittest.TestCase):
def test_operator_attr_exception(self):
dyn_f = routes.DynamicFilter(MagicMock(), MagicMock())
op = 'invalid_method'
with self.assertRaises(Exception): dyn_f.operator_attr(MockClass(), op)
def test_operator_attr_return(self):
dyn_f = routes.DynamicFilter(MagicMock(), MagicMock())
op = 'mock_method'
self.assertEqual(dyn_f.operator_attr(MockClass(), op), op)
def test_operator_attr_underscore(self):
dyn_f = routes.DynamicFilter(MagicMock(), MagicMock())
op = 'mock_method'
existing_op = 'mock_method_'
self.assertEqual(dyn_f.operator_attr(MockClassUnderscore(), op), existing_op)
def test_operator_attr_double_underscore(self):
dyn_f = routes.DynamicFilter(MagicMock(), MagicMock())
op = 'mock_method'
existing_op = '__mock_method__'
self.assertEqual(dyn_f.operator_attr(MockClassDoubleUnderscore(), op), existing_op)
@patch.object(routes.DynamicFilter, 'operator_attr')
def test_filter_query_return(self, mock_operator_attr):
"""Return mock query with filter applied."""
query_str = 'query'
filter_str = '_mock_filter_value'
expected_final_query = query_str + filter_str
mock_operator_attr.return_value = "mock_method"
dyn_f = routes.DynamicFilter(MockQuery(query_str), MockClass())
dyn_f.filter_query([('mock_column', 'mock_method', filter_str)])
self.assertEqual(dyn_f.query.val, expected_final_query)
@patch.object(routes.DynamicFilter, 'operator_attr')
def test_filter_multi_query_return(self, mock_operator_attr):
"""Return mock query with >1 filter applied."""
query_str = 'query'
filter_str = '_mock_filter_value'
filter_str2 = 'second_value'
expected_final_query = query_str + filter_str + filter_str2
mock_operator_attr.return_value = "mock_method"
dyn_f = routes.DynamicFilter(MockQuery(query_str), MockClass())
[('mock_column', 'mock_method', filter_str),
('mock_column', 'mock_method', filter_str2)])
self.assertEqual(dyn_f.query.val, expected_final_query)
def test_filter_query_invalid_input(self):
dyn_f = routes.DynamicFilter(MockQuery('query'), MockClass())
with self.assertRaises(Exception): dyn_f.filter_query(
[('mock_column', 'mock_method', '_mock_filter_value', 'extra val')])
def test_filter_query_invalid_method(self):
dyn_f = routes.DynamicFilter(MockQuery('query'), MockClass())
with self.assertRaises(Exception): dyn_f.filter_query(
[('mock_column', 'invalid_method', '_mock_filter_value')])
......@@ -74,55 +74,76 @@ class BoreholeResource(ResourceBase):
def get(self, borehole_id):
class FilterQuery(object):
def __init__(self, query):
self.query = query
def _filter_query(self, query_name, orm_tablename, orm_paramname, filter_op):
class DynamicFilter(ResourceBase):
#operator_methods = {'egt': 'egt_query_filter',
# 'elt': 'elt_query_filter',
# 'eq': 'eq_query_filter',
# 'neq': 'neq_query_filter'}
query_param = query_params.get(query_name)
if query_param:
orm_methodname = getattr(orm, orm_tablename)
orm_param = getattr(orm_methodname, prm_paramname)
Dynamic filtering of query.
#getattr(self, operator_methods[filter_op])(orm_param, query_param)
getattr(self, filter_op)(orm_param, query_param)
raise ValueError('No filter method exists for: {}'.format(filter_op))
#if filter_op == 'egt':
# self.egt_query_filter(orm_param, query_parameter)
#elif filter_op == 'elt':
# self.elt_query_filter(orm_param, query_parameter)
#elif filter_op == 'eq':
# self.eq_query_filter(orm_param, query_parameter)
#elif filter_op == 'neq':
# self.neq_query_filter(orm_param, query_parameter)
# raise ValueError('No filter method exists for: {}'.format(filter_op))
dyn_query = DynamicFilter(query, orm.BoreholeSection)
dyn_query.filter_query([('m_starttime', 'eq', datetime(...))])
def __init__(self, query, orm_class):
self.query = query
self.orm_class = orm_class
def egt_query_filter(self, orn_param, query_param):
query = self.query.filter(orm_param >= query_param)
def operator_attr(self, column, op):
Returns method associated with an comparison operator.
If op, op_ or __op__ does not exist, Exception returned.
:returns type: str.
def elt_query_filter(self, query_name, orm_tablename, orm_paramname, filter_op):
query = self.query.filter(orm_param <= query_param)
return list(filter(
lambda e: hasattr(column, e % op),
['%s', '%s_', '__%s__']))[0] % op
except IndexError:
raise Exception('Invalid filter operator: %s' % op)
def filter_query(self, filter_condition):
Update self.query based on filter_condition.
:param filter_condition: list, ie: [(key,operator,value)]
operator examples:
eq for ==
lt for <
ge for >=
in for in_
like for like
value can be list or a string.
key must belong in self.orm_class.
def eq_query_filter(self, query_name, orm_tablename, orm_paramname, filter_op):
query = self.query.filter(orm_param == query_param)
def neq_query_filter(self, query_name, orm_tablename, orm_paramname, filter_op):
query = self.query.filter(orm_param != query_param)
for f in filter_condition:
key, op, value = f
except ValueError:
raise Exception('Invalid filter input: %s' % f)
column = getattr(self.orm_class, key)
if not column:
raise Exception('Invalid filter column: %s' % key)
if op == 'in':
if isinstance(value, list):
filt = column.in_(value)
filt = column.in_(value.split(','))
attr = self.operator_attr(self, column, op)
if value == 'null':
value = None
print(column, attr, value)
filt = getattr(column, attr)(value)
self.query = self.query.filter(filt)
def return_query(self):
return self.query
class BoreholeHydraulicDataListResource(ResourceBase):
......@@ -133,6 +154,7 @@ class BoreholeHydraulicDataListResource(ResourceBase):
locations=("query", ))
def get(self, borehole_id, **query_params):
borehole_id = decode_publicid(borehole_id)
......@@ -150,10 +172,12 @@ class BoreholeHydraulicDataListResource(ResourceBase):
resp = BoreholeSchema().dumps(resp)
return make_response(resp, settings.MIMETYPE_JSON)
def _process_request(self, session, borehole_id=None, section_id=None,
if not borehole_id:
raise ValueError(f"Invalid borehole identifier: {borehole_id!r}")
......@@ -162,7 +186,7 @@ class BoreholeHydraulicDataListResource(ResourceBase):
filter_statement = FilterQuery(query)
dynamic_query = DynamicFilter(query, orm.BoreholeSection)
# XXX(damb): Emulate QuakeML type Epoch (though on DB level it is
# defined as QuakeML type OpenEpoch
......@@ -180,10 +204,13 @@ class BoreholeHydraulicDataListResource(ResourceBase):
(orm.BoreholeSection.m_endtime == None)).\
filter(orm.HydraulicSample.m_datetime_value <= endtime)
# TODO(damb): Add additional filter criteria
# XXX(lsarson): Filter None first or query will fail due to type differences.
dynamic_query.filter_query([('m_starttime', 'ne', None),
('m_starttime', 'ge', query_params.get('starttime'))])
# TODO(lsarson): Think about if endtime not defined.
dynamic_query.filter_query(['m_endtime', 'le', query_params.get('endtime')])
# TODO(lsarson): Add additional filter criteria. Just test out this for now.
return query.\
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment