Commit 5a66f4a7 authored by sarsonl's avatar sarsonl
Browse files

Addition of unit tests and refactoring code

Some refactoring of code was required to make the code cleaner.

Added more validation to the schema based on what input data is allowed.

Centralised the column prefix m_

Added more docstrings.
parent f875d98b
This diff is collapsed.
"""
HYDWS datamodel ORM representation.
.. module:: orm
:synopsis: HYDWS datamodel ORM representation.
.. moduleauthor:: Laura Sarson <laura.sarson@sed.ethz.ch>
"""
from operator import itemgetter
......@@ -12,8 +16,9 @@ from sqlalchemy import select, func
from hydws.db.base import (ORMBase, CreationInfoMixin, RealQuantityMixin,
TimeQuantityMixin, EpochMixin,
LiteratureSourceMixin, PublicIDMixin)
from hydws.server import settings
PREFIX = 'm_'
# XXX(damb): The implementation of the entities below is based on the QuakeML
# and the SC3 DB model naming conventions. As a consequence,
......@@ -22,22 +27,28 @@ PREFIX = 'm_'
# Note, that this fact inherently leads to huge tables containing lots of
# columns.
# XXX(sarsonl): Do we want to restrict nullable values on the db level,
# or just schema.
try:
PREFIX = settings.HYDWS_PREFIX
except AttributeError:
PREFIX = ''
class Borehole(CreationInfoMixin('CreationInfo',
parent_prefix='creationinfo_',
global_column_prefix=PREFIX,
used=False),
LiteratureSourceMixin('LiteratureSource',
parent_prefix='literaturesource_',
global_column_prefix=PREFIX,
used=False),
RealQuantityMixin('longitude', global_column_prefix=PREFIX, value_nullable=False),
RealQuantityMixin('latitude', global_column_prefix=PREFIX, value_nullable=False),
RealQuantityMixin('depth', global_column_prefix=PREFIX),
RealQuantityMixin('bedrockdepth', global_column_prefix=PREFIX),
RealQuantityMixin('measureddepth', global_column_prefix=PREFIX),
PublicIDMixin(global_column_prefix=PREFIX),
RealQuantityMixin('longitude',
value_nullable=False),
RealQuantityMixin('latitude',
value_nullable=False),
RealQuantityMixin('depth'),
RealQuantityMixin('bedrockdepth'),
RealQuantityMixin('measureddepth'),
PublicIDMixin(),
ORMBase):
"""
ORM representation of a borehole. The attributes are in accordance with
......@@ -52,18 +63,16 @@ class Borehole(CreationInfoMixin('CreationInfo',
cascade='all, delete-orphan', lazy='noload', order_by='BoreholeSection.topdepth_value')
class BoreholeSection(EpochMixin('Epoch', epoch_type='open',
column_prefix='m_'),
RealQuantityMixin('toplongitude', global_column_prefix=PREFIX),
RealQuantityMixin('toplatitude', global_column_prefix=PREFIX),
RealQuantityMixin('topdepth', global_column_prefix=PREFIX),
RealQuantityMixin('bottomlongitude', global_column_prefix=PREFIX),
RealQuantityMixin('bottomlatitude', global_column_prefix=PREFIX),
RealQuantityMixin('bottomdepth', global_column_prefix=PREFIX),
RealQuantityMixin('holediameter', global_column_prefix=PREFIX),
RealQuantityMixin('casingdiameter', global_column_prefix=PREFIX),
PublicIDMixin(global_column_prefix=PREFIX),
class BoreholeSection(EpochMixin('Epoch', epoch_type='open'),
RealQuantityMixin('toplongitude'),
RealQuantityMixin('toplatitude'),
RealQuantityMixin('topdepth'),
RealQuantityMixin('bottomlongitude'),
RealQuantityMixin('bottomlatitude'),
RealQuantityMixin('bottomdepth'),
RealQuantityMixin('holediameter'),
RealQuantityMixin('casingdiameter'),
PublicIDMixin(),
ORMBase):
"""
ORM representation of a borehole. The attributes are in accordance with
......@@ -74,13 +83,13 @@ class BoreholeSection(EpochMixin('Epoch', epoch_type='open',
*Quantities* are implemented as `QuakeML
<https://quake.ethz.ch/quakeml>`_ quantities.
"""
topclosed = Column('{}topclosed'.format(PREFIX), Boolean, nullable=False)
bottomclosed = Column('{}bottomclosed'.format(PREFIX), Boolean, nullable=False)
sectiontype = Column('{}sectiontype'.format(PREFIX), String)
casingtype = Column('{}casingtype'.format(PREFIX), String)
description = Column('{}description'.format(PREFIX), String)
topclosed = Column(f'{PREFIX}topclosed', Boolean, nullable=False)
bottomclosed = Column(f'{PREFIX}bottomclosed', Boolean, nullable=False)
sectiontype = Column(f'{PREFIX}sectiontype', String)
casingtype = Column(f'{PREFIX}casingtype', String)
description = Column(f'{PREFIX}description', String)
borehole_oid = Column('{}borehole_oid'.format(PREFIX), Integer,
borehole_oid = Column(f'{PREFIX}borehole_oid', Integer,
ForeignKey('borehole._oid'))
_borehole = relationship("Borehole", back_populates="_sections")
......@@ -90,17 +99,17 @@ class BoreholeSection(EpochMixin('Epoch', epoch_type='open',
order_by='HydraulicSample.datetime_value')
class HydraulicSample(TimeQuantityMixin('datetime', global_column_prefix=PREFIX, value_nullable=False),
RealQuantityMixin('bottomtemperature', global_column_prefix=PREFIX),
RealQuantityMixin('bottomflow', global_column_prefix=PREFIX),
RealQuantityMixin('bottompressure', global_column_prefix=PREFIX),
RealQuantityMixin('toptemperature', global_column_prefix=PREFIX),
RealQuantityMixin('topflow', global_column_prefix=PREFIX),
RealQuantityMixin('toppressure', global_column_prefix=PREFIX),
RealQuantityMixin('fluiddensity', global_column_prefix=PREFIX),
RealQuantityMixin('fluidviscosity', global_column_prefix=PREFIX),
RealQuantityMixin('fluidph', global_column_prefix=PREFIX),
PublicIDMixin(global_column_prefix=PREFIX),
class HydraulicSample(TimeQuantityMixin('datetime', value_nullable=False),
RealQuantityMixin('bottomtemperature'),
RealQuantityMixin('bottomflow'),
RealQuantityMixin('bottompressure'),
RealQuantityMixin('toptemperature'),
RealQuantityMixin('topflow'),
RealQuantityMixin('toppressure'),
RealQuantityMixin('fluiddensity'),
RealQuantityMixin('fluidviscosity'),
RealQuantityMixin('fluidph'),
PublicIDMixin(),
ORMBase):
"""
Represents an hydraulics sample. The definition is based on `QuakeML
......@@ -111,9 +120,9 @@ class HydraulicSample(TimeQuantityMixin('datetime', global_column_prefix=PREFIX,
*Quantities* are implemented as `QuakeML
<https://quake.ethz.ch/quakeml>`_ quantities.
"""
fluidcomposition = Column('{}fluidcomposition'.format(PREFIX), String)
fluidcomposition = Column(f'{PREFIX}fluidcomposition', String)
_section = relationship("BoreholeSection", back_populates="_hydraulics")
boreholesection_oid = Column('{}boreholesection_oid'.format(PREFIX),
boreholesection_oid = Column(f'{PREFIX}boreholesection_oid',
Integer, ForeignKey('boreholesection._oid'))
"""
SQLAlchemy object actions including filter, order by, paginate,
limit.
.. module:: query_filters
:synopsis: Contains class to interact with SQLAlchemy Query
object including filter and paginate.
.. moduleauthor:: Laura Sarson <laura.sarson@sed.ethz.ch>
"""
from sqlalchemy.orm.exc import NoResultFound
from hydws.db.orm import Borehole, HydraulicSample
from hydws.db.orm import Borehole, BoreholeSection, HydraulicSample
# Mapping of columns to comparison operator and input parameter.
# [(orm column, operator, input comparison value)]
# Filter on hydraulics fields:
# Mapping of orm table columns to comparison operator and input values.
# [(orm attr, operator, input comparison value)]
# operator examples:
# eq for ==
# lt for <
# ge for >=
# in for in_
# like for like
#
# input comparison value can be list or a string.
# operator must belong in orm attr as op, op_, __op__
filter_hydraulics = [
('datetime', 'ge', 'starttime'),
('datetime', 'le', 'endtime'),
('toptemperature', 'ge', 'mintoptemperature'),
('toptemperature', 'le', 'maxtoptemperature'),
('bottomtemperature', 'ge', 'minbottomtemperature'),
('bottomtemperature', 'le', 'maxbottomtemperature'),
('toppressure', 'ge', 'mintoppressure'),
('toppressure', 'le', 'maxtoppressure'),
('bottompressure', 'ge', 'minbottompressure'),
('bottompressure', 'le', 'maxbottompressure'),
('topflow', 'ge', 'mintopflow'),
('topflow', 'le', 'maxtopflow'),
('bottomflow', 'ge', 'minbottomflow'),
('bottomflow', 'le', 'maxbottomflow'),
('fluiddensity', 'ge', 'minfluiddensity'),
('fluiddensity', 'le', 'maxfluiddensity'),
('fluidviscosity', 'ge', 'minfluidviscosity'),
('fluidviscosity', 'le', 'maxfluidviscosity'),
('fluidph', 'ge', 'minfluidph'),
('fluidph', 'le', 'maxfluidph')]
('datetime_value', 'ge', 'starttime'),
('datetime_value', 'le', 'endtime'),
('toptemperature_value', 'ge', 'mintoptemperature'),
('toptemperature_value', 'le', 'maxtoptemperature'),
('bottomtemperature_value', 'ge', 'minbottomtemperature'),
('bottomtemperature_value', 'le', 'maxbottomtemperature'),
('toppressure_value', 'ge', 'mintoppressure'),
('toppressure_value', 'le', 'maxtoppressure'),
('bottompressure_value', 'ge', 'minbottompressure'),
('bottompressure_value', 'le', 'maxbottompressure'),
('topflow_value', 'ge', 'mintopflow'),
('topflow_value', 'le', 'maxtopflow'),
('bottomflow_value', 'ge', 'minbottomflow'),
('bottomflow_value', 'le', 'maxbottomflow'),
('fluiddensity_value', 'ge', 'minfluiddensity'),
('fluiddensity_value', 'le', 'maxfluiddensity'),
('fluidviscosity_value', 'ge', 'minfluidviscosity'),
('fluidviscosity_value', 'le', 'maxfluidviscosity'),
('fluidph_value', 'ge', 'minfluidph'),
('fluidph_value', 'le', 'maxfluidph')]
filter_boreholes = [
('latitude', 'ge', 'minlatitude'),
('latitude', 'le', 'maxlatitude'),
('longitude', 'ge', 'minlongitude'),
('longitude', 'le', 'maxlongitude')]
('latitude_value', 'ge', 'minlatitude'),
('latitude_value', 'le', 'maxlatitude'),
('longitude_value', 'ge', 'minlongitude'),
('longitude_value', 'le', 'maxlongitude')]
class DynamicQuery(object):
......@@ -44,8 +57,12 @@ class DynamicQuery(object):
Dynamic filtering and of query.
Example:
dyn_query = DynamicQuery(query, orm.BoreholeSection)
dyn_query.filter_query([('m_starttime', 'eq', datetime(...))])
dq = DynamicQuery(session.query)
dq.filter_query([('m_starttime', 'eq', datetime(...))], 'borehole')
results = dq.return_all()
:param query: sqlalchemy query to manipulate.
:type query: sqlalchemy.orm.query.Query()
"""
......@@ -54,120 +71,94 @@ class DynamicQuery(object):
self.page = 1
def return_all(self):
"""Returns all results from query.
:rtype: list
"""
try:
return self.query.all()
except NoResultFound:
except NoResultFound as err:
return None
def paginate_query(self, limit, page=None, error_flag=False):
"""Paginate used to return a subset of results, starting from
offset*limit to offset*limit + limit.
To be used instead of self.return_all()
:returns type: Pagination object. Use .items to get similar
response to .all()
:returns: Pagination of query. Use .items to get similar
response to .return_all()
:rtype: Pagination object
"""
if not page:
page = self.page
return self.query.paginate(page, limit, error_flag)
def limit_by_query(self, limit):
"""Append a limit_by clause to the query.
"""
self.query = self.query.limit_by(limit)
def operator_attr(self, obj, op):
"""Returns method associated with an comparison operator
If one of op, op_, __op__ do not exist, Exception raised
def order_by_query(self, orm_class, key, ascending=True):
"""Append an order_by clause to the query.
:param obj: Object used to find existing operator methods
:type obj: Class or class instance.
:param str op: Operator to find method for, e.g. 'eq'
:orm_class: str name of ORM class included in query.
:param key: str name of column belonging to orm_class.
:return: Method that exists ob obj associted with op
:rtype: str
:raises: Exception
"""
try:
column = getattr(orm_class, key)
if not ascending:
column = column.desc()
except AttributeError:
raise Exception('Key: {} is not an attribute of class: {}'.\
format(key, orm_class))
try:
self.query = self.query.order_by(column)
except KeyError:
raise Exception('Invalid column: %s' % key)
def operator_attr(self, column, op):
"""
Returns method associated with an comparison operator.
If op, op_ or __op__ does not exist, Exception returned.
obj_methods = [op, f"{op}_", f"__{op}__"]
existing_methods = [m for m in obj_methods
if hasattr(obj, m)]
if existing_methods:
return existing_methods[0]
else:
raise Exception(f"Invalid operator: {op}")
:returns type: str.
def filter_query(self, query_params, filter_level):
"""Update self.query with chained filters based
on query_params
"""
try:
return list(filter(
lambda e: hasattr(column, e % op),
['%s', '%s_', '__%s__']))[0] % op
except IndexError:
raise Exception('Invalid filter operator: %s' % op)
def filter_query(self, query_params, filter_level,
key_suffix="_value"):
"""
Update self.query based on filter_condition.
:param filter_condition: list, ie: [(key,operator,value)]
operator examples:
eq for ==
lt for <
ge for >=
in for in_
like for like
value can be list or a string.
key must belong in self.orm_class.
:param query_params: values to filter query results
:type query_params: dict
:params filter_level: orm table level to use to filter query,
one of ("hydraulic", "borehole")
:type filter_level: str
:raises: Exception
"""
if filter_level == 'hydraulic':
if filter_level == "hydraulic":
orm_class = HydraulicSample
filter_condition = filter_hydraulics
elif filter_level == 'borehole':
elif filter_level == "borehole":
orm_class = Borehole
filter_condition = filter_boreholes
else:
raise Exception(f'filter level not handled: {filter_level}')
for f in filter_condition:
for filter_tuple in filter_condition:
try:
key_basename, op, param_name = f
key, op, param_name = filter_tuple
except ValueError:
raise Exception('Invalid filter input: %s' % f)
if key_suffix:
key = "{}{}".format(key_basename, key_suffix)
else:
key = key_basename
#put in try statement?
raise Exception(f"Invalid filter input: {filter_tuple}")
param_value = query_params.get(param_name)
if not param_value:
continue
# todo: check column type against value type
# and if they don't match then error?
try:
column = getattr(orm_class, key)
except AttributeError:
raise Exception('Invalid filter column: %s' % key)
if op == 'in':
raise Exception(f"Invalid filter column: {key}")
if op == "in":
if isinstance(value, list):
filt = column.in_(param_value)
else:
filt = column.in_(param_value.split(','))
filt = column.in_(param_value.split(","))
else:
attr = self.operator_attr(column, op)
filt = getattr(column, attr)(param_value)
print(filt, column, attr, param_value)
self.query = self.query.filter(filt)
self.query = self.query.filter(filt)
......@@ -23,8 +23,8 @@ HYDWS_OFORMATS = (HYDWS_DEFAULT_OFORMAT, 'xml')
HYDWS_DEFAULT_LEVEL = 'section'
HYDWS_SECTION_LEVELS = (HYDWS_DEFAULT_LEVEL, 'borehole')
HYDWS_HYDRAULIC_LEVELS = (HYDWS_DEFAULT_LEVEL, 'borehole', 'hydraulic')
HYDWS_SECTIONS_ORDER_BY = 'topdepth_value'
HYDWS_HYDRAULICS_ORDER_BY = 'datetime_value'
HYDWS_PREFIX = 'm_'
MIMETYPE_JSON = 'application/json'
MIMETYPE_TEXT = 'text/plain'
ERROR_MIMETYPE = MIMETYPE_TEXT
......
"""
HYDWS datamodel ORM entity de-/serialization facilities.
"""
import logging
import datetime
from marshmallow import Schema, fields, post_dump, pre_load, missing, validate
from marshmallow.utils import get_value
from collections import defaultdict
from functools import reduce, partial
from operator import getitem
.. module:: schema
:synopsis: HYDWS datamodel ORM entity de-/serialization facilities..
from hydws.server.v1.ostream import schema_literaturesource as sdc
from hydws.db.orm import Borehole
_ATTR_PREFIX = 'm_'
.. moduleauthor:: Laura Sarson <laura.sarson@sed.ethz.ch>
"""
import logging
from functools import partial
from marshmallow import Schema, fields, post_dump, pre_load, validate, validates_schema
ValidateLatitude = validate.Range(min=-90., max=90.)
......@@ -25,10 +16,13 @@ ValidatePositive = validate.Range(min=0.)
ValidateConfidenceLevel = validate.Range(min=0., max=100.)
ValidateCelcius = validate.Range(min=-273.15)
Datetime = fields.DateTime(format='iso')
Datetime = partial(fields.DateTime, format='iso')
DatetimeRequired = partial(Datetime, required=True)
Degree = partial(fields.Float)
Latitude = partial(Degree, validate=ValidateLatitude)
RequiredLatitude = partial(Latitude, required=True)
Longitude = partial(Degree, validate=ValidateLongitude)
RequiredLongitude = partial(Longitude, required=True)
Uncertainty = partial(fields.Float, validate=ValidatePositive)
ConfidenceLevel = partial(fields.Float, validate=ValidateConfidenceLevel)
Depth = partial(fields.Float, validate=ValidatePositive)
......@@ -53,7 +47,7 @@ class SchemaBase(Schema):
Filter out fields with empty (e.g. :code:`None`, :code:`[], etc.)
values.
"""
return {k: v for k, v in data.items() if v}
return {k: v for k, v in data.items() if v or isinstance(v, (int, float))}
@classmethod
def _flatten_dict(cls, data, sep='_'):
......@@ -117,7 +111,6 @@ class CreationInfoSchema(SchemaBase):
creationinfo_license = fields.String()
class LSCreatorPersonSchema(SchemaBase):
"""Schema implementation of literature source and creation info
defined levels.
......@@ -261,7 +254,7 @@ class HydraulicSampleSchema(SchemaBase):
"""Schema implementation of an hydraulic data sample.
"""
datetime_value = Datetime
datetime_value = DatetimeRequired()
datetime_uncertainty = Uncertainty()
datetime_loweruncertainty = Uncertainty()
datetime_upperuncertainty = Uncertainty()
......@@ -328,9 +321,9 @@ class SectionSchema(SchemaBase):
"""Schema implementation of a borehole section.
"""
publicid = fields.String()
starttime = Datetime
endtime = Datetime
publicid = fields.String(required=True)
starttime = DatetimeRequired()
endtime = Datetime()
toplongitude_value = Longitude()
toplongitude_uncertainty = Uncertainty()
......@@ -382,9 +375,8 @@ class SectionSchema(SchemaBase):
casingdiameter_upperuncertainty = Uncertainty()
casingdiameter_confidencelevel = ConfidenceLevel()
topclosed = fields.Boolean()
bottomclosed = fields.Boolean()
topclosed = fields.Boolean(required=True)
bottomclosed = fields.Boolean(required=True)
sectiontype = fields.String()
casingtype = fields.String()
description = fields.String()
......@@ -392,20 +384,29 @@ class SectionSchema(SchemaBase):
hydraulics = fields.Nested(HydraulicSampleSchema, many=True,
attribute='_hydraulics')
@validates_schema
def validate_temporal_constraints(self, data):
"""Validation of temporal constraints."""
starttime = data.get('starttime')
endtime = data.get('endtime')
now = datetime.datetime.utcnow()
if starttime and endtime and starttime >= endtime:
raise ValidationError(
'endtime must be greater than starttime')
class BoreholeSchema(LiteratureSourceCreationInfoSchema, SchemaBase):
"""Schema implementation of a borehole."""
# TODO(damb): Provide a hierarchical implementation of sub_types; create
# them dynamically (see: e.g. QuakeMLQuantityField)
publicid = fields.String()
longitude_value = Longitude()
longitude_value = RequiredLongitude()
longitude_uncertainty = Uncertainty()
longitude_loweruncertainty = Uncertainty()
longitude_upperuncertainty = Uncertainty()
longitude_confidencelevel = ConfidenceLevel()
latitude_value = Latitude()
latitude_value = RequiredLatitude()
latitude_uncertainty = Uncertainty()
latitude_loweruncertainty = Uncertainty()
latitude_upperuncertainty = Uncertainty()
......
"""
HYDWS parser related facilities.
.. module:: parser
:synopsis: HYDWS parser related facilities.
.. moduleauthor:: Laura Sarson <laura.sarson@sed.ethz.ch>
"""
import datetime
import functools
from collections import OrderedDict
from marshmallow import (Schema, fields, pre_load, validates_schema,
validate, ValidationError)
from hydws.server import settings
from hydws.server.misc import from_fdsnws_datetime, fdsnws_isoformat
import logging
Format = functools.partial(
fields.String,
......@@ -56,12 +61,72 @@ class GeneralSchema(Schema):
"""
Common HYDWS parser schema
"""
LOGGER = 'hydws.server.v1.parserschema'
nodata = NoData()
format = Format()
class Meta:
strict = True
ordered = True
class TimeConstraintsSchemaMixin(Schema):
"""
Schema for time related query parameters.
"""
starttime = FDSNWSDateTime(format='fdsnws')
start = fields.Str(load_only=True)
endtime = FDSNWSDateTime(format='fdsnws', allow_none=True)
end = fields.Str(load_only=True)
@pre_load
def merge_keys(self, data):
"""
Merge alternative field parameter values.
.. note::
The default :py:mod:`webargs` parser does not provide this feature
by default such that :code:`load_only` field parameters are
separated handled.
"""
_mappings = [
('start', 'starttime'),
('end', 'endtime')]