Commit c7c4929b authored by sarsonl's avatar sarsonl
Browse files

Real values held in real value table

Rather than have real values with associated value, uncertainty etc.
heald in underscored seperated attributes, now there is a defined orm
model for this. The lofic of querying and the ourput schema had to
change how these columns are queried and returned. Tests have been
updated, and postman tests pass.
parent 31365e07
This diff is collapsed.
......@@ -6,19 +6,18 @@
"""
from operator import itemgetter
from sqlalchemy import Column, String, Boolean, Integer, ForeignKey
from sqlalchemy.orm import relationship
from sqlalchemy.orm import relationship, backref
from sqlalchemy.ext.hybrid import hybrid_property
from sqlalchemy import select, func
from sqlalchemy import select
from hydws.db.base import (ORMBase, RealQuantityMixin, LiteratureSource, CreationInfo,
TimeQuantityMixin, EpochMixin, PublicIDMixin)
from hydws.db.base import (ORMBase, LiteratureSource,
CreationInfo, EpochMixin,
PublicIDMixin, FloatQuantity, TimeQuantity)
from hydws.server import settings
# XXX(damb): The implementation of the entities below is based on the QuakeML
# and the SC3 DB model naming conventions. As a consequence,
# sub-(sub-(...)types (e.g. CreationInfo, LiteratureSource, Comment, etc. )
......@@ -34,14 +33,8 @@ try:
except AttributeError:
PREFIX = ''
class Borehole( RealQuantityMixin('longitude',
value_nullable=False),
RealQuantityMixin('latitude',
value_nullable=False),
RealQuantityMixin('depth'),
RealQuantityMixin('bedrockdepth'),
RealQuantityMixin('measureddepth'),
PublicIDMixin(),
class Borehole(PublicIDMixin(),
ORMBase):
"""
ORM representation of a borehole. The attributes are in accordance with
......@@ -52,27 +45,61 @@ class Borehole( RealQuantityMixin('longitude',
*Quantities* are implemented as `QuakeML
<https://quake.ethz.ch/quakeml>`_ quantities.
"""
_sections = relationship("BoreholeSection", back_populates="_borehole",
cascade='all, delete-orphan', lazy='noload', order_by='BoreholeSection.topdepth_value')
_longitude_oid = Column(Integer, ForeignKey('floatquantity._oid'),
nullable=False)
_longitude = relationship(
"FloatQuantity", backref=backref(
"_borehole_longitude", uselist=False),
foreign_keys=[_longitude_oid], lazy='joined')
_latitude_oid = Column(Integer, ForeignKey('floatquantity._oid'),
nullable=False)
_latitude = relationship(
"FloatQuantity", backref=backref(
"_borehole_latitude", uselist=False),
foreign_keys=[_latitude_oid], lazy='joined')
_depth_oid = Column(Integer, ForeignKey('floatquantity._oid'))
_depth = relationship(
"FloatQuantity", backref=backref(
"_borehole_depth", uselist=False),
foreign_keys=[_depth_oid], lazy='joined')
_bedrockdepth_oid = Column(Integer, ForeignKey('floatquantity._oid'))
_bedrockdepth = relationship(
"FloatQuantity", backref=backref(
"_borehole_bedrockdepth", uselist=False),
foreign_keys=[_bedrockdepth_oid], lazy='joined')
_literaturesource_oid = Column(Integer, ForeignKey('literaturesource._oid'))
_literaturesource = relationship("LiteratureSource", uselist=False, foreign_keys=[_literaturesource_oid])
_measureddepth_oid = Column(Integer, ForeignKey('floatquantity._oid'))
_measureddepth = relationship(
"FloatQuantity", backref=backref(
"_borehole_measureddepth", uselist=False),
foreign_keys=[_measureddepth_oid], lazy='joined')
_sections = relationship(
"BoreholeSection", back_populates="_borehole",
cascade='all,delete-orphan', lazy='noload',
order_by="BoreholeSection.topdepth_values")
_literaturesource_oid = Column(Integer,
ForeignKey('literaturesource._oid'))
_literaturesource = relationship(
"LiteratureSource", backref=backref(
"_borehole_literaturesource", uselist=False),
foreign_keys=[_literaturesource_oid])
_creationinfo_oid = Column(Integer, ForeignKey('creationinfo._oid'))
_creationinfo = relationship("CreationInfo", uselist=False, foreign_keys=[_creationinfo_oid])
_creationinfo = relationship(
"CreationInfo", backref=backref(
"_borehole_creationinfo", uselist=False),
foreign_keys=[_creationinfo_oid])
description = Column(f'{PREFIX}description', String)
class BoreholeSection(EpochMixin('Epoch', epoch_type='open'),
RealQuantityMixin('toplongitude'),
RealQuantityMixin('toplatitude'),
RealQuantityMixin('topdepth'),
RealQuantityMixin('bottomlongitude'),
RealQuantityMixin('bottomlatitude'),
RealQuantityMixin('bottomdepth'),
RealQuantityMixin('holediameter'),
RealQuantityMixin('casingdiameter'),
PublicIDMixin(),
ORMBase):
PublicIDMixin(), ORMBase):
"""
ORM representation of a borehole. The attributes are in accordance with
`QuakeML <https://quake.ethz.ch/quakeml/>`_.
......@@ -88,27 +115,74 @@ class BoreholeSection(EpochMixin('Epoch', epoch_type='open'),
casingtype = Column(f'{PREFIX}casingtype', String)
description = Column(f'{PREFIX}description', String)
_toplongitude_oid = Column(Integer, ForeignKey('floatquantity._oid'))
_toplongitude = relationship(
"FloatQuantity", backref=backref(
"_boreholesection_toplongitude", uselist=False),
foreign_keys=[_toplongitude_oid], lazy='joined')
_toplatitude_oid = Column(Integer, ForeignKey('floatquantity._oid'))
_toplatitude = relationship(
"FloatQuantity", backref=backref(
"_boreholesection_toplatitude", uselist=False),
foreign_keys=[_toplatitude_oid], lazy='joined')
_topdepth_oid = Column(Integer, ForeignKey('floatquantity._oid'))
_topdepth = relationship(
"FloatQuantity", backref=backref(
"_boreholesection_topdepth", uselist=False),
foreign_keys=[_topdepth_oid], lazy='joined')
_bottomlongitude_oid = Column(Integer, ForeignKey('floatquantity._oid'))
_bottomlongitude = relationship(
"FloatQuantity", backref=backref(
"_boreholesection_bottomlongitude", uselist=False),
foreign_keys=[_bottomlongitude_oid], lazy='joined')
_bottomlatitude_oid = Column(Integer, ForeignKey('floatquantity._oid'))
_bottomlatitude = relationship(
"FloatQuantity", backref=backref(
"_boreholesection_bottomlatitude", uselist=False),
foreign_keys=[_bottomlatitude_oid], lazy='joined')
_bottomdepth_oid = Column(Integer, ForeignKey('floatquantity._oid'))
_bottomdepth = relationship(
"FloatQuantity", backref=backref(
"_boreholesection_bottomdepth", uselist=False),
foreign_keys=[_bottomdepth_oid], lazy='joined')
_holediameter_oid = Column(Integer, ForeignKey('floatquantity._oid'))
_holediameter = relationship(
"FloatQuantity", backref=backref(
"_boreholesection_holediameter", uselist=False),
foreign_keys=[_holediameter_oid], lazy='joined')
_casingdiameter_oid = Column(Integer, ForeignKey('floatquantity._oid'))
_casingdiameter = relationship(
"FloatQuantity", backref=backref(
"_boreholesection_casingdiameter", uselist=False),
foreign_keys=[_casingdiameter_oid], lazy='joined')
borehole_oid = Column(f'{PREFIX}borehole_oid', Integer,
ForeignKey('borehole._oid'))
_borehole = relationship("Borehole", back_populates="_sections")
_hydraulics = relationship("HydraulicSample", back_populates="_section",
lazy='noload',
order_by='HydraulicSample.datetime_value')
class HydraulicSample(TimeQuantityMixin('datetime', value_nullable=False),
RealQuantityMixin('bottomtemperature'),
RealQuantityMixin('bottomflow'),
RealQuantityMixin('bottompressure'),
RealQuantityMixin('toptemperature'),
RealQuantityMixin('topflow'),
RealQuantityMixin('toppressure'),
RealQuantityMixin('fluiddensity'),
RealQuantityMixin('fluidviscosity'),
RealQuantityMixin('fluidph'),
PublicIDMixin(),
_hydraulics = relationship(
"HydraulicSample", back_populates="_section", lazy='noload',
order_by="HydraulicSample.datetime_values")
@hybrid_property
def topdepth_values(self):
return self._topdepth.value
@topdepth_values.expression
def topdepth_values(cls):
return select([FloatQuantity.value]).\
where(cls._topdepth_oid == FloatQuantity._oid).as_scalar()
class HydraulicSample(PublicIDMixin(),
ORMBase):
"""
Represents an hydraulics sample. The definition is based on `QuakeML
......@@ -125,3 +199,72 @@ class HydraulicSample(TimeQuantityMixin('datetime', value_nullable=False),
boreholesection_oid = Column(f'{PREFIX}boreholesection_oid',
Integer, ForeignKey('boreholesection._oid'))
_bottomtemperature_oid = Column(Integer, ForeignKey('floatquantity._oid'))
_bottomtemperature = relationship(
"FloatQuantity", backref=backref(
"_hydraulicsample_bottomtemperature", uselist=False),
foreign_keys=[_bottomtemperature_oid], lazy='joined')
_bottomflow_oid = Column(Integer, ForeignKey('floatquantity._oid'))
_bottomflow = relationship(
"FloatQuantity", backref=backref(
"_hydraulicsample_bottomflow", uselist=False),
foreign_keys=[_bottomflow_oid], lazy='joined')
_bottompressure_oid = Column(Integer, ForeignKey('floatquantity._oid'))
_bottompressure = relationship(
"FloatQuantity", backref=backref(
"_hydraulicsample_bottompressure", uselist=False),
foreign_keys=[_bottompressure_oid], lazy='joined')
_toptemperature_oid = Column(Integer, ForeignKey('floatquantity._oid'))
_toptemperature = relationship(
"FloatQuantity", backref=backref(
"_hydraulicsample_toptemperature", uselist=False),
foreign_keys=[_toptemperature_oid], lazy='joined')
_topflow_oid = Column(Integer, ForeignKey('floatquantity._oid'))
_topflow = relationship(
"FloatQuantity", backref=backref(
"_hydraulicsample_topflow", uselist=False),
foreign_keys=[_topflow_oid], lazy='joined')
_toppressure_oid = Column(Integer, ForeignKey('floatquantity._oid'))
_toppressure = relationship(
"FloatQuantity", backref=backref(
"_hydraulicsample_toppressure", uselist=False),
foreign_keys=[_toppressure_oid], lazy='joined')
_fluiddensity_oid = Column(Integer, ForeignKey('floatquantity._oid'))
_fluiddensity = relationship(
"FloatQuantity", backref=backref(
"_hydraulicsample_fluiddensity", uselist=False),
foreign_keys=[_fluiddensity_oid], lazy='joined')
_fluidviscosity_oid = Column(Integer, ForeignKey('floatquantity._oid'))
_fluidviscosity = relationship(
"FloatQuantity", backref=backref(
"_hydraulicsample_fluidviscosity", uselist=False),
foreign_keys=[_fluidviscosity_oid], lazy='joined')
_fluidph_oid = Column(Integer, ForeignKey('floatquantity._oid'))
_fluidph = relationship(
"FloatQuantity", backref=backref(
"_hydraulicsample_fluidph", uselist=False),
foreign_keys=[_fluidph_oid], lazy='joined')
_datetime_oid = Column(Integer, ForeignKey('timequantity._oid'))
_datetime = relationship(
"TimeQuantity", backref=backref(
"_hydraulicsample_datetime", uselist=False),
foreign_keys=[_datetime_oid], lazy='joined')
@hybrid_property
def datetime_values(self):
return self._datetime.value
@datetime_values.expression
def datetime_values(cls):
return select([TimeQuantity.value]).\
where(cls._datetime_oid == TimeQuantity._oid).as_scalar()
......@@ -11,7 +11,9 @@ from sqlalchemy import or_
from sqlalchemy.orm.exc import NoResultFound
from hydws.db.orm import Borehole, BoreholeSection, HydraulicSample
from hydws.db.base import TimeQuantity, FloatQuantity
REAL_VALUE_MODELS = [TimeQuantity, FloatQuantity]
# Mapping of orm table columns to comparison operator and input values.
# [(orm attr, operator, input comparison value)]
......@@ -24,53 +26,60 @@ from hydws.db.orm import Borehole, BoreholeSection, HydraulicSample
#
# input comparison value can be list or a string.
# operator must belong in orm attr as op, op_, __op__
FILTER_HYDRAULICS = [
('datetime_value', 'ge', 'starttime'),
('datetime_value', 'le', 'endtime'),
('toptemperature_value', 'ge', 'mintoptemperature'),
('toptemperature_value', 'le', 'maxtoptemperature'),
('bottomtemperature_value', 'ge', 'minbottomtemperature'),
('bottomtemperature_value', 'le', 'maxbottomtemperature'),
('toppressure_value', 'ge', 'mintoppressure'),
('toppressure_value', 'le', 'maxtoppressure'),
('bottompressure_value', 'ge', 'minbottompressure'),
('bottompressure_value', 'le', 'maxbottompressure'),
('topflow_value', 'ge', 'mintopflow'),
('topflow_value', 'le', 'maxtopflow'),
('bottomflow_value', 'ge', 'minbottomflow'),
('bottomflow_value', 'le', 'maxbottomflow'),
('fluiddensity_value', 'ge', 'minfluiddensity'),
('fluiddensity_value', 'le', 'maxfluiddensity'),
('fluidviscosity_value', 'ge', 'minfluidviscosity'),
('fluidviscosity_value', 'le', 'maxfluidviscosity'),
('fluidph_value', 'ge', 'minfluidph'),
('fluidph_value', 'le', 'maxfluidph')]
('datetime', 'ge', 'starttime'),
('datetime', 'le', 'endtime'),
('toptemperature', 'ge', 'mintoptemperature'),
('toptemperature', 'le', 'maxtoptemperature'),
('bottomtemperature', 'ge', 'minbottomtemperature'),
('bottomtemperature', 'le', 'maxbottomtemperature'),
('toppressure', 'ge', 'mintoppressure'),
('toppressure', 'le', 'maxtoppressure'),
('bottompressure', 'ge', 'minbottompressure'),
('bottompressure', 'le', 'maxbottompressure'),
('topflow', 'ge', 'mintopflow'),
('topflow', 'le', 'maxtopflow'),
('bottomflow', 'ge', 'minbottomflow'),
('bottomflow', 'le', 'maxbottomflow'),
('fluiddensity', 'ge', 'minfluiddensity'),
('fluiddensity', 'le', 'maxfluiddensity'),
('fluidviscosity', 'ge', 'minfluidviscosity'),
('fluidviscosity', 'le', 'maxfluidviscosity'),
('fluidph', 'ge', 'minfluidph'),
('fluidph', 'le', 'maxfluidph')]
FILTER_SECTIONS_EPOCH = [
('starttime', 'le', 'endtime'),
('endtime', 'ge', 'starttime')]
FILTER_SECTIONS = [
[('toplatitude_value', 'ge', 'minlatitude'),
('bottomlatitude_value', 'ge', 'minlatitude')],
[('toplatitude_value', 'le', 'maxlatitude'),
('bottomlatitude_value', 'le', 'maxlatitude')],
[('toplongitude_value', 'ge', 'minlongitude'),
('bottomlongitude_value', 'ge', 'minlongitude')],
[('toplongitude_value', 'le', 'maxlongitude'),
('longitude_valuee', 'le', 'maxlongitude')]]
[('toplatitude', 'ge', 'minlatitude'),
('bottomlatitude', 'ge', 'minlatitude')],
[('toplatitude', 'le', 'maxlatitude'),
('bottomlatitude', 'le', 'maxlatitude')],
[('toplongitude', 'ge', 'minlongitude'),
('bottomlongitude', 'ge', 'minlongitude')],
[('toplongitude', 'le', 'maxlongitude'),
('longitude', 'le', 'maxlongitude')],
('casingdiameter', 'ge', 'mincasingdiameter'),
('casingdiameter', 'le', 'maxcasingdiameter'),
('holediameter', 'ge', 'minholediameter'),
('holediameter', 'le', 'maxholediameter'),
('topdepth', 'ge', 'mintopdepth'),
('topdepth', 'le', 'maxtopdepth'),
('bottomdepth', 'ge', 'minbottomdepth'),
('bottomdepth', 'le', 'maxbottomdepth'),
('topclosed', 'eq', 'topclosed'),
('bottomclosed', 'eq', 'bottomclosed'),
('casingtype', 'eq', 'casingtype'),
('sectiontype', 'eq', 'sectiontype')]
FILTER_BOREHOLES = [
('latitude_value', 'ge', 'minlatitude'),
('latitude_value', 'le', 'maxlatitude'),
('longitude_value', 'ge', 'minlongitude'),
('longitude_value', 'le', 'maxlongitude')]
#class InvalidOperator(ErrorWithTraceback):
# def __init__(self,*args,**kwargs):
# Exception.__init__(self,*args,**kwargs)
('latitude', 'ge', 'minlatitude'),
('latitude', 'le', 'maxlatitude'),
('longitude', 'ge', 'minlongitude'),
('longitude', 'le', 'maxlongitude')]
class DynamicQuery(object):
......@@ -95,7 +104,7 @@ class DynamicQuery(object):
:rtype: list
"""
try:
try:
return self.query.all()
except NoResultFound as err:
return None
......@@ -106,10 +115,10 @@ class DynamicQuery(object):
:rtype: dict
"""
#MultipleResultsFound from sqlalchemy.orm.exc
# TODO (sarsonl) Handle MultipleResultsFound from sqlalchemy.orm.exc
return self.query.one_or_none()
def format_results(self, order_column=None, limit=None, offset=None):
def format_results(self, order_columns=None, limit=None, offset=None):
"""
Return a subset of results of size limit
and with an offset if required.
......@@ -117,8 +126,8 @@ class DynamicQuery(object):
:param limit: Limit to number of results returned.
:type query: sqlalchemy.orm.query.Query()
"""
if order_column:
self.query = self.query.order_by(order_column)
if order_columns:
self.query = self.query.order_by(*order_columns)
if limit:
self.query = self.query.limit(limit)
if offset:
......@@ -139,7 +148,7 @@ class DynamicQuery(object):
"""
obj_methods = [op, f"{op}_", f"__{op}__"]
existing_methods = [m for m in obj_methods
if hasattr(obj, m)]
if hasattr(obj, m)]
if existing_methods:
return existing_methods[0]
else:
......@@ -152,7 +161,7 @@ class DynamicQuery(object):
This requires BoreholeSection starttime and endtime values to
include None values if no value has been set for them.
:param column: Attribute name of ORM table to filter on.
:param column: Attribute of ORM table to filter on.
:type column: str
:params attr: Attribute name of operator to use in evaluation.
:type filter_level: str
......@@ -161,13 +170,37 @@ class DynamicQuery(object):
:type filter_level: matches type of values stored in column.
:return: Method to evaluate ORM column
e.g. getattr(col, operator)(param value)
e.g. column_value >= 22
:type: Column evaluation method.
"""
eq_attr = self.operator_attr(column, 'eq')
filt = or_((getattr(column, attr)(param_value)),
(getattr(column, eq_attr)(None)))
filt = or_((getattr(column, attr)(param_value)),
(getattr(column, eq_attr)(None)))
return filt
def filter_backref(self, real_value_model, parent_column,
attr, param_value):
"""
Filter on real values associated to the ORM model by a
relationship.
:param real_value_model: ORM class which contains a 'value'
column.
:param parent_column: Attribute of parent ORM table to filter on.
:type column: str
:params attr: Attribute name of operator to use in evaluation.
:type filter_level: str
:params param_value: Value of input query parameter to filter
column on.
:type filter_level: matches type of values stored in column.
:return: Method to evaluate ORM column
e.g. column_value >= 22
:type: Column evaluation method.
"""
filt = parent_column.has(
getattr(real_value_model.value, attr)(param_value))
return filt
def filter_level(self, query_params, filter_level):
......@@ -190,7 +223,6 @@ class DynamicQuery(object):
filter_condition = {"borehole": FILTER_BOREHOLES}
elif filter_level == "section":
orm_class = BoreholeSection
filter_condition = {"section_epoch": FILTER_SECTIONS_EPOCH,
"section": FILTER_SECTIONS}
else:
......@@ -211,7 +243,8 @@ class DynamicQuery(object):
self.query = self.query.filter(or_(*filt_list))
else:
filt = self.get_filter( filter_clause, filter_name, query_params,orm_class)
filt = self.get_filter(filter_clause, filter_name,
query_params, orm_class)
if filt is None:
continue
......@@ -232,35 +265,57 @@ class DynamicQuery(object):
:return: Method to evaluate ORM column
e.g. getattr(col, operator)(param value)
:type: Column evaluation method or None if no param value exists.
:type: Column evaluation method or None if no param value exists.
"""
key, op, param_name, param_value = self.get_query_param(filter_clause, query_params)
key, op, param_name, param_value = self.get_query_param(filter_clause,
query_params)
if param_value:
return self.filter_query(query_params, filter_name, key, op, param_name, param_value, orm_class)
return self.filter_query(query_params, filter_name, key, op,
param_name, param_value, orm_class)
else:
return None
def get_query_param(self, filter_clause, query_params):
try:
key, op, param_name = filter_clause
except ValueError as err:
raise Exception(f"Invalid filter input")
param_value = query_params.get(param_name)
return key, op, param_name, param_value
def filter_query(self, query_params, filter_name, key, op, param_name, param_value, orm_class):
try:
column = getattr(orm_class, key)
except AttributeError:
raise Exception(f"Invalid filter column: {key}")
def filter_query(self, query_params, filter_name, key, op, param_name,
param_value, orm_class):
if hasattr(orm_class, key):
column = getattr(orm_class, key)
else:
backref_name = "_{}_{}".format(orm_class.__tablename__, key)
parent_column_name = "_{}".format(key)
parent_column = getattr(orm_class, parent_column_name)
column = None
for real_value_model in REAL_VALUE_MODELS:
if (hasattr(real_value_model, backref_name)
and hasattr(orm_class, parent_column_name)):
has_parent = True
backref_column = getattr(real_value_model, backref_name)
column = getattr(real_value_model, "value")
filter_name = "backref"
selected_model = real_value_model
if not column:
raise Exception(
"There is an unmapped backref: {}, {}".\
format(real_value_model, REAL_VALUE_MODELS))
# (sarsonl) Currently 'in' is unused and untested.
# Still requires handling for the backref quantities.
if op == "in":
if isinstance(value, list):
if isinstance(param_value, list):
filt = column.in_(param_value)
else:
filt = column.in_(param_value.split(","))
......@@ -268,6 +323,9 @@ class DynamicQuery(object):
attr = self.operator_attr(column, op)
if filter_name == "section_epoch":
filt = self.filter_section_epoch(column, attr, param_value)
elif filter_name == "backref":
filt = self.filter_backref(selected_model, parent_column,
attr, param_value)
else:
filt = getattr(column, attr)(param_value)
......
......@@ -347,7 +347,7 @@ class SectionHydraulicProcessRequestTestcase(unittest.TestCase):
returnval = bhlr._process_request(session, bh1_publicid_encoded, sec1_publicid_encoded, **params)
mock_dynquery.return_value.format_results.assert_called_with(
limit=10, offset=None, order_column=hydsample.datetime_value)
limit=10, offset=None, order_columns=(hydsample.datetime_values,))
def test_borehole_hyd_process_request_raises(
self, mock_dynquery, mock_hydsamples, mock_in_borehole, hydsample):
......
......@@ -25,24 +25,20 @@ ls0 = base.LiteratureSource(author='Charles Dickens', _creator=creator0,)
bh0 = orm.Borehole(
publicid='smi:ch.ethz.sed/bh/11111111-e4a0-4692-bf29-33b5591eb2d43',
depth_value=1000,
latitude_value=10.66320713,
latitude_uncertainty=0.5368853227,
longitude_value=10.66320713,
longitude_uncertainty=0.7947170871,
bedrockdepth_value=0,
_depth = base.FloatQuantity(value=1000,),
_latitude = base.FloatQuantity(value=10.66320713, uncertainty=0.5368853227),
_longitude = base.FloatQuantity(value=10.66320713, uncertainty=0.7947170871),
_bedrockdepth = base.FloatQuantity(value=0,),
_literaturesource=ls0,
)
bh1 = orm.Borehole(
publicid='smi:ch.ethz.sed/bh/11111111-e4a0-4692-bf29-33b5591eb798',
depth_value=1000,
latitude_value=10.66320713,
latitude_uncertainty=0.5368853227,
longitude_value=10.66320713,