Commit f875d98b authored by sarsonl's avatar sarsonl
Browse files

Reimplementation of schema logic and parsing data

Cross-code change to the way the schemas have been implemented and where
they are used. Each fields that will be output from a schema is uniquely
referenced so that the same schema can be used for validataion on
deserialization and also for producing json output.

Added measured_depth to the borehole level.

Using a custom parser with strict validation on the input query
parameters, as when using webargs, inputing a parameter that is not
included in the schema would not return an error. I have added the tests
for the strict.py parser file but these do not pass yet.
parent 3658862d
......@@ -136,8 +136,7 @@ def _create_creationinfo_map(parent_prefix=None, global_column_prefix=None, colu
@declared_attr
def _creationtime(cls):
return Column('%screationtime' % column_prefix, DateTime,
default=datetime.datetime.utcnow())
return Column('%screationtime' % column_prefix, DateTime)
func_map.append(('%screationtime'% parent_prefix, _creationtime))
@declared_attr
......@@ -745,10 +744,10 @@ def LiteratureSourceMixin(name, parent_prefix=None, global_column_prefix=None, c
`SQLAlchemy <https://www.sqlalchemy.org/>`_ mixin emulating type
:code:`LiteratureSource` from `QuakeML <https://quake.ethz.ch/quakeml/>`_.
"""
if not parent_prefix:
parent_prefix = ''
return type(name, (object,),
_create_literaturesource_map(parent_prefix=parent_prefix, global_column_prefix=global_column_prefix, used=used))
......@@ -773,7 +772,7 @@ def PublicIDMixin(name='', parent_prefix=None, column_prefix=None, global_column
@declared_attr
def _publicid(cls):
return Column('%spublicid' % column_prefix, String)
return Column('%spublicid' % column_prefix, String, nullable=False)
return type(name, (object,), {'%spublicid' % parent_prefix: _publicid})
......@@ -882,7 +881,7 @@ UniqueOpenEpochMixin = EpochMixin('Epoch', epoch_type='open',
column_prefix='')
def QuantityMixin(name, quantity_type, column_prefix=None, global_column_prefix=None):
def QuantityMixin(name, quantity_type, column_prefix=None, global_column_prefix=None, value_nullable=True):
"""
Mixin factory for common :code:`Quantity` types from
`QuakeML <https://quake.ethz.ch/quakeml/>`_.
......@@ -937,10 +936,8 @@ def QuantityMixin(name, quantity_type, column_prefix=None, global_column_prefix=
@declared_attr
def _value(cls):
#return {'%svalue' % name: Column('%svalue' % column_prefix, sql_type,
# nullable=False)}
return Column('%svalue' % column_prefix, sql_type,
nullable=False)
nullable=value_nullable)
return _value
if 'int' == quantity_type:
......
......@@ -23,16 +23,20 @@ PREFIX = 'm_'
# columns.
class Borehole(CreationInfoMixin('CreationInfo',
parent_prefix='creationinfo_',
global_column_prefix=PREFIX),
global_column_prefix=PREFIX,
used=False),
LiteratureSourceMixin('LiteratureSource',
parent_prefix='literaturesource_',
global_column_prefix=PREFIX),
RealQuantityMixin('longitude', global_column_prefix=PREFIX),
RealQuantityMixin('latitude', global_column_prefix=PREFIX),
global_column_prefix=PREFIX,
used=False),
RealQuantityMixin('longitude', global_column_prefix=PREFIX, value_nullable=False),
RealQuantityMixin('latitude', global_column_prefix=PREFIX, value_nullable=False),
RealQuantityMixin('depth', global_column_prefix=PREFIX),
RealQuantityMixin('bedrockdepth', global_column_prefix=PREFIX),
RealQuantityMixin('measureddepth', global_column_prefix=PREFIX),
PublicIDMixin(global_column_prefix=PREFIX),
ORMBase):
"""
......@@ -45,7 +49,7 @@ class Borehole(CreationInfoMixin('CreationInfo',
<https://quake.ethz.ch/quakeml>`_ quantities.
"""
_sections = relationship("BoreholeSection", back_populates="_borehole",
cascade='all, delete-orphan')
cascade='all, delete-orphan', lazy='noload', order_by='BoreholeSection.topdepth_value')
......@@ -70,19 +74,23 @@ class BoreholeSection(EpochMixin('Epoch', epoch_type='open',
*Quantities* are implemented as `QuakeML
<https://quake.ethz.ch/quakeml>`_ quantities.
"""
topclosed = Column('{}topclosed'.format(PREFIX), Boolean)
bottomclosed = Column('{}bottomclosed'.format(PREFIX), Boolean)
topclosed = Column('{}topclosed'.format(PREFIX), Boolean, nullable=False)
bottomclosed = Column('{}bottomclosed'.format(PREFIX), Boolean, nullable=False)
sectiontype = Column('{}sectiontype'.format(PREFIX), String)
casingtype = Column('{}casingtype'.format(PREFIX), String)
description = Column('{}description'.format(PREFIX), String)
borehole_oid = Column('{}borehole_oid'.format(PREFIX), Integer, ForeignKey('borehole._oid'))
borehole_oid = Column('{}borehole_oid'.format(PREFIX), Integer,
ForeignKey('borehole._oid'))
_borehole = relationship("Borehole", back_populates="_sections")
_hydraulics = relationship("HydraulicSample", back_populates="_section")
_hydraulics = relationship("HydraulicSample", back_populates="_section",
lazy='noload',
order_by='HydraulicSample.datetime_value')
class HydraulicSample(TimeQuantityMixin('datetime', global_column_prefix=PREFIX),
class HydraulicSample(TimeQuantityMixin('datetime', global_column_prefix=PREFIX, value_nullable=False),
RealQuantityMixin('bottomtemperature', global_column_prefix=PREFIX),
RealQuantityMixin('bottomflow', global_column_prefix=PREFIX),
RealQuantityMixin('bottompressure', global_column_prefix=PREFIX),
......@@ -105,5 +113,7 @@ class HydraulicSample(TimeQuantityMixin('datetime', global_column_prefix=PREFIX)
"""
fluidcomposition = Column('{}fluidcomposition'.format(PREFIX), String)
boreholesection_oid = Column('boreholesection_oid'.format(PREFIX), Integer, ForeignKey('boreholesection._oid'))
_section = relationship("BoreholeSection", back_populates="_hydraulics")
boreholesection_oid = Column('{}boreholesection_oid'.format(PREFIX),
Integer, ForeignKey('boreholesection._oid'))
......@@ -3,6 +3,40 @@ SQLAlchemy object actions including filter, order by, paginate,
limit.
"""
from sqlalchemy.orm.exc import NoResultFound
from hydws.db.orm import Borehole, HydraulicSample
# Mapping of columns to comparison operator and input parameter.
# [(orm column, operator, input comparison value)]
# Filter on hydraulics fields:
filter_hydraulics = [
('datetime', 'ge', 'starttime'),
('datetime', 'le', 'endtime'),
('toptemperature', 'ge', 'mintoptemperature'),
('toptemperature', 'le', 'maxtoptemperature'),
('bottomtemperature', 'ge', 'minbottomtemperature'),
('bottomtemperature', 'le', 'maxbottomtemperature'),
('toppressure', 'ge', 'mintoppressure'),
('toppressure', 'le', 'maxtoppressure'),
('bottompressure', 'ge', 'minbottompressure'),
('bottompressure', 'le', 'maxbottompressure'),
('topflow', 'ge', 'mintopflow'),
('topflow', 'le', 'maxtopflow'),
('bottomflow', 'ge', 'minbottomflow'),
('bottomflow', 'le', 'maxbottomflow'),
('fluiddensity', 'ge', 'minfluiddensity'),
('fluiddensity', 'le', 'maxfluiddensity'),
('fluidviscosity', 'ge', 'minfluidviscosity'),
('fluidviscosity', 'le', 'maxfluidviscosity'),
('fluidph', 'ge', 'minfluidph'),
('fluidph', 'le', 'maxfluidph')]
filter_boreholes = [
('latitude', 'ge', 'minlatitude'),
('latitude', 'le', 'maxlatitude'),
('longitude', 'ge', 'minlongitude'),
('longitude', 'le', 'maxlongitude')]
class DynamicQuery(object):
"""
......@@ -79,7 +113,7 @@ class DynamicQuery(object):
except IndexError:
raise Exception('Invalid filter operator: %s' % op)
def filter_query(self, orm_class, query_params, filter_condition,
def filter_query(self, query_params, filter_level,
key_suffix="_value"):
"""
Update self.query based on filter_condition.
......@@ -95,6 +129,16 @@ class DynamicQuery(object):
key must belong in self.orm_class.
"""
if filter_level == 'hydraulic':
orm_class = HydraulicSample
filter_condition = filter_hydraulics
elif filter_level == 'borehole':
orm_class = Borehole
filter_condition = filter_boreholes
else:
raise Exception(f'filter level not handled: {filter_level}')
for f in filter_condition:
try:
key_basename, op, param_name = f
......@@ -123,6 +167,7 @@ class DynamicQuery(object):
else:
attr = self.operator_attr(column, op)
filt = getattr(column, attr)(param_value)
print(filt, column, attr, param_value)
self.query = self.query.filter(filt)
......@@ -23,6 +23,8 @@ HYDWS_OFORMATS = (HYDWS_DEFAULT_OFORMAT, 'xml')
HYDWS_DEFAULT_LEVEL = 'section'
HYDWS_SECTION_LEVELS = (HYDWS_DEFAULT_LEVEL, 'borehole')
HYDWS_HYDRAULIC_LEVELS = (HYDWS_DEFAULT_LEVEL, 'borehole', 'hydraulic')
HYDWS_SECTIONS_ORDER_BY = 'topdepth_value'
HYDWS_HYDRAULICS_ORDER_BY = 'datetime_value'
MIMETYPE_JSON = 'application/json'
MIMETYPE_TEXT = 'text/plain'
ERROR_MIMETYPE = MIMETYPE_TEXT
......
# -*- coding: utf-8 -*-
# -----------------------------------------------------------------------------
# This is <strict.py>
# -----------------------------------------------------------------------------
#
# This file is part of EIDA NG webservices.
#
# EIDA NG webservices is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# EDIA NG webservices is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
# ----
#
# Copyright (c) Sven Marti (ETH), Daniel Armbruster (ETH), Fabian Euchner (ETH)
#
# REVISION AND CHANGES
# 2018/06/05 V0.1 Sven Marti
# =============================================================================
"""
Keywordparser facilities for EIDA NG webservices.
"""
from __future__ import (absolute_import, division, print_function,
unicode_literals)
from builtins import * # noqa
import functools
import inspect
import logging
from webargs.flaskparser import parser as flaskparser
from marshmallow import Schema, exceptions
# Dummy value.
MAX_POST_CONTENT_LENGTH = 1024*10240
FDSNWS_QUERY_VALUE_SEPARATOR_CHAR = '='
class ValidationError(exceptions.ValidationError):
"""ValidationError: {}."""
def _callable_or_raise(obj):
"""
Makes sure an object is callable if it is not ``None``. If not
callable, a ValueError is raised.
"""
if obj and not callable(obj):
raise ValueError("{0!r} is not callable.".format(obj))
return obj
# _callable_or_raise ()
# -----------------------------------------------------------------------------
class KeywordParser(object):
"""
Base class for keyword parsers.
"""
LOGGER = 'strict.keywordparser'
__location_map__ = {
'query': 'parse_querystring',
'form': 'parse_form'}
def __init__(self, error_handler=None):
self.error_callback = _callable_or_raise(error_handler)
self.logger = logging.getLogger(self.LOGGER)
# __init__ ()
@staticmethod
def _parse_arg_keys(arg_dict):
"""
:param dict arg_dict: Dictionary like structure to be parsed
:returns: Tuple with argument keys
:rtype: tuple
"""
return tuple(arg_dict.keys())
# _parse_arg_keys ()
@staticmethod
def _parse_postfile(postfile):
"""
Parse all argument keys from a POST request file.
:param str postfile: Postfile content
:returns: Tuple with parsed keys.
:rtype: tuple
"""
argmap = {}
for line in postfile.split('\n'):
_line = line.split(
FDSNWS_QUERY_VALUE_SEPARATOR_CHAR)
if len(_line) != 2:
continue
if all(w == '' for w in _line):
raise ValidationError('RTFM :)')
argmap[_line[0]] = _line[1]
return tuple(argmap.keys())
# _parse_postfile ()
def parse_querystring(self, req):
"""
Parse argument keys from :code:`req.args`.
:param req: Request object to be parsed
:type req: :py:class:`flask.Request`
"""
return self._parse_arg_keys(req.args)
# parse_querystring ()
def parse_form(self, req):
"""
:param req: Request object to be parsed
:type req: :py:class:`flask.Request`
"""
try:
parsed_list = self._parse_postfile(self._get_data(req))
except ValidationError as err:
if self.error_callback:
self.error_callback(err, req)
else:
self.handle_error(err, req)
return parsed_list
# parse_form ()
def get_default_request(self):
"""
Template function for getting the default request.
"""
raise NotImplementedError
# get_default_request ()
def parse(self, func, schemas, locations):
"""
Validate request query parameters.
:param schemas: Marshmallow Schemas to validate request after
:type schemas: tuple/list of :py:class:`marshmallow.Schema`
or :py:class:`marshmallow.Schema`
:param locations:
:type locations: tuple of str
Calls `handle_error` with :py:class:`ValidationError`.
"""
req = self.get_default_request()
if inspect.isclass(schemas):
schemas = [schemas()]
elif isinstance(schemas, Schema):
schemas = [schemas]
valid_fields = set()
print("parse ##########")
for schema in [s() if inspect.isclass(s) else s for s in schemas]:
valid_fields.update(schema.fields.keys())
print("#########################", schema.fields.keys())
parsers = []
for l in locations:
try:
f = self.__location_map__[l]
if inspect.isfunction(f):
function = f
else:
function = getattr(self, f)
parsers.append(function)
except KeyError:
raise ValueError('Invalid location: {!r}'.format(l))
@functools.wraps(func)
def decorator(*args, **kwargs):
req_args = set()
for f in parsers:
req_args.update(f(req))
invalid_args = req_args.difference(valid_fields)
print('#######', valid_fields, req_args)
if invalid_args:
err = ValidationError(
'Invalid request query parameters: {}'.format(
invalid_args))
if self.error_callback:
self.error_callback(err, req)
else:
self.handle_error(err, req)
return func(*args, **kwargs)
# decorator ()
return decorator
# parse ()
def with_strict_args(self, schemas, locations=None):
"""
Wrapper of :py:func:`parse`.
"""
return functools.partial(self.parse,
schemas=schemas,
locations=locations)
# with_strict_args ()
def _get_data(self, req, as_text=True,
max_content_length=MAX_POST_CONTENT_LENGTH):
"""
Savely reads the buffered incoming data from the client.
:param req: Request the raw data is read from
:type req: :py:class:`flask.Request`
:param bool as_text: If set to :code:`True` the return value will be a
decoded unicode string.
:param int max_content_length: Max bytes accepted
:returns: Byte string or rather unicode string, respectively. Depending
on the :code:`as_text` parameter.
"""
if req.content_length > max_content_length:
err = ValidationError(
'Request too large: {} bytes > {} bytes '.format(
req.content_length, max_content_length))
if self.error_callback:
self.error_callback(err, req)
else:
self.handle_error(err, req)
return req.get_data(cache=True, as_text=as_text)
# _get_data ()
def handle_error(self, error, req):
"""
Called if an error occurs while parsing strict args.
By default, just logs and raises ``error``.
:param Exception error: an Error to be handled
:param Request req: request object
:raises: error
:rtype: Exception
"""
self.logger.error(error)
raise error
# handle_error ()
def error_handler(self, func):
"""
Decorator that registers a custom error handling function. The
function should received the raised error, request object used
to parse the request. Overrides the parser's ``handle_error``
method.
Example: ::
from strict import flask_keywordparser
class CustomError(Exception):
pass
@flask_keywordparser.error_handler
def handle_error(error, req):
raise CustomError(error.messages)
:param callable func: The error callback to register.
"""
self.error_callback = func
return func
# class KeywordParser
# -----------------------------------------------------------------------------
class FlaskKeywordParser(KeywordParser):
"""
Flask implementation of :py:class:`KeywordParser`.
"""
def get_default_request(self):
"""
Returns the flask default request
:returns: :py:class:`flask.Request`
"""
return flaskparser.get_default_request()
# get_default_request ()
# class FlaskKeywordParser
flask_keywordparser = FlaskKeywordParser()
with_strict_args = flask_keywordparser.with_strict_args
# ---- END OF <strict.py> ----
This diff is collapsed.
......@@ -59,6 +59,11 @@ class GeneralSchema(Schema):
nodata = NoData()
format = Format()
class Meta:
strict = True
ordered = True
class LocationConstraintsSchemaMixin(Schema):
"""
Query parameters for boreholes, location specific.
......@@ -88,15 +93,14 @@ class LocationConstraintsSchemaMixin(Schema):
if minlongitude and minlongitude > -180.0:
raise ValidationError('minlongitude greater than -180 degrees')
if maxlatitude and minlatitude:
if maxlatitude < minlatitude:
raise ValidationError('maxlatitude must be greater than'
'minlatitude')
if maxlatitude and minlatitude and maxlatitude < minlatitude:
raise ValidationError('maxlatitude must be greater than'
'minlatitude')
if maxlongitude and minlongitude and maxlongitude < minlongitude:
raise ValidationError('maxlongitude must be greater than'
'minlongitude')
if maxlongitude and minlongitude:
if maxlongitude < minlongitude:
raise ValidationError('maxlongitude must be greater than'
'minlongitude')
class HydraulicsSchemaMixin(Schema):
"""
Query parameters for hydraulics data.
......@@ -120,7 +124,7 @@ class HydraulicsSchemaMixin(Schema):
minfluidph = fields.Float()
maxfluidph = fields.Float()
limit = fields.Integer()
offset = fields.Integer()
page = fields.Integer()
#XXX Todo sarsonl add constraints vaidation.
class TimeConstraintsSchemaMixin(Schema):
......@@ -179,11 +183,6 @@ class TimeConstraintsSchemaMixin(Schema):
raise ValidationError(
'endtime must be greater than starttime')
# validate_temporal_constraints ()
class Meta:
strict = True
ordered = True
class BoreholeHydraulicSampleListResourceSchema(TimeConstraintsSchemaMixin,
......@@ -195,6 +194,7 @@ class BoreholeHydraulicSampleListResourceSchema(TimeConstraintsSchemaMixin,
"""
level = LevelHydraulic()
class BoreholeListResourceSchema(TimeConstraintsSchemaMixin,
LocationConstraintsSchemaMixin,
GeneralSchema):
......@@ -205,8 +205,10 @@ class BoreholeListResourceSchema(TimeConstraintsSchemaMixin,
"""
level = LevelSection()
class SectionHydraulicSampleListResourceSchema(TimeConstraintsSchemaMixin,
GeneralSchema):
HydraulicsSchemaMixin,
GeneralSchema):
"""
Handle optional query parameters for call returning hydraulics
data for specified borehole id and section id.
......
......@@ -3,67 +3,34 @@ HYDWS resources.
"""
import logging
from sqlalchemy import literal
from flask_restful import Api, Resource
from sqlalchemy.orm.exc import NoResultFound
from webargs.flaskparser import use_kwargs
from marshmallow import fields
from sqlalchemy.orm import subqueryload, eagerload, joinedload, contains_eager, lazyload
from hydws import __version__
from hydws.db import orm
from hydws.db.orm import Borehole, BoreholeSection, HydraulicSample
from hydws.server import db, settings
from hydws.server.errors import FDSNHTTPError
from hydws.server.misc import (with_fdsnws_exception_handling, decode_publicid,
make_response)
from hydws.server.v1 import blueprint
from hydws.server.v1.ostream.schema import (BoreholeSchema,
BoreholeSectionSchema,
BoreholeSectionHydraulicSampleSchema,
SectionHydraulicSampleSchema,
HydraulicSampleSchema)
from hydws.server.v1.parser import (
BoreholeHydraulicSampleListResourceSchema,
BoreholeListResourceSchema,
SectionHydraulicSampleListResourceSchema)
from hydws.server.query_filters import DynamicQuery
from hydws.server.strict import with_strict_args
api_v1 = Api(blueprint)