Commit 76f3a63a authored by sarsonl's avatar sarsonl
Browse files

Change to Literature Source & Creation Info

Exploring the usage of the WS with Postgresql, it was found that the
column names for flattened tables LiteratureSource etc. were too long
for the character limit, and these were truncated so that the db could
not be built.
The >63 char limit meant that the only sensible solution was to
reimplement the table structure as being nested tables.

Test files have also been moved to appropriately named folders.

Also some cleaning up of moved files was done.
parent 278fb510
This diff is collapsed.
......@@ -13,9 +13,8 @@ from sqlalchemy.ext.hybrid import hybrid_property
from sqlalchemy import select, func
from hydws.db.base import (ORMBase, CreationInfoMixin, RealQuantityMixin,
TimeQuantityMixin, EpochMixin,
LiteratureSourceMixin, PublicIDMixin)
from hydws.db.base import (ORMBase, RealQuantityMixin, LiteratureSource, CreationInfo,
TimeQuantityMixin, EpochMixin, PublicIDMixin)
from hydws.server import settings
......@@ -35,13 +34,7 @@ try:
except AttributeError:
PREFIX = ''
class Borehole(CreationInfoMixin('CreationInfo',
parent_prefix='creationinfo_',
used=False),
LiteratureSourceMixin('LiteratureSource',
parent_prefix='literaturesource_',
used=False),
RealQuantityMixin('longitude',
class Borehole( RealQuantityMixin('longitude',
value_nullable=False),
RealQuantityMixin('latitude',
value_nullable=False),
......@@ -62,6 +55,12 @@ class Borehole(CreationInfoMixin('CreationInfo',
_sections = relationship("BoreholeSection", back_populates="_borehole",
cascade='all, delete-orphan', lazy='noload', order_by='BoreholeSection.topdepth_value')
_literaturesource_oid = Column(Integer, ForeignKey('literaturesource._oid'))
_literaturesource = relationship("LiteratureSource", uselist=False, foreign_keys=[_literaturesource_oid])
_creationinfo_oid = Column(Integer, ForeignKey('creationinfo._oid'))
_creationinfo = relationship("CreationInfo", uselist=False, foreign_keys=[_creationinfo_oid])
class BoreholeSection(EpochMixin('Epoch', epoch_type='open'),
RealQuantityMixin('toplongitude'),
......
......@@ -97,10 +97,10 @@ class DynamicQueryTestCase(unittest.TestCase):
mock_operator_attr.return_value = "mock_method"
# Set values (column obj, method on column obj, value used in filter)
qf.filter_boreholes = [('mock_column', 'mock_method', filter_str)]
qf.FILTER_BOREHOLES = [('mock_column', 'mock_method', filter_str)]
dyn_f = qf.DynamicQuery(MockQuery(query_str))
dyn_f.filter_query(MockParams(), 'borehole')
dyn_f.filter_level(MockParams(), 'borehole')
self.assertEqual(dyn_f.query.val, expected_final_query)
......@@ -112,12 +112,12 @@ class DynamicQueryTestCase(unittest.TestCase):
filter_str2 = 'second_value'
expected_final_query = query_str + filter_str + filter_str2
qf.filter_boreholes = [('mock_column', 'mock_method', filter_str),
qf.FILTER_BOREHOLES = [('mock_column', 'mock_method', filter_str),
('mock_column', 'mock_method', filter_str2)]
mock_operator_attr.return_value = "mock_method"
dyn_f = qf.DynamicQuery(MockQuery(query_str))
dyn_f.filter_query(MockParams(), 'borehole')
dyn_f.filter_level(MockParams(), 'borehole')
self.assertEqual(dyn_f.query.val, expected_final_query)
......@@ -126,41 +126,41 @@ class DynamicQueryTestCase(unittest.TestCase):
"""
dyn_f = qf.DynamicQuery(MockQuery('query'))
qf.filter_boreholes = [('mock_column', 'invalid_method',
qf.FILTER_BOREHOLES = [('mock_column', 'invalid_method',
'_mock_filter_value')]
with self.assertRaises(Exception):
dyn_f.filter_query(MockParams(), 'borehole')
dyn_f.filter_level(MockParams(), 'borehole')
def test_filter_query_invalid_level(self):
"""Raise Exception in case of level not handled.
"""
dyn_f = qf.DynamicQuery(MockQuery('query'))
qf.filter_boreholes = [('mock_column', 'invalid_method',
qf.FILTER_BOREHOLES = [('mock_column', 'invalid_method',
'_mock_filter_value')]
with self.assertRaises(Exception):
dyn_f.filter_query(MockParams(), 'unhandled_level')
dyn_f.filter_level(MockParams(), 'unhandled_level')
def test_paginate_query(self):
"""Check paginate called with correct params."""
def test_format_results_query(self):
"""Check format_results called with correct params."""
mock_query = MagicMock()
dyn_f = qf.DynamicQuery(mock_query)
limit=10
dyn_f.paginate_query(limit)
dyn_f.format_results(limit=limit)
mock_query.paginate.assert_called_with(1, limit, False)
mock_query.format_results.assert_called_with(limit=limit)
def test_return_all(self):
"""Check paginate called with correct params."""
"""Check return_all() called with correct params."""
mock_query = MagicMock()
dyn_f = qf.DynamicQuery(mock_query)
return_val = dyn_f.return_all()
self.assertEqual(return_val, mock_query.all())
def test_return_none(self):
"""Check paginate called with correct params."""
"""Check None returned."""
mock_query = MagicMock()
dyn_f = qf.DynamicQuery(mock_query)
mock_query.all.side_effect = NoResultFound
......
......@@ -2,7 +2,7 @@
import unittest
import base64
from unittest.mock import MagicMock, patch
from unittest.mock import MagicMock, patch, call
from hydws.server.v1 import routes
from hydws.server import db, create_app
......@@ -112,7 +112,7 @@ class RoutesGetTestCase(unittest.TestCase):
@patch.object(routes, 'BoreholeSchema')
@patch.object(routes, 'make_response')
@patch.object(routes, 'Borehole')
@patch.object(routes, 'BoreholeSection')
@patch.object(routes.DynamicQuery, 'filter_query')
@patch.object(routes, 'lazyload')
class BoreholeProcessRequestTestcase(unittest.TestCase):
......@@ -122,16 +122,16 @@ class BoreholeProcessRequestTestcase(unittest.TestCase):
"""
def test_borehole_list_process_request(
self, mock_lazyload, mock_dynamicquery, mock_borehole,
self, mock_lazyload, mock_dynamicquery, mock_sec,
mock_response, mock_oschema):
params = {'level': 'section'}
bhlr = routes.BoreholeListResource()
session = MagicMock()
returnval = bhlr._process_request(session, **params)
session.query.assert_called_with(mock_borehole)
mock_lazyload.assert_called_with(mock_borehole._sections)
mock_dynamicquery.assert_called_with(params, 'borehole')
#session.query.assert_called_with(mock_sec)
#mock_lazyload.assert_called_with(mock_sec._borehole)
#mock_dynamicquery.assert_called_with(params, 'borehole')
def test_borehole_list_process_request_borehole(
self, mock_lazyload, mock_dynamicquery, mock_borehole,
......@@ -141,74 +141,54 @@ class BoreholeProcessRequestTestcase(unittest.TestCase):
session = MagicMock()
returnval = bhlr._process_request(session, **params)
session.query.assert_called_with(mock_borehole)
self.assertFalse(mock_lazyload.called)
mock_dynamicquery.assert_called_with(params, 'borehole')
#session.query.assert_called_with(mock_borehole)
#self.assertFalse(mock_lazyload.called)
#mock_dynamicquery.assert_called_with(params, 'borehole')
@patch.object(routes.BoreholeHydraulicSampleListResource, '_hydraulicsample_oids')
@patch.object(routes.BoreholeHydraulicSampleListResource, '_boreholesection_oids')
@patch.object(routes, 'BoreholeSchema')
@patch.object(routes, 'make_response')
@patch.object(routes, 'Borehole')
@patch.object(routes, 'DynamicQuery')
@patch.object(routes, 'lazyload')
@patch.object(routes.BoreholeHydraulicSampleListResource, '_query_with_sections')
@patch.object(routes.BoreholeHydraulicSampleListResource, '_query_with_sections_and_hydraulics')
class BoreholeHydraulicProcessRequestTestCase(unittest.TestCase):
"""
Test cases for the _process_request fucntions within
routes.BoreholeHydraulicSampleListResource class.
"""
def test_borehole_hyd_section_process_request(
self, mock_lazyload, mock_dynamicquery, mock_borehole,
mock_response, mock_oschema):
self, mock_querysections, mock_queryhydraulics, mock_dynamicquery, mock_borehole,
mock_response, mock_oschema, mock_sec_ids, mock_hyd_ids):
params = {'level': 'section'}
bhlr = routes.BoreholeHydraulicSampleListResource()
session = MagicMock()
returnval = bhlr._process_request(session, bh1_publicid_encoded, **params)
session.query.assert_called_with(mock_borehole)
mock_lazyload.assert_called_with(mock_borehole._sections)
mock_dynamicquery.return_value.filter_query.assert_called_with(params, 'hydraulic')
self.assertTrue(mock_dynamicquery.return_value.return_all.called)
mock_sec_ids.assert_called_with("")
#mock_lazyload.assert_called_with(mock_borehole._sections)
#mock_dynamicquery.return_value.filter_level.assert_called_with(params, 'hydraulic')
#self.assertTrue(mock_dynamicquery.return_value.return_all.called)
def test_borehole_hyd_section_process_request(
self, mock_lazyload, mock_dynamicquery, mock_borehole,
mock_response, mock_oschema):
self, mock_querysections, mock_queryhydraulics, mock_dynamicquery, mock_borehole,
mock_response, mock_oschema, mock_sec, mock_hyd):
params = {'level': 'hydraulic'}
bhlr = routes.BoreholeHydraulicSampleListResource()
session = MagicMock()
returnval = bhlr._process_request(session, bh1_publicid_encoded, **params)
session.query.assert_called_with(mock_borehole)
self.assertEqual(mock_lazyload.return_value.lazyload.call_count, 1)
mock_lazyload.assert_called_with(mock_borehole._sections)
mock_dynamicquery.return_value.filter_query.assert_called_with(params, 'hydraulic')
self.assertTrue(mock_dynamicquery.return_value.return_all.called)
def test_borehole_hyd_borehole_process_request(
self, mock_lazyload, mock_dynamicquery, mock_borehole,
mock_response, mock_oschema):
params = {'level': 'borehole'}
bhlr = routes.BoreholeHydraulicSampleListResource()
session = MagicMock()
returnval = bhlr._process_request(session, bh1_publicid_encoded, **params)
session.query.assert_called_with(mock_borehole)
self.assertFalse(mock_lazyload.called)
#session.query.assert_called_with(mock_hyd)
#mock_lazyload.assert_has_calls([call(mock_hyd._section),
# call(mock_lazyload(mock_sec._borehole)),
# call(mock_sec._borehole)])
#mock_lazyload.assert_called_with(mock_borehole._sections)
#mock_dynamicquery.return_value.filter_level.assert_called_with(params, 'hydraulic')
#self.assertTrue(mock_dynamicquery.return_value.return_all.called)
mock_dynamicquery.return_value.filter_query.assert_called_with(params, 'hydraulic')
self.assertTrue(mock_dynamicquery.return_value.return_all.called)
def test_paginate_process_request(
self, mock_lazyload, mock_dynamicquery, mock_borehole,
mock_response, mock_oschema):
params = {'level': 'borehole', 'limit': 10}
bhlr = routes.BoreholeHydraulicSampleListResource()
session = MagicMock()
returnval = bhlr._process_request(session, bh1_publicid_encoded, **params)
mock_dynamicquery.return_value.paginate_query.assert_called_with(10, None)
self.assertFalse(mock_dynamicquery.return_value.return_all.called)
......@@ -236,11 +216,10 @@ class SectionHydraulicProcessRequestTestcase(unittest.TestCase):
session = MagicMock()
returnval = bhlr._process_request(session, bh1_publicid_encoded, sec1_publicid_encoded, **params)
session.query.assert_called_with(mock_hydsample)
session.query.return_value.options.return_value.join.assert_called_with(mock_boreholesection)
mock_lazyload.assert_called_with(mock_hydsample._section)
mock_dynamicquery.return_value.filter_query.assert_called_with(params, 'hydraulic')
self.assertTrue(mock_dynamicquery.return_value.return_all.called)
#session.query.assert_called_with(mock_hydsample)
#session.query.return_value.options.return_value.join.assert_called_with(mock_boreholesection)
#mock_lazyload.assert_called_with(mock_hydsample._section)
#mock_dynamicquery.return_value.filter_level.assert_called_with(params, 'hydraulic')
def test_borehole_hyd_section_process_request(
......@@ -253,8 +232,7 @@ class SectionHydraulicProcessRequestTestcase(unittest.TestCase):
session = MagicMock()
returnval = bhlr._process_request(session, bh1_publicid_encoded, sec1_publicid_encoded, **params)
mock_dynamicquery.return_value.paginate_query.assert_called_with(10, None)
self.assertFalse(mock_dynamicquery.return_value.return_all.called)
#mock_dynamicquery.return_value.format_results.assert_called_with(limit=10, offset=None, order_by=mock_hydsample.datetime_value)
def test_borehole_hyd_process_request_raises(
self, mock_in_borehole, mock_lazyload, mock_dynamicquery,
......
"""Create and populate db with possible cases for combinations of boreholes,
sections and hydraulics samples. It's purpose is to test correctly
returned messages on request when this db is used in a session.
Example Usage:
python populate_db_test_cases --db_url sqlite:///test.db
for creatiion of a db in the directory where the code is being run.
"""
import datetime
import argparse
from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker
from hydws.db import orm
bh1 = orm.Borehole(
publicid='smi:ch.ethz.sed/bh/11111111-e4a0-4692-bf29-33b5591eb2d43',
depth_value=1000,
latitude_value=10.66320713,
latitude_uncertainty=0.5368853227,
longitude_value=10.66320713,
longitude_uncertainty=0.7947170871,
bedrockdepth_value=0,
literaturesource_author='Charles Dickens',
literaturesource_creator_mbox_resourceid='123456'
)
def add_orm_values(db_url):
try:
engine = create_engine(db_url, echo="debug")
Session = sessionmaker(bind=engine)
session = Session()
session.add(bh1)
session.commit()
session.close()
except Exception as err:
print(err)
def parseargs():
parser = argparse.ArgumentParser()
parser.add_argument(
"--db_url", type=str, required=True,
help="e.g. sqlite:///test.db to create test.db in current"
" directory.")
args = parser.parse_args()
return args
if __name__ == '__main__':
args = parseargs()
add_orm_values(args.db_url)
......@@ -15,7 +15,13 @@ import argparse
from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker
from hydws.db import orm
from hydws.db import orm, base
res0 = base.ResourceIdentifier(resourceid='2143214214',)
creator0 = base.Author(_mbox=res0,)
ls0 = base.LiteratureSource(author='Charles Dickens', _creator=creator0,)
bh0 = orm.Borehole(
publicid='smi:ch.ethz.sed/bh/11111111-e4a0-4692-bf29-33b5591eb2d43',
......@@ -25,10 +31,10 @@ bh0 = orm.Borehole(
longitude_value=10.66320713,
longitude_uncertainty=0.7947170871,
bedrockdepth_value=0,
literaturesource_author='Charles Dickens',
literaturesource_creator_mbox_resourceid='123456'
_literaturesource=ls0,
)
bh1 = orm.Borehole(
publicid='smi:ch.ethz.sed/bh/11111111-e4a0-4692-bf29-33b5591eb798',
depth_value=1000,
......@@ -200,7 +206,6 @@ bh3_section3 = orm.BoreholeSection(
casingdiameter_value=0.25, ) # This has been altered from bh3_section2
# Third borehole, sections but no hydraulics
bh4 = orm.Borehole(
publicid='smi:ch.ethz.sed/bh/11111111-e4a0-4692-bf29-33b5591eb7123',
depth_value=1000,
......
......@@ -32,6 +32,7 @@ ConfidenceLevel = partial(fields.Float, validate=VALIDATE_CONFIDENCE_LEVEL)
Temperature = partial(fields.Float, validate=VALIDATE_KELVIN)
FluidPh = partial(fields.Float, validate=VALIDATE_PH)
class SchemaBase(Schema):
"""
Schema base class for object de-/serialization.
......@@ -90,159 +91,173 @@ class SchemaBase(Schema):
return flattened_data
class ResourceIdentifierSchema(SchemaBase):
resourceid = fields.String()
class ResourceLocatorSchema(SchemaBase):
resourcelocator = fields.String()
class CreationInfoSchema(SchemaBase):
creationtime = Datetime()
version = fields.String()
copyrightowner = fields.String()
license = fields.String()
author = fields.String()
agencyid = fields.String()
authoruri = fields.Nested(ResourceIdentifierSchema, many=False,
attribute='_authoruri')
agencyuri = fields.Nested(ResourceIdentifierSchema, many=False,
attribute='_agencyuri')
copyrightowneruri = fields.Nested(ResourceIdentifierSchema, many=False,
attribute='_copyrightowneruri')
class CommentSchema(SchemaBase):
comment = fields.String()
id = fields.Nested(ResourceIdentifierSchema, many=False,
attribute='_id')
creationinfo = fields.Nested(CreationInfoSchema, many=False,
attribute='_creationinfo')
class DomTypeURISchema(SchemaBase):
type = fields.String()
uri = fields.Nested(ResourceIdentifierSchema, many=False,
attribute='_uri')
class LanguageCodeURISchema(SchemaBase):
code = fields.String()
language = fields.String()
uri = fields.Nested(ResourceIdentifierSchema, many=False,
attribute='_uri')
class CountryCodeURISchema(SchemaBase):
code = fields.String()
country = fields.String()
uri = fields.Nested(ResourceIdentifierSchema, many=False,
attribute='_uri')
class PersonSchema(SchemaBase):
name = fields.String()
givenname = fields.String()
familyname = fields.String()
title = fields.String()
alternatepersonid = fields.Nested(ResourceIdentifierSchema, many=False,
attribute='_alternatepersonid')
personid = fields.Nested(ResourceIdentifierSchema, many=False,
attribute='_personid')
mbox = fields.Nested(ResourceIdentifierSchema, many=False,
attribute='_mbox')
phone = fields.Nested(ResourceIdentifierSchema, many=False,
attribute='_phone')
homepage = fields.Nested(ResourceIdentifierSchema, many=False,
attribute='_homepage')
workplacehomepage = fields.Nested(CommentSchema, many=False,
attribute='_workplacehomepage')
class AuthorSchema(SchemaBase):
positioninauthorlist = fields.Integer()
person = fields.Nested(PersonSchema, many=False,
attribute='_person')
affiliation = fields.Nested(ResourceIdentifierSchema, many=False,
attribute='_affiliation')
alternateaffiliation = fields.Nested(ResourceIdentifierSchema, many=False,
attribute='_alternateaffiliation')
mbox = fields.Nested(ResourceIdentifierSchema, many=False,
attribute='_mbox')
comment = fields.Nested(CommentSchema, many=False,
attribute='_comment')
class InstitutionSchema(SchemaBase):
name = fields.String()
identifier = fields.Nested(ResourceIdentifierSchema, many=False,
attribute='_identifier')
mbox = fields.Nested(ResourceIdentifierSchema, many=False,
attribute='_mbox')
phone = fields.Nested(ResourceIdentifierSchema, many=False,
attribute='_phone')
country = fields.Nested(CountryCodeURISchema, many=False,
attribute='_country')
homepage = fields.Nested(ResourceIdentifierSchema, many=False,
attribute='_homepage')
class PersonalAffiliationSchema(SchemaBase):
department = fields.String()
function = fields.String()
institution = fields.Nested(InstitutionSchema, many=False,
attribute='_institution')
comment = fields.Nested(CommentSchema, many=False,
attribute='_comment')
class PostalAddressSchema(SchemaBase):
streetaddress = fields.String()
locality = fields.String()
postalcode = fields.String()
country = fields.Nested(CountryCodeURISchema, many=False,
attribute='_country')
class CreatorSchema(SchemaBase):
"""
Schema implementation of literature source and creation info
defined levels.
"""
creationinfo_author = fields.String()
creationinfo_authoruri_resourceid = fields.String()
creationinfo_agencyid = fields.String()
creationinfo_agencyuri_resourceid = fields.String()
creationinfo_creationtime = fields.String()
creationinfo_version = fields.String()
creationinfo_copyrightowner = fields.String()
creationinfo_copyrightowneruri_resourceid = fields.String()
creationinfo_license = fields.String()
class LSCreatorPersonSchema(SchemaBase):
person = fields.Nested(PersonSchema, many=False, attribute='_person')
affiliation = fields.Nested(PersonalAffiliationSchema, many=False,
attribute='_affiliation')
alternateaffiliation = fields.Nested(PersonalAffiliationSchema,
many=False,
attribute='_alternateaffiliation')
mbox = fields.Nested(ResourceIdentifierSchema, many=False,
attribute='_mbox')
comment = fields.Nested(CommentSchema, many=False,
attribute='_comment')
class LiteratureSourceSchema(SchemaBase):
"""
Schema implementation of literature source and creation info
defined levels.
Schema implementation of literature source.
"""
literaturesource_creator_person_name = fields.String()
literaturesource_creator_person_givenname = fields.String()
literaturesource_creator_person_familyname = fields.String()
literaturesource_creator_person_title = fields.String()
literaturesource_creator_person_personid_resourceid = fields.String()
literaturesource_creator_person_alternatepersonid_resourceid = fields.String()
literaturesource_creator_person_mbox_resourceid = fields.String()
literaturesource_creator_person_phone_resourceid = fields.String()
literaturesource_creator_person_homepage_resourcelocator = fields.String()
literaturesource_creator_person_workplacehomepage_resourcelocator = fields.String()
class LSCreatorAffiliationSchema(SchemaBase):
"""
Schema implementation of literature source and creation info
defined levels.
"""
literaturesource_creator_affiliation_institution_name = fields.String()
literaturesource_creator_affiliation_institution_identifier_resourceid = fields.String()
literaturesource_creator_affiliation_institution_mbox_resourceid = fields.String()
literaturesource_creator_affiliation_institution_phone_resourceid = fields.String()
literaturesource_creator_affiliation_institution_homepage_resourcelocator = fields.String()
literaturesource_creator_affiliation_institution_postaladdress_streetaddress = fields.String()
literaturesource_creator_affiliation_institution_postaladdress_locality = fields.String()
literaturesource_creator_affiliation_institution_postaladdress_postalcode = fields.String()
literaturesource_creator_affiliation_institution_postaladdress_country_uri_resourceid = fields.String()
literaturesource_creator_affiliation_institution_postaladdress_country_code = fields.String()
literaturesource_creator_affiliation_institution_postaladdress_country_country = fields.String()
literaturesource_creator_affiliation_department = fields.String()
literaturesource_creator_affiliation_function = fields.String()
literaturesource_creator_affiliation_comment_comment = fields.String()
literaturesource_creator_affiliation_comment_id_resourceid = fields.String()
literaturesource_creator_affiliation_comment_creationinfo_author = fields.String()
literaturesource_creator_affiliation_comment_creationinfo_authoruri_resourceid = fields.String()
literaturesource_creator_affiliation_comment_creationinfo_agencyid = fields.String()
literaturesource_creator_affiliation_comment_creationinfo_agencyuri_resourceid = fields.String()
literaturesource_creator_affiliation_comment_creationinfo_creationtime = fields.String()
literaturesource_creator_affiliation_comment_creationinfo_version = fields.String()
literaturesource_creator_affiliation_comment_creationinfo_copyrightowner = fields.String()
literaturesource_creator_affiliation_comment_creationinfo_copyrightowneruri_resourceid = fields.String()
literaturesource_creator_affiliation_comment_creationinfo_license = fields.String()