Commit 39e051f6 authored by sarsonl's avatar sarsonl
Browse files

Fixing of unit tests and addition to READMS.md

The unit tests have now been fixed to run without error and some
information has been added to the readme file, but this has gaps. The
app file has had an extra warning added correctly informing the user
that a config file does not exist rather than the config section not
existing.
parent 76f3a63a
......@@ -10,6 +10,89 @@ The installation of the package is possible by means of invoking
pip install -e .
```
This will create sym links to the python entry points 'hydws-db-init' and 'hydws-test'.
Note, that encapsulating the installation by means of a [virtual
environment](https://docs.python.org/3/tutorial/venv.html) is strongly
recommended.
## Initialize HYDWS
```
hydws-db-init <db_url>
```
Run the above command. To be run locally with a sqlite db, db_url=sqlite:///<absolute_path>/<name_db_file>
The database now contains tables ready for population.
A postgresql database served with Apache2 and wsgi will be used for production, and the setup for this is handled with Docker.
## Setup with Docker
<Docker setup here>
## Test local installation
The following command should not be used for production, but instead for testing.
A sqlite database file exists already populated with data:
db_url = sqlite:///<path to repo>/hydws/server/v1/data/test.db
```
hydws-test --logging-conf <path_to_logging.conf> <db_url>
```
where logging.conf exists in the repository. By default the port used is 5000.
Testing examples can be found with the Postman test collection.
## API usage
The full specification for the HYDWS API can be found here: <link to spec>
Some examples demonstrate the usage with the data using db_url=test.db
To return an array of JSON objects representing boreholes:
```
curl "http://localhost:5000/v1/boreholes?&level=borehole"
```
To return a JSON object representing a borehole with all borehole sections and all hydraulics:
```
curl "http://localhost:5000/v1/boreholes/c21pOmNoLmV0aHouc2VkL2JoLzExMTExMTExLWU0YTAtNDY5Mi1iZjI5LTMzYjU1OTFlYjc5OA==?level=hydraulic"
```
Note: the borehole publicid: c21pOmNoLmV0... is encoded with base64. This is decoded within the application.
The command to encode/decode strings with python can be found below:
```
import base64
encoded borehole publicid = base64.b64encode(b'<borehole publicid>')
decoded borehole publicid = base64.b64decode(b'<encoded borehole publicid>')
```
To return a JSON array of all hydraulics from a specific borehole and specific section:
```
curl "http://localhost:5000/v1/boreholes/c21pOmNoLmV0aHouc2VkL2JoLzExMTExMTExLWU0YTAtNDY5Mi1iZjI5LTMzYjU1OTFlYjc5OA==/sections/c21pOmNoLmV0aHouc2VkL2JoL3NlY3Rpb24vMTExMTExMTEtOGQ4OS00ZjEzLTk1ZTctNTI2YWRlNzNjYzhi/hydraulics?"
```
Note: Both the borehole publicid and section publicid are base64 encoded as noted above.
### API call response body
The response body will be JSON, or an array of JSON objects.
### Postman tests
Postman is an API development and testing tool. A collection of tests has been created for HYDWS. To run the tests, follow the instructions to initialize and run a local instance.
The button below can be pressed to run the tests:
[![Run in Postman](https://run.pstmn.io/button.svg)](https://app.getpostman.com/run-collection/3da8e474b7ecf5b8e1e0)
## Unit tests
Unit tests can be run from the top level of the repository using the pytest command.
......@@ -150,7 +150,9 @@ class DynamicQueryTestCase(unittest.TestCase):
limit=10
dyn_f.format_results(limit=limit)
mock_query.format_results.assert_called_with(limit=limit)
mock_query.limit.assert_called_with(limit)
self.assertFalse(mock_query.offset.called)
self.assertFalse(mock_query.order_by.called)
def test_return_all(self):
"""Check return_all() called with correct params."""
......
"""Unit testing for hydws.server.v1.routes.py."""
import unittest
import base64
from datetime import datetime
from unittest.mock import MagicMock, patch, call
from hydws.server.v1 import routes
from hydws.server import db, create_app
......@@ -54,67 +56,135 @@ class RoutesGetTestCase(unittest.TestCase):
@patch.object(routes, 'BoreholeListResourceSchema')
@patch.object(routes, 'BoreholeSchema')
@patch.object(routes, 'make_response')
def test_borehole_list_get(self, mock_response, mock_oschema, mock_schema, mock_process_request):
# Mock the query parameters.
mock_schema.return_value.dump.return_value = {'f': 10}
def test_borehole_list_get_level_sec(
self, mock_response, mock_oschema, mock_schema,
mock_process_request):
"""Test level=section for BoreholeListResource."""
routes.db = self.db
mock_dumps = mock_oschema.return_value.dumps
mock_dumps.return_value = MagicMock()
with self.client as c:
response = c.get('/v1/boreholes?')
mock_process_request.assert_called_with(self.db.session, f=10)
response = c.get('/v1/boreholes?maxlatitude=10&level=section')
mock_process_request.assert_called_with(
self.db.session, format='json', level='section',
maxlatitude=10.0, nodata=204)
self.assertTrue(mock_oschema.called)
self.assertTrue(mock_dumps.called)
self.assertTrue(mock_response.called)
@patch.object(routes.BoreholeHydraulicSampleListResource, '_process_request')
@patch.object(routes.BoreholeListResource, '_process_request')
@patch.object(routes, 'BoreholeListResourceSchema')
@patch.object(routes, 'BoreholeSchema')
@patch.object(routes, 'make_response')
def test_borehole_list_get_level_bh(
self, mock_response, mock_oschema, mock_schema,
mock_process_request):
"""Test level=borehole for BoreholeListResource."""
routes.db = self.db
with self.client as c:
response = c.get('/v1/boreholes?maxlatitude=10&level=borehole')
mock_process_request.assert_called_with(
self.db.session, format='json', level='borehole',
maxlatitude=10.0, nodata=204)
self.assertTrue(mock_oschema.called)
self.assertTrue(mock_response.called)
@patch.object(routes.BoreholeHydraulicSampleListResource,
'_process_request')
@patch.object(routes, 'BoreholeHydraulicSampleListResource')
@patch.object(routes, 'BoreholeSchema')
@patch.object(routes, 'make_response')
def test_borehole_hydraulic_list_get_bh_level(
self, mock_response, mock_oschema, mock_schema,
mock_process_request):
"""Test level=borehole for BoreholeHydraulicSampleListResource."""
routes.db = self.db
with self.client as c:
response = c.get('/v1/boreholes/{}?level=borehole'.\
format(bh1_publicid_encoded))
self.assertTrue(mock_process_request.called)
mock_process_request.assert_called_with(
self.db.session, borehole_id=bh1_publicid,
format='json', level='borehole', nodata=204)
self.assertTrue(mock_oschema.called)
self.assertTrue(mock_response.called)
@patch.object(routes.BoreholeHydraulicSampleListResource,
'_process_request')
@patch.object(routes, 'BoreholeHydraulicSampleListResourceSchema')
@patch.object(routes, 'BoreholeSchema')
@patch.object(routes, 'make_response')
def test_borehole_hydraulic_list_get(self, mock_response, mock_oschema, mock_schema, mock_process_request):
# Mock the query parameters.
mock_schema.return_value.dump.return_value = {'f': 10}
def test_borehole_hydraulic_list_get_sec_level(
self, mock_response, mock_oschema, mock_schema,
mock_process_request):
"""Test level=section for BoreholeHydraulicSampleListResource."""
routes.db = self.db
mock_dumps = mock_oschema.return_value.dumps
mock_dumps.return_value = MagicMock()
with self.client as c:
response = c.get('/v1/boreholes/{}?'.format(bh1_publicid_encoded))
response = c.get(
'/v1/boreholes/{}?starttime=2019-01-01&level=section'.\
format(bh1_publicid_encoded))
self.assertTrue(mock_process_request.called)
mock_process_request.assert_called_with(self.db.session, borehole_id=bh1_publicid, f=10)
mock_process_request.assert_called_with(
self.db.session, borehole_id=bh1_publicid,
format='json', level='section',
starttime=datetime(2019, 1, 1), nodata=204)
self.assertTrue(mock_oschema.called)
self.assertTrue(mock_dumps.called)
self.assertTrue(mock_response.called)
@patch.object(routes.BoreholeHydraulicSampleListResource,
'_process_request')
@patch.object(routes, 'BoreholeHydraulicSampleListResourceSchema')
@patch.object(routes, 'BoreholeSchema')
@patch.object(routes, 'make_response')
def test_borehole_hydraulic_list_get_hyd_level(
self, mock_response, mock_oschema, mock_schema,
mock_process_request):
"""Test level=hydraulic for BoreholeHydraulicSampleListResource."""
routes.db = self.db
with self.client as c:
response = c.get(
'/v1/boreholes/{}?maxfluidph=10.0&level=hydraulic'.\
format(bh1_publicid_encoded))
self.assertTrue(mock_process_request.called)
mock_process_request.assert_called_with(
self.db.session, borehole_id=bh1_publicid,
format='json', level='hydraulic',
maxfluidph=10., nodata=204)
self.assertTrue(mock_oschema.called)
self.assertTrue(mock_response.called)
@patch.object(routes.SectionHydraulicSampleListResource, '_process_request')
@patch.object(routes, 'SectionHydraulicSampleListResourceSchema')
@patch.object(routes, 'HydraulicSampleSchema')
@patch.object(routes, 'make_response')
def test_hydraulic_list_get(self, mock_response, mock_oschema, mock_schema, mock_process_request):
def test_hydraulic_list_get(self, mock_response, mock_oschema,
mock_schema, mock_process_request):
"""Test SectionHydraulicSampleListResource."""
# Mock the query parameters.
mock_schema.return_value.dump.return_value = {'f': 10}
routes.db = self.db
mock_dumps = mock_oschema.return_value.dumps
mock_dumps.return_value = MagicMock()
with self.client as c:
response = c.get('/v1/boreholes/{}/sections/{}/hydraulics?'.format(bh1_publicid_encoded,
sec1_publicid_encoded))
print(response)
response = c.get(
'/v1/boreholes/{}/sections/{}/hydraulics?maxfluidph=10.0'.\
format(bh1_publicid_encoded,
sec1_publicid_encoded))
self.assertTrue(mock_process_request.called)
mock_process_request.assert_called_with(self.db.session,
borehole_id=bh1_publicid,
section_id=sec1_publicid, f=10)
borehole_id=bh1_publicid, section_id=sec1_publicid,
format='json', maxfluidph=10., nodata=204)
self.assertTrue(mock_oschema.called)
self.assertTrue(mock_dumps.called)
self.assertTrue(mock_response.called)
@patch.object(routes, 'BoreholeSchema')
@patch.object(routes, 'make_response')
@patch.object(routes, 'boreholesection_oids')
@patch.object(routes, 'query_with_sections')
@patch.object(routes, 'Borehole')
@patch.object(routes, 'BoreholeSection')
@patch.object(routes.DynamicQuery, 'filter_query')
@patch.object(routes, 'lazyload')
@patch.object(routes.DynamicQuery, 'filter_level')
class BoreholeProcessRequestTestcase(unittest.TestCase):
"""
Test cases for the _process_request fucntions within
......@@ -122,84 +192,133 @@ class BoreholeProcessRequestTestcase(unittest.TestCase):
"""
def test_borehole_list_process_request(
self, mock_lazyload, mock_dynamicquery, mock_sec,
mock_response, mock_oschema):
self, mock_dynamicquery, mock_sec, mock_bh,
mock_query_section, mock_sec_oids):
params = {'level': 'section'}
bhlr = routes.BoreholeListResource()
session = MagicMock()
mock_sec_oids.return_value = ['1']
returnval = bhlr._process_request(session, **params)
self.assertTrue(mock_sec_oids.called)
self.assertTrue(mock_query_section.called)
#session.query.assert_called_with(mock_sec)
#mock_lazyload.assert_called_with(mock_sec._borehole)
#mock_dynamicquery.assert_called_with(params, 'borehole')
mock_dynamicquery.assert_called_with(params, 'borehole')
def test_borehole_list_process_request_borehole(
self, mock_lazyload, mock_dynamicquery, mock_borehole,
mock_response, mock_oschema):
self, mock_dynamicquery, mock_sec, mock_bh,
mock_query_section, mock_sec_oids):
params = {'level': 'borehole'}
bhlr = routes.BoreholeListResource()
session = MagicMock()
returnval = bhlr._process_request(session, **params)
self.assertFalse(mock_sec_oids.called)
self.assertFalse(mock_query_section.called)
mock_dynamicquery.assert_called_with(params, 'borehole')
#session.query.assert_called_with(mock_borehole)
#self.assertFalse(mock_lazyload.called)
#mock_dynamicquery.assert_called_with(params, 'borehole')
@patch.object(routes.BoreholeHydraulicSampleListResource, '_hydraulicsample_oids')
@patch.object(routes.BoreholeHydraulicSampleListResource, '_boreholesection_oids')
@patch.object(routes, 'BoreholeSchema')
@patch.object(routes, 'hydraulicsample_oids')
@patch.object(routes, 'boreholesection_oids')
@patch.object(routes, 'make_response')
@patch.object(routes, 'Borehole')
@patch.object(routes, 'DynamicQuery')
@patch.object(routes.BoreholeHydraulicSampleListResource, '_query_with_sections')
@patch.object(routes.BoreholeHydraulicSampleListResource, '_query_with_sections_and_hydraulics')
@patch.object(routes, 'query_with_sections')
@patch.object(routes, 'query_with_sections_and_hydraulics')
class BoreholeHydraulicProcessRequestTestCase(unittest.TestCase):
"""
Test cases for the _process_request fucntions within
routes.BoreholeHydraulicSampleListResource class.
"""
def test_borehole_hyd_section_process_request(
self, mock_querysections, mock_queryhydraulics, mock_dynamicquery, mock_borehole,
mock_response, mock_oschema, mock_sec_ids, mock_hyd_ids):
def test_process_request_level_section(
self, mock_queryhydraulics, mock_querysections,
mock_dynamicquery, mock_borehole,
mock_response, mock_sec_ids, mock_hyd_ids):
"""
Test level=section for BoreholeHydraulicSampleListResource
with sections available
"""
params = {'level': 'section'}
bhlr = routes.BoreholeHydraulicSampleListResource()
session = MagicMock()
mock_sec_ids.return_value = ['1']
mock_querysections.return_value.filter.return_value = 'query'
returnval = bhlr._process_request(session, bh1_publicid_encoded, **params)
mock_sec_ids.assert_called_with("")
#mock_lazyload.assert_called_with(mock_borehole._sections)
#mock_dynamicquery.return_value.filter_level.assert_called_with(params, 'hydraulic')
#self.assertTrue(mock_dynamicquery.return_value.return_all.called)
def test_borehole_hyd_section_process_request(
self, mock_querysections, mock_queryhydraulics, mock_dynamicquery, mock_borehole,
mock_response, mock_oschema, mock_sec, mock_hyd):
self.assertTrue(mock_sec_ids.called)
self.assertTrue(mock_querysections.called)
self.assertFalse(mock_hyd_ids.called)
self.assertFalse(mock_queryhydraulics.called)
mock_dynamicquery.assert_called_with('query')
self.assertTrue(mock_dynamicquery.return_value.return_one.called)
def test_process_request_level_section_no_sections(
self, mock_queryhydraulics, mock_querysections,
mock_dynamicquery, mock_borehole,
mock_response, mock_sec_ids, mock_hyd_ids):
"""
Test level=section for BoreholeHydraulicSampleListResource
with no sections available
"""
params = {'level': 'section'}
bhlr = routes.BoreholeHydraulicSampleListResource()
session = MagicMock()
mock_sec_ids.return_value = []
returnval = bhlr._process_request(session, bh1_publicid_encoded, **params)
self.assertTrue(mock_sec_ids.called)
self.assertFalse(mock_querysections.called)
self.assertFalse(mock_hyd_ids.called)
self.assertFalse(mock_queryhydraulics.called)
self.assertTrue(mock_dynamicquery.return_value.return_one.called)
def test_process_request_level_hydraulic(
self, mock_queryhydraulics, mock_querysections,
mock_dynamicquery, mock_borehole,
mock_response, mock_sec_ids, mock_hyd_ids):
"""
Test level=hydraulic for BoreholeHydraulicSampleListResource
with sections and hydraulics available
"""
params = {'level': 'hydraulic'}
bhlr = routes.BoreholeHydraulicSampleListResource()
session = MagicMock()
mock_sec_ids.return_value = ['1']
mock_hyd_ids.return_value = ['1']
mock_queryhydraulics.return_value.filter.return_value = 'query'
returnval = bhlr._process_request(
session, bh1_publicid_encoded, **params)
self.assertTrue(mock_sec_ids.called)
self.assertFalse(mock_querysections.called)
self.assertTrue(mock_hyd_ids.called)
self.assertTrue(mock_queryhydraulics.called)
mock_dynamicquery.assert_called_with('query')
self.assertTrue(mock_dynamicquery.return_value.return_one.called)
def test_process_request_level_hydraulic_no_hydraulics(
self, mock_queryhydraulics, mock_querysections,
mock_dynamicquery, mock_borehole,
mock_response, mock_sec_ids, mock_hyd_ids):
"""
Test level=section for BoreholeHydraulicSampleListResource
with sections but no hydraulics available
"""
params = {'level': 'hydraulic'}
bhlr = routes.BoreholeHydraulicSampleListResource()
session = MagicMock()
mock_sec_ids.return_value = ['1']
mock_hyd_ids.return_value = []
mock_querysections.return_value.filter.return_value = 'query'
returnval = bhlr._process_request(session, bh1_publicid_encoded, **params)
self.assertTrue(mock_sec_ids.called)
self.assertTrue(mock_querysections.called)
self.assertTrue(mock_hyd_ids.called)
self.assertFalse(mock_queryhydraulics.called)
mock_dynamicquery.assert_called_with('query')
self.assertTrue(mock_dynamicquery.return_value.return_one.called)
#session.query.assert_called_with(mock_hyd)
#mock_lazyload.assert_has_calls([call(mock_hyd._section),
# call(mock_lazyload(mock_sec._borehole)),
# call(mock_sec._borehole)])
#mock_lazyload.assert_called_with(mock_borehole._sections)
#mock_dynamicquery.return_value.filter_level.assert_called_with(params, 'hydraulic')
#self.assertTrue(mock_dynamicquery.return_value.return_all.called)
@patch.object(routes, 'BoreholeSchema')
@patch.object(routes, 'make_response')
@patch.object(routes, 'HydraulicSample')
@patch.object(routes, 'BoreholeSection')
@patch.object(routes, 'section_in_borehole')
@patch.object(routes, 'query_hydraulicsamples')
@patch.object(routes, 'DynamicQuery')
@patch.object(routes, 'lazyload')
@patch.object(routes.SectionHydraulicSampleListResource,
'_section_in_borehole')
class SectionHydraulicProcessRequestTestcase(unittest.TestCase):
"""
Test cases for the _process_request fucntions within
......@@ -207,37 +326,31 @@ class SectionHydraulicProcessRequestTestcase(unittest.TestCase):
"""
def test_borehole_hyd_process_request_limit(
self, mock_in_borehole, mock_lazyload, mock_dynamicquery,
mock_boreholesection, mock_hydsample, mock_response,
mock_oschema):
self, mock_dynquery, mock_hydsamples, mock_in_borehole, hydsample):
params = {}
mock_in_borehole.return_value = True
mock_in_borehole.return_value = ['1']
bhlr = routes.SectionHydraulicSampleListResource()
session = MagicMock()
returnval = bhlr._process_request(session, bh1_publicid_encoded, sec1_publicid_encoded, **params)
returnval = bhlr._process_request(session, bh1_publicid_encoded,
sec1_publicid_encoded, **params)
#session.query.assert_called_with(mock_hydsample)
#session.query.return_value.options.return_value.join.assert_called_with(mock_boreholesection)
#mock_lazyload.assert_called_with(mock_hydsample._section)
#mock_dynamicquery.return_value.filter_level.assert_called_with(params, 'hydraulic')
mock_dynquery.assert_called_with(mock_hydsamples())
mock_dynquery.return_value.filter_level.assert_called_with(params, 'hydraulic')
def test_borehole_hyd_section_process_request(
self, mock_in_borehole, mock_lazyload, mock_dynamicquery,
mock_boreholesection, mock_hydsample, mock_response,
mock_oschema):
self, mock_dynquery, mock_hydsamples, mock_in_borehole, hydsample):
params = {'limit': 10}
mock_in_borehole.return_value = True
bhlr = routes.SectionHydraulicSampleListResource()
session = MagicMock()
returnval = bhlr._process_request(session, bh1_publicid_encoded, sec1_publicid_encoded, **params)
#mock_dynamicquery.return_value.format_results.assert_called_with(limit=10, offset=None, order_by=mock_hydsample.datetime_value)
mock_dynquery.return_value.format_results.assert_called_with(
limit=10, offset=None, order_column=hydsample.datetime_value)
def test_borehole_hyd_process_request_raises(
self, mock_in_borehole, mock_lazyload, mock_dynamicquery,
mock_boreholesection, mock_hydsample, mock_response,
mock_oschema):
self, mock_dynquery, mock_hydsamples, mock_in_borehole, hydsample):
params = {}
mock_in_borehole.return_value = False
bhlr = routes.SectionHydraulicSampleListResource()
......@@ -245,5 +358,16 @@ class SectionHydraulicProcessRequestTestcase(unittest.TestCase):
with self.assertRaises(ValueError):
bhlr._process_request(session, 'borehole_publicid', 'section_publicid', **params)
@patch.object(routes, 'DynamicQuery')
@patch.object(routes, 'lazyload')
class BoreholeSectionOIDsTestcase(unittest.TestCase):
def test_borehole_section_oids(self, mock_lazy_load, mock_dynquery):
session = MagicMock()
query_params = {}
return_val = routes.boreholesection_oids(session, **query_params)
self.assertTrue(mock_dynquery.called)
if __name__ == '__main__':
unittest.main()
......@@ -74,6 +74,87 @@ class ResourceBase(Resource):
raise NotImplementedError
def boreholesection_oids(session, borehole_id=None, **query_params):
"""
Return a list of BoreholeSection id's that match to the search
parameters given.
"""
sec_base_query = session.query(BoreholeSection).\
options(lazyload(BoreholeSection._borehole)).\
join(Borehole)
if borehole_id:
sec_base_query = sec_base_query.filter(Borehole.publicid==borehole_id)
sec_query = DynamicQuery(sec_base_query)
sec_query.filter_level(query_params, 'section')
sec_list = [i._oid for i in sec_query.return_all()]
return sec_list
# Note (sarsonl): Seaching for a better way of keeping parent
# boreholes and sections without children hydraulics. This
# is a work around where seperate calls are done to find if children
# exist before joining the tables.
def hydraulicsample_oids(session, borehole_id, **query_params):
"""
Return a list of HydraulicSample id's that match to the search
parameters given.
"""
hyd_base_query = session.query(HydraulicSample).\
options(lazyload(HydraulicSample._section).\
lazyload(BoreholeSection._borehole)).\
join(BoreholeSection).join(Borehole).\
filter(Borehole.publicid==borehole_id)
hyd_query = DynamicQuery(hyd_base_query)
hyd_query.filter_level(query_params,
'hydraulic')
hyd_query.format_results(limit=query_params.get('limit'),
offset=query_params.get('offset'))
hyd_list = [i._oid for i in hyd_query.return_all()]
return hyd_list
def query_with_sections(query, sec_list, keep_all_boreholes=True, **query_params):
query = query.options(lazyload(Borehole._sections)).\
join(BoreholeSection, isouter=True).\
options(contains_eager("_sections"))
if keep_all_boreholes:
query = query.filter(or_(BoreholeSection._oid.in_(sec_list),
Borehole._sections == None))
else:
query = query.filter(BoreholeSection._oid.in_(sec_list))
return query
def query_with_sections_and_hydraulics(query, hyd_list, **query_params):
query = query.options(lazyload(Borehole._sections).\
lazyload(BoreholeSection._hydraulics)).\
join(BoreholeSection, isouter=True).\
join(HydraulicSample, isouter=True).\
options(contains_eager("_sections").\
contains_eager("_hydraulics")).\
filter(or_(HydraulicSample._oid.in_(hyd_list),
BoreholeSection._hydraulics == None))
return query
def query_hydraulicsamples(session, section_id):
query = session.query(HydraulicSample).\
options(lazyload(HydraulicSample._section)).\
join(BoreholeSection).filter(BoreholeSection.publicid==section_id)
return query
def section_in_borehole(query, borehole_id, section_id):
boreholesection_filter = query(Borehole).\
options(lazyload(Borehole._sections)).\
filter(Borehole.publicid == borehole_id).\
filter(BoreholeSection.publicid == section_id)
section_exists = query(literal(True)).\