From fd1259e908657d972b2f34d6e73380ca1fe0e2f2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stephan=20G=C3=BCnther?= Date: Thu, 10 Mar 2016 21:10:10 +0100 Subject: [PATCH 01/44] Start developing v0.0.4 There's a corresponding `whatsnew/0004.txt` now, everybody can update when he adds something new. --- doc/whats_new.rst | 1 + doc/whatsnew/v0004.txt | 26 ++++++++++++++++++++++++++ 2 files changed, 27 insertions(+) create mode 100644 doc/whatsnew/v0004.txt diff --git a/doc/whats_new.rst b/doc/whats_new.rst index ec03ca6..24ed21e 100644 --- a/doc/whats_new.rst +++ b/doc/whats_new.rst @@ -8,6 +8,7 @@ These are new features and improvements of note in each release :local: :backlinks: top +.. include:: whatsnew/v0004.txt .. include:: whatsnew/v0003.txt .. include:: whatsnew/v0002.txt .. include:: whatsnew/v0001.txt diff --git a/doc/whatsnew/v0004.txt b/doc/whatsnew/v0004.txt new file mode 100644 index 0000000..3d5e23e --- /dev/null +++ b/doc/whatsnew/v0004.txt @@ -0,0 +1,26 @@ +v0.0.4 (TBA) ++++++++++++++++++++++++++ + +New features +############ + + +Documentation +############# + + +Testing +####### + + +Bug fixes +######### + + +Other changes +############# + + +Contributors +############ + From d4efdde580f6be3be07b250f897c718f2366d063 Mon Sep 17 00:00:00 2001 From: Guido Date: Mon, 14 Mar 2016 18:24:03 +0100 Subject: [PATCH 02/44] create new table with serial primary key --- oemof/db/tools.py | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/oemof/db/tools.py b/oemof/db/tools.py index 5821519..e0e3415 100644 --- a/oemof/db/tools.py +++ b/oemof/db/tools.py @@ -288,3 +288,39 @@ def get_windzone(conn, geometry): else: zone = 0 return zone + + +def create_empty_table_serial_primary(conn, schema, table, columns=None, id_col='id'): + r"""New database table with primary key type serial and empty columns + + + Parameters + ---------- + conn : sqlalchemy connection object + A valid connection to a database + schema : str + The database schema + table : str + The database table + columns : list, optional + Columns that are to be created + id_col : str, optional + Name of index column of database table + + Notes + ------- + Currently all created table columns will be of type `double precision`. Feel free to enhance this function by + by generalizing this aspect. + """ + + sql_str = """CREATE TABLE {schema}.{table} ({id_col} SERIAL PRIMARY KEY NOT NULL) + """.format(schema=schema, table=table, id_col=id_col) + + conn.execute(sql_str) + + # define more columns + if columns is not None: + for col in columns: + col_str = """alter table {schema}.{table} add column {col} double precision; + """.format(schema=schema, table=table, col=col) + conn.execute(col_str) \ No newline at end of file From 9a2f4f4101bedfcda7b69d5f14d4ab0a2362fd44 Mon Sep 17 00:00:00 2001 From: Guido Date: Mon, 14 Mar 2016 18:24:32 +0100 Subject: [PATCH 03/44] grant database rights --- oemof/db/tools.py | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/oemof/db/tools.py b/oemof/db/tools.py index e0e3415..0e5257e 100644 --- a/oemof/db/tools.py +++ b/oemof/db/tools.py @@ -323,4 +323,26 @@ def create_empty_table_serial_primary(conn, schema, table, columns=None, id_col= for col in columns: col_str = """alter table {schema}.{table} add column {col} double precision; """.format(schema=schema, table=table, col=col) - conn.execute(col_str) \ No newline at end of file + conn.execute(col_str) + + +def grant_db_access(conn, schema, table, role): + r"""Gives access to database users/ groups + + + Parameters + ---------- + conn : sqlalchemy connection object + A valid connection to a database + schema : str + The database schema + table : str + The database table + role : str + database role that access is granted to + + """ + grant_str = """GRANT ALL ON TABLE {schema}.{table} + TO {role} WITH GRANT OPTION;""".format(schema=schema, table=table, role=role) + + conn.execute(grant_str) \ No newline at end of file From 1df612812a1d1d03115d4b5fefeab9524711399e Mon Sep 17 00:00:00 2001 From: Guido Date: Tue, 15 Mar 2016 09:23:40 +0100 Subject: [PATCH 04/44] adjust line breaks --- oemof/db/tools.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/oemof/db/tools.py b/oemof/db/tools.py index 0e5257e..bee5ef0 100644 --- a/oemof/db/tools.py +++ b/oemof/db/tools.py @@ -290,7 +290,8 @@ def get_windzone(conn, geometry): return zone -def create_empty_table_serial_primary(conn, schema, table, columns=None, id_col='id'): +def create_empty_table_serial_primary(conn, schema, table, columns=None, + id_col='id'): r"""New database table with primary key type serial and empty columns @@ -309,19 +310,22 @@ def create_empty_table_serial_primary(conn, schema, table, columns=None, id_col= Notes ------- - Currently all created table columns will be of type `double precision`. Feel free to enhance this function by + Currently all created table columns will be of type `double precision`. + Feel free to enhance this function by by generalizing this aspect. """ - sql_str = """CREATE TABLE {schema}.{table} ({id_col} SERIAL PRIMARY KEY NOT NULL) - """.format(schema=schema, table=table, id_col=id_col) + sql_str = """CREATE TABLE {schema}.{table} ({id_col} SERIAL PRIMARY KEY + NOT NULL) + """.format(schema=schema, table=table, id_col=id_col) conn.execute(sql_str) # define more columns if columns is not None: for col in columns: - col_str = """alter table {schema}.{table} add column {col} double precision; + col_str = """alter table {schema}.{table} add column {col} + double precision; """.format(schema=schema, table=table, col=col) conn.execute(col_str) @@ -343,6 +347,7 @@ def grant_db_access(conn, schema, table, role): """ grant_str = """GRANT ALL ON TABLE {schema}.{table} - TO {role} WITH GRANT OPTION;""".format(schema=schema, table=table, role=role) + TO {role} WITH GRANT OPTION;""".format(schema=schema, table=table, + role=role) conn.execute(grant_str) \ No newline at end of file From b973a03574ba4248e61b433ed483466b7c7e163c Mon Sep 17 00:00:00 2001 From: gplssm Date: Tue, 15 Mar 2016 10:23:42 +0100 Subject: [PATCH 05/44] add information about recent changes --- doc/whatsnew/v0004.txt | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/doc/whatsnew/v0004.txt b/doc/whatsnew/v0004.txt index 3d5e23e..b64112e 100644 --- a/doc/whatsnew/v0004.txt +++ b/doc/whatsnew/v0004.txt @@ -4,6 +4,9 @@ v0.0.4 (TBA) New features ############ + * Create an empty database table with primary key type serial in `tools.py` + * Access grant to users/ group of users, see `grant_db_access` in `tools.py` + Documentation ############# @@ -24,3 +27,4 @@ Other changes Contributors ############ + * Guido Pleßmann From 0020d6b16f5f8b449d4d114ab52bc8b75c54ef6b Mon Sep 17 00:00:00 2001 From: Guido Date: Wed, 16 Mar 2016 13:39:10 +0100 Subject: [PATCH 06/44] adjust blank lines --- oemof/db/tools.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/oemof/db/tools.py b/oemof/db/tools.py index bee5ef0..e059fac 100644 --- a/oemof/db/tools.py +++ b/oemof/db/tools.py @@ -294,7 +294,6 @@ def create_empty_table_serial_primary(conn, schema, table, columns=None, id_col='id'): r"""New database table with primary key type serial and empty columns - Parameters ---------- conn : sqlalchemy connection object @@ -329,11 +328,9 @@ def create_empty_table_serial_primary(conn, schema, table, columns=None, """.format(schema=schema, table=table, col=col) conn.execute(col_str) - def grant_db_access(conn, schema, table, role): r"""Gives access to database users/ groups - Parameters ---------- conn : sqlalchemy connection object @@ -350,4 +347,5 @@ def grant_db_access(conn, schema, table, role): TO {role} WITH GRANT OPTION;""".format(schema=schema, table=table, role=role) + conn.execute(grant_str) conn.execute(grant_str) \ No newline at end of file From 68f4d976d530d321f9af204e29c7dbf76df75124 Mon Sep 17 00:00:00 2001 From: Guido Date: Wed, 16 Mar 2016 13:58:26 +0100 Subject: [PATCH 07/44] Add primary key constraint setter definition --- oemof/db/tools.py | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/oemof/db/tools.py b/oemof/db/tools.py index e059fac..c2c74f9 100644 --- a/oemof/db/tools.py +++ b/oemof/db/tools.py @@ -348,4 +348,24 @@ def grant_db_access(conn, schema, table, role): role=role) conn.execute(grant_str) - conn.execute(grant_str) \ No newline at end of file + conn.execute(grant_str) + +def add_primary_key(conn, schema, table, pk_col): + r"""Adds primary key to database table + + Parameters + ---------- + conn : sqlalchemy connection object + A valid connection to a database + schema : str + The database schema + table : str + The database table + pk_col : str + Column that primary key is applied to + + """ + sql_str = """alter table {schema}.{table} add primary key ({col})""".format( + schema=schema, table=table, col=pk_col) + + conn.execute(sql_str) \ No newline at end of file From fa90b32a4e1adf47d54d42e7516ac42ca018a366 Mon Sep 17 00:00:00 2001 From: Guido Date: Wed, 16 Mar 2016 13:58:56 +0100 Subject: [PATCH 08/44] Fix blank lines --- oemof/db/tools.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/oemof/db/tools.py b/oemof/db/tools.py index c2c74f9..1751a65 100644 --- a/oemof/db/tools.py +++ b/oemof/db/tools.py @@ -348,7 +348,7 @@ def grant_db_access(conn, schema, table, role): role=role) conn.execute(grant_str) - conn.execute(grant_str) + def add_primary_key(conn, schema, table, pk_col): r"""Adds primary key to database table @@ -368,4 +368,5 @@ def add_primary_key(conn, schema, table, pk_col): sql_str = """alter table {schema}.{table} add primary key ({col})""".format( schema=schema, table=table, col=pk_col) + conn.execute(sql_str) conn.execute(sql_str) \ No newline at end of file From eb0a5b159d9a7e54a713c24cd432b5e844ec2642 Mon Sep 17 00:00:00 2001 From: Guido Date: Wed, 16 Mar 2016 13:59:27 +0100 Subject: [PATCH 09/44] Add table owner ship changer definition --- .gitignore | 8 ++++++++ oemof/db/tools.py | 22 ++++++++++++++++++++++ 2 files changed, 30 insertions(+) diff --git a/.gitignore b/.gitignore index ae008fd..f9e8e9a 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,11 @@ +/.idea/codeStyleSettings.xml +/.idea/.name +/.idea/encodings.xml +/.idea/misc.xml +/.idea/modules.xml +/.idea/oemof.db.iml +/.idea/vcs.xml +/.idea/workspace.xml # Byte-compiled / optimized / DLL files __pycache__/ *.py[cod] diff --git a/oemof/db/tools.py b/oemof/db/tools.py index 1751a65..83580d3 100644 --- a/oemof/db/tools.py +++ b/oemof/db/tools.py @@ -369,4 +369,26 @@ def add_primary_key(conn, schema, table, pk_col): schema=schema, table=table, col=pk_col) conn.execute(sql_str) + + +def change_owner_to(conn, schema, table, role): + r"""Changes table's ownership to role + + Parameters + ---------- + conn : sqlalchemy connection object + A valid connection to a database + schema : str + The database schema + table : str + The database table + role : str + database role that access is granted to + + """ + sql_str = """ALTER TABLE {schema}.{table} + OWNER TO {role};""".format(schema=schema, + table=table, + role=role) + conn.execute(sql_str) \ No newline at end of file From 4db88710e746188476a5bf292b3c6ee92c9ce334 Mon Sep 17 00:00:00 2001 From: Guido Date: Wed, 16 Mar 2016 14:02:48 +0100 Subject: [PATCH 10/44] Remove .gitignore from index --- .gitignore | 68 ------------------------------------------------------ 1 file changed, 68 deletions(-) delete mode 100644 .gitignore diff --git a/.gitignore b/.gitignore deleted file mode 100644 index f9e8e9a..0000000 --- a/.gitignore +++ /dev/null @@ -1,68 +0,0 @@ -/.idea/codeStyleSettings.xml -/.idea/.name -/.idea/encodings.xml -/.idea/misc.xml -/.idea/modules.xml -/.idea/oemof.db.iml -/.idea/vcs.xml -/.idea/workspace.xml -# Byte-compiled / optimized / DLL files -__pycache__/ -*.py[cod] - -.spyderproject - -# C extensions -*.so - -# Distribution / packaging -.Python -env/ -build/ -_build/ -develop-eggs/ -dist/ -downloads/ -eggs/ -.eggs/ -lib/ -lib64/ -parts/ -sdist/ -var/ -*.egg-info/ -.installed.cfg -*.egg - -# PyInstaller -# Usually these files are written by a python script from a template -# before PyInstaller builds the exe, so as to inject date/other infos into it. -*.manifest -*.spec - -# Installer logs -pip-log.txt -pip-delete-this-directory.txt - -# Unit test / coverage reports -htmlcov/ -.tox/ -.coverage -.coverage.* -.cache -nosetests.xml -coverage.xml -*,cover - -# Translations -*.mo -*.pot - -# Django stuff: -*.log - -# Sphinx documentation -docs/_build/ - -# PyBuilder -target/ From c29b548eb8d0f985e206433f12a5fe98d9fa6cfe Mon Sep 17 00:00:00 2001 From: Guido Date: Tue, 12 Apr 2016 15:31:22 +0200 Subject: [PATCH 11/44] Revert "Remove .gitignore from index" This reverts commit 4db88710e746188476a5bf292b3c6ee92c9ce334. --- .gitignore | 68 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 68 insertions(+) create mode 100644 .gitignore diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..f9e8e9a --- /dev/null +++ b/.gitignore @@ -0,0 +1,68 @@ +/.idea/codeStyleSettings.xml +/.idea/.name +/.idea/encodings.xml +/.idea/misc.xml +/.idea/modules.xml +/.idea/oemof.db.iml +/.idea/vcs.xml +/.idea/workspace.xml +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] + +.spyderproject + +# C extensions +*.so + +# Distribution / packaging +.Python +env/ +build/ +_build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +*.egg-info/ +.installed.cfg +*.egg + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*,cover + +# Translations +*.mo +*.pot + +# Django stuff: +*.log + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ From 5d7817d1cc61aa98a440ec40f68b01077e08b05f Mon Sep 17 00:00:00 2001 From: Guido Date: Tue, 12 Apr 2016 15:32:55 +0200 Subject: [PATCH 12/44] change ignored files to ignoring directory --- .gitignore | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/.gitignore b/.gitignore index f9e8e9a..b88dd45 100644 --- a/.gitignore +++ b/.gitignore @@ -1,11 +1,5 @@ -/.idea/codeStyleSettings.xml -/.idea/.name -/.idea/encodings.xml -/.idea/misc.xml -/.idea/modules.xml -/.idea/oemof.db.iml -/.idea/vcs.xml -/.idea/workspace.xml +.idea/ + # Byte-compiled / optimized / DLL files __pycache__/ *.py[cod] From 14228955462e5fe3108f056d670ddb14403804e2 Mon Sep 17 00:00:00 2001 From: uwe Date: Wed, 20 Apr 2016 20:54:52 +0200 Subject: [PATCH 13/44] add doc/api Conflicts: .gitignore --- doc/api/oemof.db.rst | 54 ++++++++++++++++++++++++++++++++++++++++++++ doc/api/oemof.rst | 17 ++++++++++++++ 2 files changed, 71 insertions(+) create mode 100644 doc/api/oemof.db.rst create mode 100644 doc/api/oemof.rst diff --git a/doc/api/oemof.db.rst b/doc/api/oemof.db.rst new file mode 100644 index 0000000..4abd413 --- /dev/null +++ b/doc/api/oemof.db.rst @@ -0,0 +1,54 @@ +oemof.db package +================ + +Submodules +---------- + +oemof.db.coastdat module +------------------------ + +.. automodule:: oemof.db.coastdat + :members: + :undoc-members: + :show-inheritance: + +oemof.db.config module +---------------------- + +.. automodule:: oemof.db.config + :members: + :undoc-members: + :show-inheritance: + +oemof.db.feedin_pg module +------------------------- + +.. automodule:: oemof.db.feedin_pg + :members: + :undoc-members: + :show-inheritance: + +oemof.db.powerplants module +--------------------------- + +.. automodule:: oemof.db.powerplants + :members: + :undoc-members: + :show-inheritance: + +oemof.db.tools module +--------------------- + +.. automodule:: oemof.db.tools + :members: + :undoc-members: + :show-inheritance: + + +Module contents +--------------- + +.. automodule:: oemof.db + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/api/oemof.rst b/doc/api/oemof.rst new file mode 100644 index 0000000..c2521fe --- /dev/null +++ b/doc/api/oemof.rst @@ -0,0 +1,17 @@ +oemof package +============= + +Subpackages +----------- + +.. toctree:: + + oemof.db + +Module contents +--------------- + +.. automodule:: oemof + :members: + :undoc-members: + :show-inheritance: From 960c98b32b43e15a9c7e44baec320c1a3bba65f6 Mon Sep 17 00:00:00 2001 From: birgit Date: Mon, 25 Apr 2016 11:47:11 +0200 Subject: [PATCH 14/44] Remove creation of FixedSource instance from Feedin class --- oemof/db/feedin_pg.py | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/oemof/db/feedin_pg.py b/oemof/db/feedin_pg.py index 7f8756e..be43ecf 100644 --- a/oemof/db/feedin_pg.py +++ b/oemof/db/feedin_pg.py @@ -9,7 +9,6 @@ from . import powerplants as pg_pp from . import tools from feedinlib import powerplants as pp -from oemof.core.network.entities.components import sources as source class Feedin: @@ -34,13 +33,7 @@ def create_fixed_source(self, conn, **kwargs): df = pd.concat([pv_df.sum(axis=1), wind_df.sum(axis=1)], axis=1) feedin_df = df.rename(columns={0: 'pv_pwr', 1: 'wind_pwr'}) - for stype in feedin_df.keys(): - source.FixedSource( - uid=('FixedSrc', region.name, stype), - outputs=[obj for obj in region.entities if obj.uid == ( - 'bus', region.name, kwargs['bustype'])], - val=feedin_df[stype], - out_max=[float(cap.sum()[stype])]) + return feedin_df, cap def get_timeseries(self, conn, **kwargs): '' From 076f91f0baa29c64cfb83ff15ee7a035ee7bbdc1 Mon Sep 17 00:00:00 2001 From: birgit Date: Mon, 25 Apr 2016 11:54:51 +0200 Subject: [PATCH 15/44] Rename definition create_fixed_source to aggregate_cap_val --- oemof/db/feedin_pg.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/oemof/db/feedin_pg.py b/oemof/db/feedin_pg.py index be43ecf..0a827aa 100644 --- a/oemof/db/feedin_pg.py +++ b/oemof/db/feedin_pg.py @@ -18,7 +18,7 @@ def __init__(self): '' pass - def create_fixed_source(self, conn, **kwargs): + def aggregate_cap_val(self, conn, **kwargs): '' region = kwargs['region'] [pv_df, wind_df, cap] = self.get_timeseries( From d44c7fda9ebe0f5a23b119e009fbf39a81e2abf4 Mon Sep 17 00:00:00 2001 From: birgit Date: Mon, 25 Apr 2016 13:42:33 +0200 Subject: [PATCH 16/44] Return capacity as sum of the capacity installed in each weather cell --- oemof/db/feedin_pg.py | 1 + 1 file changed, 1 insertion(+) diff --git a/oemof/db/feedin_pg.py b/oemof/db/feedin_pg.py index 0a827aa..905d351 100644 --- a/oemof/db/feedin_pg.py +++ b/oemof/db/feedin_pg.py @@ -32,6 +32,7 @@ def aggregate_cap_val(self, conn, **kwargs): # Summerize the results to one column for pv and one for wind df = pd.concat([pv_df.sum(axis=1), wind_df.sum(axis=1)], axis=1) feedin_df = df.rename(columns={0: 'pv_pwr', 1: 'wind_pwr'}) + cap = cap.sum() return feedin_df, cap From f4526c08a230addaa027a5c754c655c9939860f2 Mon Sep 17 00:00:00 2001 From: birgit Date: Mon, 25 Apr 2016 14:54:24 +0200 Subject: [PATCH 17/44] Add docstring to definition aggregate_cap_val --- oemof/db/feedin_pg.py | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/oemof/db/feedin_pg.py b/oemof/db/feedin_pg.py index 905d351..d526640 100644 --- a/oemof/db/feedin_pg.py +++ b/oemof/db/feedin_pg.py @@ -19,7 +19,27 @@ def __init__(self): pass def aggregate_cap_val(self, conn, **kwargs): - '' + ''' + Returns the normalised feedin profile and installed capacity for + a given region. + + Parameters + ---------- + region : Region instance + region.geom : shapely.geometry object + Geo-spatial data with information for location/region-shape. The + geometry can be a polygon/multi-polygon or a point. + + Returns + ------- + feedin_df : pandas dataframe + Dataframe containing the normalised feedin profile of the given + region. Index of the dataframe is the hour of the year; columns + are 'pv_pwr' and 'wind_pwr'. + cap : pandas series + Series containing the installed capacity (in W) of PV and wind + turbines of the given region. + ''' region = kwargs['region'] [pv_df, wind_df, cap] = self.get_timeseries( conn, From dd54b557d33a5aa8b03e3e954f0e9b5a8ce65b4b Mon Sep 17 00:00:00 2001 From: birgit Date: Wed, 27 Apr 2016 16:49:00 +0200 Subject: [PATCH 18/44] Scale feedin_df to installed capacity of 1 --- oemof/db/feedin_pg.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/oemof/db/feedin_pg.py b/oemof/db/feedin_pg.py index d526640..08ef161 100644 --- a/oemof/db/feedin_pg.py +++ b/oemof/db/feedin_pg.py @@ -50,9 +50,10 @@ def aggregate_cap_val(self, conn, **kwargs): self.store_full_df(pv_df, wind_df, **kwargs) # Summerize the results to one column for pv and one for wind - df = pd.concat([pv_df.sum(axis=1), wind_df.sum(axis=1)], axis=1) - feedin_df = df.rename(columns={0: 'pv_pwr', 1: 'wind_pwr'}) cap = cap.sum() + df = pd.concat([pv_df.sum(axis=1) / cap['pv_pwr'], + wind_df.sum(axis=1) / cap['wind_pwr']], axis=1) + feedin_df = df.rename(columns={0: 'pv_pwr', 1: 'wind_pwr'}) return feedin_df, cap From ce1420e83c6875faed9b1c5bffcee4c947490557 Mon Sep 17 00:00:00 2001 From: Guido Date: Fri, 29 Apr 2016 11:39:13 +0200 Subject: [PATCH 19/44] add shapely dependency --- setup.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 6ac2cd9..f2272b8 100644 --- a/setup.py +++ b/setup.py @@ -9,4 +9,5 @@ packages=find_packages(), package_dir={'oemof': 'oemof'}, install_requires=['sqlalchemy >= 1.0', - 'keyring >= 4.0']) + 'keyring >= 4.0', + 'shapely']) From 27db7b8a7862f57c6a33ef74e2ef080a1e6d4309 Mon Sep 17 00:00:00 2001 From: Guido Date: Fri, 29 Apr 2016 11:39:36 +0200 Subject: [PATCH 20/44] change to python 3 as interpreter --- examples/feedinlib_example_coastdat.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/feedinlib_example_coastdat.py b/examples/feedinlib_example_coastdat.py index ff4f032..cac1df9 100644 --- a/examples/feedinlib_example_coastdat.py +++ b/examples/feedinlib_example_coastdat.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/python3 # -*- coding: utf-8 try: From 9b237ff4bc74dc2178bcccd92767a415e45224a9 Mon Sep 17 00:00:00 2001 From: Guido Date: Fri, 29 Apr 2016 11:40:03 +0200 Subject: [PATCH 21/44] fix wrong import of oemof_pg/oemof.db --- examples/feedinlib_example_coastdat.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/feedinlib_example_coastdat.py b/examples/feedinlib_example_coastdat.py index cac1df9..045d1bb 100644 --- a/examples/feedinlib_example_coastdat.py +++ b/examples/feedinlib_example_coastdat.py @@ -10,8 +10,8 @@ import logging from shapely import geometry as geopy -from oemof_pg import db -from oemof_pg import coastdat +from oemof import db +from oemof.db import coastdat from feedinlib import powerplants as plants from feedinlib import models From 7140111a5a326d8d065110079e59b5f90aa700bb Mon Sep 17 00:00:00 2001 From: gplssm Date: Fri, 3 Jun 2016 14:23:49 +0200 Subject: [PATCH 22/44] Add missing required package: psycopg2 --- setup.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index f2272b8..0080a7c 100644 --- a/setup.py +++ b/setup.py @@ -10,4 +10,5 @@ package_dir={'oemof': 'oemof'}, install_requires=['sqlalchemy >= 1.0', 'keyring >= 4.0', - 'shapely']) + 'shapely', + 'psycopg2']) From 5363ff427d01d7b5c3eeafade7fec35ff26ec5a7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stephan=20G=C3=BCnther?= Date: Mon, 4 Jul 2016 18:29:51 +0200 Subject: [PATCH 23/44] Allow getting the database URL --- oemof/db/__init__.py | 50 +++++++++++++++++++++++++++++++++----------- 1 file changed, 38 insertions(+), 12 deletions(-) diff --git a/oemof/db/__init__.py b/oemof/db/__init__.py index 91544e7..84fae2a 100755 --- a/oemof/db/__init__.py +++ b/oemof/db/__init__.py @@ -3,23 +3,24 @@ import keyring from . import config as cfg +def url(section="postGIS"): + """ Retrieve the URL used to connect to the database. -def engine(section="postGIS"): - """Creates engine object for database access - - If keyword argument `section` is used it requires an existing config.ini - file at the right location. + Use this if you have your own means of accessing the database and do not + want to use :func:`engine` or :func:`connection`. Parameters ---------- section : str, optional - Section (in config.ini) of targeted database containing connection - details that are used to set up connection + The `config.ini` section corresponding to the targeted database. + It should contain all the details that needed to set up a connection. Returns ------- - engine : :class:`sqlalchemy.engine.Engine` - Engine for sqlalchemy + database URL : str + The URL with which one can connect to the database. Be careful as this + will probably contain sensitive data like the username/password + combination. Notes ----- @@ -44,13 +45,38 @@ def engine(section="postGIS"): "\nExiting.") exit(-1) - return create_engine( - "postgresql+psycopg2://{user}:{passwd}@{host}:{port}/{db}".format( + return "postgresql+psycopg2://{user}:{passwd}@{host}:{port}/{db}".format( user=cfg.get(section, "username"), passwd=pw, host=cfg.get(section, "host"), db=cfg.get(section, "database"), - port=int(cfg.get(section, "port")))) + port=int(cfg.get(section, "port"))) + + +def engine(section="postGIS"): + """Creates engine object for database access + + If keyword argument `section` is used it requires an existing config.ini + file at the right location. + + Parameters + ---------- + section : str, optional + Section (in config.ini) of targeted database containing connection + details that are used to set up connection + + Returns + ------- + engine : :class:`sqlalchemy.engine.Engine` + Engine for sqlalchemy + + Notes + ----- + + For documentation on config.ini see the README section on + :ref:`configuring ` :mod:`oemof.db`. + """ + return create_engine(url(section)) def connection(section="postGIS"): From 5afe275061a57b7c75208848cd9069bc7e0f5b56 Mon Sep 17 00:00:00 2001 From: uwe Date: Thu, 25 Aug 2016 17:46:27 +0200 Subject: [PATCH 24/44] make pylint a little bit happier --- oemof/db/coastdat.py | 71 ++++++++++++++++++++++---------------------- 1 file changed, 36 insertions(+), 35 deletions(-) diff --git a/oemof/db/coastdat.py b/oemof/db/coastdat.py index afebd84..b2f4ab1 100644 --- a/oemof/db/coastdat.py +++ b/oemof/db/coastdat.py @@ -6,7 +6,7 @@ import numpy as np from pytz import timezone from datetime import datetime -from feedinlib import weather +import feedinlib.weather as weather from . import tools from shapely.wkt import loads as wkt_loads @@ -50,42 +50,42 @@ def get_weather(conn, geometry, year): return obj -def sql_weather_string(conn, geometry, year, sql_part): - ''' - Creates an sql-string to read all datasets within a given geometry. - ''' - # TODO@Günni. Replace sql-String with alchemy/GeoAlchemy - # Create string parts for where conditions - - return ''' - SELECT tsptyti.*, y.leap - FROM coastdat.year as y +def sql_weather_string(year, sql_part): + """ + Creates an sql-string to read all datasets within a given geometry. + """ + # TODO@Günni. Replace sql-String with alchemy/GeoAlchemy + # Create string parts for where conditions + + return ''' + SELECT tsptyti.*, y.leap + FROM coastdat.year as y + INNER JOIN ( + SELECT tsptyd.*, sc.time_id + FROM coastdat.scheduled as sc INNER JOIN ( - SELECT tsptyd.*, sc.time_id - FROM coastdat.scheduled as sc + SELECT tspty.*, dt.name, dt.height + FROM coastdat.datatype as dt INNER JOIN ( - SELECT tspty.*, dt.name, dt.height - FROM coastdat.datatype as dt + SELECT tsp.*, typ.type_id + FROM coastdat.typified as typ INNER JOIN ( - SELECT tsp.*, typ.type_id - FROM coastdat.typified as typ + SELECT spl.*, t.tsarray, t.id + FROM coastdat.timeseries as t INNER JOIN ( - SELECT spl.*, t.tsarray, t.id - FROM coastdat.timeseries as t - INNER JOIN ( - SELECT sps.*, l.data_id - FROM ( - {sql_part} - ) as sps - INNER JOIN coastdat.located as l - ON (sps.gid = l.spatial_id)) as spl - ON (spl.data_id = t.id)) as tsp - ON (tsp.id = typ.data_id)) as tspty - ON (tspty.type_id = dt.id)) as tsptyd - ON (tsptyd.id = sc.data_id))as tsptyti - ON (tsptyti.time_id = y.year) - where y.year = '{year}' - ;'''.format(year=year, sql_part=sql_part) + SELECT sps.*, l.data_id + FROM ( + {sql_part} + ) as sps + INNER JOIN coastdat.located as l + ON (sps.gid = l.spatial_id)) as spl + ON (spl.data_id = t.id)) as tsp + ON (tsp.id = typ.data_id)) as tspty + ON (tspty.type_id = dt.id)) as tsptyd + ON (tsptyd.id = sc.data_id))as tsptyti + ON (tsptyti.time_id = y.year) + where y.year = '{year}' + ;'''.format(year=year, sql_part=sql_part) def fetch_raw_data(sql, connection, geometry): @@ -133,13 +133,14 @@ def create_single_weather(df, geo, rename_dc): 'Create an oemof weather object for the given geometry' my_weather = weather.FeedinWeather() data_height = {} - + name = None # Create a pandas.DataFrame with the time series of the weather data set weather_df = pd.DataFrame(index=df.time_series.iloc[0].index) for row in df.iterrows(): key = rename_dc[row[1].type] weather_df[key] = row[1].time_series data_height[key] = row[1].height if not np.isnan(row[1].height) else 0 + name = row[1].gid my_weather.data = weather_df my_weather.timezone = weather_df.index.tz if geo.geom_type == 'Point': @@ -150,7 +151,7 @@ def create_single_weather(df, geo, rename_dc): my_weather.latitude = geo.centroid.y my_weather.geometry = geo my_weather.data_height = data_height - my_weather.name = row[1].gid + my_weather.name = name return my_weather From 4962aa1ced88353921f32e77917117719be5eb72 Mon Sep 17 00:00:00 2001 From: uwe Date: Thu, 25 Aug 2016 17:47:27 +0200 Subject: [PATCH 25/44] make pylint a little bit happier --- examples/feedinlib_example_coastdat.py | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/examples/feedinlib_example_coastdat.py b/examples/feedinlib_example_coastdat.py index 045d1bb..bf63a02 100644 --- a/examples/feedinlib_example_coastdat.py +++ b/examples/feedinlib_example_coastdat.py @@ -3,18 +3,17 @@ try: from matplotlib import pyplot as plt - plot_fkt = True -except: - plot_fkt = False +except ImportError: + plt = None import logging from shapely import geometry as geopy -from oemof import db +import oemof.db as db from oemof.db import coastdat -from feedinlib import powerplants as plants -from feedinlib import models +import feedinlib.powerplants as plants +import feedinlib.models as models # Feel free to remove or change these lines import warnings @@ -80,12 +79,14 @@ year = 2010 conn = db.connection() -my_weather = coastdat.get_weather( +my_weather_single = coastdat.get_weather( conn, geopy.Point(loc_berlin['longitude'], loc_berlin['latitude']), year) geo = geopy.Polygon([(12.2, 52.2), (12.2, 51.6), (13.2, 51.6), (13.2, 52.2)]) multi_weather = coastdat.get_weather(conn, geo, year) + my_weather = multi_weather[0] +# my_weather = my_weather_single # Initialise different power plants E126_power_plant = plants.WindPowerPlant(**enerconE126) @@ -101,7 +102,7 @@ E126_feedin.name = 'E126' V90_feedin.name = 'V90' -if plot_fkt: +if plt: E126_feedin.plot(legend=True) V90_feedin.plot(legend=True) plt.show() @@ -123,7 +124,7 @@ pv_feedin5.name = 'Advent' # Output -if plot_fkt: +if plt: pv_feedin4.plot(legend=True) pv_feedin5.plot(legend=True) plt.show() @@ -135,7 +136,7 @@ w_model.get_wind_pp_types() cp_values = models.SimpleWindTurbine().fetch_cp_values( wind_conv_type='ENERCON E 126 7500') -if plot_fkt: +if plt: plt.plot(cp_values.loc[0, :][2:55].index, cp_values.loc[0, :][2:55].values, '*') plt.show() From 5d1e52c805cc4da44b6565e8ec14e32bf10e1f48 Mon Sep 17 00:00:00 2001 From: uwe Date: Thu, 25 Aug 2016 17:49:40 +0200 Subject: [PATCH 26/44] fix wrong wrong coordinates of weather object The coordinates of the weather object should be the longitude and the latitude of the weather data set. Until this commit accidentily the coordinates of the request where passed to the weather module. --- oemof/db/coastdat.py | 49 ++++++++++++++++++++++---------------------- 1 file changed, 24 insertions(+), 25 deletions(-) diff --git a/oemof/db/coastdat.py b/oemof/db/coastdat.py index b2f4ab1..92fba06 100644 --- a/oemof/db/coastdat.py +++ b/oemof/db/coastdat.py @@ -28,25 +28,26 @@ def get_weather(conn, geometry, year): # Create MultiWeather # If polygon covers only one data set switch to SingleWeather sql_part = """ - SELECT sp.gid, ST_AsText(sp.geom) - FROM coastdat.cosmoclmgrid as sp + SELECT sp.gid, ST_AsText(point.geom), ST_AsText(sp.geom) + FROM coastdat.cosmoclmgrid AS sp + JOIN coastdat.spatial AS point ON (sp.gid=point.gid) WHERE st_intersects(ST_GeomFromText('{wkt}',4326), sp.geom) """.format(wkt=geometry.wkt) - df = fetch_raw_data(sql_weather_string(conn, geometry, year, sql_part), - conn, geometry) - obj = create_multi_weather(df, geometry, rename_dc) + df = fetch_raw_data(sql_weather_string(year, sql_part), conn, geometry) + obj = create_multi_weather(df, rename_dc) elif geometry.geom_type == 'Point': # Create SingleWeather sql_part = """ - SELECT sp.gid, ST_AsText(sp.geom) - FROM coastdat.cosmoclmgrid sp + SELECT sp.gid, ST_AsText(point.geom), ST_AsText(sp.geom) + FROM coastdat.cosmoclmgrid AS sp + JOIN coastdat.spatial AS point ON (sp.gid=point.gid) WHERE st_contains(sp.geom, ST_GeomFromText('{wkt}',4326)) """.format(wkt=geometry.wkt) - df = fetch_raw_data(sql_weather_string(conn, geometry, year, sql_part), - conn, geometry) - obj = create_single_weather(df, geometry, rename_dc) + df = fetch_raw_data(sql_weather_string(year, sql_part), conn, geometry) + obj = create_single_weather(df, rename_dc) else: logging.error('Unknown geometry type: {0}'.format(geometry.geom_type)) + obj = None return obj @@ -96,15 +97,16 @@ def fetch_raw_data(sql, connection, geometry): tmp_dc = {} weather_df = pd.DataFrame( connection.execute(sql).fetchall(), columns=[ - 'gid', 'geom', 'data_id', 'time_series', 'dat_id', 'type_id', - 'type', 'height', 'year', 'leap_year']).drop('dat_id', 1) + 'gid', 'geom_point', 'geom_polygon', 'data_id', 'time_series', + 'dat_id', 'type_id', 'type', 'height', 'year', 'leap_year']).drop( + 'dat_id', 1) # Get the timezone of the geometry tz = tools.tz_from_geom(connection, geometry) for ix in weather_df.index: # Convert the point of the weather location to a shapely object - weather_df.loc[ix, 'geom'] = wkt_loads(weather_df['geom'][ix]) + weather_df.loc[ix, 'geom_point'] = wkt_loads(weather_df['geom_point'][ix]) # Roll the dataset forward according to the timezone, because the # dataset is based on utc (Berlin +1, Kiev +2, London +0) @@ -129,8 +131,8 @@ def fetch_raw_data(sql, connection, geometry): return weather_df -def create_single_weather(df, geo, rename_dc): - 'Create an oemof weather object for the given geometry' +def create_single_weather(df, rename_dc): + """Create an oemof weather object for the given geometry""" my_weather = weather.FeedinWeather() data_height = {} name = None @@ -143,25 +145,22 @@ def create_single_weather(df, geo, rename_dc): name = row[1].gid my_weather.data = weather_df my_weather.timezone = weather_df.index.tz - if geo.geom_type == 'Point': - my_weather.longitude = geo.x - my_weather.latitude = geo.y - else: - my_weather.longitude = geo.centroid.x - my_weather.latitude = geo.centroid.y - my_weather.geometry = geo + my_weather.longitude = df.geom_point.iloc[0].x + my_weather.latitude = df.geom_point.iloc[0].y + my_weather.geometry = df.geom_point.iloc[0] my_weather.data_height = data_height my_weather.name = name return my_weather -def create_multi_weather(df, geo, rename_dc): - 'Create a list of oemof weather objects if the given geometry is a polygon' +def create_multi_weather(df, rename_dc): + """Create a list of oemof weather objects if the given geometry is a polygon + """ weather_list = [] # Create a pandas.DataFrame with the time series of the weather data set # for each data set and append them to a list. for gid in df.gid.unique(): gid_df = df[df.gid == gid] - obj = create_single_weather(gid_df, gid_df.geom.iloc[0], rename_dc) + obj = create_single_weather(gid_df, rename_dc) weather_list.append(obj) return weather_list From 63da624966f44d755806fe99831e8ec6f9db5580 Mon Sep 17 00:00:00 2001 From: uwe Date: Tue, 20 Sep 2016 15:20:24 +0200 Subject: [PATCH 27/44] add new tool function do read db-table as pandas DataFrame --- oemof/db/__init__.py | 1 + oemof/db/tools.py | 13 ++++++++++++- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/oemof/db/__init__.py b/oemof/db/__init__.py index 84fae2a..a89e9b6 100755 --- a/oemof/db/__init__.py +++ b/oemof/db/__init__.py @@ -2,6 +2,7 @@ from sqlalchemy import create_engine import keyring from . import config as cfg +from oemof.db.tools import db_table2pandas def url(section="postGIS"): """ Retrieve the URL used to connect to the database. diff --git a/oemof/db/tools.py b/oemof/db/tools.py index 83580d3..29bd569 100644 --- a/oemof/db/tools.py +++ b/oemof/db/tools.py @@ -10,6 +10,7 @@ """ import logging +import pandas as pd # get_polygon_from_nuts @@ -391,4 +392,14 @@ def change_owner_to(conn, schema, table, role): table=table, role=role) - conn.execute(sql_str) \ No newline at end of file + conn.execute(sql_str) + + +def db_table2pandas(conn, schema, table, columns=None): + if columns is None: + columns = '*' + sql = "SELECT {0} FROM {1}.{2};".format(columns, schema, table) + logging.debug("SQL query: {0}".format(sql)) + results = (conn.execute(sql)) + columns = results.keys() + return pd.DataFrame(results.fetchall(), columns=columns) From cd83e1158228a58d068245f2eaf1f6b429c3356e Mon Sep 17 00:00:00 2001 From: gplessm Date: Fri, 21 Oct 2016 15:45:15 +0200 Subject: [PATCH 28/44] Update version number --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 0080a7c..143ab71 100644 --- a/setup.py +++ b/setup.py @@ -3,7 +3,7 @@ from setuptools import find_packages, setup setup(name='oemof.db', - version='0.0.2', + version='0.0.4', description='The oemof database extension', namespace_package = ['oemof'], packages=find_packages(), From cfe3a04d3b2aed154dc2187fd628df9eaba169a6 Mon Sep 17 00:00:00 2001 From: gplessm Date: Fri, 21 Oct 2016 15:45:57 +0200 Subject: [PATCH 29/44] Add missing keyrings.alt dependency Close #18 --- setup.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 143ab71..228324a 100644 --- a/setup.py +++ b/setup.py @@ -11,4 +11,5 @@ install_requires=['sqlalchemy >= 1.0', 'keyring >= 4.0', 'shapely', - 'psycopg2']) + 'psycopg2', + 'keyrings.alt']) From ca1a31984202362ab551dddf141e8ccd71423a65 Mon Sep 17 00:00:00 2001 From: gplessm Date: Fri, 21 Oct 2016 15:46:07 +0200 Subject: [PATCH 30/44] Update whatsnew --- doc/whatsnew/v0004.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/whatsnew/v0004.txt b/doc/whatsnew/v0004.txt index b64112e..96100dd 100644 --- a/doc/whatsnew/v0004.txt +++ b/doc/whatsnew/v0004.txt @@ -19,6 +19,8 @@ Testing Bug fixes ######### + * Add missing keyrings.alt dependency + Other changes ############# From 2fb25b1172495a53ad8ea5ea82cc3d72c33f11d3 Mon Sep 17 00:00:00 2001 From: uvchik Date: Thu, 15 Dec 2016 13:47:30 +0100 Subject: [PATCH 31/44] remove additional roll_value because coastdat values are instantan values --- oemof/db/coastdat.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/oemof/db/coastdat.py b/oemof/db/coastdat.py index 92fba06..3fc5cbf 100644 --- a/oemof/db/coastdat.py +++ b/oemof/db/coastdat.py @@ -114,17 +114,13 @@ def fetch_raw_data(sql, connection, geometry): offset = int(utc.localize(datetime(2002, 1, 1)).astimezone( timezone(tz)).strftime("%z")[:-2]) - # Roll the dataset backwards because the first value (1. Jan, 0:00) - # contains the measurements of the hour before (coasDat2). - roll_value = offset - 1 - # Get the year and the length of the data array db_year = weather_df.loc[ix, 'year'] db_len = len(weather_df['time_series'][ix]) # Set absolute time index for the data sets to avoid errors. tmp_dc[ix] = pd.Series( - np.roll(np.array(weather_df['time_series'][ix]), roll_value), + np.roll(np.array(weather_df['time_series'][ix]), offset), index=pd.date_range(pd.datetime(db_year, 1, 1, 0), periods=db_len, freq='H', tz=tz)) weather_df['time_series'] = pd.Series(tmp_dc) From 2dc30930adaaf499248c9e06eeda252e732c893f Mon Sep 17 00:00:00 2001 From: uvchik Date: Thu, 15 Dec 2016 13:47:42 +0100 Subject: [PATCH 32/44] fix pep8 --- oemof/db/coastdat.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/oemof/db/coastdat.py b/oemof/db/coastdat.py index 3fc5cbf..fd3fc30 100644 --- a/oemof/db/coastdat.py +++ b/oemof/db/coastdat.py @@ -106,7 +106,8 @@ def fetch_raw_data(sql, connection, geometry): for ix in weather_df.index: # Convert the point of the weather location to a shapely object - weather_df.loc[ix, 'geom_point'] = wkt_loads(weather_df['geom_point'][ix]) + weather_df.loc[ix, 'geom_point'] = wkt_loads( + weather_df['geom_point'][ix]) # Roll the dataset forward according to the timezone, because the # dataset is based on utc (Berlin +1, Kiev +2, London +0) From 2f87d814a973f1e0ae456fde6d42947a5a72f017 Mon Sep 17 00:00:00 2001 From: gplessm Date: Thu, 15 Dec 2016 18:55:53 +0100 Subject: [PATCH 33/44] Add pandas as required package --- setup.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 228324a..58847b6 100644 --- a/setup.py +++ b/setup.py @@ -12,4 +12,5 @@ 'keyring >= 4.0', 'shapely', 'psycopg2', - 'keyrings.alt']) + 'keyrings.alt', + 'pandas >=0.19.1, <=0.19.1']) From 8f2977b8017f50240bc447f4b83b55320445abf7 Mon Sep 17 00:00:00 2001 From: gplessm Date: Thu, 15 Dec 2016 18:56:23 +0100 Subject: [PATCH 34/44] Fix overwritten error class --- oemof/db/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/oemof/db/__init__.py b/oemof/db/__init__.py index a89e9b6..e1d647f 100755 --- a/oemof/db/__init__.py +++ b/oemof/db/__init__.py @@ -1,4 +1,4 @@ -from configparser import NoOptionError as option, NoSectionError as section +from configparser import NoOptionError as option, NoSectionError from sqlalchemy import create_engine import keyring from . import config as cfg @@ -41,7 +41,7 @@ def url(section="postGIS"): "the oemof config or keyring." + "\nExiting.") exit(-1) - except section: + except NoSectionError: print("Unable to find the 'postGIS' section in oemof's config." + "\nExiting.") exit(-1) From 81332c5ae7237e4730b8b3630ad2a59fd1296967 Mon Sep 17 00:00:00 2001 From: gplessm Date: Thu, 15 Dec 2016 19:47:28 +0100 Subject: [PATCH 35/44] Add "ask user for password if is missing in config.ini/keyring" --- oemof/db/__init__.py | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/oemof/db/__init__.py b/oemof/db/__init__.py index e1d647f..fe2df6d 100755 --- a/oemof/db/__init__.py +++ b/oemof/db/__init__.py @@ -3,6 +3,7 @@ import keyring from . import config as cfg from oemof.db.tools import db_table2pandas +import getpass def url(section="postGIS"): """ Retrieve the URL used to connect to the database. @@ -30,17 +31,27 @@ def url(section="postGIS"): :ref:`configuring ` :mod:`oemof.db`. """ - pw = keyring.get_password(cfg.get(section, "database"), - cfg.get(section, "username")) + try: + pw = keyring.get_password(cfg.get(section, "database"), + cfg.get(section, "username")) + except NoSectionError as e: + print("There is no section {section} in your config file. Please " + "choose one available section from your config file or " + "specify a new one!".format( + section=section)) + exit(-1) + if pw is None: try: pw = cfg.get(section, "pw") except option: - print("Unable to find the database password in " + - "the oemof config or keyring." + - "\nExiting.") - exit(-1) + pw = getpass.getpass(prompt="No password available in your "\ + "keyring for database {database}. " + "\n\nEnter your password to " \ + "store it in " + "keyring:".format(database=section)) + keyring.set_password(section, cfg.get(section, "username"), pw) except NoSectionError: print("Unable to find the 'postGIS' section in oemof's config." + "\nExiting.") From 81b0a0ee5dfa17b63ae617435c1539a4ca59868e Mon Sep 17 00:00:00 2001 From: gplessm Date: Fri, 16 Dec 2016 12:46:01 +0100 Subject: [PATCH 36/44] Add optional definition of file (location) This includes some more additions regarding handling of input and error messages --- oemof/db/__init__.py | 21 +++++++++--- oemof/db/config.py | 77 +++++++++++++++++++++++++++++++++++++++++--- 2 files changed, 88 insertions(+), 10 deletions(-) diff --git a/oemof/db/__init__.py b/oemof/db/__init__.py index fe2df6d..5f2bd7a 100755 --- a/oemof/db/__init__.py +++ b/oemof/db/__init__.py @@ -5,7 +5,8 @@ from oemof.db.tools import db_table2pandas import getpass -def url(section="postGIS"): + +def url(section="postGIS", config_file=None): """ Retrieve the URL used to connect to the database. Use this if you have your own means of accessing the database and do not @@ -23,6 +24,9 @@ def url(section="postGIS"): The URL with which one can connect to the database. Be careful as this will probably contain sensitive data like the username/password combination. + config_file : str, optional + Relative of absolute of config.ini. If not specified, it tries to read + from .oemof/config.ini in your HOME dir Notes ----- @@ -31,6 +35,8 @@ def url(section="postGIS"): :ref:`configuring ` :mod:`oemof.db`. """ + cfg.load_config(config_file) + try: pw = keyring.get_password(cfg.get(section, "database"), cfg.get(section, "username")) @@ -65,7 +71,7 @@ def url(section="postGIS"): port=int(cfg.get(section, "port"))) -def engine(section="postGIS"): +def engine(section="postGIS", config_file=None): """Creates engine object for database access If keyword argument `section` is used it requires an existing config.ini @@ -76,6 +82,9 @@ def engine(section="postGIS"): section : str, optional Section (in config.ini) of targeted database containing connection details that are used to set up connection + config_file : str, optional + Relative of absolute of config.ini. If not specified, it tries to read + from .oemof/config.ini in your HOME dir Returns ------- @@ -88,10 +97,11 @@ def engine(section="postGIS"): For documentation on config.ini see the README section on :ref:`configuring ` :mod:`oemof.db`. """ - return create_engine(url(section)) + return create_engine(url(section, config_file=config_file)) -def connection(section="postGIS"): + +def connection(section="postGIS", config_file=None): """Database connection method of sqlalchemy engine object This function purely calls the `connect()` method of the engine object @@ -99,4 +109,5 @@ def connection(section="postGIS"): For description of parameters see :py:func:`engine`. """ - return engine(section=section).connect() + + return engine(section=section, config_file=config_file).connect() \ No newline at end of file diff --git a/oemof/db/config.py b/oemof/db/config.py index 481c2b9..dfc586e 100755 --- a/oemof/db/config.py +++ b/oemof/db/config.py @@ -29,7 +29,8 @@ """ -import os.path as path +import os +import logging try: import configparser as cp @@ -38,23 +39,88 @@ import ConfigParser as cp FILENAME = 'config.ini' -FILE = path.join(path.expanduser("~"), '.oemof', FILENAME) +FILE = os.path.join(os.path.expanduser("~"), '.oemof', FILENAME) cfg = cp.RawConfigParser() _loaded = False +def load_config(filename): + """ + Load data from config file to `cfg` that can be accessed by get, set + afterwards. + + Specify absolute or relative path to your config file. + + :param filename: Relative or absolute path + """ + + if filename is None: + filename = '' + + abs_filename = os.path.join(os.getcwd(), filename) + + global FILE + + # find the config file + if os.path.isfile(filename): + FILE = filename + elif os.path.isfile(abs_filename): + FILE = abs_filename + elif os.path.isfile(FILE): + pass + else: + if os.path.dirname(filename): + file_not_found = filename + else: + file_not_found = abs_filename + file_not_found_message(file_not_found) + + # load config + init(FILE) + + +def file_not_found_message(file_not_found): + """ + Show error message incl. help if file not found + + :param filename: + :return: + """ + + logging.error( + """ + Config file {file} cannot be found. Make sure this file exists! + + An exemplary section in the config file looks as follows + + [database] + username = username under which to connect to the database + database = name of the database from which to read + host = host to connect to + port = port to connect to + + For further advice, see in the docs (https://oemofdb.readthedocs.io) + how to format the config. + """.format(file=file_not_found)) + + def main(): pass -def init(): +def init(FILE): + """ + Read config file + + :param FILE: Absolute path to config file (incl. filename) + """ try: cfg.read(FILE) global _loaded _loaded = True except: - print("configfile not found.") + file_not_found_message(FILE) def get(section, key): @@ -71,8 +137,9 @@ def get(section, key): if no cast is successfull, the raw string will be returned. """ + # FILE = 'config_misc' if not _loaded: - init() + init(FILE) try: return cfg.getfloat(section, key) except Exception: From 2741618e6cdd0ede57c8c7e2a712e09124a6af6e Mon Sep 17 00:00:00 2001 From: gplessm Date: Fri, 16 Dec 2016 13:01:12 +0100 Subject: [PATCH 37/44] Fix docs --- oemof/db/config.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/oemof/db/config.py b/oemof/db/config.py index dfc586e..0bf887d 100755 --- a/oemof/db/config.py +++ b/oemof/db/config.py @@ -53,6 +53,7 @@ def load_config(filename): Specify absolute or relative path to your config file. :param filename: Relative or absolute path + :type filename: str """ if filename is None: @@ -85,7 +86,7 @@ def file_not_found_message(file_not_found): Show error message incl. help if file not found :param filename: - :return: + :type filename: str """ logging.error( @@ -114,6 +115,7 @@ def init(FILE): Read config file :param FILE: Absolute path to config file (incl. filename) + :type FILE: str """ try: cfg.read(FILE) From 1d26f8398bf59d691079412901ed1341e19dbbce Mon Sep 17 00:00:00 2001 From: gplessm Date: Fri, 16 Dec 2016 13:11:48 +0100 Subject: [PATCH 38/44] Fix copyright time period --- doc/conf.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/conf.py b/doc/conf.py index ba0be94..47fb524 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -52,7 +52,7 @@ # General information about the project. project = u'oemof.db' -copyright = u'2015, oemof developing group' +copyright = u'2015-2016, oemof developing group' author = u'Uwe Krien, oemof developing group' # The version info for the project you're documenting, acts as replacement for @@ -277,7 +277,7 @@ epub_title = u'oemof.db' epub_author = u'Uwe Krien, oemof developing group' epub_publisher = u'oemof developing group' -epub_copyright = u'2015, oemof developing group' +epub_copyright = u'2015-2016, oemof developing group' # The basename for the epub file. It defaults to the project name. #epub_basename = u'pahesmf' From 04ee93dcd0034b675dfd6d6e28c2cde6bbac9fa2 Mon Sep 17 00:00:00 2001 From: gplessm Date: Fri, 16 Dec 2016 13:17:03 +0100 Subject: [PATCH 39/44] Update whatsnew --- doc/whatsnew/v0004.txt | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/doc/whatsnew/v0004.txt b/doc/whatsnew/v0004.txt index 96100dd..26d67dc 100644 --- a/doc/whatsnew/v0004.txt +++ b/doc/whatsnew/v0004.txt @@ -1,11 +1,13 @@ -v0.0.4 (TBA) -+++++++++++++++++++++++++ +v0.0.4 (December 16, 2016) +++++++++++++++++++++++++++ New features ############ * Create an empty database table with primary key type serial in `tools.py` * Access grant to users/ group of users, see `grant_db_access` in `tools.py` + * Additional optional keyword argument to specify config file (`issue #21 `_) + * Ask for password and save store in keyring if not existent (`issue #22 `_) Documentation @@ -30,3 +32,4 @@ Contributors ############ * Guido Pleßmann + * Uwe Krien From 527cdb25ec4216ec65a49204301992e9036bc0a1 Mon Sep 17 00:00:00 2001 From: gplessm Date: Fri, 16 Dec 2016 13:38:13 +0100 Subject: [PATCH 40/44] Update README including badge to docs --- README.rst | 58 +++++++++++++++++++++++++++++++++++++++--------------- 1 file changed, 42 insertions(+), 16 deletions(-) diff --git a/README.rst b/README.rst index ddd6606..75348f1 100644 --- a/README.rst +++ b/README.rst @@ -1,36 +1,68 @@ +[![Documentation Status](https://readthedocs.org/projects/oemofdb/badge/?version=dev)](http://oemofdb.readthedocs.io/en/dev/?badge=dev) + An oemof extension to use the oemof related postgis database. To use this extension you have to have access to the oemof postgis database. -The oemof code will be released in early 2016 but you can try this extension -already with the feedinlib. If you are interested to join the oemof database -project please contact us. + Installation ++++++++++++ -Clone the repository to your local system. +Use pypi to install the latest oemof version. + +.. code:: bash + + pip3 install oemof + +If you want to have the developer version clone the repository by .. code:: bash git clone git@github.com:oemof/oemof.db.git -Then you can install it using pip3 with the -e flag. +and can install it using pip3 with the -e flag. .. code:: bash sudo pip3 install -e - .. _readme#configuration: -Configuration -+++++++++++++ +Keep `virtualenvs`_ in mind! + +.. _`keyring package`: https://virtualenv.pypa.io + +Configuration and usage ++++++++++++++++++++++++ As the purpose of this package is to facilitate usage of the ``oemof`` database, it needs to know how to connect to this database. Being part of -``oemof``, ``oemof.db`` looks for this configuration in the file ``config.ini`` -in a directory called ``.oemof`` in your home directory. +``oemof``, as fallback ``oemof.db`` always looks for this configuration in the +file ``config.ini`` in a directory called ``.oemof`` in your home directory. + +A particular config-file can either specified and accessed via + + +.. code-block:: python + + from oemof.db import cfg + + # only load config file + cfg.load_config(config_file=) + + # access config parameters + cfg.get(
, ) + +If you're interested in establishing a database connection and specify config +file connection parameters are stored in use + +.. code-block:: python + + from oemof.db import cfg + + # establish database connection with specified section and config_file + db.connection(section=
, config_file=) To configure database access this file has to have at least one dedicated section containing the necessary options, like this: @@ -62,9 +94,3 @@ where ``"database"`` and ``"username"`` have the same values as the corresponding options in ``config.ini``. .. _`keyring package`: https://pypi.python.org/pypi/keyring - -Required packages -+++++++++++++++++ - -* python3-sqlalchemy -* python3-keyring From 0c7b6914956233427b07abf32cc034b20d807e91 Mon Sep 17 00:00:00 2001 From: gplessm Date: Fri, 16 Dec 2016 13:44:01 +0100 Subject: [PATCH 41/44] Replace (not working) badge by link --- README.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.rst b/README.rst index 75348f1..b97815d 100644 --- a/README.rst +++ b/README.rst @@ -1,8 +1,8 @@ -[![Documentation Status](https://readthedocs.org/projects/oemofdb/badge/?version=dev)](http://oemofdb.readthedocs.io/en/dev/?badge=dev) - An oemof extension to use the oemof related postgis database. -To use this extension you have to have access to the oemof postgis database. +See `the documentation`_ for more information! + +.. _`the _the documentation: https://oemofdb.readthedocs.io From 378ec5c78803c581e99899ca3781bcb5f290a12d Mon Sep 17 00:00:00 2001 From: gplessm Date: Fri, 16 Dec 2016 13:45:32 +0100 Subject: [PATCH 42/44] Fix README link --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index b97815d..369eaa6 100644 --- a/README.rst +++ b/README.rst @@ -2,7 +2,7 @@ An oemof extension to use the oemof related postgis database. See `the documentation`_ for more information! -.. _`the _the documentation: https://oemofdb.readthedocs.io +.. _`the documentation: https://oemofdb.readthedocs.io From c8da7e06cf514f754b4e84352245efecd2ba64f0 Mon Sep 17 00:00:00 2001 From: gplessm Date: Fri, 16 Dec 2016 13:46:45 +0100 Subject: [PATCH 43/44] Fix readme again --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 369eaa6..9a363c9 100644 --- a/README.rst +++ b/README.rst @@ -2,7 +2,7 @@ An oemof extension to use the oemof related postgis database. See `the documentation`_ for more information! -.. _`the documentation: https://oemofdb.readthedocs.io +.. _`the documentation`: https://oemofdb.readthedocs.io From eb9aeaee6baebf0e773f3db04d61ae98f8ab334a Mon Sep 17 00:00:00 2001 From: gplessm Date: Fri, 16 Dec 2016 18:46:21 +0100 Subject: [PATCH 44/44] Update docs --- doc/api/coastdat.rst | 7 ------- doc/api/config.rst | 7 ------- doc/api/db.rst | 7 ------- doc/api/modules.rst | 9 +++------ doc/api/oemof.db.rst | 8 ++++++++ doc/api/tools.rst | 7 ------- doc/conf.py | 4 ++-- 7 files changed, 13 insertions(+), 36 deletions(-) delete mode 100644 doc/api/coastdat.rst delete mode 100644 doc/api/config.rst delete mode 100644 doc/api/db.rst delete mode 100644 doc/api/tools.rst diff --git a/doc/api/coastdat.rst b/doc/api/coastdat.rst deleted file mode 100644 index b2fede5..0000000 --- a/doc/api/coastdat.rst +++ /dev/null @@ -1,7 +0,0 @@ -coastdat module -=============== - -.. automodule:: coastdat - :members: - :undoc-members: - :show-inheritance: diff --git a/doc/api/config.rst b/doc/api/config.rst deleted file mode 100644 index f504a63..0000000 --- a/doc/api/config.rst +++ /dev/null @@ -1,7 +0,0 @@ -config module -============= - -.. automodule:: config - :members: - :undoc-members: - :show-inheritance: diff --git a/doc/api/db.rst b/doc/api/db.rst deleted file mode 100644 index 4f97568..0000000 --- a/doc/api/db.rst +++ /dev/null @@ -1,7 +0,0 @@ -db module -========= - -.. automodule:: db - :members: - :undoc-members: - :show-inheritance: diff --git a/doc/api/modules.rst b/doc/api/modules.rst index 094a7ca..580a494 100644 --- a/doc/api/modules.rst +++ b/doc/api/modules.rst @@ -1,10 +1,7 @@ -oemof_pg -======== +oemof +===== .. toctree:: :maxdepth: 4 - coastdat - config - db - tools + oemof diff --git a/doc/api/oemof.db.rst b/doc/api/oemof.db.rst index 4abd413..b24d297 100644 --- a/doc/api/oemof.db.rst +++ b/doc/api/oemof.db.rst @@ -36,6 +36,14 @@ oemof.db.powerplants module :undoc-members: :show-inheritance: +oemof.db.test_config_changes module +----------------------------------- + +.. automodule:: oemof.db.test_config_changes + :members: + :undoc-members: + :show-inheritance: + oemof.db.tools module --------------------- diff --git a/doc/api/tools.rst b/doc/api/tools.rst deleted file mode 100644 index 1fcb26f..0000000 --- a/doc/api/tools.rst +++ /dev/null @@ -1,7 +0,0 @@ -tools module -============ - -.. automodule:: tools - :members: - :undoc-members: - :show-inheritance: diff --git a/doc/conf.py b/doc/conf.py index ba0be94..de6d8e9 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -60,9 +60,9 @@ # built documents. # # The short X.Y version. -version = '0.0.8' +version = '0.0.4' # The full version, including alpha/beta/rc tags. -release = 'beta' +release = version # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages.