Merge pull request #9 from ie3-institute/df/#8-sql-coordinates-source

Sql coordinates source

Author: t-ober

.github/workflows/simple-ci.yml renamed to .github/workflows/ci.yml

@@ -16,7 +16,7 @@ jobs:
       - uses: actions/checkout@v3
       - uses: actions/setup-python@v4
         with:
-            python-version: '3.10'
+            python-version: '3.12'
       #----------------------------------------------
       #        load pip cache if cache exists
       #----------------------------------------------
@@ -40,7 +40,7 @@ jobs:
       fail-fast: true
       matrix:
         os: [ "ubuntu-latest", "macos-latest" ]
-        python-version: [ "3.10"]
+        python-version: [ "3.12"]
     runs-on: ${{ matrix.os }}
     steps:
       #----------------------------------------------
@@ -54,13 +54,38 @@ jobs:
         with:
           python-version: ${{ matrix.python-version }}
       #----------------------------------------------
+      #       Install PostgreSQL dependencies
+      #----------------------------------------------
+      - name: Install PostgreSQL (Ubuntu)
+        if: matrix.os == 'ubuntu-latest'
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y libpq-dev postgresql postgresql-contrib
+
+      - name: Install PostgreSQL (macOS)
+        if: matrix.os == 'macos-latest'
+        run: |
+          brew install postgresql@14
+          brew link --force postgresql@14
+          export PATH="/usr/local/opt/postgresql@14/bin:$PATH"
+          echo 'export PATH="/usr/local/opt/postgresql@14/bin:$PATH"' >> ~/.bash_profile
+
+      #----------------------------------------------
       #  -----  install & configure poetry  -----
       #----------------------------------------------
       - name: Install Poetry
         uses: snok/install-poetry@v1
         with:
           virtualenvs-create: true
           virtualenvs-in-project: true
+          version: 1.6.1
+
+      #----------------------------------------------
+      #       Check Poetry version
+      #----------------------------------------------
+      - name: Check Poetry version
+        run: poetry --version
+
       #----------------------------------------------
       #       load cached venv if cache exists
       #----------------------------------------------
@@ -75,12 +100,10 @@ jobs:
       #----------------------------------------------
       - name: Install dependencies
         if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
-        run: poetry install --no-interaction --no-root
-      #----------------------------------------------
-      # install your root project, if required
-      #----------------------------------------------
-      - name: Install library
-        run: poetry install --no-interaction
+        run: |
+          # Try with --no-update, fallback if not supported
+          poetry lock --no-update || poetry lock
+          poetry install --no-root --no-interaction
       #----------------------------------------------
       #    run test suite
       #----------------------------------------------

.idea/misc.xml

@@ -1,4 +1,5 @@
 # Changelog
+
 All notable changes to this project will be documented in this file.
 
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
@@ -8,10 +9,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Added
 - Implementing auto-merge for dependabot PRs [#12](https://github.com/ie3-institute/copernicusWeather2psdmWeather/issues/12)
+- Initialisation of this converter [#1](https://github.com/ie3-institute/copernicusWeather2psdmWeather/issues/1)
 
 ### Changed
 - Disabled auto-merge for minor Dependabot updates [#20](https://github.com/ie3-institute/copernicusWeather2psdmWeather/issues/20)
 
+### Removed
+
 ### Fixed
 - Fixed implementation of dependabot auto-merge feature [#14](https://github.com/ie3-institute/copernicusWeather2psdmWeather/issues/14)
 

.idea/python-template.iml

@@ -1,51 +1,36 @@
-# python-template
+# copernicusWeather2psdmWeather
 
-Python template that includes all the basics for your new shiny project.
+Converts [copernicus.eu](https://cds.climate.copernicus.eu/) netCDF4 weather data to [PowerSystemDataModel](https://github.com/ie3-institute/PowerSystemDataModel) weather data format.
 
-## Setup 
+## How To Use
 
-1. Open a new repository while using this template.
-This repository is marked as a template repository.
-Github offers you the convenient possibility to open a new repository from a such a template repository.
-See [here](https://docs.github.com/en/repositories/creating-and-managing-repositories/creating-a-repository-from-a-template) for details.
-2. Clone your new repository
-3. Rename the `python-project` package to your project name (this is where your source code should go). 
-4. Install the dependencies (preferably within a fresh virtual environment)
-5. Run `pre-commit install`
-6. Enjoy!
+### Config File
+This tool can be used by giving parameters through config file `config.yaml`.
 
-## What's in here?
+Here the parameter of the PostGreSql-Database can be assigned:
+- `db_user`: Database username, e.g. `postgres`
+- `db_password`: Database password of username, e.g. `postgres`
+- `db_port`: Database port, e.g. `5432`
+- `db_name`: Name of the database, `westfalia_2023_03_24-2023_03_28`
+- `input_dir`: Location of the CDS input files, e.g. `input/samples`
+- `file_name_base`: First part of the input files, see comment below. e.g. `westfalia_2023_03_24-2023_03_28`
 
-### Formatting
+So far we support the following format of input files:
+- netCDF4
 
-To spare you the pain of manually taking care of formatting your code and establish some consistency (remember "äußere Ordnung führt zu innerer Ordnung") it includes [Black](https://black.readthedocs.io/en/stable/) for automated formatting. 
-Execute `black .` in the terminal at the project's root directory to format your code.
+To start converter
+- Adapt config file
+- Setup PostGreSql database
+- Start `main.py`
 
-### Testing
+### netCDF4 Input files
+Climate Data Store (CDS) provides the weather data in two files, which file names ends on '_accum.nc' and '_instant.nc'. Sample data can be found in `input/samples`.
 
-To make sure your code actually does what you think it does! 
-The template includes [pytest](https://docs.pytest.org/en/7.1.x/) for that purpose. 
-Include the tests in the `tests` directory. Running `pytest` in the terminal at the project's root directory executes all tests and hopefully replaces hoping that your code works by knowing (at least so far as your actual test all relevant cases and boundary conditions). 
+#### File name base
+Weather in netCFD format are provided by Copernicus in two files ending with `_accum.nc` and `_instant.nc`.
+This parameter allows to set the file_name_base.
+E.g. `weather_data_accum.nc` and `weather_data_instant.nc`. file_name_base would be `weather_data`.
 
-### CI
-
-To give you a slap on the wrist if you did not apply the formatter or your tests don't pass there is a [Github Actions](https://github.com/features/actions) workflow at `.github/workflows/simple-ci.yml`. 
-After pushing to the remote repository it checks out your code, installs all the dependencies then runs `flake8` for any code style issues and `black --check` to check the formatting. 
-Depending on how that went you will see yourself either confronted with a friendly green tick or rather unfriendly red cross at the code window in github (or the `Actions` section of the github repository.) 
-
-### Pre-Commit
-
-To reduce slaps on the wrist there is [pre-commit](https://pre-commit.com/) having your back. 
-It installs pre-commit hooks that `black` and `flake8` before the CI gets a chance to complain.
-The types of git hooks are configured in `.pre-commit-config.yaml`.
-To install the hooks run `pre-commit install` once in the terminal at the project's root directory.
-
-### Dependabot
-
-To keep the dust off your dependencies [Dependabot](https://github.blog/2020-06-01-keep-all-your-packages-up-to-date-with-dependabot/) checks for new versions once a day (according to current cofigurations) and opens Pull-Requests in case it finds any newer versions.
-The configuration can be changed at `.github/dependabot.yml`
-Keep in mind that dependency updates can break your code.
-You safeguard yourself against this by making sure you have all your code tested and the dependency pull request does not break any of them.
 
 ## Something Missing? 
 

CHANGELOG.md

@@ -0,0 +1,9 @@
+db_user: "postgres"
+db_port: 5432
+db_password: "postgres"
+db_name: "PLACEHOLDER_DB_NAME"
+input_dir: "input"
+#Weather in netCFD format are provided by Copernicus in two files ending with `_accum.nc` and `_instant.nc`.
+#This parameter allows to set the file_name_base.
+#E.g. `weather_data_accum.nc` and `weather_data_instant.nc`. file_name_base would be `weather_data`
+file_name_base: "WEATHER_DATA_INPUT_FILE_BASE_NAME"

README.md

@@ -0,0 +1,26 @@
+import numpy as np
+from netCDF4 import Dataset
+from pypsdm.db.weather.models import Coordinate
+from sqlmodel import Session
+
+
+def create_coordinates_df(weather: Dataset, session: Session):
+    lats = np.asarray(weather.variables["latitude"])
+    lons = np.asarray(weather.variables["longitude"])
+
+    coordinates = []
+    idx_to_id = {}
+
+    coord_id = 0
+    for lat_idx, lat in enumerate(lats):
+        for lon_idx, lon in enumerate(lons):
+            coord = Coordinate.from_xy(coord_id, float(lon), float(lat))
+            coordinates.append(coord)
+
+            idx_to_id[(lat_idx, lon_idx)] = coord_id
+            coord_id += 1
+
+    session.add_all(coordinates)
+    session.commit()
+
+    return idx_to_id

config.yaml

@@ -0,0 +1 @@
+docker run --name PLACEHOLDER_DB_NAME -e POSTGRES_USER=postgres -e POSTGRES_PASSWORD=postgres -e POSTGRES_DB=PLACEHOLDER_DB_NAME -v C:\docker\PLACEHOLDER_DB_NAME/var/lib/postgresql/data -p 5432:5432 -d postgis/postgis:17-3.4