Scriptification

.envsample

@@ -0,0 +1,3 @@
+INI_FILE_PATH=config/sample.yml
+CLIENT_ID=XXX
+CLIENT_SECRET=XXX

.gitignore

@@ -1,3 +1,4 @@
+.idea
 __pycache__
 .python-version
 .tox
@@ -18,13 +19,20 @@ dist
 *.sublime-project
 *.sublime-workspace
 venv/
-manip
-test-data/
-manip.txt
 demos/
 *.epc
 *.h5
 *.xml
 *.txt
 output*
-*.log
+*.log
+
+.env
+config/*_.yml
+config/*_.yaml
+
+
+downloads
+manip
+test-data/
+manip.txt

README.md

@@ -24,46 +24,102 @@ poetry update
 poetry install
 ```
 
+## Connection to a server : 
+
+You must fill a configuration file (see. config/sample.yml):
+```yaml
+url: wss://XXX
+port: 80
+username: XXX
+password: XXX
+additional-headers:
+  - data-partition-id: osdu
+
+token-url: https://XXX
+token-grant-type: client_credentials
+token-scope: openid profile email
+token-refresh_token: XXX
+```
+
+Then you must set an environment variable to refer it : **INI_FILE_PATH**.
+Or you can fill a .env file : 
+```dotenv
+INI_FILE_PATH=config/sample.yml
+CLIENT_ID=XXX
+CLIENT_SECRET=XXX
+```
+
 ## Sample commands :
 
+### Interactive client : 
+
 ```bash
-poetry run python client --host RDDMS_HOST --port 9002 -t MY_TOKEN
+poetry run client
+```
 
-poetry run python client --host MY_HOST --port 80 --sub-path etp -t MY_TOKEN
 
-poetry run python client --host 127.0.0.1 --port 17000 --sub-path etp --username login --password passwordTest
+### Run a specific script (here *download_xmls* that downloads all xml from a server) : 
+```bash
+poetry run script download_xmls
+```
 
-poetry run python client --host 127.0.0.1 --port 5432 --username testerlogin --password passwordtester
+### Run a pre-written script :
+```bash
+poetry run file_script .\script_sample.txt
+```
+Example of a script file : 
+```bash
+getresources eml:///
+
+# a commented line, a line starting with a '#' will not be executed
+
+getdataspaces
+
+quit
 ```
 
 
 ## ETP supported commands : 
 
-When the client is connected you can send your request.
+When the *interactive* client is connected you can send your request (this commands are the same for the *script files*).
 
 This is the help menu :
 ```bash
 [XXX] : replace XXX with your value
 [XXX=Y] : replace XXX with your value, default is Y
 [[XXX]] : optional parameter
 
-Help : show this menu
+[URI] for dataspaces can sometimes be set as "eml:///dataspace('DATASPACE_NAME')" but also with only the DATASPACE_NAME.
+
+    Help : show this menu
+
+    Quit : hard quit (no CloseSession sent)
+    CloseSession : close this session
+
+    Authorize             [ [TOKEN] | ( [USERNAME] [PASSWORD]) ]
+    RequestSession
+
+    GetDataArrayMetadata  [URI] [PATH_IN_RESOURCE]
+    GetDataArray          [URI] [PATH_IN_RESOURCE]
+    GetDataSubArray       [URI] [PATH_IN_RESOURCE] [START] [COUNT]
+    PutDataArray          [DATASPACE_NAME] [EPC_FILE_PATH] [H5_FILE_PATH] [[UUIDS]]*
+    PutDataArray_filter   [DATASPACE_NAME] [EPC_FILE_PATH] [H5_FILE_PATH] [[REGEX_TYPE_FILTER]]
+
+    GetDataObject         [URI_1] [...] [URI_N]
+    DeleteDataObjects     [URI_1] [...] [URI_N]
+    PutDataObject         [FILE_PATH] [[DATASPACE_NAME]] [[UUIDS]]*
 
-Quit : hard quit (no CloseSession sent)
-CloseSession : close this session
+    GetResources          [[uri=eml:/// or notUri=DataspaceName]] [[depth=1]] [[SCOPE]]
+    GetDeletedResources   [[uri=eml:/// or notUri=DataspaceName]] [[DELETE_TIME_FILTER]] [[DATA_OBJECT_TYPES]]*
 
-GetDataArrayMetadata  [URI] [PATH_IN_RESOURCE]
-GetDataArray          [URI] [PATH_IN_RESOURCE]
-GetDataSubArray       [URI] [PATH_IN_RESOURCE] [START] [COUNT]
-PutDataArray          [[UUIDS]]* [DATASPACE_NAME] [EPC_FILE_PATH] [H5_FILE_PATH]
+    GetDataspaces
+    PutDataspace          [NAME]
+    DeleteDataspace       [NAME]*
 
-GetDataObject         [URI_1] [...] [URI_N]
-PutDataObject         [FILE_PATH] [[DATASPACE_NAME]]
-GetResources          [[uri=eml:/// or notUri=DataspaceName]] [[depth=1]] [[SCOPE]]
+    Download              [OUTPUT_FILE_PATH] [DATASPACE_NAME]
+    DownloadObject        [OUTPUT_FOLDER_PATH] [URI]
 
-GetDataspaces
-PutDataspace          [NAME]
-DeleteDataspace       [NAME]
+    GetSupportedTypes     [URI] [[COUNT=True]] [[RETURN_EMPTY_TYPES=True]] [[SCOPE=Self]]
 ```
 
 ## Configuration

config/sample.yml

@@ -0,0 +1,11 @@
+url: wss://XXX
+port: 80
+username: XXX
+password: XXX
+additional-headers:
+  - data-partition-id: osdu
+
+token-url: https://XXX
+token-grant-type: client_credentials
+token-scope: openid profile email
+token-refresh_token:

etpclient/etp/h5_handler.py

@@ -2,19 +2,20 @@
 # Copyright (c) 2022-2023 Geosiris.
 # SPDX-License-Identifier: Apache-2.0
 #
+import logging
+from typing import Dict
+
 import h5py
+from etptypes.energistics.etp.v12.protocol.data_array.put_data_arrays import (
+    PutDataArrays,
+)
 
 from etpclient.utils import (
     search_all_element_value,
-    get_xml_dict_from_path,
     get_xml_dict_from_string,
     xml_dict_get_uri,
 )
 
-from etptypes.energistics.etp.v12.protocol.data_array.put_data_arrays import (
-    PutDataArrays,
-)
-
 
 def descend_obj(obj, sep="\t"):
     """
@@ -60,25 +61,36 @@
     for path_in_hdf in search_all_element_value(
         obj_dict, "PathInExternalFile"
     ) + search_all_element_value(obj_dict, "PathInHdfFile"):
-        print(f"search in h5 {path_in_hdf}")
-        data, shape, dtype = h5_search_dataset(h5_file_path, path_in_hdf)
-        print(f"\t==> shape {shape}")
-        pda_dict = {
-            "dataArrays": {
-                "0": {
-                    "uid": {
-                        "uri": xml_dict_get_uri(obj_dict, dataspace),
-                        "pathInResource": path_in_hdf,
-                    },
-                    "array": {
-                        "dimensions": list(shape),
-                        "data": {"item": {"values": data.flatten().tolist()}},
-                    },
-                    "customData": {},
+        logging.info(f"search in h5 {path_in_hdf}")
+        try:
+            data, shape, dtype = h5_search_dataset(h5_file_path, path_in_hdf)
+            logging.info(f"\t==> shape {shape}")
+            pda_dict = {
+                "dataArrays": {
+                    "0": {
+                        "uid": {
+                            "uri": xml_dict_get_uri(obj_dict, dataspace),
+                            "pathInResource": path_in_hdf,
+                        },
+                        "array": {
+                            "dimensions": list(shape),
+                            "data": {"item": {"values": data.flatten().tolist()}},
+                        },
+                        "customData": {},
+                    }
                 }
             }
-        }
-        res.append(PutDataArrays.parse_obj(pda_dict))
+            res.append(PutDataArrays.parse_obj(pda_dict))
+        except Exception as e:
+            logging.error("Failed to find data")
+            logging.error(e)
         # res.append(h5_search_dataset(h5_file_path, path_in_hdf))
 
     return res
+
+
+def write_h5(h5_path: str, arrays: Dict):
+    with h5py.File(h5_path, "a") as f:
+        for d_name, d_array in arrays.items():
+            dset = f.create_dataset(d_name, d_array.shape, d_array.dtype)
+            dset[()] = d_array