diff --git a/CHANGES.rst b/CHANGES.rst index d435ccee97..35c77dd068 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -27,6 +27,12 @@ heasarc no data associated with that row rather than filtering it out. [#3275] +utils.tap +^^^^^^^^^ + +- Get the cookie associated to the keys JSESSIONID or SESSION due to the tap library release at ESAC. [#3289] + + Infrastructure, Utility and Other Changes and Additions ------------------------------------------------------- diff --git a/astroquery/utils/tap/conn/tapconn.py b/astroquery/utils/tap/conn/tapconn.py index bad2576654..e28e260c00 100755 --- a/astroquery/utils/tap/conn/tapconn.py +++ b/astroquery/utils/tap/conn/tapconn.py @@ -17,14 +17,14 @@ import http.client as httplib import mimetypes +import os import platform +import requests import time -import os -from astroquery.utils.tap.xmlparser import utils -from astroquery.utils.tap import taputils -from astroquery import version -import requests +from astroquery import version +from astroquery.utils.tap import taputils +from astroquery.utils.tap.xmlparser import utils __all__ = ['TapConn'] @@ -485,6 +485,22 @@ def find_header(self, headers, key): """ return taputils.taputil_find_header(headers, key) + def find_all_headers(self, headers, key): + """Searches for the specified keyword + + Parameters + ---------- + headers : HTTP(s) headers object, mandatory + HTTP(s) response headers + key : str, mandatory + header key to be searched for + + Returns + ------- + A list of requested header values or an emtpy list if no header is found + """ + return taputils.taputil_find_all_headers(headers, key) + def dump_to_file(self, output, response): """Writes the connection response into the specified output @@ -585,7 +601,7 @@ def get_file_from_header(self, headers): if content_disposition is not None: p = content_disposition.find('filename="') if p >= 0: - filename = os.path.basename(content_disposition[p+10:len(content_disposition)-1]) + filename = os.path.basename(content_disposition[p + 10:len(content_disposition) - 1]) content_encoding = self.find_header(headers, 'Content-Encoding') if content_encoding is not None: @@ -722,7 +738,7 @@ def encode_multipart(self, fields, files): def __str__(self): return f"\tHost: {self.__connHost}\n\tUse HTTPS: {self.__isHttps}" \ - f"\n\tPort: {self.__connPort}\n\tSSL Port: {self.__connPortSsl}" + f"\n\tPort: {self.__connPort}\n\tSSL Port: {self.__connPortSsl}" class ConnectionHandler: diff --git a/astroquery/utils/tap/conn/tests/DummyConnHandler.py b/astroquery/utils/tap/conn/tests/DummyConnHandler.py index 551ff7a51c..02544c8765 100644 --- a/astroquery/utils/tap/conn/tests/DummyConnHandler.py +++ b/astroquery/utils/tap/conn/tests/DummyConnHandler.py @@ -138,6 +138,9 @@ def get_file_from_header(self, headers): def find_header(self, headers, key): return taputils.taputil_find_header(headers, key) + def find_all_headers(self, headers, key): + return taputils.taputil_find_all_headers(headers, key) + def execute_table_edit(self, data, content_type="application/x-www-form-urlencoded", verbose=False): diff --git a/astroquery/utils/tap/conn/tests/test_conn.py b/astroquery/utils/tap/conn/tests/test_conn.py index 06bbc803e1..ec7696822b 100644 --- a/astroquery/utils/tap/conn/tests/test_conn.py +++ b/astroquery/utils/tap/conn/tests/test_conn.py @@ -115,3 +115,59 @@ def test_login(): assert r.get_method() == 'POST' assert r.get_context() == context assert r.get_body() == data + + +def test_find_header(): + host = "testHost" + tap = TapConn(ishttps=False, host=host) + + headers = [('Date', 'Sat, 12 Apr 2025 05:10:47 GMT'), + ('Server', 'Apache/2.4.6 (Red Hat Enterprise Linux) OpenSSL/1.0.2k-fips mod_jk/1.2.43'), + ('Set-Cookie', 'JSESSIONID=E677B51BA5C4837347D1E17D4E36647E; Path=/data-server; Secure; HttpOnly'), + ('X-Content-Type-Options', 'nosniff'), ('X-XSS-Protection', '0'), + ('Cache-Control', 'no-cache, no-store, max-age=0, must-revalidate'), ('Pragma', 'no-cache'), + ('Expires', '0'), ('X-Frame-Options', 'SAMEORIGIN'), + ('Set-Cookie', 'SESSION=ZjQ3MjIzMDAt; Path=/data-server; Secure; HttpOnly; SameSite=Lax'), + ('Transfer-Encoding', 'chunked'), ('Content-Type', 'text/plain; charset=UTF-8')] + key = 'Set-Cookie' + result = tap.find_header(headers, key) + + assert (result == "JSESSIONID=E677B51BA5C4837347D1E17D4E36647E; Path=/data-server; Secure; HttpOnly") + + +def test_find_all_headers(): + host = "testHost" + tap = TapConn(ishttps=False, host=host) + + headers = [('Date', 'Sat, 12 Apr 2025 05:10:47 GMT'), + ('Server', 'Apache/2.4.6 (Red Hat Enterprise Linux) OpenSSL/1.0.2k-fips mod_jk/1.2.43'), + ('Set-Cookie', 'JSESSIONID=E677B51BA5C4837347D1E17D4E36647E; Path=/data-server; Secure; HttpOnly'), + ('X-Content-Type-Options', 'nosniff'), ('X-XSS-Protection', '0'), + ('Cache-Control', 'no-cache, no-store, max-age=0, must-revalidate'), ('Pragma', 'no-cache'), + ('Expires', '0'), ('X-Frame-Options', 'SAMEORIGIN'), + ('Set-Cookie', 'SESSION=ZjQ3MjIzMDAtNjNiYy00Mj; Path=/data-server; Secure; HttpOnly; SameSite=Lax'), + ('Transfer-Encoding', 'chunked'), ('Content-Type', 'text/plain; charset=UTF-8')] + key = 'Set-Cookie' + result = tap.find_all_headers(headers, key) + + assert (result[0] == "JSESSIONID=E677B51BA5C4837347D1E17D4E36647E; Path=/data-server; Secure; HttpOnly") + assert (result[1] == "SESSION=ZjQ3MjIzMDAtNjNiYy00Mj; Path=/data-server; Secure; HttpOnly; SameSite=Lax") + + +def test_get_file_from_header(): + host = "testHost" + tap = TapConn(ishttps=False, host=host) + + headers = [('Date', 'Sat, 12 Apr 2025 05:10:47 GMT'), + ('Server', 'Apache/2.4.6 (Red Hat Enterprise Linux) OpenSSL/1.0.2k-fips mod_jk/1.2.43'), + ('Set-Cookie', 'JSESSIONID=E677B51BA5C4837347D1E17D4E36647E; Path=/data-server; Secure; HttpOnly'), + ('X-Content-Type-Options', 'nosniff'), ('X-XSS-Protection', '0'), + ('Cache-Control', 'no-cache, no-store, max-age=0, must-revalidate'), ('Pragma', 'no-cache'), + ('Expires', '0'), ('X-Frame-Options', 'SAMEORIGIN'), + ('Set-Cookie', 'SESSION=ZjQ3MjIzMDAtNjNiYy00Mj; Path=/data-server; Secure; HttpOnly; SameSite=Lax'), + ('Transfer-Encoding', 'chunked'), ('Content-Type', 'text/plain; charset=UTF-8'), + ('Content-Disposition', 'filename="my_file.vot.gz"'), ('Content-Encoding', "gzip")] + + result = tap.get_file_from_header(headers) + + assert (result == "my_file.vot.gz") diff --git a/astroquery/utils/tap/core.py b/astroquery/utils/tap/core.py index 4bf9a6f07e..f13d515e6d 100755 --- a/astroquery/utils/tap/core.py +++ b/astroquery/utils/tap/core.py @@ -15,11 +15,10 @@ """ import getpass import os -import tempfile -from urllib.parse import urlencode - import requests +import tempfile from astropy.table.table import Table +from urllib.parse import urlencode from astroquery import log from astroquery.utils.tap import taputils @@ -661,16 +660,23 @@ def __extract_sync_subcontext(self, location): return location[pos:] def __findCookieInHeader(self, headers, *, verbose=False): - cookies = self.__connHandler.find_header(headers, 'Set-Cookie') + cookies = self.__connHandler.find_all_headers(headers, 'Set-Cookie') if verbose: print(cookies) - if cookies is None: + if not cookies: return None else: - items = cookies.split(';') - for i in items: - if i.startswith("JSESSIONID="): - return i + for cook in cookies: + items = cook.split(';') + for item in items: + if item.startswith("SESSION="): + return item + + for cook in cookies: + items = cook.split(';') + for item in items: + if item.startswith("JSESSIONID="): + return item return None def __parseUrl(self, url, *, verbose=False): diff --git a/astroquery/utils/tap/taputils.py b/astroquery/utils/tap/taputils.py index d1382288d1..051baaf301 100644 --- a/astroquery/utils/tap/taputils.py +++ b/astroquery/utils/tap/taputils.py @@ -49,6 +49,28 @@ def taputil_find_header(headers, key): return None +def taputil_find_all_headers(headers, key): + """Searches for the specified keyword + + Parameters + ---------- + headers : HTTP(s) headers object, mandatory + HTTP(s) response headers + key : str, mandatory + header key to be searched for + + Returns + ------- + A list of requested header values or an empty list if not header is found + """ + + result = list() + for entry in headers: + if key.lower() == entry[0].lower(): + result.append(entry[1]) + return result + + def taputil_create_sorted_dict_key(dictionaryObject): """Searches for the specified keyword diff --git a/astroquery/utils/tap/tests/test_tap.py b/astroquery/utils/tap/tests/test_tap.py index 507fb6ed98..d2cdaaff4d 100644 --- a/astroquery/utils/tap/tests/test_tap.py +++ b/astroquery/utils/tap/tests/test_tap.py @@ -12,22 +12,21 @@ Created on 30 jun. 2016 """ +import gzip from pathlib import Path from unittest.mock import patch from urllib.parse import quote_plus, urlencode -import gzip import numpy as np import pytest +from astropy.table import Table from requests import HTTPError -from astroquery.utils.tap.model.tapcolumn import TapColumn - +from astroquery.utils.tap import taputils from astroquery.utils.tap.conn.tests.DummyConnHandler import DummyConnHandler from astroquery.utils.tap.conn.tests.DummyResponse import DummyResponse from astroquery.utils.tap.core import TapPlus -from astroquery.utils.tap import taputils -from astropy.table import Table +from astroquery.utils.tap.model.tapcolumn import TapColumn def read_file(filename): @@ -115,8 +114,8 @@ def test_load_tables_parameters(): def test_load_table(): - connHandler = DummyConnHandler() - tap = TapPlus(url="http://test:1111/tap", connhandler=connHandler) + conn_handler = DummyConnHandler() + tap = TapPlus(url="http://test:1111/tap", connhandler=conn_handler) # No arguments with pytest.raises(Exception): @@ -128,7 +127,7 @@ def test_load_table(): tableName = "table1" fullQualifiedTableName = f"{tableSchema}.{tableName}" tableRequest = f"tables?tables={fullQualifiedTableName}" - connHandler.set_response(tableRequest, responseLoadTable) + conn_handler.set_response(tableRequest, responseLoadTable) with pytest.raises(Exception): tap.load_table(fullQualifiedTableName) @@ -871,26 +870,26 @@ def test_rename_table(): def __find_table(schemaName, tableName, tables): - qualifiedName = f"{schemaName}.{tableName}" - for table in (tables): - if table.get_qualified_name() == qualifiedName: + qualified_name = f"{schemaName}.{tableName}" + for table in tables: + if table.get_qualified_name() == qualified_name: return table # not found: raise exception - pytest.fail(f"Table '{qualifiedName}' not found") + pytest.fail(f"Table '{qualified_name}' not found") -def __find_column(columnName, columns): - for c in (columns): - if c.name == columnName: +def __find_column(column_name, columns): + for c in columns: + if c.name == column_name: return c # not found: raise exception - pytest.fail(f"Column '{columnName}' not found") + pytest.fail(f"Column '{column_name}' not found") -def __check_column(column, description, unit, dataType, flag): +def __check_column(column, description, unit, data_type, flag): assert column.description == description assert column.unit == unit - assert column.data_type == dataType + assert column.data_type == data_type assert column.flag == flag @@ -906,11 +905,11 @@ def __check_results_column(results, columnName, description, unit, def test_login(mock_login): conn_handler = DummyConnHandler() tap = TapPlus(url="http://test:1111/tap", connhandler=conn_handler) - tap.login("user", "password") + tap.login(user="user", password="password") assert (mock_login.call_count == 1) mock_login.side_effect = HTTPError("Login error") with pytest.raises(HTTPError): - tap.login("user", "password") + tap.login(user="user", password="password") assert (mock_login.call_count == 2) @@ -923,7 +922,7 @@ def test_login_gui(mock_login_gui, mock_login): assert (mock_login_gui.call_count == 0) mock_login_gui.side_effect = HTTPError("Login error") with pytest.raises(HTTPError): - tap.login("user", "password") + tap.login(user="user", password="password") assert (mock_login.call_count == 1) @@ -951,3 +950,33 @@ def test_upload_table(): tap.upload_table(upload_resource=table, table_name=table_name) assert str(exc_info.value) == f"Table name is not allowed to contain a dot: {table_name}" + + +def test___findCookieInHeader(): + conn_handler = DummyConnHandler() + tap = TapPlus(url="http://test:1111/tap", connhandler=conn_handler) + + headers = [('Date', 'Sat, 12 Apr 2025 05:10:47 GMT'), + ('Server', 'Apache/2.4.6 (Red Hat Enterprise Linux) OpenSSL/1.0.2k-fips mod_jk/1.2.43'), + ('Set-Cookie', 'JSESSIONID=E677B51BA5C4837347D1E17D4E36647E; Path=/data-server; Secure; HttpOnly'), + ('X-Content-Type-Options', 'nosniff'), ('X-XSS-Protection', '0'), + ('Cache-Control', 'no-cache, no-store, max-age=0, must-revalidate'), ('Pragma', 'no-cache'), + ('Expires', '0'), ('X-Frame-Options', 'SAMEORIGIN'), + ('Set-Cookie', 'SESSION=ZjQ3MjIzMDAtNjNiYy00Mj; Path=/data-server; Secure; HttpOnly; SameSite=Lax'), + ('Transfer-Encoding', 'chunked'), ('Content-Type', 'text/plain; charset=UTF-8')] + + result = tap._Tap__findCookieInHeader(headers) + + assert (result == "SESSION=ZjQ3MjIzMDAtNjNiYy00Mj") + + headers = [('Date', 'Sat, 12 Apr 2025 05:10:47 GMT'), + ('Server', 'Apache/2.4.6 (Red Hat Enterprise Linux) OpenSSL/1.0.2k-fips mod_jk/1.2.43'), + ('Set-Cookie', 'JSESSIONID=E677B51BA5C4837347D1E17D4E36647E; Path=/data-server; Secure; HttpOnly'), + ('X-Content-Type-Options', 'nosniff'), ('X-XSS-Protection', '0'), + ('Cache-Control', 'no-cache, no-store, max-age=0, must-revalidate'), ('Pragma', 'no-cache'), + ('Expires', '0'), ('X-Frame-Options', 'SAMEORIGIN'), + ('Transfer-Encoding', 'chunked'), ('Content-Type', 'text/plain; charset=UTF-8')] + + result = tap._Tap__findCookieInHeader(headers) + + assert (result == "JSESSIONID=E677B51BA5C4837347D1E17D4E36647E") diff --git a/docs/gaia/gaia.rst b/docs/gaia/gaia.rst index e773bd5ad0..4fc1565059 100644 --- a/docs/gaia/gaia.rst +++ b/docs/gaia/gaia.rst @@ -867,6 +867,7 @@ The following example shows how to retrieve the DataLink products associated wit ... data_release=data_release, retrieval_type=retrieval_type, data_structure=data_structure) The DataLink products are stored inside a Python Dictionary. Each of its elements (keys) contains a one-element list that can be extracted as follows: + .. code-block:: python >>> dl_keys = [inp for inp in datalink.keys()]