diff --git a/CHANGES.rst b/CHANGES.rst index b0d7964ee2..f6171ec9c2 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -23,6 +23,12 @@ alma - Bug fix in ``footprint_to_reg`` that did not allow regions to be plotted. [#3285] + +gaia +^^^^ + +- New method cross_match_basic that simplifies the positional x-match method [#3320] + linelists.cdms ^^^^^^^^^^^^^^ diff --git a/astroquery/esa/euclid/tests/test_euclidtap.py b/astroquery/esa/euclid/tests/test_euclidtap.py index 4912c695f0..de83c6af25 100644 --- a/astroquery/esa/euclid/tests/test_euclidtap.py +++ b/astroquery/esa/euclid/tests/test_euclidtap.py @@ -266,7 +266,7 @@ def test_load_table(): responseLaunchJob = DummyResponse(200) responseLaunchJob.set_data(method='GET', context=None, body=TABLE_DATA, headers=None) - table = 'my_table' + table = 'schema.my_table' conn_handler.set_response(f"tables?tables={table}", responseLaunchJob) tap = EuclidClass(tap_plus_conn_handler=conn_handler, datalink_handler=tap_plus, show_server_messages=False) diff --git a/astroquery/gaia/core.py b/astroquery/gaia/core.py index fea988e05d..fb7a2ea494 100644 --- a/astroquery/gaia/core.py +++ b/astroquery/gaia/core.py @@ -4,14 +4,9 @@ Gaia TAP plus ============= -@author: Juan Carlos Segovia -@contact: juan.carlos.segovia@sciops.esa.int - European Space Astronomy Centre (ESAC) European Space Agency (ESA) -Created on 30 jun. 2016 -Modified on 18 Ene. 2022 by mhsarmiento """ import datetime import json @@ -795,7 +790,7 @@ def __getQuantityInput(self, value, msg): if value is None: raise ValueError(f"Missing required argument: {msg}") if not (isinstance(value, str) or isinstance(value, units.Quantity)): - raise ValueError(f"{msg} must be either a string or astropy.coordinates") + raise ValueError(f"{msg} must be either a string or astropy.coordinates: {type(value)}") if isinstance(value, str): return Quantity(value) @@ -853,15 +848,170 @@ def load_user(self, user_id, *, verbose=False): return self.is_valid_user(user_id=user_id, verbose=verbose) + def cross_match_basic(self, *, table_a_full_qualified_name, table_a_column_ra, table_a_column_dec, + table_b_full_qualified_name=MAIN_GAIA_TABLE, table_b_column_ra=MAIN_GAIA_TABLE_RA, + table_b_column_dec=MAIN_GAIA_TABLE_DEC, results_name=None, + radius=1.0, background=False, verbose=False): + """Performs a positional cross-match between the specified tables. + + This method simples the execution of the method `cross_match` since it carries out the following steps in one + step: + + 1. updates the user table metadata to flag the positional RA/Dec columns; + 2. launches a positional cross-match as an asynchronous query; + 3. returns all the columns from both tables plus the angular distance (deg) for the cross-matched sources. + + The result is a join table with the identifies of both tables and the distance (degrees), that is returned + without metadata units. If desired, units can be added using the Units package of Astropy as follows: + results[‘separation’].unit = u.degree. To speed up the cross-match, pass the biggest table to the + ``table_b_full_qualified_name`` parameter. + TAP+ only + + Parameters + ---------- + table_a_full_qualified_name : str, mandatory + a full qualified table name (i.e. schema name and table name) + table_a_column_ra : str, mandatory + the ‘ra’ column in the table table_a_full_qualified_name + table_a_column_dec : str, mandatory + the ‘dec’ column in the table table_a_full_qualified_name + table_b_full_qualified_name : str, optional, default MAIN_GAIA_TABLE + a full qualified table name (i.e. schema name and table name) + table_b_column_ra : str, optional, default MAIN_GAIA_TABLE_RA + the ‘ra’ column in the table table_b_full_qualified_name + table_b_column_dec : str, default MAIN_GAIA_TABLE_DEC + the ‘dec’ column in the table table_b_full_qualified_name + results_name : str, optional, default None + custom name defined by the user for the job that is going to be created + radius : float (arc. seconds), str or astropy.coordinate, optional, default 1.0 + radius (valid range: 0.1-10.0). For an astropy.coordinate any angular unit is valid, but its value in arc + sec must be contained within the valid range. + background : bool, optional, default 'False' + when the job is executed in asynchronous mode, this flag specifies + whether the execution will wait until results are available + verbose : bool, optional, default 'False' + flag to display information about the process + + Returns + ------- + A Job object + """ + + radius_quantity = self.__get_radius_as_quantity_arcsec(radius) + + radius_arc_sec = radius_quantity.value + + if radius_arc_sec < 0.1 or radius_arc_sec > 10.0: + raise ValueError(f"Invalid radius value. Found {radius_quantity}, valid range is: 0.1 to 10.0") + + schema_a = self.__get_schema_name(table_a_full_qualified_name) + if not schema_a: + raise ValueError(f"Schema name is empty in full qualified table: '{table_a_full_qualified_name}'") + + table_b_full_qualified_name = table_b_full_qualified_name or self.MAIN_GAIA_TABLE or conf.MAIN_GAIA_TABLE + + schema_b = self.__get_schema_name(table_b_full_qualified_name) + if not schema_b: + raise ValueError(f"Schema name is empty in full qualified table: '{table_b_full_qualified_name}'") + + table_metadata_a = self.__get_table_metadata(table_a_full_qualified_name, verbose) + + table_metadata_b = self.__get_table_metadata(table_b_full_qualified_name, verbose) + + self.__check_columns_exist(table_metadata_a, table_a_full_qualified_name, table_a_column_ra, table_a_column_dec) + + self.__update_ra_dec_columns(table_a_full_qualified_name, table_a_column_ra, table_a_column_dec, + table_metadata_a, verbose) + + self.__check_columns_exist(table_metadata_b, table_b_full_qualified_name, table_b_column_ra, table_b_column_dec) + + self.__update_ra_dec_columns(table_b_full_qualified_name, table_b_column_ra, table_b_column_dec, + table_metadata_b, verbose) + + query = ( + f"SELECT a.*, DISTANCE(a.{table_a_column_ra}, a.{table_a_column_dec}, b.{table_b_column_ra}, " + f"b.{table_b_column_dec}) AS separation, b.* " + f"FROM {table_a_full_qualified_name} AS a JOIN {table_b_full_qualified_name} AS b " + f"ON DISTANCE(a.{table_a_column_ra}, a.{table_a_column_dec}, b.{table_b_column_ra}, b.{table_b_column_dec})" + f" < {radius_quantity.to(u.deg).value}") + + return self.launch_job_async(query=query, + name=results_name, + output_file=None, + output_format="votable_gzip", + verbose=verbose, + dump_to_file=False, + background=background, + upload_resource=None, + upload_table_name=None) + + def __get_radius_as_quantity_arcsec(self, radius): + """ + transform the input radius into an astropy.Quantity in arc seconds + """ + if not isinstance(radius, units.Quantity): + radius_quantity = Quantity(value=radius, unit=u.arcsec) + else: + radius_quantity = radius.to(u.arcsec) + return radius_quantity + + def __update_ra_dec_columns(self, full_qualified_table_name, column_ra, column_dec, table_metadata, verbose): + """ + Update table metadata for the ‘ra’ and the ‘dec’ columns in the input table + """ + if full_qualified_table_name.startswith("user_"): + list_of_changes = list() + for column in table_metadata.columns: + if column.name == column_ra and column.flags != '1': + list_of_changes.append([column_ra, "flags", "Ra"]) + list_of_changes.append([column_ra, "indexed", True]) + if column.name == column_dec and column.flags != '2': + list_of_changes.append([column_dec, "flags", "Dec"]) + list_of_changes.append([column_dec, "indexed", True]) + + if list_of_changes: + TapPlus.update_user_table(self, table_name=full_qualified_table_name, list_of_changes=list_of_changes, + verbose=verbose) + + def __check_columns_exist(self, table_metadata_a, full_qualified_table_name, column_ra, column_dec): + """ + Check whether the ‘ra’ and the ‘dec’ columns exists the input table + """ + column_names = [column.name for column in table_metadata_a.columns] + if column_ra not in column_names or column_dec not in column_names: + raise ValueError( + f"Please check: columns {column_ra} or {column_dec} not available in the table '" + f"{full_qualified_table_name}'") + + def __get_table_metadata(self, full_qualified_table_name, verbose): + """ + Get the table metadata for the input table + """ + try: + table_metadata = self.load_table(table=full_qualified_table_name, verbose=verbose) + except Exception: + raise ValueError(f"Not found table '{full_qualified_table_name}' in the archive") + return table_metadata + + def __get_schema_name(self, full_qualified_table_name): + """ + Get the schema name from the full qualified table + """ + schema = taputils.get_schema_name(full_qualified_table_name) + if schema is None: + raise ValueError(f"Not found schema name in full qualified table: '{full_qualified_table_name}'") + return schema + def cross_match(self, *, full_qualified_table_name_a, full_qualified_table_name_b, results_table_name, radius=1.0, background=False, verbose=False): - """Performs a cross-match between the specified tables - The result is a join table (stored in the user storage area) - with the identifies of both tables and the distance. + """Performs a positional cross-match between the specified tables. + + The result is a join table (stored in the user storage area) with the identifies of both tables and the + distance. TAP+ only Parameters @@ -872,8 +1022,9 @@ def cross_match(self, *, full_qualified_table_name_a, a full qualified table name (i.e. schema name and table name) results_table_name : str, mandatory a table name without schema. The schema is set to the user one - radius : float (arc. seconds), optional, default 1.0 - radius (valid range: 0.1-10.0) + radius : float (arc. seconds), str or astropy.coordinate, optional, default 1.0 + radius (valid range: 0.1-10.0). For an astropy.coordinate any angular unit is valid, but its value in arc + sec must be contained within the valid range. background : bool, optional, default 'False' when the job is executed in asynchronous mode, this flag specifies whether the execution will wait until results are available @@ -884,24 +1035,26 @@ def cross_match(self, *, full_qualified_table_name_a, ------- A Job object """ - if radius < 0.1 or radius > 10.0: - raise ValueError(f"Invalid radius value. Found {radius}, valid range is: 0.1 to 10.0") - schemaA = taputils.get_schema_name(full_qualified_table_name_a) - if schemaA is None: - raise ValueError(f"Not found schema name in full qualified table A: '{full_qualified_table_name_a}'") - tableA = taputils.get_table_name(full_qualified_table_name_a) - schemaB = taputils.get_schema_name(full_qualified_table_name_b) + radius_quantity = self.__get_radius_as_quantity_arcsec(radius) + + radius_arc_sec = radius_quantity.value + + if radius_arc_sec < 0.1 or radius_arc_sec > 10.0: + raise ValueError(f"Invalid radius value. Found {radius_quantity}, valid range is: 0.1 to 10.0") + + schema_a = self.__get_schema_name(full_qualified_table_name_a) + + table_a = taputils.get_table_name(full_qualified_table_name_a) - if schemaB is None: - raise ValueError(f"Not found schema name in full qualified table B: '{full_qualified_table_name_b}'") + schema_b = self.__get_schema_name(full_qualified_table_name_b) - tableB = taputils.get_table_name(full_qualified_table_name_b) + table_b = taputils.get_table_name(full_qualified_table_name_b) if taputils.get_schema_name(results_table_name) is not None: raise ValueError("Please, do not specify schema for 'results_table_name'") - query = f"SELECT crossmatch_positional('{schemaA}','{tableA}','{schemaB}','{tableB}',{radius}, " \ + query = f"SELECT crossmatch_positional('{schema_a}','{table_a}','{schema_b}','{table_b}',{radius_arc_sec}, " \ f"'{results_table_name}') FROM dual;" name = str(results_table_name) @@ -916,10 +1069,8 @@ def cross_match(self, *, full_qualified_table_name_a, upload_resource=None, upload_table_name=None) - def launch_job(self, query, *, name=None, output_file=None, - output_format="votable_gzip", verbose=False, - dump_to_file=False, upload_resource=None, - upload_table_name=None): + def launch_job(self, query, *, name=None, output_file=None, output_format="votable_gzip", verbose=False, + dump_to_file=False, upload_resource=None, upload_table_name=None): """Launches a synchronous job Parameters diff --git a/astroquery/gaia/tests/test_gaiatap.py b/astroquery/gaia/tests/test_gaiatap.py index 1d48fc0e97..6fc79a8b9f 100644 --- a/astroquery/gaia/tests/test_gaiatap.py +++ b/astroquery/gaia/tests/test_gaiatap.py @@ -25,6 +25,7 @@ import pytest from astropy.coordinates.sky_coordinate import SkyCoord from astropy.table import Column, Table +from astropy.units import Quantity from astropy.utils.data import get_pkg_data_filename from astropy.utils.exceptions import AstropyDeprecationWarning from requests import HTTPError @@ -34,7 +35,9 @@ from astroquery.utils.commons import ASTROPY_LT_7_1_1 from astroquery.utils.tap.conn.tests.DummyConnHandler import DummyConnHandler from astroquery.utils.tap.conn.tests.DummyResponse import DummyResponse -from astroquery.utils.tap.core import TapPlus +from astroquery.utils.tap.core import TapPlus, Tap +from astroquery.utils.tap.model.tapcolumn import TapColumn +from astroquery.utils.tap.model.taptable import TapTableMeta GAIA_QUERIER = GaiaClass(show_server_messages=False) @@ -195,7 +198,6 @@ def mock_querier(): @pytest.fixture(scope="function") def mock_datalink_querier(patch_datetime_now): - assert datetime.datetime.now(datetime.timezone.utc) == FAKE_TIME conn_handler = DummyConnHandler() @@ -445,6 +447,26 @@ def cross_match_kwargs(): "results_table_name": "results"} +@pytest.fixture +def cross_match_basic_kwargs(): + return {"table_a_full_qualified_name": "schemaA.tableA", + "table_a_column_ra": "ra", + "table_a_column_dec": "dec", + "table_b_full_qualified_name": "schemaB.tableB", + "table_b_column_ra": "ra", + "table_b_column_dec": "dec"} + + +@pytest.fixture +def cross_match_basic_2_kwargs(): + return {"table_a_full_qualified_name": "user_hola.tableA", + "table_a_column_ra": "ra", + "table_a_column_dec": "dec", + "table_b_full_qualified_name": "user_hola.tableB", + "table_b_column_ra": "ra", + "table_b_column_dec": "dec"} + + def test_show_message(): print(JOB_DATA_FILE_NAME) connHandler = DummyConnHandler() @@ -833,9 +855,8 @@ def test_datalink_querier_load_data_vot_exception(mock_datalink_querier, overwri overwrite_output_file=overwrite_output_file, verbose=False) - assert str( - excinfo.value) == ( - f"{file_final} file already exists. Please use overwrite_output_file='True' to overwrite output file.") + messg = f"{file_final} file already exists. Please use overwrite_output_file='True' to overwrite output file." + assert str(excinfo.value) == messg else: mock_datalink_querier.load_data(ids=[5937083312263887616], data_release='Gaia DR3', @@ -1006,7 +1027,6 @@ def test_datalink_querier_load_data_fits(mock_datalink_querier_fits): def test_load_data_vot(monkeypatch, tmp_path, tmp_path_factory, patch_datetime_now): - assert datetime.datetime.now(datetime.timezone.utc) == FAKE_TIME now = datetime.datetime.now(datetime.timezone.utc) @@ -1086,7 +1106,6 @@ def load_data_monkeypatched(self, params_dict, output_file, verbose): def test_load_data_csv(monkeypatch, tmp_path, tmp_path_factory, patch_datetime_now): - assert datetime.datetime.now(datetime.timezone.utc) == FAKE_TIME now = datetime.datetime.now(datetime.timezone.utc) @@ -1125,7 +1144,6 @@ def load_data_monkeypatched(self, params_dict, output_file, verbose): def test_load_data_ecsv(monkeypatch, tmp_path, tmp_path_factory, patch_datetime_now): - assert datetime.datetime.now(datetime.timezone.utc) == FAKE_TIME now = datetime.datetime.now(datetime.timezone.utc) @@ -1164,7 +1182,6 @@ def load_data_monkeypatched(self, params_dict, output_file, verbose): def test_load_data_linking_parameter(monkeypatch, tmp_path, patch_datetime_now): - assert datetime.datetime.now(datetime.timezone.utc) == FAKE_TIME now = datetime.datetime.now(datetime.timezone.utc) @@ -1204,7 +1221,6 @@ def load_data_monkeypatched(self, params_dict, output_file, verbose): @pytest.mark.parametrize("linking_param", ['TRANSIT_ID', 'IMAGE_ID']) def test_load_data_linking_parameter_with_values(monkeypatch, tmp_path, linking_param, patch_datetime_now): - assert datetime.datetime.now(datetime.timezone.utc) == FAKE_TIME now = datetime.datetime.now(datetime.timezone.utc) @@ -1329,16 +1345,14 @@ def test_cross_match(background, cross_match_kwargs, mock_querier_async): "kwarg,invalid_value,error_message", [("full_qualified_table_name_a", "tableA", - "^Not found schema name in full qualified table A: 'tableA'$"), + "^Not found schema name in full qualified table: 'tableA'$"), ("full_qualified_table_name_b", "tableB", - "^Not found schema name in full qualified table B: 'tableB'$"), + "^Not found schema name in full qualified table: 'tableB'$"), ("results_table_name", "schema.results", "^Please, do not specify schema for 'results_table_name'$")]) -def test_cross_match_invalid_mandatory_kwarg( - cross_match_kwargs, kwarg, invalid_value, error_message -): +def test_cross_match_invalid_mandatory_kwarg(cross_match_kwargs, kwarg, invalid_value, error_message): cross_match_kwargs[kwarg] = invalid_value with pytest.raises(ValueError, match=error_message): GAIA_QUERIER.cross_match(**cross_match_kwargs) @@ -1348,7 +1362,7 @@ def test_cross_match_invalid_mandatory_kwarg( def test_cross_match_invalid_radius(cross_match_kwargs, radius): with pytest.raises( ValueError, - match=rf"^Invalid radius value. Found {radius}, valid range is: 0.1 to 10.0$", + match=rf"^Invalid radius value. Found {radius} arcsec, valid range is: 0.1 to 10.0$", ): GAIA_QUERIER.cross_match(**cross_match_kwargs, radius=radius) @@ -1364,6 +1378,186 @@ def test_cross_match_missing_mandatory_kwarg(cross_match_kwargs, missing_kwarg): GAIA_QUERIER.cross_match(**cross_match_kwargs) +def make_table_metadata(table_name): + tap_table = TapTableMeta() + tap_table.name = table_name + tap_column_ra = TapColumn(0) + tap_column_ra.name = "ra" + tap_table.add_column(tap_column_ra) + tap_column_dec = TapColumn(0) + tap_column_dec.name = "dec" + tap_table.add_column(tap_column_dec) + return tap_table + + +@pytest.mark.parametrize("background", [False, True]) +def test_cross_match_basic(monkeypatch, background, cross_match_basic_kwargs, mock_querier_async): + def load_table_monkeypatched(self, table, verbose): + tap_table_a = make_table_metadata("schemaA.tableA") + tap_table_b = make_table_metadata("schemaB.tableB") + + return_val = {"schemaA.tableA": tap_table_a, "schemaB.tableB": tap_table_b} + return return_val[table] + + monkeypatch.setattr(Tap, "load_table", load_table_monkeypatched) + + job = mock_querier_async.cross_match_basic(**cross_match_basic_kwargs, background=background) + assert job.async_ is True + assert job.get_phase() == "EXECUTING" if background else "COMPLETED" + assert job.failed is False + + +@pytest.mark.parametrize("background", [False, True]) +def test_cross_match_basic_2(monkeypatch, background, cross_match_basic_2_kwargs, mock_querier_async): + def load_table_monkeypatched(self, table, verbose): + tap_table_a = make_table_metadata("user_hola.tableA") + tap_table_b = make_table_metadata("user_hola.tableB") + + return_val = {"user_hola.tableA": tap_table_a, "user_hola.tableB": tap_table_b} + return return_val[table] + + def update_user_table(self, table_name, list_of_changes, verbose): + return None + + monkeypatch.setattr(Tap, "load_table", load_table_monkeypatched) + monkeypatch.setattr(TapPlus, "update_user_table", update_user_table) + + job = mock_querier_async.cross_match_basic(**cross_match_basic_2_kwargs, background=background) + assert job.async_ is True + assert job.get_phase() == "EXECUTING" if background else "COMPLETED" + assert job.failed is False + + +@pytest.mark.parametrize("background", [False, True]) +def test_cross_match_basic_3(monkeypatch, background, mock_querier_async): + mock_querier_async.MAIN_GAIA_TABLE = None + + def load_table_monkeypatched_2(self, table, verbose): + tap_table_a = make_table_metadata("user_hola.tableA") + tap_table_b = make_table_metadata("gaiadr3.gaia_source") + + return_val = {"user_hola.tableA": tap_table_a, "gaiadr3.gaia_source": tap_table_b} + return return_val[table] + + def update_user_table(self, table_name, list_of_changes, verbose): + return None + + monkeypatch.setattr(Tap, "load_table", load_table_monkeypatched_2) + monkeypatch.setattr(TapPlus, "update_user_table", update_user_table) + + job = mock_querier_async.cross_match_basic(table_a_full_qualified_name="user_hola.tableA", table_a_column_ra="ra", + table_a_column_dec="dec", background=background) + assert job.async_ is True + assert job.get_phase() == "EXECUTING" if background else "COMPLETED" + assert job.failed is False + + radius_quantity = Quantity(value=1.0, unit=u.arcsec) + job = mock_querier_async.cross_match_basic(table_a_full_qualified_name="user_hola.tableA", table_a_column_ra="ra", + table_a_column_dec="dec", radius=radius_quantity, background=background) + assert job.async_ is True + assert job.get_phase() == "EXECUTING" if background else "COMPLETED" + assert job.failed is False + + radius_quantity = Quantity(value=1.0/3600.0, unit=u.deg) + job = mock_querier_async.cross_match_basic(table_a_full_qualified_name="user_hola.tableA", table_a_column_ra="ra", + table_a_column_dec="dec", radius=radius_quantity, background=background) + assert job.async_ is True + assert job.get_phase() == "EXECUTING" if background else "COMPLETED" + assert job.failed is False + + +@pytest.mark.parametrize("background", [False, True]) +def test_cross_match_basic_wrong_column(monkeypatch, background, mock_querier_async): + mock_querier_async.MAIN_GAIA_TABLE = None + + def load_table_monkeypatched(self, table, verbose): + tap_table_a = make_table_metadata("user_hola.tableA") + tap_table_b = make_table_metadata("gaiadr3.gaia_source") + + return_val = {"user_hola.tableA": tap_table_a, "gaiadr3.gaia_source": tap_table_b} + return return_val[table] + + def update_user_table(self, table_name, list_of_changes, verbose): + return None + + monkeypatch.setattr(Tap, "load_table", load_table_monkeypatched) + monkeypatch.setattr(TapPlus, "update_user_table", update_user_table) + + error_message = "Please check: columns Wrong_ra or dec not available in the table 'user_hola.tableA'" + with pytest.raises(ValueError, match=error_message): + mock_querier_async.cross_match_basic(table_a_full_qualified_name="user_hola.tableA", + table_a_column_ra="Wrong_ra", table_a_column_dec="dec", + background=background) + + error_message = "Please check: columns ra or Wrong_dec not available in the table 'user_hola.tableA'" + with pytest.raises(ValueError, match=error_message): + mock_querier_async.cross_match_basic(table_a_full_qualified_name="user_hola.tableA", + table_a_column_ra="ra", table_a_column_dec="Wrong_dec", + background=background) + + +def test_cross_match_basic_exceptions(monkeypatch): + def load_table_monkeypatched(self, table, verbose): + raise ValueError(f"Not found schema name in full qualified table: '{table}'") + + def update_user_table(self, table_name, list_of_changes, verbose): + return None + + monkeypatch.setattr(Tap, "load_table", load_table_monkeypatched) + monkeypatch.setattr(TapPlus, "update_user_table", update_user_table) + + error_message = "Not found table 'user_hola.tableA' in the archive" + with pytest.raises(ValueError, match=error_message): + GAIA_QUERIER.cross_match_basic(table_a_full_qualified_name="user_hola.tableA", table_a_column_ra="ra", + table_a_column_dec="dec", background=True) + + # Check invalid input values + error_message = "Not found schema name in full qualified table: 'hola'" + with pytest.raises(ValueError, match=error_message): + GAIA_QUERIER.cross_match_basic(table_a_full_qualified_name="hola", table_a_column_ra="ra", + table_a_column_dec="dec") + + error_message = "Schema name is empty in full qualified table: '.table_name'" + with pytest.raises(ValueError, match=error_message): + GAIA_QUERIER.cross_match_basic(table_a_full_qualified_name=".table_name", table_a_column_ra="ra", + table_a_column_dec="dec") + + error_message = "Not found schema name in full qualified table: 'hola'" + with pytest.raises(ValueError, match=error_message): + GAIA_QUERIER.cross_match_basic(table_a_full_qualified_name="schema.table_name", table_a_column_ra="ra", + table_a_column_dec="dec", table_b_full_qualified_name="hola") + + error_message = "Schema name is empty in full qualified table: '.table_name'" + with pytest.raises(ValueError, match=error_message): + GAIA_QUERIER.cross_match_basic(table_a_full_qualified_name="schema.table_name", table_a_column_ra="ra", + table_a_column_dec="dec", table_b_full_qualified_name=".table_name") + + error_message = "Invalid radius value. Found 50.0 arcsec, valid range is: 0.1 to 10.0" + with pytest.raises(ValueError, match=error_message): + GAIA_QUERIER.cross_match_basic(table_a_full_qualified_name="schema.table_name", table_a_column_ra="ra", + table_a_column_dec="dec", table_b_full_qualified_name="schema.table_name", + radius=50.0) + + error_message = "Invalid radius value. Found 0.01 arcsec, valid range is: 0.1 to 10.0" + with pytest.raises(ValueError, match=error_message): + GAIA_QUERIER.cross_match_basic(table_a_full_qualified_name="schema.table_name", table_a_column_ra="ra", + table_a_column_dec="dec", table_b_full_qualified_name="schema.table_name", + radius=0.01) + + radius_quantity = Quantity(value=0.01, unit=u.arcsec) + with pytest.raises(ValueError, match=error_message): + GAIA_QUERIER.cross_match_basic(table_a_full_qualified_name="schema.table_name", table_a_column_ra="ra", + table_a_column_dec="dec", table_b_full_qualified_name="schema.table_name", + radius=radius_quantity) + + radius_quantity = Quantity(value=1.0, unit=u.deg) + error_message = "Invalid radius value. Found 3600.0 arcsec, valid range is: 0.1 to 10.0" + with pytest.raises(ValueError, match=error_message): + GAIA_QUERIER.cross_match_basic(table_a_full_qualified_name="schema.table_name", table_a_column_ra="ra", + table_a_column_dec="dec", table_b_full_qualified_name="schema.table_name", + radius=radius_quantity) + + @patch.object(TapPlus, 'login') def test_login(mock_login): conn_handler = DummyConnHandler() diff --git a/astroquery/utils/tap/core.py b/astroquery/utils/tap/core.py index 7475e290d8..c137b10a56 100755 --- a/astroquery/utils/tap/core.py +++ b/astroquery/utils/tap/core.py @@ -181,18 +181,30 @@ def load_table(self, table, *, verbose=False): """ if table is None: raise ValueError("Table name is required") + + schema = taputils.get_schema_name(table) + if schema is None: + raise ValueError(f"Not found schema name in full qualified table: '{table}'") + if verbose: print(f"Retrieving table '{table}'") + response = self.__connHandler.execute_tapget(f"tables?tables={table}", verbose=verbose) + if verbose: print(response.status, response.reason) + self.__connHandler.check_launch_response_status(response, verbose, 200) + if verbose: print(f"Parsing table '{table}'...") + tsp = TableSaxParser() tsp.parseData(response) + if verbose: print("Done.") + return tsp.get_table() def __load_tables(self, *, only_names=False, include_shared_tables=False, verbose=False): diff --git a/astroquery/utils/tap/model/taptable.py b/astroquery/utils/tap/model/taptable.py index 2499f81ada..7a178d8a8d 100755 --- a/astroquery/utils/tap/model/taptable.py +++ b/astroquery/utils/tap/model/taptable.py @@ -34,7 +34,7 @@ def get_qualified_name(self): Returns ------- - The the qualified TAP table name (schema+table) + The qualified TAP table name (schema+table) """ if '.' in self.name: return self.name diff --git a/astroquery/utils/tap/tests/test_tap.py b/astroquery/utils/tap/tests/test_tap.py index 9d31e42979..8e1afd0346 100644 --- a/astroquery/utils/tap/tests/test_tap.py +++ b/astroquery/utils/tap/tests/test_tap.py @@ -9,7 +9,6 @@ """ import gzip import os - from pathlib import Path from unittest.mock import patch from urllib.parse import quote_plus, urlencode @@ -19,7 +18,6 @@ from astropy.io.registry import IORegistryError from astropy.table import Table from astropy.utils.data import get_pkg_data_filename - from requests import HTTPError from astroquery.utils.tap import taputils @@ -118,7 +116,8 @@ def test_load_table(): tap = TapPlus(url="http://test:1111/tap", connhandler=conn_handler) # No arguments - with pytest.raises(Exception): + error_message = ".*load_table\\(\\) missing 1 required positional argument: 'table'$" + with pytest.raises(TypeError, match=error_message): tap.load_table() responseLoadTable = DummyResponse(500) @@ -129,11 +128,12 @@ def test_load_table(): tableRequest = f"tables?tables={fullQualifiedTableName}" conn_handler.set_response(tableRequest, responseLoadTable) - with pytest.raises(Exception): + error_message = "^Error 500" + with pytest.raises(Exception, match=error_message): tap.load_table(fullQualifiedTableName) responseLoadTable.set_status_code(200) - table = tap.load_table(fullQualifiedTableName) + table = tap.load_table(fullQualifiedTableName, verbose=True) assert table is not None assert table.description == 'Table1 desc' columns = table.columns @@ -144,6 +144,19 @@ def test_load_table(): __check_column(col, 'Table1 Column2 desc', '', 'INTEGER', None) +def test_load_table_exceptions(): + conn_handler = DummyConnHandler() + tap = TapPlus(url="http://test:1111/tap", connhandler=conn_handler) + + error_message = "Table name is required" + with pytest.raises(ValueError, match=error_message): + tap.load_table(None) + + error_message = "Not found schema name in full qualified table: 'only_table_name'" + with pytest.raises(ValueError, match=error_message): + tap.load_table("only_table_name") + + def test_launch_sync_job(): conn_handler = DummyConnHandler() tap = TapPlus(url="http://test:1111/tap", connhandler=conn_handler) @@ -778,7 +791,7 @@ def test_get_current_column_values_for_update(): def test_update_user_table(): - tableName = 'table' + tableName = 'schema.table' conn_handler = DummyConnHandler() tap = TapPlus(url="http://test:1111/tap", connhandler=conn_handler) dummyResponse = DummyResponse(200) diff --git a/docs/gaia/gaia.rst b/docs/gaia/gaia.rst index ee8cc161ec..ed33da1f2f 100644 --- a/docs/gaia/gaia.rst +++ b/docs/gaia/gaia.rst @@ -82,7 +82,7 @@ This query searches for all the objects contained in an arbitrary rectangular pr WARNING: This method implements the ADQL BOX function that is deprecated in the latest version of the standard (ADQL 2.1, see: https://ivoa.net/documents/ADQL/20231107/PR-ADQL-2.1-20231107.html#tth_sEc4.2.9). -It is possible to choose which data release to query, by default the Gaia DR3 catalogue is used. For example:: +It is possible to choose which data release to query, by default the Gaia DR3 catalogue is used. For example .. doctest-remote-data:: @@ -738,15 +738,23 @@ We can type the following:: 2.6. Cross match ^^^^^^^^^^^^^^^^ -It is possible to run a geometric cross-match between the RA/Dec coordinates of two tables -using the crossmatch function provided by the archive. In order to do so the user must be -logged in. This is required because the cross match operation will generate a join table -in the user private area. That table contains the identifiers of both tables and the separation, -in degrees, between RA/Dec coordinates of each source in the first table and its associated -source in the second table. Later, the table can be used to obtain the actual data from both tables. +It is possible to run a geometric cross-match between the RA/Dec coordinates of two tables using the crossmatch function +provided by the archive. To do so, the user must be logged in. This is necessary as the cross-match process will create +a join table within the user's private space. That table includes the identifiers from both tables and the angular +separation, in degrees, between the RA/Dec coordinates of each source in the first table and its corresponding source +in the second table. -In order to perform a cross match, both tables must have defined RA and Dec columns -(Ra/Dec column flags must be set: see previous section to know how to assign those flags). +The cross-match requires 3 steps: + +1. Update the user table metadata to flag the positional RA/Dec columns using the dedicated method +`~astroquery.utils.tap.core.TapPlus.update_user_table`, as both tables must have defined RA and Dec columns. See +previous section to learn how to assign those flags; + +2. Launch the built-in cross-match method `~astroquery.gaia.GaiaClass.cross_match`, which creates a table in the user's +private area; + +3. Subsequently, this table can be employed to retrieve the data from both tables, launching an ADQL query with the +"launch_job" or "launch_job_async" method. The following example uploads a table and then, the table is used in a cross match:: @@ -758,13 +766,13 @@ The following example uploads a table and then, the table is used in a cross mat >>> # the table can be referenced as . >>> full_qualified_table_name = 'user_.my_sources' >>> xmatch_table_name = 'xmatch_table' - >>> Gaia.cross_match(full_qualified_table_name_a=full_qualified_table_name, + >>> job = Gaia.cross_match(full_qualified_table_name_a=full_qualified_table_name, ... full_qualified_table_name_b='gaiadr3.gaia_source', ... results_table_name=xmatch_table_name, radius=1.0) - -Once you have your cross match finished, you can obtain the results:: - +The cross-match launches an asynchronous query that saves the results at the user's private area, so it depends on the +user files quota. This table only contains 3 columns: first table column id, second table column id and the angular +separation (degrees) between each source. Once you have your cross match finished, you can access to this table:: >>> xmatch_table = 'user_.' + xmatch_table_name >>> query = (f"SELECT c.separation*3600 AS separation_arcsec, a.*, b.* FROM gaiadr3.gaia_source AS a, " @@ -775,6 +783,35 @@ Once you have your cross match finished, you can obtain the results:: Cross-matching catalogues is one of the most popular operations executed in the Gaia archive. +The previous 3-step cross-match can be executed in one step by the following method:: + + >>> from astroquery.gaia import Gaia + >>> Gaia.login() + >>> job = Gaia.cross_match_basic(table_a_full_qualified_name=full_qualified_table_name, table_a_column_ra='raj2000', + table_a_column_dec='dej2000', table_b_full_qualified_name='gaiadr3.gaia_source', + table_b_column_ra='ra', table_b_column_dec='dec, radius=1.0, background=True): + >>> print(job) + Jobid: 1611860482314O + Phase: COMPLETED + Owner: None + Output file: 1611860482314O-result.vot.gz + Results: None + >>> result = job.get_results() + +This method updates the user table metadata to flag the positional RA/Dec columns and launches the positional +cross-match as an asynchronous query. Unlike the previous 3-step cross-match method, the returned job provides direct +access to the output of the cross-match information: for each matched source, all the columns from the input tables plus +the angular distance (degrees). Therefore, the size of the output can be quite large. + +By default, this method targets the main catalogue of the Gaia DR3 ("gaiadr3.gaia_source") using a cone search radius +of 1.0 arcseconds. Therefore, the above example can also be simplified as follows:: + + >>> from astroquery.gaia import Gaia + >>> Gaia.login() + >>> job = Gaia.cross_match_basic(table_a_full_qualified_name=full_qualified_table_name, table_a_column_ra='raj2000', + table_a_column_dec='dej2000') + >>> result = job.get_results() + 2.7. Tables sharing ^^^^^^^^^^^^^^^^^^^