From ff3308f0ac788701bcb1cd2964e301dd9b78f270 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Brigitta=20Sip=C5=91cz?= Date: Tue, 3 Jun 2025 12:44:40 -0700 Subject: [PATCH 1/6] ENH: do not include image metadata tables in catalog list by default --- astroquery/ipac/irsa/core.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/astroquery/ipac/irsa/core.py b/astroquery/ipac/irsa/core.py index 6ebac5d2a9..35abbe65d1 100644 --- a/astroquery/ipac/irsa/core.py +++ b/astroquery/ipac/irsa/core.py @@ -304,7 +304,7 @@ def query_region(self, coordinates=None, *, catalog=None, spatial='Cone', return response.to_table() @deprecated_renamed_argument("cache", None, since="0.4.7") - def list_catalogs(self, *, full=False, filter=None, cache=False): + def list_catalogs(self, *, full=False, filter=None, include_metadata_tables=False, cache=False): """ Return information of available IRSA catalogs. @@ -316,8 +316,17 @@ def list_catalogs(self, *, full=False, filter=None, cache=False): filter : str or None If specified we only return catalogs when their catalog_name contains the filter string. + include_metadata_tables : bool + If True returns not just the catalogs but all table holdings including the image metadata tables. + These are not suitable for spatial queries with e.g. ``query_region``. """ - tap_tables = self.query_tap("SELECT * FROM TAP_SCHEMA.tables").to_table() + + if include_metadata_tables: + more_filtering = "" + else: + more_filtering = "where irsa_dbms=21" + + tap_tables = self.query_tap(f"SELECT * FROM TAP_SCHEMA.tables {more_filtering}").to_table() if filter: mask = [filter in name for name in tap_tables['table_name']] From 2a3f17d204a91d22a89b9adea6921babe14af62e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Brigitta=20Sip=C5=91cz?= Date: Tue, 3 Jun 2025 12:45:22 -0700 Subject: [PATCH 2/6] TST: adding and updating tests for include_metadata_tables --- .../ipac/irsa/tests/test_irsa_remote.py | 20 +++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/astroquery/ipac/irsa/tests/test_irsa_remote.py b/astroquery/ipac/irsa/tests/test_irsa_remote.py index 8ea0dce4aa..daae48e035 100644 --- a/astroquery/ipac/irsa/tests/test_irsa_remote.py +++ b/astroquery/ipac/irsa/tests/test_irsa_remote.py @@ -72,14 +72,26 @@ def test_list_columns(self): def test_list_catalogs(self): catalogs = Irsa.list_catalogs() # Number of available catalogs may change over time, test only for significant drop. - # (at the time of writing there are 933 tables in the list). - assert len(catalogs) > 900 + # (at the time of writing there are 645 tables in the list). + assert len(catalogs) > 640 assert isinstance(catalogs, dict) + catalogs_full = Irsa.list_catalogs(full=True) + assert isinstance(catalogs_full, Table) + def test_list_catalogs_filter(self): - spitzer_catalogs = Irsa.list_catalogs(filter='spitzer') + spitzer_catalogs = Irsa.list_catalogs(filter='allwise') + + assert len(spitzer_catalogs) == 13 + + def test_list_catalogs_metadata(self): + catalogs = Irsa.list_catalogs(filter='wise') + all_tables = Irsa.list_catalogs(filter='wise', include_metadata_tables=True) + + assert len(catalogs) < len(all_tables) - assert len(spitzer_catalogs) == 142 + assert 'wise.wise_allwise_p3am_cdd' not in catalogs + assert 'wise.wise_allwise_p3am_cdd' in all_tables @pytest.mark.parametrize('servicetype', (None, 'sia', 'ssa')) def test_list_collections(self, servicetype): From 86335aa7131873f88c8b59847e9b6671a6ac8a78 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Brigitta=20Sip=C5=91cz?= Date: Tue, 3 Jun 2025 12:57:39 -0700 Subject: [PATCH 3/6] DOC: fix failing (but unrelated) doctests --- docs/ipac/irsa/irsa.rst | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/docs/ipac/irsa/irsa.rst b/docs/ipac/irsa/irsa.rst index 3af91ec3c5..86c04e83d0 100644 --- a/docs/ipac/irsa/irsa.rst +++ b/docs/ipac/irsa/irsa.rst @@ -234,14 +234,15 @@ available about the columns in a `~astropy.table.Table`. .. doctest-remote-data:: >>> from astroquery.ipac.irsa import Irsa - >>> Irsa.list_columns(catalog="allwise_p3as_psd") - {'designation': 'WISE source designation', + >>> Irsa.list_columns(catalog="allwise_p3as_psd") # doctest: +IGNORE_OUTPUT + {... + 'designation': 'WISE source designation', 'ra': 'right ascension (J2000)', 'dec': 'declination (J2000)', 'sigra': 'uncertainty in RA', 'sigdec': 'uncertainty in DEC', ... - 'htm20': 'HTM20 spatial index key'} + } Async queries @@ -400,8 +401,9 @@ Spitzer. >>> from astropy.table import unique >>> unique(arp220_spectra, keys='dataid_collection')['dataid_collection'] - + goals + herschel_herus sofia_fifils spitzer_irsenh spitzer_sha @@ -415,7 +417,7 @@ will return a `~astropy.table.Table`. >>> from astroquery.ipac.irsa import Irsa >>> Irsa.list_collections(servicetype='SSA') - +
collection object ------------------------ @@ -437,6 +439,7 @@ will return a `~astropy.table.Table`. irts iso_sws sofia_exes + sofia_exes_enh sofia_fifils sofia_flitecam sofia_forcast @@ -454,6 +457,7 @@ will return a `~astropy.table.Table`. spitzer_sha spitzer_sings spitzer_ssgss + swas Other Configurations From 0b7654dfbbcf174ef21afc4c516f31c068a2c60e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Brigitta=20Sip=C5=91cz?= Date: Tue, 3 Jun 2025 18:23:34 -0700 Subject: [PATCH 4/6] ENH: filter out all non-spatial tables, too --- astroquery/ipac/irsa/core.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/astroquery/ipac/irsa/core.py b/astroquery/ipac/irsa/core.py index 35abbe65d1..6ab09c09e7 100644 --- a/astroquery/ipac/irsa/core.py +++ b/astroquery/ipac/irsa/core.py @@ -324,7 +324,9 @@ def list_catalogs(self, *, full=False, filter=None, include_metadata_tables=Fals if include_metadata_tables: more_filtering = "" else: - more_filtering = "where irsa_dbms=21" + # Filter out non-spatial catalogs and metadata tables with + # irsa_pos=y and irsa_dbms=21 + more_filtering = "WHERE irsa_dbms=21 AND irsa_pos='y'" tap_tables = self.query_tap(f"SELECT * FROM TAP_SCHEMA.tables {more_filtering}").to_table() From 5ce6ac4839c4123c279f79afbac9effd798edb9f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Brigitta=20Sip=C5=91cz?= Date: Tue, 3 Jun 2025 18:24:25 -0700 Subject: [PATCH 5/6] TST: updating tests --- astroquery/ipac/irsa/tests/test_irsa_remote.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/astroquery/ipac/irsa/tests/test_irsa_remote.py b/astroquery/ipac/irsa/tests/test_irsa_remote.py index daae48e035..648a7d8bcb 100644 --- a/astroquery/ipac/irsa/tests/test_irsa_remote.py +++ b/astroquery/ipac/irsa/tests/test_irsa_remote.py @@ -72,17 +72,17 @@ def test_list_columns(self): def test_list_catalogs(self): catalogs = Irsa.list_catalogs() # Number of available catalogs may change over time, test only for significant drop. - # (at the time of writing there are 645 tables in the list). - assert len(catalogs) > 640 + # (at the time of writing there are 521 catalogs in the list). + assert len(catalogs) > 520 assert isinstance(catalogs, dict) catalogs_full = Irsa.list_catalogs(full=True) assert isinstance(catalogs_full, Table) def test_list_catalogs_filter(self): - spitzer_catalogs = Irsa.list_catalogs(filter='allwise') + allwise_catalogs = Irsa.list_catalogs(filter='allwise') - assert len(spitzer_catalogs) == 13 + assert len(allwise_catalogs) == 4 def test_list_catalogs_metadata(self): catalogs = Irsa.list_catalogs(filter='wise') From e3eca6e72c97e51d3b9f3a89a513e369799f216f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Brigitta=20Sip=C5=91cz?= Date: Tue, 3 Jun 2025 18:30:34 -0700 Subject: [PATCH 6/6] DOC: adding changelog --- CHANGES.rst | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index b0d7964ee2..2165374b23 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -44,6 +44,14 @@ imcce - Changing RuntimeError to NoResultsWarning when an empty result is returned. [#3307] +ipac.irsa +^^^^^^^^^ + +- Fix ``list_catalogs`` to not include image metadata tables, only + catalogs. The ``include_metadata_tables`` keyword argument allows opting + in to return all TAP tables, including non-spatial and metadata ones, + too. [#3334] + SIMBAD ^^^^^^ @@ -76,10 +84,10 @@ mast - Fix bug in ``utils.remove_duplicate_products`` that does not retain the order of the products in an input table. [#3314] -- Added ``return_uri_map`` parameter to ``Observations.get_cloud_uris`` to return a mapping of the input data product URIs +- Added ``return_uri_map`` parameter to ``Observations.get_cloud_uris`` to return a mapping of the input data product URIs to the returned cloud URIs. [#3314] -- Added ``verbose`` parameter to ``Observations.get_cloud_uris`` to control whether warnings are logged when a product cannot +- Added ``verbose`` parameter to ``Observations.get_cloud_uris`` to control whether warnings are logged when a product cannot be found in the cloud. [#3314]