From 7fc6dbdda6ecc74ebad8ae7af36130e0ba6afee5 Mon Sep 17 00:00:00 2001 From: Sean Lake Date: Thu, 25 Apr 2019 09:44:03 +0800 Subject: [PATCH 01/12] Fix bug in list_missions() Resubmitting bug fix I accidentally deleted. --- astroquery/ibe/core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/astroquery/ibe/core.py b/astroquery/ibe/core.py index 15e20dba81..fb59b6ff2a 100644 --- a/astroquery/ibe/core.py +++ b/astroquery/ibe/core.py @@ -276,7 +276,7 @@ def list_missions(self, cache=True): root = BeautifulSoup(response.text) links = root.findAll('a') - missions = [os.path.basename(a.attrs['href']) for a in links] + missions = [os.path.basename(a.attrs['href']).rstrip('/') for a in links] self._missions = missions return missions From ff4988c4221896c357d397308597dd7bb7bec17c Mon Sep 17 00:00:00 2001 From: Sean Lake Date: Thu, 25 Apr 2019 10:19:08 +0800 Subject: [PATCH 02/12] break a long line --- astroquery/ibe/core.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/astroquery/ibe/core.py b/astroquery/ibe/core.py index fb59b6ff2a..0202bdc5dd 100644 --- a/astroquery/ibe/core.py +++ b/astroquery/ibe/core.py @@ -276,7 +276,8 @@ def list_missions(self, cache=True): root = BeautifulSoup(response.text) links = root.findAll('a') - missions = [os.path.basename(a.attrs['href']).rstrip('/') for a in links] + missions = [os.path.basename(a.attrs['href']).rstrip('/') \ + for a in links] self._missions = missions return missions From 75af6f7d43e035b46080f101fe76df4a53dcaab1 Mon Sep 17 00:00:00 2001 From: Sean Lake Date: Thu, 25 Apr 2019 10:21:54 +0800 Subject: [PATCH 03/12] remove redundant backslash (PEP 8) --- astroquery/ibe/core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/astroquery/ibe/core.py b/astroquery/ibe/core.py index 0202bdc5dd..f4d8563fe9 100644 --- a/astroquery/ibe/core.py +++ b/astroquery/ibe/core.py @@ -276,7 +276,7 @@ def list_missions(self, cache=True): root = BeautifulSoup(response.text) links = root.findAll('a') - missions = [os.path.basename(a.attrs['href']).rstrip('/') \ + missions = [os.path.basename(a.attrs['href']).rstrip('/') for a in links] self._missions = missions From 04902fcc7a0fb4ac65ae8361fab1d75899098c6b Mon Sep 17 00:00:00 2001 From: Sean Lake Date: Thu, 25 Apr 2019 10:38:31 +0800 Subject: [PATCH 04/12] Fix placement of rstrip --- astroquery/ibe/core.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/astroquery/ibe/core.py b/astroquery/ibe/core.py index f4d8563fe9..5947bd1dc3 100644 --- a/astroquery/ibe/core.py +++ b/astroquery/ibe/core.py @@ -276,7 +276,8 @@ def list_missions(self, cache=True): root = BeautifulSoup(response.text) links = root.findAll('a') - missions = [os.path.basename(a.attrs['href']).rstrip('/') + + missions = [os.path.basename(a.attrs['href'].rstrip('/')) for a in links] self._missions = missions From 0766865b85ec54cf4dd23dbf6f043d91da0c17aa Mon Sep 17 00:00:00 2001 From: Sean Lake Date: Thu, 25 Apr 2019 10:46:52 +0800 Subject: [PATCH 05/12] fix whitespace PEP 8 issues --- astroquery/ibe/core.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/astroquery/ibe/core.py b/astroquery/ibe/core.py index 5947bd1dc3..8a3868699d 100644 --- a/astroquery/ibe/core.py +++ b/astroquery/ibe/core.py @@ -276,8 +276,8 @@ def list_missions(self, cache=True): root = BeautifulSoup(response.text) links = root.findAll('a') - - missions = [os.path.basename(a.attrs['href'].rstrip('/')) + + missions = [os.path.basename(a.attrs['href'].rstrip('/')) for a in links] self._missions = missions From 0c4c7c7c5e95d78b86b606594323d71e819f23f9 Mon Sep 17 00:00:00 2001 From: Sean Lake Date: Thu, 25 Apr 2019 13:10:09 +0800 Subject: [PATCH 06/12] Fix ibe's list_datasets, list_tables, and get_columns methods --- astroquery/ibe/__init__.py | 9 ++++----- astroquery/ibe/core.py | 39 ++++++++++++-------------------------- 2 files changed, 16 insertions(+), 32 deletions(-) diff --git a/astroquery/ibe/__init__.py b/astroquery/ibe/__init__.py index a36371fc49..f1d7baa3e4 100644 --- a/astroquery/ibe/__init__.py +++ b/astroquery/ibe/__init__.py @@ -24,12 +24,11 @@ class Conf(_config.ConfigNamespace): dataset = _config.ConfigItem( 'images', - ('Default data set. See, for example, ' - 'http://irsa.ipac.caltech.edu/ibe/search/ptf for options.')) + ('This option is meaningless in the context of IRSA\'s ibe.')) table = _config.ConfigItem( - 'level1', - ('Default table. See, for example, ' - 'http://irsa.ipac.caltech.edu/ibe/search/ptf/images for options.')) + 'ptf.ptf_procimg', + ('Default table. Select the desired mission at ' + 'http://irsa.ipac.caltech.edu/ibe/search/ for options.')) timeout = _config.ConfigItem( 60, 'Time limit for connecting to the IRSA server.') diff --git a/astroquery/ibe/core.py b/astroquery/ibe/core.py index 8a3868699d..974dea327a 100644 --- a/astroquery/ibe/core.py +++ b/astroquery/ibe/core.py @@ -15,6 +15,7 @@ import astropy.coordinates as coord from astropy.table import Table +from astropy.io.ascii.html import HTML import six from ..exceptions import InvalidQueryError @@ -285,7 +286,8 @@ def list_missions(self, cache=True): def list_datasets(self, mission=None, cache=True): """ - For a given mission, list the available datasets + For a given mission, list the available datasets. + This level has no meaning in IRSA's ibe service. Parameters ---------- @@ -301,24 +303,9 @@ def list_datasets(self, mission=None, cache=True): datasets : list A list of dataset names """ - if mission is None: - mission = self.MISSION - if mission not in self.list_missions(): - raise ValueError("Invalid mission specified: {0}." - "Must be one of: {1}" - .format(mission, self.list_missions())) - - url = "{URL}search/{mission}/".format(URL=self.URL, mission=mission) - response = self._request('GET', url, timeout=self.TIMEOUT, - cache=cache) - - root = BeautifulSoup(response.text) - links = root.findAll('a') - datasets = [a.text for a in links - if a.attrs['href'].count('/') >= 4 # shown as '..'; ignore - ] + # This level - return datasets + return [ "images" ] def list_tables(self, mission=None, dataset=None, cache=True): """ @@ -360,9 +347,7 @@ def list_tables(self, mission=None, dataset=None, cache=True): .format(dataset, mission, self.list_datasets(mission, cache=True))) - url = "{URL}search/{mission}/{dataset}/".format(URL=self.URL, - mission=mission, - dataset=dataset) + url = "{URL}search/{mission}/".format(URL=self.URL, mission=mission) response = self._request('GET', url, timeout=self.TIMEOUT, cache=cache) @@ -388,10 +373,9 @@ def show_docs(self, mission=None, dataset=None, table=None): The table to be queried (if not the default table). """ - url = "{URL}docs/{mission}/{dataset}/{table}".format( + url = "{URL}docs/{mission}/{table}".format( URL=self.URL, mission=mission or self.MISSION, - dataset=dataset or self.DATASET, table=table or self.TABLE) return webbrowser.open(url) @@ -415,19 +399,20 @@ def get_columns(self, mission=None, dataset=None, table=None): A table containing a description of the columns """ - url = "{URL}search/{mission}/{dataset}/{table}".format( + url = "{URL}search/{mission}/{table}/".format( URL=self.URL, mission=mission or self.MISSION, - dataset=dataset or self.DATASET, table=table or self.TABLE) response = self._request( - 'GET', url, {'FORMAT': 'METADATA'}, timeout=self.TIMEOUT) + 'GET', url, {'FORMAT': 'METADATA'}, + timeout=self.TIMEOUT) # Raise exception, if request failed response.raise_for_status() + HTMLtoTable = HTML() - return Table.read(response.text, format='ipac', guess=False) + return HTMLtoTable.read(response.text) Ibe = IbeClass() From 516bc5eeb79d79f0786702885c11af962749b006 Mon Sep 17 00:00:00 2001 From: Sean Lake Date: Thu, 25 Apr 2019 13:20:22 +0800 Subject: [PATCH 07/12] remove extraneous comment --- astroquery/ibe/core.py | 1 - 1 file changed, 1 deletion(-) diff --git a/astroquery/ibe/core.py b/astroquery/ibe/core.py index 974dea327a..b197820aea 100644 --- a/astroquery/ibe/core.py +++ b/astroquery/ibe/core.py @@ -303,7 +303,6 @@ def list_datasets(self, mission=None, cache=True): datasets : list A list of dataset names """ - # This level return [ "images" ] From 6c61f32c14405e4829a78549c535bc17b2877e81 Mon Sep 17 00:00:00 2001 From: Sean Lake Date: Thu, 25 Apr 2019 16:53:44 +0800 Subject: [PATCH 08/12] get query_region_async minimally functional. --- astroquery/ibe/__init__.py | 3 ++- astroquery/ibe/core.py | 40 ++++++++++++++++++++++++++++---------- 2 files changed, 32 insertions(+), 11 deletions(-) diff --git a/astroquery/ibe/__init__.py b/astroquery/ibe/__init__.py index f1d7baa3e4..2fa02f21c7 100644 --- a/astroquery/ibe/__init__.py +++ b/astroquery/ibe/__init__.py @@ -14,8 +14,9 @@ class Conf(_config.ConfigNamespace): Configuration parameters for `astroquery.ibe`. """ + # For some reason the IBE in the URL is case sensitive server = _config.ConfigItem( - 'http://irsa.ipac.caltech.edu/ibe/', + 'http://irsa.ipac.caltech.edu/IBE/', 'Name of the IBE server to use.') mission = _config.ConfigItem( 'ptf', diff --git a/astroquery/ibe/core.py b/astroquery/ibe/core.py index b197820aea..7e76a5a78a 100644 --- a/astroquery/ibe/core.py +++ b/astroquery/ibe/core.py @@ -10,6 +10,8 @@ from __future__ import print_function, division import os +import sys + import webbrowser from bs4 import BeautifulSoup @@ -25,6 +27,14 @@ __all__ = ['Ibe', 'IbeClass'] +if sys.version_info[0] >= 3: + decode = lambda x: x.decode() + encode = lambda x: bytes(x, "ascii") + +else: + decode = lambda x: x + encode = lambda x: x + class IbeClass(BaseQuery): URL = conf.server @@ -108,7 +118,8 @@ def query_region( # Raise exception, if request failed response.raise_for_status() - return Table.read(response.text, format='ipac', guess=False) + return commons.parse_votable( + encode(response.text)).get_first_table().to_table() def query_region_sia(self, coordinate=None, mission=None, dataset=None, table=None, width=None, @@ -157,8 +168,10 @@ def query_region_async( SQL-like query string. Required if ``coordinates`` is absent. mission : str The mission to be used (if not the default mission). + Note: this kwarg is not actually used here. dataset : str The dataset to be used (if not the default dataset). + Note: this option is meaningless to the ibe service. table : str The table to be queried (if not the default table). columns : str, list @@ -196,7 +209,8 @@ def query_region_async( The action to perform at the server. The default is ``'search'``, which returns a table of the available data. ``'data'`` requires advanced path construction that is not yet supported. ``'sia'`` - provides access to the 'simple image access' IVOA protocol + is here for legacy reasons. This function is built around IRSA's + VO TAP API. IRSA's SIA API is completely different. Returns ------- @@ -221,8 +235,17 @@ def query_region_async( "The action='data' option is a placeholder for future " + "functionality.") + if action == "sia": + raise ValueError( + "The action='sia' is not implemented for IRSA's IBE " + "interface because IRSA's SIA interface is radically " + "different.") + + args = { - 'INTERSECT': intersect + 'INTERSECT': intersect, + 'TABLE': table or self.TABLE, + 'RESPONSEFORMAT': "VOTABLE" } # Note: in IBE, if 'mcen' argument is present, it is true. @@ -248,14 +271,11 @@ def query_region_async( columns = columns.split() args['columns'] = ','.join(columns) - url = "{URL}{action}/{mission}/{dataset}/{table}".format( - URL=self.URL, - action=action, - mission=mission or self.MISSION, - dataset=dataset or self.DATASET, - table=table or self.TABLE) + url = self.URL.rstrip("/") + + response = self._request('GET', url, args, timeout=self.TIMEOUT) - return self._request('GET', url, args, timeout=self.TIMEOUT) + return response def list_missions(self, cache=True): """ From a17f56b73345ab8f5ec0aa99161e42b258de6031 Mon Sep 17 00:00:00 2001 From: Sean Lake Date: Thu, 25 Apr 2019 17:09:46 +0800 Subject: [PATCH 09/12] Fix PEP8 issues --- astroquery/ibe/__init__.py | 2 +- astroquery/ibe/core.py | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/astroquery/ibe/__init__.py b/astroquery/ibe/__init__.py index 2fa02f21c7..d570b9ea49 100644 --- a/astroquery/ibe/__init__.py +++ b/astroquery/ibe/__init__.py @@ -16,7 +16,7 @@ class Conf(_config.ConfigNamespace): # For some reason the IBE in the URL is case sensitive server = _config.ConfigItem( - 'http://irsa.ipac.caltech.edu/IBE/', + 'http://irsa.ipac.caltech.edu/IBE/', 'Name of the IBE server to use.') mission = _config.ConfigItem( 'ptf', diff --git a/astroquery/ibe/core.py b/astroquery/ibe/core.py index 7e76a5a78a..ecfed20c7c 100644 --- a/astroquery/ibe/core.py +++ b/astroquery/ibe/core.py @@ -30,7 +30,7 @@ if sys.version_info[0] >= 3: decode = lambda x: x.decode() encode = lambda x: bytes(x, "ascii") - + else: decode = lambda x: x encode = lambda x: x @@ -240,7 +240,6 @@ def query_region_async( "The action='sia' is not implemented for IRSA's IBE " "interface because IRSA's SIA interface is radically " "different.") - args = { 'INTERSECT': intersect, @@ -324,7 +323,7 @@ def list_datasets(self, mission=None, cache=True): A list of dataset names """ - return [ "images" ] + return ["images"] def list_tables(self, mission=None, dataset=None, cache=True): """ From 62ce28b6fa6c74e4e0f39b2a834c10213de2fe2f Mon Sep 17 00:00:00 2001 From: Sean Lake Date: Fri, 26 Apr 2019 09:26:22 +0800 Subject: [PATCH 10/12] Remove version checking hack --- astroquery/ibe/core.py | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/astroquery/ibe/core.py b/astroquery/ibe/core.py index ecfed20c7c..2aa3a53b22 100644 --- a/astroquery/ibe/core.py +++ b/astroquery/ibe/core.py @@ -10,7 +10,6 @@ from __future__ import print_function, division import os -import sys import webbrowser from bs4 import BeautifulSoup @@ -27,15 +26,6 @@ __all__ = ['Ibe', 'IbeClass'] -if sys.version_info[0] >= 3: - decode = lambda x: x.decode() - encode = lambda x: bytes(x, "ascii") - -else: - decode = lambda x: x - encode = lambda x: x - - class IbeClass(BaseQuery): URL = conf.server MISSION = conf.mission @@ -119,7 +109,7 @@ def query_region( response.raise_for_status() return commons.parse_votable( - encode(response.text)).get_first_table().to_table() + response.content).get_first_table().to_table() def query_region_sia(self, coordinate=None, mission=None, dataset=None, table=None, width=None, From 51f2abe8db7c5ec2d2bf577b7d3478a07c35f616 Mon Sep 17 00:00:00 2001 From: Sean Lake Date: Fri, 26 Apr 2019 10:09:08 +0800 Subject: [PATCH 11/12] Remove `dataset` parameter from `config`. --- astroquery/ibe/__init__.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/astroquery/ibe/__init__.py b/astroquery/ibe/__init__.py index d570b9ea49..056cfa396b 100644 --- a/astroquery/ibe/__init__.py +++ b/astroquery/ibe/__init__.py @@ -23,9 +23,6 @@ class Conf(_config.ConfigNamespace): ('Default mission. See, for example, ' 'http://irsa.ipac.caltech.edu/ibe/search/ for options.')) - dataset = _config.ConfigItem( - 'images', - ('This option is meaningless in the context of IRSA\'s ibe.')) table = _config.ConfigItem( 'ptf.ptf_procimg', ('Default table. Select the desired mission at ' From b505817d98e27c3117a67d0c788ddaaf38f0653a Mon Sep 17 00:00:00 2001 From: Sean Lake Date: Fri, 26 Apr 2019 10:14:27 +0800 Subject: [PATCH 12/12] Trim code Remove `dataset` everywhere, and `mission` kwargs where not relevant. --- astroquery/ibe/core.py | 75 ++++++++---------------------------------- 1 file changed, 13 insertions(+), 62 deletions(-) diff --git a/astroquery/ibe/core.py b/astroquery/ibe/core.py index 2aa3a53b22..b02aa514a7 100644 --- a/astroquery/ibe/core.py +++ b/astroquery/ibe/core.py @@ -29,12 +29,11 @@ class IbeClass(BaseQuery): URL = conf.server MISSION = conf.mission - DATASET = conf.dataset TABLE = conf.table TIMEOUT = conf.timeout def query_region( - self, coordinate=None, where=None, mission=None, dataset=None, + self, coordinate=None, where=None, mission=None, table=None, columns=None, width=None, height=None, intersect='OVERLAPS', most_centered=False): """ @@ -57,10 +56,6 @@ def query_region( `~astropy.coordinates.SkyCoord`. Required if ``where`` is absent. where : str SQL-like query string. Required if ``coordinates`` is absent. - mission : str - The mission to be used (if not the default mission). - dataset : str - The dataset to be used (if not the default dataset). table : str The table to be queried (if not the default table). columns : str, list @@ -101,8 +96,8 @@ def query_region( A table containing the results of the query """ response = self.query_region_async( - coordinate=coordinate, where=where, mission=mission, - dataset=dataset, table=table, columns=columns, width=width, + coordinate=coordinate, where=where, + table=table, columns=columns, width=width, height=height, intersect=intersect, most_centered=most_centered) # Raise exception, if request failed @@ -112,7 +107,7 @@ def query_region( response.content).get_first_table().to_table() def query_region_sia(self, coordinate=None, mission=None, - dataset=None, table=None, width=None, + table=None, width=None, height=None, intersect='OVERLAPS', most_centered=False): """ @@ -120,8 +115,8 @@ def query_region_sia(self, coordinate=None, mission=None, details. The returned table will include a list of URLs. """ response = self.query_region_async( - coordinate=coordinate, mission=mission, - dataset=dataset, table=table, width=width, + coordinate=coordinate, + table=table, width=width, height=height, intersect=intersect, most_centered=most_centered, action='sia') @@ -132,7 +127,7 @@ def query_region_sia(self, coordinate=None, mission=None, response.text).get_first_table().to_table() def query_region_async( - self, coordinate=None, where=None, mission=None, dataset=None, + self, coordinate=None, where=None, table=None, columns=None, width=None, height=None, action='search', intersect='OVERLAPS', most_centered=False): @@ -156,12 +151,6 @@ def query_region_async( `~astropy.coordinates.SkyCoord`. Required if ``where`` is absent. where : str SQL-like query string. Required if ``coordinates`` is absent. - mission : str - The mission to be used (if not the default mission). - Note: this kwarg is not actually used here. - dataset : str - The dataset to be used (if not the default dataset). - Note: this option is meaningless to the ibe service. table : str The table to be queried (if not the default table). columns : str, list @@ -226,7 +215,7 @@ def query_region_async( "functionality.") if action == "sia": - raise ValueError( + raise NotImplementedError( "The action='sia' is not implemented for IRSA's IBE " "interface because IRSA's SIA interface is radically " "different.") @@ -293,33 +282,10 @@ def list_missions(self, cache=True): return missions - def list_datasets(self, mission=None, cache=True): + def list_tables(self, mission=None, cache=True): """ - For a given mission, list the available datasets. - This level has no meaning in IRSA's ibe service. - - Parameters - ---------- - mission : str - A mission name. Must be one of the valid missions from - `~astroquery.ibe.IbeClass.list_missions`. Defaults to the - configured Mission - cache : bool - Cache the query result - - Returns - ------- - datasets : list - A list of dataset names - """ - - return ["images"] - - def list_tables(self, mission=None, dataset=None, cache=True): - """ - For a given mission and dataset (see - `~.astroquery.ibe.IbeClass.list_missions`, - `~astroquery.ibe.IbeClass.list_datasets`), return the list of valid + For a given mission (see + `~.astroquery.ibe.IbeClass.list_missions`), return the list of valid table names to query. Parameters @@ -328,9 +294,6 @@ def list_tables(self, mission=None, dataset=None, cache=True): A mission name. Must be one of the valid missions from `~.astroquery.ibe.IbeClass.list_missions`. Defaults to the configured Mission - dataset : str - A dataset name. Must be one of the valid dataset from - ``list_datsets(mission)``. Defaults to the configured Dataset cache : bool Cache the query result @@ -341,20 +304,12 @@ def list_tables(self, mission=None, dataset=None, cache=True): """ if mission is None: mission = self.MISSION - if dataset is None: - dataset = self.DATASET if mission not in self.list_missions(): raise ValueError("Invalid mission specified: {0}." "Must be one of: {1}" .format(mission, self.list_missions())) - if dataset not in self.list_datasets(mission, cache=cache): - raise ValueError("Invalid dataset {0} specified for mission {1}." - "Must be one of: {2}" - .format(dataset, mission, - self.list_datasets(mission, cache=True))) - url = "{URL}search/{mission}/".format(URL=self.URL, mission=mission) response = self._request('GET', url, timeout=self.TIMEOUT, cache=cache) @@ -367,7 +322,7 @@ def list_tables(self, mission=None, dataset=None, cache=True): # def get_data(self, **kwargs): # return self.query_region_async(retrieve_data=True, **kwargs) - def show_docs(self, mission=None, dataset=None, table=None): + def show_docs(self, mission=None, table=None): """ Open the documentation for a given table in a web browser. @@ -375,8 +330,6 @@ def show_docs(self, mission=None, dataset=None, table=None): ---------- mission : str The mission to be used (if not the default mission). - dataset : str - The dataset to be used (if not the default dataset). table : str The table to be queried (if not the default table). """ @@ -388,7 +341,7 @@ def show_docs(self, mission=None, dataset=None, table=None): return webbrowser.open(url) - def get_columns(self, mission=None, dataset=None, table=None): + def get_columns(self, mission=None, table=None): """ Get the schema for a given table. @@ -396,8 +349,6 @@ def get_columns(self, mission=None, dataset=None, table=None): ---------- mission : str The mission to be used (if not the default mission). - dataset : str - The dataset to be used (if not the default dataset). table : str The table to be queried (if not the default table).