Implementation, docs, tests

Author: snbianco

astroquery/mast/missions.py

@@ -475,8 +475,16 @@ def filter_products(self, products, *, extension=None, **filters):
             Column-based filters to apply to the products table.
 
             Each keyword corresponds to a column name in the table, with the argument being one or more
-            acceptable values for that column. AND logic is applied between filters, OR logic within
-            each filter set. For example: type="science", extension=["fits", "jpg"]
+            acceptable values for that column. AND logic is applied between filters.
+
+            Within each column's filter set:
+
+            - Positive (non-negated) values are combined with OR logic.
+            - Any negated values (prefixed with "!") are combined with AND logic against the ORed positives.
+              This results in: (NOT any_negatives) AND (any_positives)
+              Examples:
+                ``file_suffix=['A', 'B', '!C']`` → (file_suffix != C) AND (file_suffix == A OR file_suffix == B)
+                ``size=['!14400', '<20000']`` → (size != 14400) AND (size < 20000)
 
             For columns with numeric data types (int or float), filter values can be expressed
             in several ways:

astroquery/mast/observations.py

@@ -560,10 +560,16 @@ def filter_products(self, products, *, mrp_only=False, extension=None, **filters
             `here <https://masttest.stsci.edu/api/v0/_productsfields.html>`__.
 
             Each keyword corresponds to a column name in the table, with the argument being one or more
-            acceptable values for that column. AND logic is applied between filters, OR logic within
-            each filter set.
+            acceptable values for that column. AND logic is applied between filters.
 
-            For example: type="science", extension=["fits", "jpg"]
+            Within each column's filter set:
+
+            - Positive (non-negated) values are combined with OR logic.
+            - Any negated values (prefixed with "!") are combined with AND logic against the ORed positives.
+              This results in: (NOT any_negatives) AND (any_positives)
+              Examples:
+                ``productType=['A', 'B', '!C']`` → (productType != C) AND (productType == A OR productType == B)
+                ``size=['!14400', '<20000']`` → (size != 14400) AND (size < 20000)
 
             For columns with numeric data types (int or float), filter values can be expressed
             in several ways:

astroquery/mast/tests/test_mast.py

@@ -374,7 +374,7 @@ def test_missions_filter_products(patch_post):
     filtered = mast.MastMissions.filter_products(products, extension='fits')
     assert len(filtered) > 0
 
-    # Numeric filtering
+    # -------- Numeric filtering tests --------
     # Single integer value
     filtered = mast.MastMissions.filter_products(products, size=11520)
     assert all(filtered['size'] == 11520)
@@ -404,6 +404,40 @@ def test_missions_filter_products(patch_post):
     filtered = mast.MastMissions.filter_products(products, size=[14400, '>20000'])
     assert all((filtered['size'] == 14400) | (filtered['size'] > 20000))
 
+    # -------- Negative operator tests --------
+    # Negate a single numeric value
+    filtered = mast.MastMissions.filter_products(products, size='!11520')
+    assert all(filtered['size'] != 11520)
+
+    # Negate a comparison
+    filtered = mast.MastMissions.filter_products(products, size='!<15000')
+    assert all(filtered['size'] >= 15000)
+
+    # Negate one element in a list with one other condition
+    filtered = mast.MastMissions.filter_products(products, size=['!14400', '>20000'])
+    assert all((filtered['size'] != 14400) & (filtered['size'] > 20000))
+
+    # Negate one element in a list with two other conditions
+    filtered = mast.MastMissions.filter_products(products, size=['!14400', '<20000', '>30000'])
+    assert all((filtered['size'] != 14400) & (filtered['size'] < 20000) | (filtered['size'] > 30000))
+
+    # Negate a range
+    filtered = mast.MastMissions.filter_products(products, size='!14400..17280')
+    assert all(~((filtered['size'] >= 14400) & (filtered['size'] <= 17280)))
+
+    # Negate a string match
+    filtered = mast.MastMissions.filter_products(products, category='!CALIBRATED')
+    assert all(filtered['category'] != 'CALIBRATED')
+
+    # Negate one string in a list
+    filtered = mast.MastMissions.filter_products(products, category=['!CALIBRATED', 'UNCALIBRATED'])
+    assert all((filtered['category'] != 'CALIBRATED') & (filtered['category'] == 'UNCALIBRATED'))
+
+    # Negate two strings in a list
+    filtered = mast.MastMissions.filter_products(products, category=['!CALIBRATED', '!UNCALIBRATED'])
+    assert all((filtered['category'] != 'CALIBRATED') & (filtered['category'] != 'UNCALIBRATED'))
+    # ------------------------------------------
+
     with pytest.raises(InvalidQueryError, match="Could not parse numeric filter 'invalid' for column 'size'"):
         # Invalid filter value
         mast.MastMissions.filter_products(products, size='invalid')

astroquery/mast/utils.py

@@ -347,7 +347,41 @@ def remove_duplicate_products(data_products, uri_key):
     return unique_products
 
 
-def parse_numeric_product_filter(val):
+def _combine_positive_negative_masks(mask_funcs):
+    """
+    Combines a list of mask functions into a single mask according to:
+    - OR logic among positive masks
+    - AND logic among negative masks applied after positives
+
+    Parameters
+    ----------
+    mask_funcs : list of tuples (func, is_negated)
+        Each element is a tuple where:
+        - func: a callable that takes an array and returns a boolean mask
+        - is_negated: boolean, True if the mask should be negated before combining
+
+    Returns
+    -------
+    combined_mask : np.ndarray
+        Combined boolean mask.
+    """
+    def combined(col):
+        positive_masks = [f(col) for f, neg in mask_funcs if not neg]
+        negative_masks = [~f(col) for f, neg in mask_funcs if neg]
+
+        # Use OR logic between positive masks
+        pos_mask = np.logical_or.reduce(positive_masks) if positive_masks else np.ones(len(col), dtype=bool)
+
+        # Use AND logic between negative masks
+        neg_mask = np.logical_and.reduce(negative_masks) if negative_masks else np.ones(len(col), dtype=bool)
+
+        # Use AND logic to combine positive and negative masks
+        return pos_mask & neg_mask
+
+    return combined
+
+
+def parse_numeric_product_filter(vals):
     """
     Parses a numeric product filter value and returns a function that can be used to filter
     a column of a product table.
@@ -369,30 +403,35 @@ def parse_numeric_product_filter(val):
     # Regular expression to match range patterns
     range_pattern = re.compile(r'[+-]?(\d+(\.\d*)?|\.\d+)\.\.[+-]?(\d+(\.\d*)?|\.\d+)')
 
-    def single_condition(cond):
-        """Helper function to create a condition function for a single value."""
-        if isinstance(cond, (int, float)):
-            return lambda col: col == float(cond)
-        if cond.startswith('>='):
-            return lambda col: col >= float(cond[2:])
-        elif cond.startswith('<='):
-            return lambda col: col <= float(cond[2:])
-        elif cond.startswith('>'):
-            return lambda col: col > float(cond[1:])
-        elif cond.startswith('<'):
-            return lambda col: col < float(cond[1:])
-        elif range_pattern.fullmatch(cond):
-            start, end = map(float, cond.split('..'))
+    def base_condition(cond_str):
+        """Create a mask function for a numeric condition string (no negation handling here)."""
+        if isinstance(cond_str, (int, float)):
+            return lambda col: col == float(cond_str)
+        elif cond_str.startswith('>='):
+            return lambda col: col >= float(cond_str[2:])
+        elif cond_str.startswith('<='):
+            return lambda col: col <= float(cond_str[2:])
+        elif cond_str.startswith('>'):
+            return lambda col: col > float(cond_str[1:])
+        elif cond_str.startswith('<'):
+            return lambda col: col < float(cond_str[1:])
+        elif range_pattern.fullmatch(cond_str):
+            start, end = map(float, cond_str.split('..'))
             return lambda col: (col >= start) & (col <= end)
         else:
-            return lambda col: col == float(cond)
+            return lambda col: col == float(cond_str)
 
-    if isinstance(val, list):
-        # If val is a list, create a condition for each value and combine them with logical OR
-        conditions = [single_condition(v) for v in val]
-        return lambda col: np.logical_or.reduce([cond(col) for cond in conditions])
-    else:
-        return single_condition(val)
+    vals = [vals] if not isinstance(vals, list) else vals
+    mask_funcs = []
+    for v in vals:
+        # Check if the value is negated and strip the negation if present
+        is_negated = isinstance(v, str) and v.startswith('!')
+        v = v[1:] if is_negated else v
+
+        func = base_condition(v)
+        mask_funcs.append((func, is_negated))
+
+    return _combine_positive_negative_masks(mask_funcs)
 
 
 def apply_column_filters(products, filters):
@@ -426,11 +465,20 @@ def apply_column_filters(products, filters):
             except ValueError:
                 raise InvalidQueryError(f"Could not parse numeric filter '{vals}' for column '{colname}'.")
         else:  # Assume string or list filter
-            if isinstance(vals, str):
-                vals = [vals]
-            this_mask = np.isin(col_data, vals)
+            vals = [vals] if isinstance(vals, str) else vals
+            mask_funcs = []
+            for val in vals:
+                # Check if the value is negated and strip the negation if present
+                is_negated = isinstance(val, str) and val.startswith('!')
+                v = val[1:] if is_negated else val
+
+                def func(col, v=v):
+                    return np.isin(col, [v])
+                mask_funcs.append((func, is_negated))
+
+            this_mask = _combine_positive_negative_masks(mask_funcs)(col_data)
 
-        # Combine the current column mask with the overall mask
+        # AND logic across different columns
         col_mask &= this_mask
 
     return col_mask

docs/mast/mast_missions.rst

@@ -242,7 +242,15 @@ In many cases, you will not need to download every product that is associated wi
 `~astroquery.mast.MastMissionsClass.filter_products` function allows for filtering based on file extension (``extension``)
 and any other of the product fields.
 
-The **AND** operation is performed for a list of filters, and the **OR** operation is performed within a filter set. 
+The **AND** operation is applied between filters, and the **OR** operation is applied within each filter set, except in the case of negated values.
+
+A filter value can be negated by prefiing it with ``!``, meaning that rows matching that value will be excluded from the results.
+When any negated value is present in a filter set, any positive values in that set are combined with **OR** logic, and the negated 
+values are combined with **AND** logic against the positives. 
+
+For example:
+  - ``file_suffix=['A', 'B', '!C']`` → (file_suffix != C) AND (file_suffix == A OR file_suffix == B)
+  - ``size=['!14400', '<20000']`` → (size != 14400) AND (size < 20000)
 
 For columns with numeric data types (``int`` or ``float``), filter values can be expressed in several ways:
   - A single number: ``size=100``

docs/mast/mast_obsquery.rst

@@ -285,7 +285,15 @@ In many cases, you will not need to download every product that is associated wi
 `~astroquery.mast.ObservationsClass.filter_products` function allows for filtering based on minimum recommended products
 (``mrp_only``), file extension (``extension``), and any other of the `CAOM products fields <https://mast.stsci.edu/api/v0/_productsfields.html>`_.
 
-The **AND** operation is performed for a list of filters, and the **OR** operation is performed within a filter set. 
+The **AND** operation is applied between filters, and the **OR** operation is applied within each filter set, except in the case of negated values.
+
+A filter value can be negated by prefiing it with ``!``, meaning that rows matching that value will be excluded from the results.
+When any negated value is present in a filter set, any positive values in that set are combined with **OR** logic, and the negated 
+values are combined with **AND** logic against the positives. 
+
+For example:
+  - ``productType=['A', 'B', '!C']`` → (productType != C) AND (productType == A OR productType == B)
+  - ``size=['!14400', '<20000']`` → (size != 14400) AND (size < 20000)
 
 For columns with numeric data types (``int`` or ``float``), filter values can be expressed in several ways:
   - A single number: ``size=100``