Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions clarsach/__init__.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
#!/usr/bin/env python
'''Clarsach'''
from . import spectrum
from .spectrum import XSpectrum, calculate_flux_spectrum
from .respond import RMF, ARF
from .models import *
from .models import Powerlaw
18 changes: 16 additions & 2 deletions clarsach/respond.py
Original file line number Diff line number Diff line change
Expand Up @@ -260,6 +260,7 @@ def _load_arf(self, filename):
self.e_high = np.array(data.field("ENERG_HI"))
self.e_unit = data.columns["ENERG_LO"].unit
self.specresp = np.array(data.field("SPECRESP"))
self.exposure = hdr["EXPOSURE"]

if "FRACEXPO" in data.columns.names:
self.fracexpo = data["FRACEXPO"]
Expand All @@ -268,7 +269,7 @@ def _load_arf(self, filename):

return

def apply_arf(self, spec):
def apply_arf(self, spec, apply_exp=True, apply_frac_exp=True):
"""
Fold the spectrum through the ARF.

Expand All @@ -281,6 +282,12 @@ def apply_arf(self, spec):
spec : numpy.ndarray
The (model) spectrum to be folded

apply_exp : bool, default True
If True, apply exposure to ARF-corrected spectrum

apply_frac_exp : bool, default True
If True, apply fractional exposure to ARF-corrected spectrum

Returns
-------
s_arf : numpy.ndarray
Expand All @@ -292,4 +299,11 @@ def apply_arf(self, spec):
"be of same size as the " \
"ARF array."

return np.array(spec) * self.specresp
spec_corr = np.array(spec) * self.specresp

if apply_exp:
spec_corr *= self.exposure
if apply_frac_exp:
spec_corr *= self.fracexpo

return spec_corr
103 changes: 91 additions & 12 deletions clarsach/spectrum.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,16 +4,64 @@
from clarsach.respond import RMF, ARF
from astropy.io import fits

__all__ = ['XSpectrum']
__all__ = ['XSpectrum', 'calculate_flux_spectrum']

ALLOWED_UNITS = ['keV','angs','angstrom','kev']
ALLOWED_TELESCOPES = ['HETG','ACIS']

CONST_HC = 12.398418573430595 # Copied from ISIS, [keV angs]
UNIT_LABELS = dict(zip(ALLOWED_UNITS, ['Energy (keV)', 'Wavelength (angs)']))

def calculate_flux_spectrum(spec, counts=None):
"""
Convert a spectrum from units of photon counts to
units of photon flux.

**Warning**: You should do this for plotting *only*.
Converting from flux to counts, as is done in the detector,
involves a convolution with the response matrix of the detector.
Deconvolving this correctly is very hard, and not done in this
function. This is for visualization purposes only!

Parameters
----------
spec : clarsach.XSpectrum object
An XSpectrum object containing the X-ray spectrum

counts : iterable
An array with counts to be converted to the flux
If this is None, use the `counts` attribute of the `spec`
object.

Returns
-------
flux_spectrum:
The flux spectrum

"""

# make a flat spectrum so that I can integrate
# ARF and RMF only
flat_model = np.ones_like(spec.counts) * (spec.rmf.energ_hi -
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is what I'm working on in xpysis.

I am certain that you should not multiply by (energ_hi - energ_lo). When I do this with the test data on Mrk 421, it gives me something that I know is an order of magnitude off from the true values

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

To see how this method for unfolding data performs, see this notebook:
https://github.com/eblur/xpysis/blob/master/notebooks/Example1_fit_HETG_powerlaw.ipynb

Copy link
Owner Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Regarding the multiplication by (energ_hi - energ_lo): I don't know how to square that circle, because I have a case where the exact opposite is true: if I don't multiply by the bin width, I end up with something that is no longer a power law, because my power law and the fact that the energy bins are logarithmic messes up the slope.
I don't know how to make sense of that.

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Oh my. Is it instrument dependent? What data set are you using where you do need to multiply by the bin width?

Copy link
Owner Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Well, right now simulated Chandra/HETG data, actually, so this is the same instrument.

Here's a test I just did: I took the simulated Chandra/HETG spectrum (the one that's just a power law), and applied both methods to it (once without multiplying by the energy bin, and once with), and then plotted that with a comparison to the code that the sherpa manual suggests using.

Here's the plot:
flux_model_comparison

I can see that this is about an order of magnitude off from what sherpa calculates (and sherpa doesn't have those weird features), but the one where I don't multiply by the bin width is off, too, and it has a different slope. So, if sherpa is right, currently neither of our methods is working correctly. If sherpa is wrong, then I don't know at all anymore what's going on.

Victoria and I tested this by eye when I was at ESTEC: the code I used (which is similar to what I implemented in this PR) seemed to give the right answer (where "right" == "the same as ISIS").

I am super confused.

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Those features (e.g. the bump around 2 keV) should definitely not be there. Sherpa is right in this case.

If the fitting is still working (i.e. giving us a power law with photon index = 2), then I'm inclined to say it's our method of unfolding the data. If the fitting is not working any more, then it's something wrong with the way we are applying the response.

Thank you for testing this, as I have never been able to install sherpa properly and can only use ISIS as a comparison.

Copy link
Collaborator

@eblur eblur Jul 14, 2017

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

By the way, did you notice PR #16 ? If you have not yet reviewed this change, this could be the problem.

(but even after fixing it in my own code, I still get those weird features around 2 keV, which are clearly artifacts from the ARF)

Copy link
Owner Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Well, so I'm pretty sure it's the unfolding, not the application of the ARF/RMF, because I tested those on simulated spectra with a direct comparison to sherpa, and that gives me correct results for all instruments I tested so far.

Here's a slide I found in a etalk by David Huenemoerder that deals with this:
screen shot 2017-07-14 at 17 30 25

I find that incredibly confusing, though. The words sound like we should be integrating: if S(E) = 1, then we want the counts per unit flux, then we should be integrating S(E) in order to get something that's in the same units as the data. But the equation makes it look like we shouldn't be integrating. He does mention, though, that you can find spurious features. I've been trying to figure out how exactly sherpa computes this plot, but so far, I've not managed to track it down within the source code.

Do you think it would help if we talked this through per telecon rather that GitHub issues next week?

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes, let's make some time to skype about this. I think it's easier in person. Plus we may as well talk about some other considered changes. I am going home soon, so next week is good. I'll email you on Monday.

Copy link
Owner Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Okay, I will go home soon, too, but first, two observations:
(1) The weird features in the spectrum are caused by applying frac_expo. I don't know why, but when I don't apply frac_expo, they go away.
(2) It looks like there should definitely not be a np.sum in the expression for the RMF calculation. The reference implementation that I used to compare with some of the simulated data sets (NICER, HEXTE, Chandra etc) doesn't have a sum.
If those two conditions are fulfilled, I get the same spectrum as sherpa does.

spec.rmf.energ_lo)

# now apply ARF to the flat model
m_arf = spec.arf.apply_arf(flat_model)
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Use spec.apply_resp(flat_model) here instead of apply_arf and apply_resp.

apply_resp takes into account fracexpo. It is not specific to Chandra. For ARF files where the "FRACEXPO" column does not exist, fracexpo is set to 1.0 and doesn't really do anything.


# apply RMF to the flat model
m_rmf = spec.rmf.apply_rmf(m_arf)

# divide the observed counts by the flat model with the ARF/RMF applied
if counts is None:
flux_spec = spec.counts / m_rmf / spec.exposure
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If apply_resp is used, you will not need the spec.exposure term here.

else:
flux_spec = counts / m_rmf / spec.exposure

return flux_spec

# Not a very smart reader, but it works for HETG
class XSpectrum(object):

def __init__(self, filename, telescope='HETG'):
assert telescope in ALLOWED_TELESCOPES

Expand All @@ -35,11 +83,30 @@ def __store_path(self, filename):
return

def apply_resp(self, mflux):
# Given a model flux spectrum, apply the response
mrate = self.arf.apply_arf(mflux) # phot/s per bin
mrate *= self.exposure * self.arf.fracexpo # phot per bin
result = self.rmf.apply_rmf(mrate) # counts per bin
return result
"""
Given a model flux spectrum, apply the response, both ARF and RMF.

Parameters
----------
mflux : iterable
list or array with the model flux spectrum

Returns
-------
m_rmf: numpy.ndarray
The counts spectrum with responses applied

"""
# For some instruments, the ARF could not exist
if self.arf is not None:
mrate = self.arf.apply_arf(mflux, apply_exp=True,
apply_frac_exp=True) # phot/s per bin
else:
mrate = mflux

m_rmf = self.rmf.apply_rmf(mrate) # counts per bin

return m_rmf

@property
def bin_mid(self):
Expand Down Expand Up @@ -76,14 +143,24 @@ def hard_set_units(self, unit):
self.counts = new_cts
self.bin_unit = unit

def plot(self, ax, xunit='keV', **kwargs):
def plot(self, ax, xunit='keV', yunit="counts", **kwargs):
lo, hi, mid, cts = self._change_units(xunit)
counts_err = np.sqrt(cts)
ax.errorbar(mid, cts, yerr=counts_err,
ls='', marker=None, color='k', capsize=0, alpha=0.5)
ax.step(lo, cts, where='post', **kwargs)
counts_err = np.sqrt(cts)
if yunit == "counts":
ax.errorbar(mid, cts, yerr=counts_err,
ls='', marker=None, color='k', capsize=0, alpha=0.5)
ax.set_ylabel('Counts')
ax.step(lo, cts, where='post', **kwargs)

elif yunit == "flux":
flux = calculate_flux_spectrum(self)
flux_err = calculate_flux_spectrum(self, counts_err)
ax.errorbar(mid, flux, yerr=flux_err,
ls='', marker=None, color='k', capsize=0, alpha=0.5)
ax.set_ylabel('Flux')
ax.step(lo, flux, where='post', **kwargs)

ax.set_xlabel(UNIT_LABELS[xunit])
ax.set_ylabel('Counts')

def _read_chandra(self, filename):
this_dir = os.path.dirname(os.path.abspath(filename))
Expand All @@ -99,3 +176,5 @@ def _read_chandra(self, filename):
self.arf = ARF(self.arf_file)
self.exposure = ff[1].header['EXPOSURE'] # seconds
ff.close()

self.flux = calculate_flux_spectrum(self, None)
3 changes: 3 additions & 0 deletions data/arfs/aciss_heg1_cy19.garf
Git LFS file not shown
3 changes: 3 additions & 0 deletions data/rmfs/aciss_heg1_cy19.grmf
Git LFS file not shown
Loading