Skip to content

Commit db663c9

Browse files
DavidT3DavidT3
authored
Merge pull request #1407 from DavidT3/bug/esassInputPathsTooLong
Bug/esass input paths too long
2 parents 52cd3d6 + ade55f3 commit db663c9

File tree

3 files changed

+83
-72
lines changed

3 files changed

+83
-72
lines changed

‎xga/generate/esass/spec.py

Lines changed: 70 additions & 63 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
# This code is a part of X-ray: Generate and Analyse (XGA), a module designed for the XMM Cluster Survey (XCS).
2-
# Last modified by David J Turner (turne540@msu.edu) 03/07/2025, 10:55. Copyright (c) The Contributors
2+
# Last modified by David J Turner (turne540@msu.edu) 28/07/2025, 11:02. Copyright (c) The Contributors
33

44
import os
55
from copy import deepcopy, copy
@@ -97,10 +97,10 @@ def _append_spec_info(evt_list):
9797
check_sp = source.get_spectra(outer_radii[s_ind], obs_id, inst, inner_radii[s_ind], group_spec,
9898
min_counts, min_sn, telescope='erosita')
9999
exists = True
100-
100+
101101
except NoProductAvailableError:
102102
exists = False
103-
103+
104104
if exists and check_sp.usable and not force_gen:
105105
continue
106106

@@ -124,29 +124,34 @@ def _append_spec_info(evt_list):
124124
if use_combine_obs and (len(source.obs_ids['erosita']) > 1):
125125
# The files produced by this function will now be stored in the combined directory.
126126
final_dest_dir = OUTPUT + "erosita/combined/"
127-
# FIXME: randint doesn't like 100_000_000 because its a float, replacing with 100_000_000
128-
rand_ident = randint(0, 100_000_000) # 100_000_000)
127+
rand_ident = randint(0, 100_000_000)
129128
# Makes absolutely sure that the random integer hasn't already been used
130129
while len([f for f in os.listdir(final_dest_dir)
131130
if str(rand_ident) in f.split(OUTPUT+"erosita/combined/")[-1]]) != 0:
132131
rand_ident = randint(0, 100_000_000)
133132

134133
dest_dir = os.path.join(final_dest_dir, "temp_srctool_{}".format(rand_ident))
135134

136-
else:
137-
# Sets up the file names of the output files, adding a random number so that the
138-
# function for generating annular spectra doesn't clash and try to use the same folder
135+
else:
136+
# Sets up working directory, adding a random number so that parallel processes can't clash.
139137
# The temporary region files necessary to generate eROSITA spectra (if contaminating sources are
140138
# being removed) will be written to a different temporary folder using the same random identifier.
141139
rand_ident = randint(0, 100_000_000)
142140
dest_dir = OUTPUT + "erosita/" + "{o}/{i}_{n}_temp_{r}/".format(o=obs_id, i=inst, n=source_name,
143141
r=rand_ident)
144-
# If something got interrupted and the temp directory still exists, this will remove it
142+
# If something got interrupted and the temp directory still exists, and it somehow has the same
143+
# random number, this will remove it
145144
if os.path.exists(dest_dir):
146145
rmtree(dest_dir)
147146

147+
# The temporary directory is made, and we also set up a symlink to the relevant event list - this is
148+
# to help fix issue #1400. We found that event list paths over 204 characters long cause errors when
149+
# trying to generate spectra for eROSITA (eSASS4DR1)
148150
os.mkdir(dest_dir)
149-
151+
# The temporary directory will now have a symlink to the relevant event list, with the symlink name
152+
# the same as the actual event list
153+
os.symlink(evt_list.path, os.path.join(dest_dir, os.path.basename(evt_list.path)))
154+
150155
# If there is no match to a region, the source region returned by this method will be None,
151156
# and if the user wants to generate spectra from region files, we have to ignore that observations
152157
# TODO ASSUMPTION6 source.source_back_regions will have a telescope parameter
@@ -175,7 +180,7 @@ def _append_spec_info(evt_list):
175180
# TODO implement the detector map
176181
# This creates a detection map for the source and background region
177182
# map_path = _det_map_creation(outer_radii[s_ind], source, obs_id, inst)
178-
183+
179184
# Setting up file names that include the extra variables
180185
if group_spec and min_counts is not None:
181186
extra_file_name = "_mincnt{c}".format(c=min_counts)
@@ -197,7 +202,7 @@ def _append_spec_info(evt_list):
197202
rmf_str = "ra{ra}_dec{dec}_ri{ri}_ro{ro}_grp{gr}{ex}.rmf"
198203
rmf_str = prefix + rmf_str
199204
arf_str = "ra{ra}_dec{dec}_ri{ri}_ro{ro}_grp{gr}{ex}.arf"
200-
arf_str = prefix + arf_str
205+
arf_str = prefix + arf_str
201206
b_spec_str = "ra{ra}_dec{dec}_ri{ri}_ro{ro}_grp{gr}{ex}_backspec.fits"
202207
b_spec_str = prefix + b_spec_str
203208
b_rmf_str = "ra{ra}_dec{dec}_ri{ri}_ro{ro}_grp{gr}{ex}_backspec.rmf"
@@ -209,31 +214,31 @@ def _append_spec_info(evt_list):
209214
no_grp_spec_str = "ra{ra}_dec{dec}_ri{ri}_ro{ro}{ex}_spec_not_grouped.fits"
210215
no_grp_spec_str = prefix + no_grp_spec_str
211216
no_grp_spec = no_grp_spec_str.format(ra=source.default_coord[0].value,
212-
dec=source.default_coord[1].value, ri=src_inn_rad_str,
213-
ro=src_out_rad_str, ex=extra_file_name)
217+
dec=source.default_coord[1].value, ri=src_inn_rad_str,
218+
ro=src_out_rad_str, ex=extra_file_name)
214219

215220
# Making the strings of the XGA formatted names that we will rename the outputs of srctool to
216221
spec = spec_str.format(ra=source.default_coord[0].value,
217-
dec=source.default_coord[1].value, ri=src_inn_rad_str, ro=src_out_rad_str,
218-
ex=extra_file_name, gr=group_spec)
222+
dec=source.default_coord[1].value, ri=src_inn_rad_str, ro=src_out_rad_str,
223+
ex=extra_file_name, gr=group_spec)
219224

220225
rmf = rmf_str.format(ra=source.default_coord[0].value,
221-
dec=source.default_coord[1].value, ri=src_inn_rad_str, ro=src_out_rad_str,
222-
ex=extra_file_name, gr=group_spec)
226+
dec=source.default_coord[1].value, ri=src_inn_rad_str, ro=src_out_rad_str,
227+
ex=extra_file_name, gr=group_spec)
223228
arf = arf_str.format(ra=source.default_coord[0].value,
224-
dec=source.default_coord[1].value, ri=src_inn_rad_str, ro=src_out_rad_str,
225-
ex=extra_file_name, gr=group_spec)
229+
dec=source.default_coord[1].value, ri=src_inn_rad_str, ro=src_out_rad_str,
230+
ex=extra_file_name, gr=group_spec)
226231

227232
b_spec = b_spec_str.format(ra=source.default_coord[0].value,
228-
dec=source.default_coord[1].value, ri=src_inn_rad_str, ro=src_out_rad_str,
229-
ex=extra_file_name, gr=group_spec)
233+
dec=source.default_coord[1].value, ri=src_inn_rad_str, ro=src_out_rad_str,
234+
ex=extra_file_name, gr=group_spec)
230235
b_rmf = b_rmf_str.format(ra=source.default_coord[0].value,
231-
dec=source.default_coord[1].value, ri=src_inn_rad_str, ro=src_out_rad_str,
232-
ex=extra_file_name, gr=group_spec)
236+
dec=source.default_coord[1].value, ri=src_inn_rad_str, ro=src_out_rad_str,
237+
ex=extra_file_name, gr=group_spec)
233238
b_arf = b_arf_str.format(ra=source.default_coord[0].value,
234-
dec=source.default_coord[1].value, ri=src_inn_rad_str, ro=src_out_rad_str,
235-
ex=extra_file_name, gr=group_spec)
236-
239+
dec=source.default_coord[1].value, ri=src_inn_rad_str, ro=src_out_rad_str,
240+
ex=extra_file_name, gr=group_spec)
241+
237242
# These file names are for the debug images of the source and background images, they will not be loaded
238243
# in as a XGA products, but exist purely to check by eye if necessary
239244
dim = "{o}_{i}_{n}_ra{ra}_dec{dec}_ri{ri}_ro{ro}_debug.fits".format(o=obs_id, i=inst, n=source_name,
@@ -242,9 +247,9 @@ def _append_spec_info(evt_list):
242247
ri=src_inn_rad_str,
243248
ro=src_out_rad_str)
244249
b_dim = ("{o}_{i}_{n}_ra{ra}_dec{dec}_ri{ri}_ro{ro}_"
245-
"back_debug.fits").format(o=obs_id, i=inst, n=source_name, ra=source.default_coord[0].value,
246-
dec=source.default_coord[1].value, ri=src_inn_rad_str,
247-
ro=src_out_rad_str)
250+
"back_debug.fits").format(o=obs_id, i=inst, n=source_name, ra=source.default_coord[0].value,
251+
dec=source.default_coord[1].value, ri=src_inn_rad_str,
252+
ro=src_out_rad_str)
248253

249254
# TODO ADD MANY MORE COMMENTS
250255
coord_str = "icrs;{ra}, {dec}".format(ra=source.default_coord[0].value,
@@ -265,28 +270,30 @@ def _append_spec_info(evt_list):
265270
group_scale = ''
266271

267272
# Fills out the srctool command to make the main and background spectra
273+
# WE NOTE THAT, BECAUSE WE CREATE EVENT LIST SYMLINKS TO SOLVE ISSUE #1400, THE EVENT LIST PATHS
274+
# ARE JUST THE BASE FILENAME OF THE EVENT LIST
268275
if isinstance(source, ExtendedSource):
269276
try:
270277
if use_combine_obs and (len(source.obs_ids['erosita']) > 1):
271-
im = source.get_combined_images(lo_en=Quantity(0.2, 'keV'), hi_en=Quantity(10.0, 'keV'),
278+
im = source.get_combined_images(lo_en=Quantity(0.2, 'keV'), hi_en=Quantity(10.0, 'keV'),
272279
telescope='erosita')
273280
else:
274281
# We only need the image path for extended source generation
275282
im = source.get_images(obs_id, lo_en=Quantity(0.2, 'keV'), hi_en=Quantity(10.0, 'keV'),
276-
telescope='erosita')
283+
telescope='erosita')
277284
# We have a slightly different command for extended and point sources
278-
s_cmd_str = ext_srctool_cmd.format(d=dest_dir, ef=evt_list.path, sc=coord_str, reg=src_reg_str,
279-
i=inst_no, ts=t_step, em=im.path, et=et)
285+
s_cmd_str = ext_srctool_cmd.format(d=dest_dir, ef=os.path.basename(evt_list.path), sc=coord_str,
286+
reg=src_reg_str, i=inst_no, ts=t_step, em=im.path, et=et)
280287
except:
281288
raise ValueError(f"it was this sources {source.name}")
282289

283290
else:
284-
s_cmd_str = pnt_srctool_cmd.format(d=dest_dir, ef=evt_list.path, sc=coord_str, reg=src_reg_str,
285-
i=inst_no, ts=t_step)
291+
s_cmd_str = pnt_srctool_cmd.format(d=dest_dir, ef=os.path.basename(evt_list.path), sc=coord_str,
292+
reg=src_reg_str, i=inst_no, ts=t_step)
286293

287294
# TODO FIGURE OUT WHAT TO DO ABOUT THE TIMESTEP
288-
sb_cmd_str = bckgr_srctool_cmd.format(ef=evt_list.path, sc=coord_str, breg=bsrc_reg_str,
289-
i=inst_no, ts=t_step*4)
295+
sb_cmd_str = bckgr_srctool_cmd.format(ef=os.path.basename(evt_list.path), sc=coord_str, breg=bsrc_reg_str,
296+
i=inst_no, ts=t_step * 4)
290297
# Filling out the grouping command
291298
grp_cmd_str = grp_cmd.format(infi=no_grp_spec, of=spec, gt=group_type, gs=group_scale)
292299

@@ -355,25 +362,25 @@ def _append_spec_info(evt_list):
355362
if os.path.exists(OUTPUT + 'erosita/' + obs_id + '/temp_regs_{i}'.format(i=rand_ident)):
356363
# Removing this directory
357364
cmd_str += ";rm -r temp_regs_{i}".format(i=rand_ident)
358-
365+
359366
cmds.append(cmd_str) # Adds the full command to the set
360-
367+
361368
final_paths.append(os.path.join(OUTPUT, "erosita", obs_id, spec))
362369
extra_info.append({"inner_radius": inn_rad_degrees, "outer_radius": out_rad_degrees,
363-
"rmf_path": os.path.join(OUTPUT, "erosita", obs_id, rmf),
364-
"arf_path": os.path.join(OUTPUT, "erosita", obs_id, arf),
365-
"b_spec_path": os.path.join(OUTPUT, "erosita", obs_id, b_spec),
366-
"b_rmf_path": os.path.join(OUTPUT, "erosita", obs_id, b_rmf),
367-
"b_arf_path": os.path.join(OUTPUT, "erosita", obs_id, b_arf),
368-
"obs_id": obs_id, "instrument": inst,
369-
"central_coord": source.default_coord,
370-
"grouped": group_spec,
371-
"min_counts": min_counts,
372-
"min_sn": min_sn,
373-
"over_sample": None,
374-
"from_region": from_region,
375-
"telescope": "erosita"})
376-
370+
"rmf_path": os.path.join(OUTPUT, "erosita", obs_id, rmf),
371+
"arf_path": os.path.join(OUTPUT, "erosita", obs_id, arf),
372+
"b_spec_path": os.path.join(OUTPUT, "erosita", obs_id, b_spec),
373+
"b_rmf_path": os.path.join(OUTPUT, "erosita", obs_id, b_rmf),
374+
"b_arf_path": os.path.join(OUTPUT, "erosita", obs_id, b_arf),
375+
"obs_id": obs_id, "instrument": inst,
376+
"central_coord": source.default_coord,
377+
"grouped": group_spec,
378+
"min_counts": min_counts,
379+
"min_sn": min_sn,
380+
"over_sample": None,
381+
"from_region": from_region,
382+
"telescope": "erosita"})
383+
377384
# TODO MORE COMMENTS
378385

379386
# TODO This will change in a future release, so that the user can control it - see issue #1113. The definitions
@@ -393,7 +400,7 @@ def _append_spec_info(evt_list):
393400
# This function supports passing both individual sources and sets of sources
394401
if isinstance(sources, BaseSource):
395402
sources = [sources]
396-
403+
397404
if combine_obs:
398405
# This requires combined event lists - this function will generate them
399406
evtool_combine_evts(sources)
@@ -411,7 +418,7 @@ def _append_spec_info(evt_list):
411418
min_counts = int(min_counts)
412419
if min_sn is not None:
413420
min_sn = float(min_sn)
414-
421+
415422
# Checking user has passed a grouping argument if group spec is true
416423
if all([o is not None for o in [min_counts, min_sn]]):
417424
raise eSASSInputInvalid("Only one grouping option can passed, you can't group both by"
@@ -420,7 +427,7 @@ def _append_spec_info(evt_list):
420427
elif group_spec and all([o is None for o in [min_counts, min_sn]]):
421428
raise eSASSInputInvalid("If you set group_spec=True, you must supply a grouping option, either min_counts"
422429
" or min_sn.")
423-
430+
424431
# Sets up the extra part of the storage key name depending on if grouping is enabled
425432
if group_spec and min_counts is not None:
426433
extra_name = "_mincnt{}".format(min_counts)
@@ -464,7 +471,7 @@ def _append_spec_info(evt_list):
464471
pnt_srctool_cmd = 'cd {d}; srctool eventfiles="{ef}" srccoord="{sc}" todo="SPEC ARF RMF"' \
465472
' srcreg="{reg}" exttype="POINT" tstep={ts}' \
466473
' insts={i} psftype="2D_PSF"'
467-
474+
468475
# You can't control the whole name of the output of srctool, so this renames it to the XGA format
469476
rename_cmd = 'mv srctoolout_{i_no}??_{type}* {nn}'
470477
# Having a string to remove the 'merged' spectra that srctool outputs, even when you only request one instrument
@@ -514,7 +521,7 @@ def _append_spec_info(evt_list):
514521
# once the eSASS cmd has run
515522
sources_extras.append(np.array(extra_info))
516523
sources_types.append(np.full(sources_cmds[-1].shape, fill_value="spectrum"))
517-
524+
518525
# then we can continue with the rest of the sources
519526
continue
520527

@@ -545,7 +552,7 @@ def _append_spec_info(evt_list):
545552
ri=src_inn_rad_str, ro=src_out_rad_str, gr=group_spec)
546553
else:
547554
spec_storage_name = "region"
548-
555+
549556
# Adds on the extra information about grouping to the storage key
550557
spec_storage_name += extra_name
551558

@@ -557,7 +564,7 @@ def _append_spec_info(evt_list):
557564
# that are defined above
558565
_append_spec_info(evt_list)
559566

560-
567+
561568
else:
562569
# getting Eventlist product
563570
evt_list = source.get_products("combined_events", just_obj=True, telescope="erosita")[0]
@@ -678,7 +685,7 @@ def srctool_spectrum(sources: Union[BaseSource, BaseSample], outer_radius: Union
678685
inner_radius: Union[str, Quantity] = Quantity(0, 'arcsec'), group_spec: bool = True,
679686
min_counts: int = 5, min_sn: float = None, num_cores: int = NUM_CORES,
680687
disable_progress: bool = False, combine_tm: bool = True, combine_obs: bool = True, force_gen: bool = False):
681-
688+
682689
"""
683690
A wrapper for all the eSASS and Heasoft processes necessary to generate an eROSITA spectrum that can be analysed
684691
in XSPEC. Every observation associated with this source, and every instrument associated with that
@@ -940,7 +947,7 @@ def esass_spectrum_set(sources: Union[BaseSource, BaseSample], radii: Union[List
940947
split_a = copy(interim_extras[p_ind]['arf_path']).split('/')
941948
# split_ba = copy(interim_extras[p_ind]['b_arf_path']).split('/')
942949
split_bs = copy(interim_extras[p_ind]['b_spec_path']).split('/')
943-
950+
944951
new_arf = split_a[-1].replace('.arf', "_ident{si}_{ai}".format(si=set_id, ai=r_ind)) + ".arf"
945952
# new_b_arf = split_ba[-1].replace('_back.arf', "_ident{si}_{ai}".format(si=set_id, ai=r_ind)) \
946953
# + "_back.arf"

0 commit comments

Comments
 (0)