2023-05-06T18:30:05Z INFO __main__ 253: 
CONFIG

[Paths]
tm_archive = /data/stix/SOLSOC/from_edds/tm/processing
fits_archive = /data/stix/out/fits_v1.0.0
spice_kernels = /data/stix/spice/kernels
soop_files = /data/stix/SOLSOC/from_soc

[Logging]
stop_on_error = False

[IDLBridge]
enabled = True
gsw_path = /opt/STIX-GSW
batchsize = 30

[Pipeline]
status_server_port = 12388
parallel_batchsize_l0 = 100
parallel_batchsize_l1 = 100
parallel_batchsize_l2 = 100
log_level = INFO
log_dir = /data/stix/out/fits_v1.0.0/logging
error_mail_send = True
error_mail_receivers = nicky.hochmuth@ateleris.ch, shane.maloney@tcd.ie
error_mail_smpt_host = lmailer.fhnw.ch
error_mail_smpt_port = 25
error_mail_sender = nicky.hochmuth@fhnw.ch
start_with_unprocessed = False

[Publish]
db_file = /home/stixcore/soar_pub/published.sqlite
include_levels = L0, L1
waiting_period = 21d
target_dir = /home/solsoc/to_soc
report_mail_receivers = nicky.hochmuth@ateleris.ch, shane.maloney@tcd.ie
report_mail_send = True
same_esa_name_dir = /data/stix/out/esa_conflicts
rid_lut_file = ./stixcore/data/publish/rid_lut.csv
rid_lut_file_update_url = https://datacenter.stix.i4ds.net/api/bsd/info/
batch_size = 20000
fits_dir = /data/stix/out/fits_v1.0.0
target_host = pub026.cs.technik.fhnw.ch
supplement_report = /home/stixcore/soar_pub/supplement_report.csv

[SOOP]
endpoint = https://solarorbiter.esac.esa.int/soopkitchen/api
user = smaloney
password = xxxx
soop_files_download = ./stixcore/data/soop


2023-05-06T18:30:05Z INFO __main__ 270: 
SINGLETONS

SOOPManager: /data/stix/SOLSOC/from_soc
SPICE: /data/stix/spice/kernels/mk/solo_ANC_soc-flown-mk_V110_20230504_001.tm
IDBManager: /home/stixcore/STIXCore/stixcore/data/idb
Versions:
[{'label': '2.26.36', 'path': '/home/stixcore/STIXCore/stixcore/data/idb/v2.26.36', 'version': ['2', '26', '36']}, {'label': '2.26.32', 'path': '/home/stixcore/STIXCore/stixcore/data/idb/v2.26.32', 'version': ['2', '26', '32']}, {'label': '2.26.34', 'path': '/home/stixcore/STIXCore/stixcore/data/idb/v2.26.34', 'version': ['2', '26', '34']}, {'label': '2.26.33', 'path': '/home/stixcore/STIXCore/stixcore/data/idb/v2.26.33', 'version': ['2', '26', '33']}, {'label': '2.26.31', 'path': '/home/stixcore/STIXCore/stixcore/data/idb/v2.26.31', 'version': ['2', '26', '31']}, {'label': '2.26.35', 'path': '/home/stixcore/STIXCore/stixcore/data/idb/v2.26.35', 'version': ['2', '26', '35']}]
Force version: None
History:
IntervalTree([Interval(0.0, 640198039.0, '2.26.32'), Interval(640198039.0, 640265396.0, '2.26.31'), Interval(640265396.0, 642159993.0, '2.26.32'), Interval(642159993.0, 643123725.0, '2.26.33'), Interval(643123725.0, 677774249.0, '2.26.34'), Interval(677774249.0, 692380776.0, '2.26.35'), Interval(692380776.0, 4294967295.0, '2.26.36')])

2023-05-06T18:30:05Z INFO stixcore.ephemeris.manager 160: LOADING NEW META KERNEL: /data/stix/spice/kernels/mk/solo_ANC_soc-flown-mk_V110_20230505_001.tm
2023-05-06T18:30:06Z INFO __main__ 204: new spice kernel detected and loaded
2023-05-06T18:30:06Z INFO stixcore.processing.TMTCtoLB 30: Processing file: SOCPacketFile('/data/stix/SOLSOC/from_edds/tm/processing/STIX_TM_04PM_BatchRequest.PktTmRaw.SOL.0.2023.121.21.07.32.949.xrCM@2023.124.16.00.01.172.1.xml')
2023-05-06T18:30:06Z INFO stixcore.io.fits.processors 404: Fits file solo_LB_stix-5-2-21551_0736387200_V01.fits exists appending data
2023-05-06T18:30:06Z INFO stixcore.io.fits.processors 404: Fits file solo_LB_stix-1-1_0736300800_V01.fits exists appending data
2023-05-06T18:30:06Z INFO stixcore.io.fits.processors 404: Fits file solo_LB_stix-1-7_0736300800_V01.fits exists appending data
2023-05-06T18:30:06Z INFO stixcore.io.fits.processors 435: Writing fits file to /data/stix/out/fits_v1.0.0/LB/5/2/21551/solo_LB_stix-5-2-21551_0736387200_V01.fits
2023-05-06T18:30:06Z INFO stixcore.io.fits.processors 435: Writing fits file to /data/stix/out/fits_v1.0.0/LB/1/1/solo_LB_stix-1-1_0736300800_V01.fits
2023-05-06T18:30:06Z INFO stixcore.io.fits.processors 404: Fits file solo_LB_stix-1-1_0736387200_V01.fits exists appending data
2023-05-06T18:30:06Z INFO stixcore.io.fits.processors 435: Writing fits file to /data/stix/out/fits_v1.0.0/LB/1/7/solo_LB_stix-1-7_0736300800_V01.fits
2023-05-06T18:30:06Z INFO stixcore.io.fits.processors 404: Fits file solo_LB_stix-1-7_0736387200_V01.fits exists appending data
2023-05-06T18:30:06Z INFO stixcore.io.fits.processors 404: Fits file solo_LB_stix-3-25-2_0736300800_V01.fits exists appending data
2023-05-06T18:30:06Z INFO stixcore.io.fits.processors 435: Writing fits file to /data/stix/out/fits_v1.0.0/LB/1/1/solo_LB_stix-1-1_0736387200_V01.fits
2023-05-06T18:30:06Z INFO stixcore.io.fits.processors 435: Writing fits file to /data/stix/out/fits_v1.0.0/LB/1/7/solo_LB_stix-1-7_0736387200_V01.fits
2023-05-06T18:30:06Z INFO stixcore.io.fits.processors 435: Writing fits file to /data/stix/out/fits_v1.0.0/LB/3/25/2/solo_LB_stix-3-25-2_0736300800_V01.fits
2023-05-06T18:30:07Z INFO stixcore.io.fits.processors 435: Writing fits file to /data/stix/out/fits_v1.0.0/LB/3/25/2/solo_LB_stix-3-25-2_0736387200_V01.fits
2023-05-06T18:30:07Z INFO stixcore.io.fits.processors 435: Writing fits file to /data/stix/out/fits_v1.0.0/LB/3/25/2/solo_LB_stix-3-25-2_0736473600_V01.fits
2023-05-06T18:30:07Z INFO __main__ 207: generated LB files: 
{PosixPath('/data/stix/out/fits_v1.0.0/LB/1/1/solo_LB_stix-1-1_0736300800_V01.fits'),
 PosixPath('/data/stix/out/fits_v1.0.0/LB/1/1/solo_LB_stix-1-1_0736387200_V01.fits'),
 PosixPath('/data/stix/out/fits_v1.0.0/LB/1/7/solo_LB_stix-1-7_0736300800_V01.fits'),
 PosixPath('/data/stix/out/fits_v1.0.0/LB/1/7/solo_LB_stix-1-7_0736387200_V01.fits'),
 PosixPath('/data/stix/out/fits_v1.0.0/LB/3/25/2/solo_LB_stix-3-25-2_0736300800_V01.fits'),
 PosixPath('/data/stix/out/fits_v1.0.0/LB/3/25/2/solo_LB_stix-3-25-2_0736387200_V01.fits'),
 PosixPath('/data/stix/out/fits_v1.0.0/LB/3/25/2/solo_LB_stix-3-25-2_0736473600_V01.fits'),
 PosixPath('/data/stix/out/fits_v1.0.0/LB/5/2/21551/solo_LB_stix-5-2-21551_0736387200_V01.fits')}
2023-05-06T18:30:07Z INFO stixcore.idb.idb 522: IDB loaded from /home/stixcore/STIXCore/stixcore/data/idb/v2.26.36/idb.sqlite
2023-05-06T18:30:07Z INFO stixcore.idb.idb 522: IDB loaded from /home/stixcore/STIXCore/stixcore/data/idb/v2.26.36/idb.sqlite
2023-05-06T18:30:07Z INFO stixcore.idb.idb 522: IDB loaded from /home/stixcore/STIXCore/stixcore/data/idb/v2.26.36/idb.sqlite
2023-05-06T18:30:07Z INFO stixcore.idb.idb 522: IDB loaded from /home/stixcore/STIXCore/stixcore/data/idb/v2.26.36/idb.sqlite
2023-05-06T18:30:07Z INFO stixcore.ephemeris.manager 160: LOADING NEW META KERNEL: /data/stix/spice/kernels/mk/solo_ANC_soc-flown-mk_V110_20230505_001.tm
2023-05-06T18:30:07Z INFO stixcore.ephemeris.manager 160: LOADING NEW META KERNEL: /data/stix/spice/kernels/mk/solo_ANC_soc-flown-mk_V110_20230505_001.tm
2023-05-06T18:30:07Z INFO stixcore.ephemeris.manager 160: LOADING NEW META KERNEL: /data/stix/spice/kernels/mk/solo_ANC_soc-flown-mk_V110_20230505_001.tm
2023-05-06T18:30:07Z INFO stixcore.ephemeris.manager 160: LOADING NEW META KERNEL: /data/stix/spice/kernels/mk/solo_ANC_soc-flown-mk_V110_20230505_001.tm
2023-05-06T18:30:07Z INFO stixcore.processing.LBtoL0 80: processing file: /data/stix/out/fits_v1.0.0/LB/1/1/solo_LB_stix-1-1_0736387200_V01.fits
2023-05-06T18:30:07Z INFO stixcore.processing.LBtoL0 80: processing file: /data/stix/out/fits_v1.0.0/LB/5/2/21551/solo_LB_stix-5-2-21551_0736387200_V01.fits
2023-05-06T18:30:07Z INFO stixcore.processing.LBtoL0 80: processing file: /data/stix/out/fits_v1.0.0/LB/1/7/solo_LB_stix-1-7_0736300800_V01.fits
2023-05-06T18:30:07Z INFO stixcore.io.fits.processors 484: Fits file solo_L0_stix-tc-verify-1_0736387200_V01.fits exists appending data
2023-05-06T18:30:07Z INFO stixcore.io.fits.processors 484: Fits file solo_L0_stix-events-2-21551_0736387200_V01.fits exists appending data
2023-05-06T18:30:07Z INFO stixcore.io.fits.processors 484: Fits file solo_L0_stix-tc-verify-7_0736300800_V01.fits exists appending data
2023-05-06T18:30:07Z INFO stixcore.processing.LBtoL0 80: processing file: /data/stix/out/fits_v1.0.0/LB/3/25/2/solo_LB_stix-3-25-2_0736387200_V01.fits
2023-05-06T18:30:07Z INFO stixcore.processing.LBtoL0 80: processing file: /data/stix/out/fits_v1.0.0/LB/1/1/solo_LB_stix-1-1_0736300800_V01.fits
2023-05-06T18:30:07Z INFO stixcore.processing.LBtoL0 80: processing file: /data/stix/out/fits_v1.0.0/LB/1/7/solo_LB_stix-1-7_0736387200_V01.fits
2023-05-06T18:30:07Z INFO stixcore.io.fits.processors 484: Fits file solo_L0_stix-tc-verify-1_0736300800_V01.fits exists appending data
2023-05-06T18:30:07Z INFO stixcore.io.fits.processors 484: Fits file solo_L0_stix-tc-verify-7_0736387200_V01.fits exists appending data
2023-05-06T18:30:14Z INFO stixcore.processing.LBtoL0 80: processing file: /data/stix/out/fits_v1.0.0/LB/3/25/2/solo_LB_stix-3-25-2_0736300800_V01.fits
2023-05-06T18:30:21Z INFO stixcore.io.fits.processors 484: Fits file solo_L0_stix-hk-maxi_0736300800_V01.fits exists appending data
2023-05-06T18:30:21Z INFO stixcore.processing.LBtoL0 80: processing file: /data/stix/out/fits_v1.0.0/LB/3/25/2/solo_LB_stix-3-25-2_0736473600_V01.fits
2023-05-06T18:30:22Z INFO __main__ 211: generated L0 files: 
[PosixPath('/data/stix/out/fits_v1.0.0/L0/3/25/2/solo_L0_stix-hk-maxi_0736387200_V01.fits'),
 PosixPath('/data/stix/out/fits_v1.0.0/L0/1/1/solo_L0_stix-tc-verify-1_0736300800_V01.fits'),
 PosixPath('/data/stix/out/fits_v1.0.0/L0/1/7/solo_L0_stix-tc-verify-7_0736387200_V01.fits'),
 PosixPath('/data/stix/out/fits_v1.0.0/L0/1/7/solo_L0_stix-tc-verify-7_0736300800_V01.fits'),
 PosixPath('/data/stix/out/fits_v1.0.0/L0/3/25/2/solo_L0_stix-hk-maxi_0736473600_V01.fits'),
 PosixPath('/data/stix/out/fits_v1.0.0/L0/3/25/2/solo_L0_stix-hk-maxi_0736300800_V01.fits'),
 PosixPath('/data/stix/out/fits_v1.0.0/L0/5/2/21551/solo_L0_stix-events-2-21551_0736387200_V01.fits'),
 PosixPath('/data/stix/out/fits_v1.0.0/L0/1/1/solo_L0_stix-tc-verify-1_0736387200_V01.fits')]
2023-05-06T18:30:22Z INFO stixcore.ephemeris.manager 160: LOADING NEW META KERNEL: /data/stix/spice/kernels/mk/solo_ANC_soc-flown-mk_V110_20230505_001.tm
2023-05-06T18:30:22Z INFO stixcore.ephemeris.manager 160: LOADING NEW META KERNEL: /data/stix/spice/kernels/mk/solo_ANC_soc-flown-mk_V110_20230505_001.tm
2023-05-06T18:30:22Z INFO stixcore.ephemeris.manager 160: LOADING NEW META KERNEL: /data/stix/spice/kernels/mk/solo_ANC_soc-flown-mk_V110_20230505_001.tm
2023-05-06T18:30:22Z INFO stixcore.ephemeris.manager 160: LOADING NEW META KERNEL: /data/stix/spice/kernels/mk/solo_ANC_soc-flown-mk_V110_20230505_001.tm
2023-05-06T18:30:23Z INFO stixcore.processing.L0toL1 89: processing file: /data/stix/out/fits_v1.0.0/L0/1/1/solo_L0_stix-tc-verify-1_0736300800_V01.fits
2023-05-06T18:30:23Z INFO stixcore.processing.L0toL1 89: processing file: /data/stix/out/fits_v1.0.0/L0/1/7/solo_L0_stix-tc-verify-7_0736387200_V01.fits
2023-05-06T18:30:23Z INFO stixcore.processing.L0toL1 89: processing file: /data/stix/out/fits_v1.0.0/L0/5/2/21551/solo_L0_stix-events-2-21551_0736387200_V01.fits
2023-05-06T18:30:23Z INFO stixcore.io.fits.processors 772: Fits file solo_L1_stix-tc-verify-1_20230503_V01.fits exists appending data
2023-05-06T18:30:23Z INFO stixcore.io.fits.processors 772: Fits file solo_L1_stix-tc-verify-7_20230503_V01.fits exists appending data
2023-05-06T18:30:23Z INFO stixcore.io.fits.processors 772: Fits file solo_L1_stix-events-2-21551_20230503_V01.fits exists appending data
2023-05-06T18:30:23Z INFO stixcore.soop.manager 482: No soops found for time: 2023-05-03 00:00:01.044889 - 2023-05-03 00:02:53.045193
2023-05-06T18:30:23Z INFO stixcore.soop.manager 482: No soops found for time: 2023-05-03 00:00:01.029477 - 2023-05-03 00:02:53.029782
2023-05-06T18:30:23Z INFO stixcore.soop.manager 482: No soops found for time: 2023-05-03 06:29:03.621264 - 2023-05-03 06:29:03.621264
2023-05-06T18:30:23Z INFO stixcore.soop.manager 489: No observations found for time: 2023-05-03 06:29:03.621264 - 2023-05-03 06:29:03.621264 : SoopObservationType.ALL
2023-05-06T18:30:23Z INFO stixcore.processing.L0toL1 89: processing file: /data/stix/out/fits_v1.0.0/L0/3/25/2/solo_L0_stix-hk-maxi_0736387200_V01.fits
2023-05-06T18:30:23Z INFO stixcore.processing.L0toL1 89: processing file: /data/stix/out/fits_v1.0.0/L0/1/7/solo_L0_stix-tc-verify-7_0736300800_V01.fits
2023-05-06T18:30:23Z INFO stixcore.processing.L0toL1 89: processing file: /data/stix/out/fits_v1.0.0/L0/1/1/solo_L0_stix-tc-verify-1_0736387200_V01.fits
2023-05-06T18:30:23Z INFO stixcore.io.fits.processors 772: Fits file solo_L1_stix-tc-verify-1_20230503_V01.fits exists appending data
2023-05-06T18:30:23Z INFO stixcore.io.fits.processors 772: Fits file solo_L1_stix-tc-verify-7_20230503_V01.fits exists appending data
2023-05-06T18:30:23Z INFO stixcore.soop.manager 482: No soops found for time: 2023-05-03 00:00:01.029477 - 2023-05-03 00:02:53.029782
2023-05-06T18:30:23Z INFO stixcore.soop.manager 482: No soops found for time: 2023-05-03 00:00:01.044889 - 2023-05-03 00:02:53.045193
2023-05-06T18:30:24Z INFO stixcore.soop.manager 482: No soops found for time: 2023-05-03 00:03:38.731019 - 2023-05-03 23:59:22.263460
2023-05-06T18:30:24Z INFO stixcore.soop.manager 482: No soops found for time: 2023-05-04 00:00:26.263573 - 2023-05-04 00:02:34.263800
2023-05-06T18:30:24Z INFO stixcore.processing.L0toL1 89: processing file: /data/stix/out/fits_v1.0.0/L0/3/25/2/solo_L0_stix-hk-maxi_0736473600_V01.fits
2023-05-06T18:30:25Z INFO stixcore.io.fits.processors 772: Fits file solo_L1_stix-hk-maxi_20230504_V01.fits exists appending data
2023-05-06T18:30:25Z INFO stixcore.soop.manager 482: No soops found for time: 2023-05-04 00:00:26.263573 - 2023-05-04 03:38:02.222409
2023-05-06T18:30:25Z INFO stixcore.processing.L0toL1 89: processing file: /data/stix/out/fits_v1.0.0/L0/3/25/2/solo_L0_stix-hk-maxi_0736300800_V01.fits
2023-05-06T18:30:26Z INFO stixcore.io.fits.processors 772: Fits file solo_L1_stix-hk-maxi_20230502_V01.fits exists appending data
2023-05-06T18:30:26Z INFO stixcore.soop.manager 482: No soops found for time: 2023-05-02 00:00:27.211793 - 2023-05-02 23:59:22.732245
2023-05-06T18:30:27Z INFO stixcore.io.fits.processors 772: Fits file solo_L1_stix-hk-maxi_20230503_V01.fits exists appending data
2023-05-06T18:30:27Z INFO stixcore.soop.manager 482: No soops found for time: 2023-05-03 00:00:26.732358 - 2023-05-03 23:59:22.263460
2023-05-06T18:30:28Z INFO __main__ 215: generated L1 files: 
[PosixPath('/data/stix/out/fits_v1.0.0/L1/2023/05/02/HK/solo_L1_stix-hk-maxi_20230502_V01.fits'),
 PosixPath('/data/stix/out/fits_v1.0.0/L1/2023/05/03/TC-VERIFY/solo_L1_stix-tc-verify-1_20230503_V01.fits'),
 PosixPath('/data/stix/out/fits_v1.0.0/L1/2023/05/03/HK/solo_L1_stix-hk-maxi_20230503_V01.fits'),
 PosixPath('/data/stix/out/fits_v1.0.0/L1/2023/05/03/EVENTS/solo_L1_stix-events-2-21551_20230503_V01.fits'),
 PosixPath('/data/stix/out/fits_v1.0.0/L1/2023/05/04/HK/solo_L1_stix-hk-maxi_20230504_V01.fits'),
 PosixPath('/data/stix/out/fits_v1.0.0/L1/2023/05/03/TC-VERIFY/solo_L1_stix-tc-verify-7_20230503_V01.fits')]
2023-05-06T18:30:28Z INFO stixcore.ephemeris.manager 160: LOADING NEW META KERNEL: /data/stix/spice/kernels/mk/solo_ANC_soc-flown-mk_V110_20230505_001.tm
2023-05-06T18:30:28Z INFO stixcore.ephemeris.manager 160: LOADING NEW META KERNEL: /data/stix/spice/kernels/mk/solo_ANC_soc-flown-mk_V110_20230505_001.tm
2023-05-06T18:30:28Z INFO stixcore.ephemeris.manager 160: LOADING NEW META KERNEL: /data/stix/spice/kernels/mk/solo_ANC_soc-flown-mk_V110_20230505_001.tm
2023-05-06T18:30:28Z INFO stixcore.ephemeris.manager 160: LOADING NEW META KERNEL: /data/stix/spice/kernels/mk/solo_ANC_soc-flown-mk_V110_20230505_001.tm
2023-05-06T18:30:28Z INFO stixcore.io.fits.processors 858: no writeout of L2 tc-verify-1 FITS files.
2023-05-06T18:30:28Z INFO stixcore.io.fits.processors 858: no writeout of L2 events-2-21551 FITS files.
2023-05-06T18:30:28Z INFO stixcore.io.fits.processors 858: no writeout of L2 tc-verify-7 FITS files.
2023-05-06T18:30:29Z INFO stixcore.io.fits.processors 858: no writeout of L2 hk-maxi FITS files.
2023-05-06T18:30:30Z INFO stixcore.io.fits.processors 858: no writeout of L2 hk-maxi FITS files.
2023-05-06T18:30:30Z INFO stixcore.io.fits.processors 858: no writeout of L2 hk-maxi FITS files.
2023-05-06T18:30:30Z INFO stixcore.products.level2.housekeepingL2 229: calling IDL for hk files:
2023-05-06T18:30:30Z INFO stixcore.products.level2.housekeepingL2 231:     /data/stix/out/fits_v1.0.0/L1/2023/05/02/HK/solo_L1_stix-hk-maxi_20230502_V01.fits
2023-05-06T18:30:30Z INFO stixcore.products.level2.housekeepingL2 231:     /data/stix/out/fits_v1.0.0/L1/2023/05/03/HK/solo_L1_stix-hk-maxi_20230503_V01.fits
2023-05-06T18:30:30Z INFO stixcore.products.level2.housekeepingL2 231:     /data/stix/out/fits_v1.0.0/L1/2023/05/04/HK/solo_L1_stix-hk-maxi_20230504_V01.fits
2023-05-06T18:33:04Z INFO stixcore.products.level2.housekeepingL2 254: returning from IDL
2023-05-06T18:33:04Z INFO stixcore.products.level2.housekeepingL2 258: IDL postprocessing HK file: b'/data/stix/out/fits_v1.0.0/L1/2023/05/02/HK/solo_L1_stix-hk-maxi_20230502_V01.fits'
2023-05-06T18:33:06Z INFO stixcore.io.fits.processors 772: Fits file solo_L2_stix-aux-ephemeris_20230502_V01.fits exists appending data
2023-05-06T18:33:06Z INFO stixcore.soop.manager 482: No soops found for time: 2023-05-02 00:00:27.211793 - 2023-05-02 23:59:22.734579
2023-05-06T18:33:06Z INFO stixcore.products.level2.housekeepingL2 258: IDL postprocessing HK file: b'/data/stix/out/fits_v1.0.0/L1/2023/05/03/HK/solo_L1_stix-hk-maxi_20230503_V01.fits'
2023-05-06T18:33:08Z INFO stixcore.soop.manager 482: No soops found for time: 2023-05-03 00:00:26.732358 - 2023-05-03 23:59:22.265139
2023-05-06T18:33:08Z INFO stixcore.products.level2.housekeepingL2 258: IDL postprocessing HK file: b'/data/stix/out/fits_v1.0.0/L1/2023/05/04/HK/solo_L1_stix-hk-maxi_20230504_V01.fits'
2023-05-06T18:33:09Z ERROR stixcore.processing.L1toL2 68: error
concurrent.futures.process._RemoteTraceback: 
"""
Traceback (most recent call last):
  File "/usr/lib/python3.9/concurrent/futures/process.py", line 246, in _process_worker
    r = call_item.fn(*call_item.args, **call_item.kwargs)
  File "/home/stixcore/STIXCore/stixcore/processing/L1toL2.py", line 106, in process_type
    all_files.extend(idlprocessor.process())
  File "/home/stixcore/STIXCore/stixcore/processing/sswidl.py", line 205, in process
    files.extend(task.run(self.fits_processor))
  File "/home/stixcore/STIXCore/stixcore/processing/sswidl.py", line 153, in run
    self._results = self.postprocessing(results, fits_processor)
  File "/home/stixcore/STIXCore/stixcore/products/level2/housekeepingL2.py", line 289, in postprocessing
    orient, dist, car, heeq = Spice.instance.get_auxiliary_positional_data(date=d)
  File "/home/stixcore/STIXCore/stixcore/ephemeris/manager.py", line 303, in get_auxiliary_positional_data
    cmat, _ = spiceypy.ckgp(SOLAR_ORBITER_STIX_ILS_FRAME_ID, sc, 1.0, 'SOLO_SUN_RTN')
  File "/home/stixcore/STIXCore/venv/lib/python3.9/site-packages/spiceypy/spiceypy.py", line 138, in with_errcheck
    res = f(*args, **kwargs)
  File "/home/stixcore/STIXCore/venv/lib/python3.9/site-packages/spiceypy/spiceypy.py", line 158, in wrapper
    raise NotFoundError(
spiceypy.utils.exceptions.NotFoundError: Spice returns not found for function: ckgp
"""

The above exception was the direct cause of the following exception:

Traceback (most recent call last):
  File "/home/stixcore/STIXCore/stixcore/processing/L1toL2.py", line 65, in process_fits_files
    new_files = job.result()
  File "/usr/lib/python3.9/concurrent/futures/_base.py", line 439, in result
    return self.__get_result()
  File "/usr/lib/python3.9/concurrent/futures/_base.py", line 391, in __get_result
    raise self._exception
spiceypy.utils.exceptions.NotFoundError: Spice returns not found for function: ckgp
2023-05-06T18:33:09Z INFO __main__ 219: generated L2 files: 
[]
2023-05-08T20:58:06Z INFO __main__ 253: 
CONFIG

[Paths]
tm_archive = /data/stix/SOLSOC/from_edds/tm/processing
fits_archive = /data/stix/out/fits_v1.0.0
spice_kernels = /data/stix/spice/kernels
soop_files = /data/stix/SOLSOC/from_soc

[Logging]
stop_on_error = False

[IDLBridge]
enabled = True
gsw_path = /opt/STIX-GSW
batchsize = 30

[Pipeline]
status_server_port = 12388
parallel_batchsize_l0 = 100
parallel_batchsize_l1 = 100
parallel_batchsize_l2 = 100
log_level = INFO
log_dir = /data/stix/out/fits_v1.0.0/logging
error_mail_send = True
error_mail_receivers = nicky.hochmuth@ateleris.ch, shane.maloney@tcd.ie
error_mail_smpt_host = lmailer.fhnw.ch
error_mail_smpt_port = 25
error_mail_sender = nicky.hochmuth@fhnw.ch
start_with_unprocessed = False

[Publish]
db_file = /home/stixcore/soar_pub/published.sqlite
include_levels = L0, L1
waiting_period = 21d
target_dir = /home/solsoc/to_soc
report_mail_receivers = nicky.hochmuth@ateleris.ch, shane.maloney@tcd.ie
report_mail_send = True
same_esa_name_dir = /data/stix/out/esa_conflicts
rid_lut_file = ./stixcore/data/publish/rid_lut.csv
rid_lut_file_update_url = https://datacenter.stix.i4ds.net/api/bsd/info/
batch_size = 20000
fits_dir = /data/stix/out/fits_v1.0.0
target_host = pub026.cs.technik.fhnw.ch
supplement_report = /home/stixcore/soar_pub/supplement_report.csv
include_products = ql,hk,sci,cal
log_dir = /data/stix/out/fits_v1.0.0/logging

[SOOP]
endpoint = https://solarorbiter.esac.esa.int/soopkitchen/api
user = smaloney
password = xxxx
soop_files_download = ./stixcore/data/soop


2023-05-08T20:58:06Z INFO __main__ 270: 
SINGLETONS

SOOPManager: /data/stix/SOLSOC/from_soc
SPICE: /data/stix/spice/kernels/mk/solo_ANC_soc-flown-mk_V110_20230507_001.tm
IDBManager: /home/stixcore/STIXCore/stixcore/data/idb
Versions:
[{'label': '2.26.36', 'path': '/home/stixcore/STIXCore/stixcore/data/idb/v2.26.36', 'version': ['2', '26', '36']}, {'label': '2.26.32', 'path': '/home/stixcore/STIXCore/stixcore/data/idb/v2.26.32', 'version': ['2', '26', '32']}, {'label': '2.26.34', 'path': '/home/stixcore/STIXCore/stixcore/data/idb/v2.26.34', 'version': ['2', '26', '34']}, {'label': '2.26.33', 'path': '/home/stixcore/STIXCore/stixcore/data/idb/v2.26.33', 'version': ['2', '26', '33']}, {'label': '2.26.31', 'path': '/home/stixcore/STIXCore/stixcore/data/idb/v2.26.31', 'version': ['2', '26', '31']}, {'label': '2.26.35', 'path': '/home/stixcore/STIXCore/stixcore/data/idb/v2.26.35', 'version': ['2', '26', '35']}]
Force version: None
History:
IntervalTree([Interval(0.0, 640198039.0, '2.26.32'), Interval(640198039.0, 640265396.0, '2.26.31'), Interval(640265396.0, 642159993.0, '2.26.32'), Interval(642159993.0, 643123725.0, '2.26.33'), Interval(643123725.0, 677774249.0, '2.26.34'), Interval(677774249.0, 692380776.0, '2.26.35'), Interval(692380776.0, 4294967295.0, '2.26.36')])

2023-05-08T20:58:06Z INFO stixcore.processing.TMTCtoLB 30: Processing file: SOCPacketFile('/data/stix/SOLSOC/from_edds/tm/processing/STIX_TM_04PM_BatchRequest.PktTmRaw.SOL.0.2023.121.21.07.32.949.xrCM@2023.124.16.00.01.172.1.xml')
2023-05-08T20:58:07Z INFO stixcore.io.fits.processors 404: Fits file solo_LB_stix-5-2-21551_0736387200_V01.fits exists appending data
2023-05-08T20:58:07Z INFO stixcore.io.fits.processors 404: Fits file solo_LB_stix-1-7_0736300800_V01.fits exists appending data
2023-05-08T20:58:07Z INFO stixcore.io.fits.processors 404: Fits file solo_LB_stix-1-1_0736300800_V01.fits exists appending data
2023-05-08T20:58:07Z INFO stixcore.io.fits.processors 435: Writing fits file to /data/stix/out/fits_v1.0.0/LB/5/2/21551/solo_LB_stix-5-2-21551_0736387200_V01.fits
2023-05-08T20:58:07Z INFO stixcore.io.fits.processors 435: Writing fits file to /data/stix/out/fits_v1.0.0/LB/1/1/solo_LB_stix-1-1_0736300800_V01.fits
2023-05-08T20:58:07Z INFO stixcore.io.fits.processors 435: Writing fits file to /data/stix/out/fits_v1.0.0/LB/1/7/solo_LB_stix-1-7_0736300800_V01.fits
2023-05-08T20:58:07Z INFO stixcore.io.fits.processors 404: Fits file solo_LB_stix-1-1_0736387200_V01.fits exists appending data
2023-05-08T20:58:07Z INFO stixcore.io.fits.processors 404: Fits file solo_LB_stix-1-7_0736387200_V01.fits exists appending data
2023-05-08T20:58:07Z INFO stixcore.io.fits.processors 404: Fits file solo_LB_stix-3-25-2_0736300800_V01.fits exists appending data
2023-05-08T20:58:07Z INFO stixcore.io.fits.processors 435: Writing fits file to /data/stix/out/fits_v1.0.0/LB/1/1/solo_LB_stix-1-1_0736387200_V01.fits
2023-05-08T20:58:07Z INFO stixcore.io.fits.processors 435: Writing fits file to /data/stix/out/fits_v1.0.0/LB/1/7/solo_LB_stix-1-7_0736387200_V01.fits
2023-05-08T20:58:07Z INFO stixcore.io.fits.processors 435: Writing fits file to /data/stix/out/fits_v1.0.0/LB/3/25/2/solo_LB_stix-3-25-2_0736300800_V01.fits
2023-05-08T20:58:07Z INFO stixcore.io.fits.processors 404: Fits file solo_LB_stix-3-25-2_0736387200_V01.fits exists appending data
2023-05-08T20:58:07Z INFO stixcore.io.fits.processors 435: Writing fits file to /data/stix/out/fits_v1.0.0/LB/3/25/2/solo_LB_stix-3-25-2_0736387200_V01.fits
2023-05-08T20:58:07Z INFO stixcore.io.fits.processors 404: Fits file solo_LB_stix-3-25-2_0736473600_V01.fits exists appending data
2023-05-08T20:58:07Z INFO stixcore.io.fits.processors 435: Writing fits file to /data/stix/out/fits_v1.0.0/LB/3/25/2/solo_LB_stix-3-25-2_0736473600_V01.fits
2023-05-08T20:58:08Z INFO __main__ 207: generated LB files: 
{PosixPath('/data/stix/out/fits_v1.0.0/LB/1/1/solo_LB_stix-1-1_0736300800_V01.fits'),
 PosixPath('/data/stix/out/fits_v1.0.0/LB/1/1/solo_LB_stix-1-1_0736387200_V01.fits'),
 PosixPath('/data/stix/out/fits_v1.0.0/LB/1/7/solo_LB_stix-1-7_0736300800_V01.fits'),
 PosixPath('/data/stix/out/fits_v1.0.0/LB/1/7/solo_LB_stix-1-7_0736387200_V01.fits'),
 PosixPath('/data/stix/out/fits_v1.0.0/LB/3/25/2/solo_LB_stix-3-25-2_0736300800_V01.fits'),
 PosixPath('/data/stix/out/fits_v1.0.0/LB/3/25/2/solo_LB_stix-3-25-2_0736387200_V01.fits'),
 PosixPath('/data/stix/out/fits_v1.0.0/LB/3/25/2/solo_LB_stix-3-25-2_0736473600_V01.fits'),
 PosixPath('/data/stix/out/fits_v1.0.0/LB/5/2/21551/solo_LB_stix-5-2-21551_0736387200_V01.fits')}
2023-05-08T20:58:08Z INFO stixcore.idb.idb 522: IDB loaded from /home/stixcore/STIXCore/stixcore/data/idb/v2.26.36/idb.sqlite
2023-05-08T20:58:08Z INFO stixcore.idb.idb 522: IDB loaded from /home/stixcore/STIXCore/stixcore/data/idb/v2.26.36/idb.sqlite
2023-05-08T20:58:08Z INFO stixcore.idb.idb 522: IDB loaded from /home/stixcore/STIXCore/stixcore/data/idb/v2.26.36/idb.sqlite
2023-05-08T20:58:08Z INFO stixcore.ephemeris.manager 160: LOADING NEW META KERNEL: /data/stix/spice/kernels/mk/solo_ANC_soc-flown-mk_V110_20230507_001.tm
2023-05-08T20:58:08Z INFO stixcore.ephemeris.manager 160: LOADING NEW META KERNEL: /data/stix/spice/kernels/mk/solo_ANC_soc-flown-mk_V110_20230507_001.tm
2023-05-08T20:58:08Z INFO stixcore.ephemeris.manager 160: LOADING NEW META KERNEL: /data/stix/spice/kernels/mk/solo_ANC_soc-flown-mk_V110_20230507_001.tm
2023-05-08T20:58:08Z INFO stixcore.idb.idb 522: IDB loaded from /home/stixcore/STIXCore/stixcore/data/idb/v2.26.36/idb.sqlite
2023-05-08T20:58:08Z INFO stixcore.ephemeris.manager 160: LOADING NEW META KERNEL: /data/stix/spice/kernels/mk/solo_ANC_soc-flown-mk_V110_20230507_001.tm
2023-05-08T20:58:08Z INFO stixcore.processing.LBtoL0 80: processing file: /data/stix/out/fits_v1.0.0/LB/1/1/solo_LB_stix-1-1_0736387200_V01.fits
2023-05-08T20:58:08Z INFO stixcore.processing.LBtoL0 80: processing file: /data/stix/out/fits_v1.0.0/LB/1/7/solo_LB_stix-1-7_0736387200_V01.fits
2023-05-08T20:58:08Z INFO stixcore.processing.LBtoL0 80: processing file: /data/stix/out/fits_v1.0.0/LB/5/2/21551/solo_LB_stix-5-2-21551_0736387200_V01.fits
2023-05-08T20:58:08Z INFO stixcore.io.fits.processors 484: Fits file solo_L0_stix-tc-verify-1_0736387200_V01.fits exists appending data
2023-05-08T20:58:08Z INFO stixcore.io.fits.processors 484: Fits file solo_L0_stix-tc-verify-7_0736387200_V01.fits exists appending data
2023-05-08T20:58:08Z INFO stixcore.io.fits.processors 484: Fits file solo_L0_stix-events-2-21551_0736387200_V01.fits exists appending data
2023-05-08T20:58:08Z INFO stixcore.processing.LBtoL0 80: processing file: /data/stix/out/fits_v1.0.0/LB/3/25/2/solo_LB_stix-3-25-2_0736300800_V01.fits
2023-05-08T20:58:08Z INFO stixcore.processing.LBtoL0 80: processing file: /data/stix/out/fits_v1.0.0/LB/1/7/solo_LB_stix-1-7_0736300800_V01.fits
2023-05-08T20:58:08Z INFO stixcore.processing.LBtoL0 80: processing file: /data/stix/out/fits_v1.0.0/LB/1/1/solo_LB_stix-1-1_0736300800_V01.fits
2023-05-08T20:58:08Z INFO stixcore.io.fits.processors 484: Fits file solo_L0_stix-tc-verify-7_0736300800_V01.fits exists appending data
2023-05-08T20:58:08Z INFO stixcore.io.fits.processors 484: Fits file solo_L0_stix-tc-verify-1_0736300800_V01.fits exists appending data
2023-05-08T20:58:14Z INFO stixcore.io.fits.processors 484: Fits file solo_L0_stix-hk-maxi_0736300800_V01.fits exists appending data
2023-05-08T20:58:15Z INFO stixcore.processing.LBtoL0 80: processing file: /data/stix/out/fits_v1.0.0/LB/3/25/2/solo_LB_stix-3-25-2_0736473600_V01.fits
2023-05-08T20:58:16Z INFO stixcore.io.fits.processors 484: Fits file solo_L0_stix-hk-maxi_0736473600_V01.fits exists appending data
2023-05-08T20:58:16Z INFO stixcore.processing.LBtoL0 80: processing file: /data/stix/out/fits_v1.0.0/LB/3/25/2/solo_LB_stix-3-25-2_0736387200_V01.fits
2023-05-08T20:58:22Z INFO stixcore.io.fits.processors 484: Fits file solo_L0_stix-hk-maxi_0736387200_V01.fits exists appending data
2023-05-08T20:58:23Z INFO __main__ 211: generated L0 files: 
[PosixPath('/data/stix/out/fits_v1.0.0/L0/1/7/solo_L0_stix-tc-verify-7_0736387200_V01.fits'),
 PosixPath('/data/stix/out/fits_v1.0.0/L0/1/7/solo_L0_stix-tc-verify-7_0736300800_V01.fits'),
 PosixPath('/data/stix/out/fits_v1.0.0/L0/3/25/2/solo_L0_stix-hk-maxi_0736300800_V01.fits'),
 PosixPath('/data/stix/out/fits_v1.0.0/L0/5/2/21551/solo_L0_stix-events-2-21551_0736387200_V01.fits'),
 PosixPath('/data/stix/out/fits_v1.0.0/L0/1/1/solo_L0_stix-tc-verify-1_0736387200_V01.fits'),
 PosixPath('/data/stix/out/fits_v1.0.0/L0/3/25/2/solo_L0_stix-hk-maxi_0736387200_V01.fits'),
 PosixPath('/data/stix/out/fits_v1.0.0/L0/3/25/2/solo_L0_stix-hk-maxi_0736473600_V01.fits'),
 PosixPath('/data/stix/out/fits_v1.0.0/L0/1/1/solo_L0_stix-tc-verify-1_0736300800_V01.fits')]
2023-05-08T20:58:23Z INFO stixcore.ephemeris.manager 160: LOADING NEW META KERNEL: /data/stix/spice/kernels/mk/solo_ANC_soc-flown-mk_V110_20230507_001.tm
2023-05-08T20:58:23Z INFO stixcore.ephemeris.manager 160: LOADING NEW META KERNEL: /data/stix/spice/kernels/mk/solo_ANC_soc-flown-mk_V110_20230507_001.tm
2023-05-08T20:58:23Z INFO stixcore.ephemeris.manager 160: LOADING NEW META KERNEL: /data/stix/spice/kernels/mk/solo_ANC_soc-flown-mk_V110_20230507_001.tm
2023-05-08T20:58:23Z INFO stixcore.ephemeris.manager 160: LOADING NEW META KERNEL: /data/stix/spice/kernels/mk/solo_ANC_soc-flown-mk_V110_20230507_001.tm
2023-05-08T20:58:23Z INFO stixcore.processing.L0toL1 89: processing file: /data/stix/out/fits_v1.0.0/L0/1/1/solo_L0_stix-tc-verify-1_0736387200_V01.fits
2023-05-08T20:58:23Z INFO stixcore.processing.L0toL1 89: processing file: /data/stix/out/fits_v1.0.0/L0/1/7/solo_L0_stix-tc-verify-7_0736387200_V01.fits
2023-05-08T20:58:23Z INFO stixcore.processing.L0toL1 89: processing file: /data/stix/out/fits_v1.0.0/L0/5/2/21551/solo_L0_stix-events-2-21551_0736387200_V01.fits
2023-05-08T20:58:23Z INFO stixcore.io.fits.processors 772: Fits file solo_L1_stix-tc-verify-1_20230503_V01.fits exists appending data
2023-05-08T20:58:23Z INFO stixcore.io.fits.processors 772: Fits file solo_L1_stix-tc-verify-7_20230503_V01.fits exists appending data
2023-05-08T20:58:23Z INFO stixcore.io.fits.processors 772: Fits file solo_L1_stix-events-2-21551_20230503_V01.fits exists appending data
2023-05-08T20:58:23Z INFO stixcore.soop.manager 482: No soops found for time: 2023-05-03 00:00:01.029477 - 2023-05-03 00:02:53.029782
2023-05-08T20:58:23Z INFO stixcore.soop.manager 482: No soops found for time: 2023-05-03 00:00:01.044889 - 2023-05-03 00:02:53.045193
2023-05-08T20:58:23Z INFO stixcore.soop.manager 482: No soops found for time: 2023-05-03 06:29:03.621264 - 2023-05-03 06:29:03.621264
2023-05-08T20:58:23Z INFO stixcore.soop.manager 489: No observations found for time: 2023-05-03 06:29:03.621264 - 2023-05-03 06:29:03.621264 : SoopObservationType.ALL
2023-05-08T20:58:23Z INFO stixcore.processing.L0toL1 89: processing file: /data/stix/out/fits_v1.0.0/L0/3/25/2/solo_L0_stix-hk-maxi_0736300800_V01.fits
2023-05-08T20:58:23Z INFO stixcore.processing.L0toL1 89: processing file: /data/stix/out/fits_v1.0.0/L0/1/7/solo_L0_stix-tc-verify-7_0736300800_V01.fits
2023-05-08T20:58:23Z INFO stixcore.processing.L0toL1 89: processing file: /data/stix/out/fits_v1.0.0/L0/1/1/solo_L0_stix-tc-verify-1_0736300800_V01.fits
2023-05-08T20:58:23Z INFO stixcore.io.fits.processors 772: Fits file solo_L1_stix-tc-verify-7_20230503_V01.fits exists appending data
2023-05-08T20:58:23Z INFO stixcore.io.fits.processors 772: Fits file solo_L1_stix-tc-verify-1_20230503_V01.fits exists appending data
2023-05-08T20:58:23Z INFO stixcore.soop.manager 482: No soops found for time: 2023-05-03 00:00:01.029477 - 2023-05-03 00:02:53.029782
2023-05-08T20:58:23Z INFO stixcore.soop.manager 482: No soops found for time: 2023-05-03 00:00:01.044889 - 2023-05-03 00:02:53.045193
2023-05-08T20:58:24Z INFO stixcore.io.fits.processors 772: Fits file solo_L1_stix-hk-maxi_20230502_V01.fits exists appending data
2023-05-08T20:58:24Z INFO stixcore.soop.manager 482: No soops found for time: 2023-05-02 00:00:27.211793 - 2023-05-02 23:59:22.734579
2023-05-08T20:58:25Z INFO stixcore.io.fits.processors 772: Fits file solo_L1_stix-hk-maxi_20230503_V01.fits exists appending data
2023-05-08T20:58:25Z INFO stixcore.soop.manager 482: No soops found for time: 2023-05-03 00:00:26.732358 - 2023-05-03 23:59:22.265139
2023-05-08T20:58:26Z INFO stixcore.processing.L0toL1 89: processing file: /data/stix/out/fits_v1.0.0/L0/3/25/2/solo_L0_stix-hk-maxi_0736387200_V01.fits
2023-05-08T20:58:26Z INFO stixcore.io.fits.processors 772: Fits file solo_L1_stix-hk-maxi_20230503_V01.fits exists appending data
2023-05-08T20:58:27Z INFO stixcore.soop.manager 482: No soops found for time: 2023-05-03 00:00:26.732358 - 2023-05-03 23:59:22.265139
2023-05-08T20:58:27Z INFO stixcore.io.fits.processors 772: Fits file solo_L1_stix-hk-maxi_20230504_V01.fits exists appending data
2023-05-08T20:58:27Z INFO stixcore.soop.manager 482: No soops found for time: 2023-05-04 00:00:26.263573 - 2023-05-04 03:38:02.226666
2023-05-08T20:58:28Z INFO stixcore.processing.L0toL1 89: processing file: /data/stix/out/fits_v1.0.0/L0/3/25/2/solo_L0_stix-hk-maxi_0736473600_V01.fits
2023-05-08T20:58:28Z INFO stixcore.io.fits.processors 772: Fits file solo_L1_stix-hk-maxi_20230504_V01.fits exists appending data
2023-05-08T20:58:28Z INFO stixcore.soop.manager 482: No soops found for time: 2023-05-04 00:00:26.263573 - 2023-05-04 03:38:02.226666
2023-05-08T20:58:28Z INFO __main__ 215: generated L1 files: 
[PosixPath('/data/stix/out/fits_v1.0.0/L1/2023/05/03/EVENTS/solo_L1_stix-events-2-21551_20230503_V01.fits'),
 PosixPath('/data/stix/out/fits_v1.0.0/L1/2023/05/02/HK/solo_L1_stix-hk-maxi_20230502_V01.fits'),
 PosixPath('/data/stix/out/fits_v1.0.0/L1/2023/05/03/HK/solo_L1_stix-hk-maxi_20230503_V01.fits'),
 PosixPath('/data/stix/out/fits_v1.0.0/L1/2023/05/03/TC-VERIFY/solo_L1_stix-tc-verify-1_20230503_V01.fits'),
 PosixPath('/data/stix/out/fits_v1.0.0/L1/2023/05/03/TC-VERIFY/solo_L1_stix-tc-verify-7_20230503_V01.fits'),
 PosixPath('/data/stix/out/fits_v1.0.0/L1/2023/05/04/HK/solo_L1_stix-hk-maxi_20230504_V01.fits')]
2023-05-08T20:58:29Z INFO stixcore.ephemeris.manager 160: LOADING NEW META KERNEL: /data/stix/spice/kernels/mk/solo_ANC_soc-flown-mk_V110_20230507_001.tm
2023-05-08T20:58:29Z INFO stixcore.ephemeris.manager 160: LOADING NEW META KERNEL: /data/stix/spice/kernels/mk/solo_ANC_soc-flown-mk_V110_20230507_001.tm
2023-05-08T20:58:29Z INFO stixcore.ephemeris.manager 160: LOADING NEW META KERNEL: /data/stix/spice/kernels/mk/solo_ANC_soc-flown-mk_V110_20230507_001.tm
2023-05-08T20:58:29Z INFO stixcore.ephemeris.manager 160: LOADING NEW META KERNEL: /data/stix/spice/kernels/mk/solo_ANC_soc-flown-mk_V110_20230507_001.tm
2023-05-08T20:58:29Z INFO stixcore.io.fits.processors 858: no writeout of L2 tc-verify-1 FITS files.
2023-05-08T20:58:29Z INFO stixcore.io.fits.processors 858: no writeout of L2 events-2-21551 FITS files.
2023-05-08T20:58:29Z INFO stixcore.io.fits.processors 858: no writeout of L2 tc-verify-7 FITS files.
2023-05-08T20:58:30Z INFO stixcore.io.fits.processors 858: no writeout of L2 hk-maxi FITS files.
2023-05-08T20:58:31Z INFO stixcore.io.fits.processors 858: no writeout of L2 hk-maxi FITS files.
2023-05-08T20:58:32Z INFO stixcore.io.fits.processors 858: no writeout of L2 hk-maxi FITS files.
2023-05-08T20:58:32Z INFO stixcore.products.level2.housekeepingL2 229: calling IDL for hk files:
2023-05-08T20:58:32Z INFO stixcore.products.level2.housekeepingL2 231:     /data/stix/out/fits_v1.0.0/L1/2023/05/02/HK/solo_L1_stix-hk-maxi_20230502_V01.fits
2023-05-08T20:58:32Z INFO stixcore.products.level2.housekeepingL2 231:     /data/stix/out/fits_v1.0.0/L1/2023/05/03/HK/solo_L1_stix-hk-maxi_20230503_V01.fits
2023-05-08T20:58:32Z INFO stixcore.products.level2.housekeepingL2 231:     /data/stix/out/fits_v1.0.0/L1/2023/05/04/HK/solo_L1_stix-hk-maxi_20230504_V01.fits
2023-05-08T21:02:08Z INFO stixcore.products.level2.housekeepingL2 254: returning from IDL
2023-05-08T21:02:08Z INFO stixcore.products.level2.housekeepingL2 258: IDL postprocessing HK file: b'/data/stix/out/fits_v1.0.0/L1/2023/05/02/HK/solo_L1_stix-hk-maxi_20230502_V01.fits'
2023-05-08T21:02:10Z INFO stixcore.io.fits.processors 772: Fits file solo_L2_stix-aux-ephemeris_20230502_V01.fits exists appending data
2023-05-08T21:02:10Z INFO stixcore.soop.manager 482: No soops found for time: 2023-05-02 00:00:27.211793 - 2023-05-02 23:59:22.734579
2023-05-08T21:02:10Z INFO stixcore.products.level2.housekeepingL2 258: IDL postprocessing HK file: b'/data/stix/out/fits_v1.0.0/L1/2023/05/03/HK/solo_L1_stix-hk-maxi_20230503_V01.fits'
2023-05-08T21:02:13Z INFO stixcore.io.fits.processors 772: Fits file solo_L2_stix-aux-ephemeris_20230503_V01.fits exists appending data
2023-05-08T21:02:13Z INFO stixcore.soop.manager 482: No soops found for time: 2023-05-03 00:00:26.732358 - 2023-05-03 23:59:22.265139
2023-05-08T21:02:13Z INFO stixcore.products.level2.housekeepingL2 258: IDL postprocessing HK file: b'/data/stix/out/fits_v1.0.0/L1/2023/05/04/HK/solo_L1_stix-hk-maxi_20230504_V01.fits'
2023-05-08T21:02:13Z ERROR stixcore.processing.L1toL2 68: error
concurrent.futures.process._RemoteTraceback: 
"""
Traceback (most recent call last):
  File "/usr/lib/python3.9/concurrent/futures/process.py", line 246, in _process_worker
    r = call_item.fn(*call_item.args, **call_item.kwargs)
  File "/home/stixcore/STIXCore/stixcore/processing/L1toL2.py", line 106, in process_type
    all_files.extend(idlprocessor.process())
  File "/home/stixcore/STIXCore/stixcore/processing/sswidl.py", line 205, in process
    files.extend(task.run(self.fits_processor))
  File "/home/stixcore/STIXCore/stixcore/processing/sswidl.py", line 153, in run
    self._results = self.postprocessing(results, fits_processor)
  File "/home/stixcore/STIXCore/stixcore/products/level2/housekeepingL2.py", line 289, in postprocessing
    orient, dist, car, heeq = Spice.instance.get_auxiliary_positional_data(date=d)
  File "/home/stixcore/STIXCore/stixcore/ephemeris/manager.py", line 303, in get_auxiliary_positional_data
    cmat, _ = spiceypy.ckgp(SOLAR_ORBITER_STIX_ILS_FRAME_ID, sc, 1.0, 'SOLO_SUN_RTN')
  File "/home/stixcore/STIXCore/venv/lib/python3.9/site-packages/spiceypy/spiceypy.py", line 138, in with_errcheck
    res = f(*args, **kwargs)
  File "/home/stixcore/STIXCore/venv/lib/python3.9/site-packages/spiceypy/spiceypy.py", line 158, in wrapper
    raise NotFoundError(
spiceypy.utils.exceptions.NotFoundError: Spice returns not found for function: ckgp
"""

The above exception was the direct cause of the following exception:

Traceback (most recent call last):
  File "/home/stixcore/STIXCore/stixcore/processing/L1toL2.py", line 65, in process_fits_files
    new_files = job.result()
  File "/usr/lib/python3.9/concurrent/futures/_base.py", line 439, in result
    return self.__get_result()
  File "/usr/lib/python3.9/concurrent/futures/_base.py", line 391, in __get_result
    raise self._exception
spiceypy.utils.exceptions.NotFoundError: Spice returns not found for function: ckgp
2023-05-08T21:02:13Z INFO __main__ 219: generated L2 files: 
[]