From d7b2414d283f19eed01d77cddcb99b64dab81044 Mon Sep 17 00:00:00 2001 From: Harrison Date: Tue, 23 Jun 2026 09:24:33 -0400 Subject: [PATCH 01/11] Harrison #118: PMCL - Remove unused code --- imap_l3_processing/models.py | 9 ---- imap_l3_processing/utils.py | 33 +------------ tests/hit/l3/test_models.py | 4 +- tests/swapi/l3b/test_models.py | 4 +- tests/test_utils.py | 84 +--------------------------------- 5 files changed, 7 insertions(+), 127 deletions(-) diff --git a/imap_l3_processing/models.py b/imap_l3_processing/models.py index 0fa646d61..8a1bf7d3a 100644 --- a/imap_l3_processing/models.py +++ b/imap_l3_processing/models.py @@ -25,15 +25,6 @@ class InputMetadata: def logical_source(self): return f"imap_{self.instrument}_{self.data_level}_{self.descriptor}" - def to_upstream_data_dependency(self, descriptor: str): - return UpstreamDataDependency(self.instrument, self.data_level, self.start_date, self.end_date, self.version, - descriptor, self.repointing) - - -@dataclass -class UpstreamDataDependency(InputMetadata): - pass - @dataclass class DataProductVariable: diff --git a/imap_l3_processing/utils.py b/imap_l3_processing/utils.py index 3af8f33f7..ec3573d7f 100644 --- a/imap_l3_processing/utils.py +++ b/imap_l3_processing/utils.py @@ -26,7 +26,7 @@ HealPixIntensityMapData, HealPixSpectralIndexMapData, RectangularSpectralIndexDataProduct, \ RectangularIntensityDataProduct, HealPixSpectralIndexDataProduct, HealPixIntensityDataProduct, MapDataProduct, \ ISNBackgroundSubtractedDataProduct, ISNBackgroundSubtractedMapData -from imap_l3_processing.models import UpstreamDataDependency, DataProduct, MagData, InputMetadata +from imap_l3_processing.models import DataProduct, MagData, InputMetadata from imap_l3_processing.ultra.models import UltraL1CPSet, UltraGlowsL3eData from imap_l3_processing.version import VERSION @@ -173,37 +173,6 @@ def format_time(t: Optional[datetime]) -> Optional[str]: return None -def download_dependency(dependency: UpstreamDataDependency) -> Path: - files_to_download = [result['file_path'] for result in - imap_data_access.query(instrument=dependency.instrument, - data_level=dependency.data_level, - descriptor=dependency.descriptor, - start_date=format_time(dependency.start_date), - end_date=format_time(dependency.end_date), - version=dependency.version - )] - if len(files_to_download) != 1: - raise ValueError(f"{files_to_download}. Expected one file to download, found {len(files_to_download)}.") - - return imap_data_access.download(files_to_download[0]) - - -def download_dependency_with_repointing(dependency: UpstreamDataDependency) -> (Path, int): - files_with_repointing_to_download = [(result['file_path'], result['repointing']) for result in - imap_data_access.query(instrument=dependency.instrument, - data_level=dependency.data_level, - descriptor=dependency.descriptor, - start_date=format_time(dependency.start_date), - end_date=format_time(dependency.end_date), - version=dependency.version - )] - if len(files_with_repointing_to_download) != 1: - raise ValueError( - f"{[file[0] for file in files_with_repointing_to_download]}. Expected one file to download, found {len(files_with_repointing_to_download)}.") - repointing_number = files_with_repointing_to_download[0][1] - return imap_data_access.download(files_with_repointing_to_download[0][0]), repointing_number - - def download_external_dependency(dependency_url: str, file_path: Path) -> Optional[Path]: try: response = requests.get(dependency_url) diff --git a/tests/hit/l3/test_models.py b/tests/hit/l3/test_models.py index 287bffc29..821945278 100644 --- a/tests/hit/l3/test_models.py +++ b/tests/hit/l3/test_models.py @@ -7,7 +7,7 @@ from imap_l3_processing.hit.l3 import models from imap_l3_processing.hit.l3.models import HitDirectEventDataProduct, HitL1Data -from imap_l3_processing.models import DataProductVariable, UpstreamDataDependency +from imap_l3_processing.models import DataProductVariable, InputMetadata from tests.swapi.cdf_model_test_case import CdfModelTestCase @@ -21,7 +21,7 @@ def tearDown(self) -> None: os.remove('test_cdf.cdf') def test_direct_events_to_data_product_variables_with_multiple_events(self): - input_metadata = UpstreamDataDependency( + input_metadata = InputMetadata( instrument="HIT", data_level="L3A", start_date=datetime.min, diff --git a/tests/swapi/l3b/test_models.py b/tests/swapi/l3b/test_models.py index 34478e772..ee8f22016 100644 --- a/tests/swapi/l3b/test_models.py +++ b/tests/swapi/l3b/test_models.py @@ -5,7 +5,7 @@ from uncertainties.unumpy import uarray from imap_l3_processing.constants import FIVE_MINUTES_IN_NANOSECONDS -from imap_l3_processing.models import UpstreamDataDependency +from imap_l3_processing.models import InputMetadata from imap_l3_processing.swapi.l3a.models import EPOCH_CDF_VAR_NAME, EPOCH_DELTA_CDF_VAR_NAME from imap_l3_processing.swapi.l3b.models import SwapiL3BCombinedVDF, COMBINED_SOLAR_WIND_DIFFERENTIAL_FLUX_CDF_VAR_NAME, \ COMBINED_SOLAR_WIND_DIFFERENTIAL_FLUX_DELTA_CDF_VAR_NAME, SOLAR_WIND_ENERGY_CDF_VAR_NAME, \ @@ -16,7 +16,7 @@ class TestModels(CdfModelTestCase): def test_combined_vdf_data_products(self): - input_metadata = UpstreamDataDependency("swapi", "l3b", + input_metadata = InputMetadata("swapi", "l3b", datetime(2024, 9, 8), datetime(2024, 9, 9), "v001", "") diff --git a/tests/test_utils.py b/tests/test_utils.py index bc16c8a0f..16f476c9b 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -19,8 +19,8 @@ from imap_l3_processing.models import InputMetadata from imap_l3_processing.swapi.l3a.models import SwapiL3AlphaSolarWindData from imap_l3_processing.swapi.quality_flags import SwapiL3Flags -from imap_l3_processing.utils import format_time, download_dependency, read_mag_data, save_data, \ - download_external_dependency, download_dependency_with_repointing, \ +from imap_l3_processing.utils import format_time, read_mag_data, save_data, \ + download_external_dependency, \ combine_glows_l3e_with_l1c_pointing, furnish_local_spice, get_spice_parent_file_names, furnish_spice_metakernel, \ SpiceKernelTypes, FurnishMetakernelOutput, read_cdf_parents, get_dependency_paths_by_descriptor, filter_bad_days from imap_l3_processing.version import VERSION @@ -362,64 +362,6 @@ def test_format_time(self): actual_time = format_time(None) self.assertEqual(None, actual_time) - @patch('imap_l3_processing.utils.imap_data_access') - def test_download_dependency(self, mock_data_access): - dependency = InputMetadata("swapi", "l2", datetime(2024, 9, 17), datetime(2024, 9, 18), "v2", - "descriptor") - query_dictionary = [{'file_path': "imap_swapi_l2_descriptor-fake-menlo-444_20240917_v2.cdf", - 'second_entry': '12345'}] - mock_data_access.query.return_value = query_dictionary - - path = download_dependency(dependency) - - mock_data_access.query.assert_called_once_with(instrument=dependency.instrument, - data_level=dependency.data_level, - descriptor=dependency.descriptor, - start_date="20240917", - end_date="20240918", - version='v2') - mock_data_access.download.assert_called_once_with("imap_swapi_l2_descriptor-fake-menlo-444_20240917_v2.cdf") - - self.assertIs(path, mock_data_access.download.return_value) - - @patch('imap_l3_processing.utils.imap_data_access') - def test_download_dependency_with_repointing(self, mock_data_access): - dependency = InputMetadata("glows", "l2", datetime(2024, 9, 17), datetime(2024, 9, 18), "v002", - "hist") - query_dictionary = [{'file_path': "imap_glows_l2_hist_20240917-repoint00001_v002.cdf", - 'repointing': 1, - 'third_entry': '12345'}] - mock_data_access.query.return_value = query_dictionary - - path, repointing = download_dependency_with_repointing(dependency) - - mock_data_access.query.assert_called_once_with(instrument=dependency.instrument, - data_level=dependency.data_level, - descriptor=dependency.descriptor, - start_date="20240917", - end_date="20240918", - version='v002') - mock_data_access.download.assert_called_once_with("imap_glows_l2_hist_20240917-repoint00001_v002.cdf") - self.assertEqual(path, mock_data_access.download.return_value) - self.assertEqual(1, repointing) - - @patch('imap_l3_processing.utils.imap_data_access') - def test_download_dependency_with_repointing_throws_if_no_files_or_more_than_one_found(self, mock_data_access): - dependency = InputMetadata("glows", "l2", datetime(2024, 9, 17), datetime(2024, 9, 18), "v002", - "hist") - - for return_values in ([], [{'file_path': "a", 'repointing': ''}, {'file_path': "b", 'repointing': ''}]): - with self.subTest(return_values): - mock_data_access.query.return_value = return_values - with self.assertRaises(Exception) as cm: - download_dependency_with_repointing(dependency) - expected_files_to_download = [dict_entry['file_path'] for dict_entry in return_values] - mock_data_access.download.assert_not_called() - - self.assertEqual( - f"{expected_files_to_download}. Expected one file to download, found {len(return_values)}.", - str(cm.exception)) - @patch("imap_l3_processing.utils.requests") @patch('builtins.open') def test_download_external_dependency(self, mock_open_file, mock_requests): @@ -450,28 +392,6 @@ def test_download_external_dependency_error_case(self, mock_requests): returned = download_external_dependency(expected_url, expected_filename) self.assertIsNone(returned) - @patch('imap_l3_processing.utils.imap_data_access') - def test_download_dependency_throws_value_error_if_not_one_file_returned(self, mock_data_access): - dependency = InputMetadata("swapi", "l2", datetime(2024, 9, 17), datetime(2024, 9, 18), "v2", - "descriptor") - query_dictionary_more_than_one_file = [{'file_path': "imap_swapi_l2_descriptor-fake-menlo-444_20240917_v2.cdf", - 'second_entry': '12345'}, {"file_path": "extra_value"}] - query_dictionary_less_than_one_file = [] - - cases = [("2", query_dictionary_more_than_one_file), - ("0", query_dictionary_less_than_one_file)] - - for case, query_dictionary in cases: - with self.subTest(case): - mock_data_access.query.return_value = query_dictionary - expected_files_to_download = [dict_entry['file_path'] for dict_entry in query_dictionary] - with self.assertRaises(Exception) as cm: - download_dependency(dependency) - - self.assertEqual( - f"{expected_files_to_download}. Expected one file to download, found {case}.", - str(cm.exception)) - def test_read_l1d_mag_data(self): file_name_as_str = "test_cdf.cdf" file_name_as_path = Path(file_name_as_str) From 18687582a7ea12953a08a7e24b253c9a1838d199 Mon Sep 17 00:00:00 2001 From: Harrison Date: Wed, 24 Jun 2026 11:51:15 -0400 Subject: [PATCH 02/11] Harrison 3295 #118: PMCL/KJON - Parse version when given in dependency file. Update GLOWS L3b to handle new major.minor version format. --- imap_l3_data_processor.py | 19 ++- imap_l3_processing/glows/glows_processor.py | 14 +- .../glows/l3bc/glows_l3bc_dependencies.py | 5 +- .../glows/l3bc/glows_l3bc_initializer.py | 16 ++- pyproject.toml | 2 +- .../glows/l3bc/science/test_generate_l3bc.py | 8 +- .../glows/l3bc/test_glows_l3bc_initializer.py | 53 ++++--- tests/glows/test_glows_processor.py | 36 +++-- tests/test_imap_l3_data_processor.py | 132 +++++++++++++++++- uv.lock | 10 +- 10 files changed, 226 insertions(+), 69 deletions(-) diff --git a/imap_l3_data_processor.py b/imap_l3_data_processor.py index 2b0bd73a8..7e9d08c11 100644 --- a/imap_l3_data_processor.py +++ b/imap_l3_data_processor.py @@ -1,4 +1,5 @@ import argparse +import json import logging import re from datetime import datetime @@ -6,6 +7,7 @@ import imap_data_access import spiceypy +from imap_data_access.file_validation import Version from imap_data_access.processing_input import ProcessingInputCollection from imap_l3_processing.codice.l3.hi.codice_hi_processor import CodiceHiProcessor @@ -38,7 +40,7 @@ def _parse_cli_arguments(): parser.add_argument("--start-date") parser.add_argument("--end-date", required=False) parser.add_argument("--repointing", required=False) - parser.add_argument("--version") + parser.add_argument("--version", required=False) parser.add_argument("--dependency") parser.add_argument( "--upload-to-sdc", @@ -67,7 +69,18 @@ def imap_l3_processor(): args.dependency = f.read() processing_input_collection = ProcessingInputCollection() - processing_input_collection.deserialize(args.dependency) + + parsed_dependency = json.loads(args.dependency) + if isinstance(parsed_dependency, list): + version = args.version + processing_input_collection.deserialize(args.dependency) + else: + version_numbers = list(parsed_dependency["version"].values()) + match version_numbers: + case [{"major_version": major, "minor_version": minor}]: + version = str(Version(major, minor)) + case _: + raise ValueError("Expected only a single version to be specified in the dependency.") repointing_number = None if args.repointing is not None: @@ -81,7 +94,7 @@ def imap_l3_processor(): args.data_level, _convert_to_datetime(args.start_date), _convert_to_datetime(args.end_date or args.start_date), - args.version, descriptor=args.descriptor, repointing=repointing_number) + version, descriptor=args.descriptor, repointing=repointing_number) if args.instrument in ["hi", "lo", "ultra"] and args.data_level == 'l3' and not parse_map_descriptor( args.descriptor): initializer_class, processor_class, descriptors = { diff --git a/imap_l3_processing/glows/glows_processor.py b/imap_l3_processing/glows/glows_processor.py index 9533fa33f..e0ce8de82 100644 --- a/imap_l3_processing/glows/glows_processor.py +++ b/imap_l3_processing/glows/glows_processor.py @@ -12,7 +12,7 @@ from zipfile import ZipFile, ZIP_DEFLATED import numpy as np -from imap_data_access.file_validation import generate_imap_file_path, ScienceFilePath, AncillaryFilePath +from imap_data_access.file_validation import generate_imap_file_path, ScienceFilePath, AncillaryFilePath, Version from imap_data_access.processing_input import ProcessingInputCollection from imap_l3_processing.constants import TEMP_CDF_FOLDER_PATH @@ -63,14 +63,14 @@ def process(self): return products elif self.input_metadata.data_level == "l3b": products_list = [] - - l3bc_initializer_data: GlowsL3BCInitializerData = GlowsL3BCInitializer.get_crs_to_process(self.dependencies) + major_version = Version.from_version(self.input_metadata.version).major + l3bc_initializer_data: GlowsL3BCInitializerData = GlowsL3BCInitializer.get_crs_to_process(self.dependencies, major_version) if len(l3bc_initializer_data.l3bc_dependencies) > 0: logger.info("Found CRs to Process L3BC:") for dep in l3bc_initializer_data.l3bc_dependencies: l3a_file_names = [l3a_d["filename"] for l3a_d in dep.l3a_data] - logger.info(f"\t{dep.carrington_rotation_number}, v{dep.version:03}: {l3a_file_names}") + logger.info(f"\t{dep.carrington_rotation_number}, {dep.version}: {l3a_file_names}") else: logger.info("No CRs to process for B/C") @@ -133,7 +133,7 @@ def add_spin_angle_delta(data: dict, ancillary_files: dict) -> dict: @staticmethod def archive_dependencies(l3bc_deps: GlowsL3BCDependencies, external_dependencies: ExternalDependencies) -> Path: start_date = l3bc_deps.start_date.strftime("%Y%m%d") - zip_path = TEMP_CDF_FOLDER_PATH / f"imap_glows_l3b-archive_{start_date}_v{l3bc_deps.version:03}.zip" + zip_path = TEMP_CDF_FOLDER_PATH / f"imap_glows_l3b-archive_{start_date}_{l3bc_deps.version}.zip" json_filename = "cr_to_process.json" with ZipFile(zip_path, "w", ZIP_DEFLATED) as file: file.write(external_dependencies.lyman_alpha_path, "lyman_alpha_composite.nc") @@ -171,9 +171,9 @@ def process_l3bc(processor, initializer_data: GlowsL3BCInitializerData): continue l3b_metadata = InputMetadata("glows", "l3b", dependency.start_date, dependency.end_date, - f"v{dependency.version:03}", GLOWS_L3B_DESCRIPTOR) + str(dependency.version), GLOWS_L3B_DESCRIPTOR) l3c_metadata = InputMetadata("glows", "l3c", dependency.start_date, dependency.end_date, - f"v{dependency.version:03}", GLOWS_L3C_DESCRIPTOR) + str(dependency.version), GLOWS_L3C_DESCRIPTOR) l3b_data_product = GlowsL3BIonizationRate.from_instrument_team_dictionary(l3b_data, l3b_metadata) l3c_data_product = GlowsL3CSolarWind.from_instrument_team_dictionary(l3c_data, l3c_metadata) diff --git a/imap_l3_processing/glows/l3bc/glows_l3bc_dependencies.py b/imap_l3_processing/glows/l3bc/glows_l3bc_dependencies.py index cf2b6de0e..c71c7236b 100644 --- a/imap_l3_processing/glows/l3bc/glows_l3bc_dependencies.py +++ b/imap_l3_processing/glows/l3bc/glows_l3bc_dependencies.py @@ -3,6 +3,7 @@ from pathlib import Path import imap_data_access +from imap_data_access.file_validation import Version from imap_l3_processing.glows.l3a.utils import create_glows_l3a_dictionary_from_cdf from imap_l3_processing.glows.l3bc.models import CRToProcess, ExternalDependencies @@ -10,7 +11,7 @@ @dataclass class GlowsL3BCDependencies: - version: int + version: Version carrington_rotation_number: int start_date: datetime end_date: datetime @@ -24,7 +25,7 @@ def l3a_file_names(self): return [l3a['filename'] for l3a in self.l3a_data] @classmethod - def download_from_cr_to_process(cls, cr_to_process: CRToProcess, version: int, + def download_from_cr_to_process(cls, cr_to_process: CRToProcess, version: Version, external_dependencies: ExternalDependencies, repointing_file_path: Path): external_files = { 'f107_raw_data': external_dependencies.f107_index_file_path, diff --git a/imap_l3_processing/glows/l3bc/glows_l3bc_initializer.py b/imap_l3_processing/glows/l3bc/glows_l3bc_initializer.py index 7cba5a01e..cd7fd7587 100644 --- a/imap_l3_processing/glows/l3bc/glows_l3bc_initializer.py +++ b/imap_l3_processing/glows/l3bc/glows_l3bc_initializer.py @@ -6,6 +6,7 @@ import imap_data_access from imap_data_access import ScienceFilePath, ProcessingInputCollection, RepointInput +from imap_data_access.file_validation import Version from imap_processing.spice.repoint import set_global_repoint_table_paths from spacepy.pycdf import CDF @@ -37,7 +38,7 @@ class GlowsL3BCInitializerData: class GlowsL3BCInitializer: @staticmethod - def get_crs_to_process(dependencies: ProcessingInputCollection) -> GlowsL3BCInitializerData: + def get_crs_to_process(dependencies: ProcessingInputCollection, major_version_to_process: int) -> GlowsL3BCInitializerData: [repoint_file] = dependencies.get_file_paths(data_type=RepointInput.data_type) repoint_downloaded_path = imap_data_access.download(repoint_file) set_global_repoint_table_paths([repoint_downloaded_path]) @@ -108,7 +109,7 @@ def get_crs_to_process(dependencies: ProcessingInputCollection) -> GlowsL3BCInit ) if version := GlowsL3BCInitializer.should_process_cr_candidate(cr_candidate, l3bs_by_cr, - external_dependencies): + external_dependencies, major_version_to_process): l3bc_dependencies = GlowsL3BCDependencies.download_from_cr_to_process(cr_candidate, version, external_dependencies, repoint_downloaded_path) @@ -128,7 +129,7 @@ def get_crs_to_process(dependencies: ProcessingInputCollection) -> GlowsL3BCInit @staticmethod def should_process_cr_candidate(cr_candidate: CRToProcess, l3bs_by_cr: dict[int, str], - external_dependencies: ExternalDependencies) -> Optional[int]: + external_dependencies: ExternalDependencies, major_version: int|None) -> Optional[Version]: if not cr_candidate.buffer_time_has_elapsed_since_cr(): logger.warning(f"Not enough time has elapsed for cr {cr_candidate.cr_rotation_number}") return None @@ -139,11 +140,14 @@ def should_process_cr_candidate(cr_candidate: CRToProcess, l3bs_by_cr: dict[int, match l3bs_by_cr.get(cr_candidate.cr_rotation_number): case None: - return 1 + return Version(major_version, 1) case l3b_file_name: l3b_parents = read_cdf_parents(l3b_file_name) - if not cr_candidate.pipeline_dependency_file_names().issubset(l3b_parents): - return int(ScienceFilePath(l3b_file_name).version[1:]) + 1 + existing_version = Version.from_version(ScienceFilePath(l3b_file_name).version) + major_version_updated = major_version != existing_version.major + inputs_changed = not cr_candidate.pipeline_dependency_file_names().issubset(l3b_parents) + if major_version_updated or inputs_changed: + return Version(major_version, existing_version.minor+1) return None @staticmethod diff --git a/pyproject.toml b/pyproject.toml index 8144ca0e2..b4cf591ad 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -11,7 +11,7 @@ dependencies = [ "astropy==6.1.7", "astropy_healpix==1.1.2", "bitstring==4.3.0", - "imap-data-access>=0.37.0", + "imap-data-access>=0.42.0.dev0", "lmfit==1.3.3", "netCDF4>=1.6", "numdifftools==0.9.41", diff --git a/tests/glows/l3bc/science/test_generate_l3bc.py b/tests/glows/l3bc/science/test_generate_l3bc.py index c34a36184..e77d3f18a 100644 --- a/tests/glows/l3bc/science/test_generate_l3bc.py +++ b/tests/glows/l3bc/science/test_generate_l3bc.py @@ -3,6 +3,8 @@ from unittest import TestCase from unittest.mock import sentinel +from imap_data_access.file_validation import Version + from imap_l3_processing.glows.l3a.utils import create_glows_l3a_dictionary_from_cdf from imap_l3_processing.glows.l3bc.cannot_process_carrington_rotation_error import CannotProcessCarringtonRotationError from imap_l3_processing.glows.l3bc.glows_l3bc_dependencies import GlowsL3BCDependencies @@ -38,7 +40,7 @@ def test_generate_l3bc_integration(self): carrington_rotation_number=cr, start_date=datetime.now(), end_date=datetime.now() + timedelta(days=1), - version=1, + version=Version(1,1), repointing_file_path=sentinel.repointing_file_path ) @@ -91,7 +93,7 @@ def test_generate_l3bc_integration_all_data_in_bad_season(self): carrington_rotation_number=cr, start_date=datetime.now(), end_date=datetime.now() + timedelta(days=1), - version=1, + version=Version(1,1), repointing_file_path=sentinel.repointing_file_path ) @@ -151,7 +153,7 @@ def test_generate_l3bc_appends_used_l3a_files_to_header(self): carrington_rotation_number=cr, start_date=datetime.now(), end_date=datetime.now() + timedelta(days=1), - version=1, + version=Version(1,1), repointing_file_path=sentinel.repointing_file_path ) l3b, l3c = generate_l3bc(dependencies) diff --git a/tests/glows/l3bc/test_glows_l3bc_initializer.py b/tests/glows/l3bc/test_glows_l3bc_initializer.py index 017893183..a132de81c 100644 --- a/tests/glows/l3bc/test_glows_l3bc_initializer.py +++ b/tests/glows/l3bc/test_glows_l3bc_initializer.py @@ -5,6 +5,7 @@ from unittest.mock import Mock, patch, call, sentinel from imap_data_access import ProcessingInputCollection, RepointInput +from imap_data_access.file_validation import Version from imap_l3_processing.glows.l3bc.glows_l3bc_initializer import GlowsL3BCInitializer, GlowsL3BCInitializerData from imap_l3_processing.glows.l3bc.models import CRToProcess, ExternalDependencies @@ -95,8 +96,12 @@ def test_get_crs_to_process(self, mock_should_process_cr_candidate, mock_group_l 2092: "imap_glows_l3c_sw-profile_20100201-cr02092_v001.cdf" } + major_version_to_process = 3 + l3bc_initializer_data = GlowsL3BCInitializer.get_crs_to_process( - ProcessingInputCollection(RepointInput("imap_2026_269_05.repoint.csv"))) + ProcessingInputCollection(RepointInput("imap_2026_269_05.repoint.csv")), + major_version_to_process + ) mock_download.assert_called_once() mock_set_global_repoint_table_paths.assert_called_with([mock_download.return_value]) @@ -150,8 +155,8 @@ def test_get_crs_to_process(self, mock_should_process_cr_candidate, mock_group_l ) mock_should_process_cr_candidate.assert_has_calls([ - call(expected_cr_candidate_1, expected_l3bs_by_cr, mock_fetch_external_deps.return_value), - call(expected_cr_candidate_2, expected_l3bs_by_cr, mock_fetch_external_deps.return_value) + call(expected_cr_candidate_1, expected_l3bs_by_cr, mock_fetch_external_deps.return_value, major_version_to_process), + call(expected_cr_candidate_2, expected_l3bs_by_cr, mock_fetch_external_deps.return_value, major_version_to_process) ], any_order=False) mock_l3bc_deps_from_cr.assert_called_once_with(expected_cr_candidate_1, 2, @@ -197,7 +202,7 @@ def query_that_returns_empty_list_for_missing_ancillary(**kwargs): mock_query.side_effect = query_that_returns_empty_list_for_missing_ancillary actual_crs_to_process = GlowsL3BCInitializer.get_crs_to_process( - ProcessingInputCollection(RepointInput("imap_2026_269_05.repoint.csv"))) + ProcessingInputCollection(RepointInput("imap_2026_269_05.repoint.csv")), sentinel.major_version) mock_query.assert_has_calls([ call(instrument="glows", data_level="l3a", descriptor="hist", version="latest"), @@ -254,7 +259,7 @@ def test_get_crs_to_process_returns_no_crs_when_missing_external_deps(self, mock ] actual_crs_to_process = GlowsL3BCInitializer.get_crs_to_process( - ProcessingInputCollection(RepointInput("imap_2026_269_05.repoint.csv"))) + ProcessingInputCollection(RepointInput("imap_2026_269_05.repoint.csv")), sentinel.major_version) mock_query.assert_has_calls([ call(instrument="glows", data_level="l3a", descriptor="hist", version="latest"), @@ -297,7 +302,7 @@ def test_should_process_cr_excludes_crs_within_buffer_period(self): omni2_data_path=Path("omni2_data_path"), ) - self.assertIsNone(GlowsL3BCInitializer.should_process_cr_candidate(cr_candidate, {}, external_dependencies)) + self.assertIsNone(GlowsL3BCInitializer.should_process_cr_candidate(cr_candidate, {}, external_dependencies, 1)) def test_should_process_cr_no_existing_l3b(self): @@ -320,9 +325,9 @@ def test_should_process_cr_no_existing_l3b(self): omni2_data_path=Path("omni2_data_path"), ) - actual_version = GlowsL3BCInitializer.should_process_cr_candidate(cr_candidate, {}, external_dependencies) + actual_version = GlowsL3BCInitializer.should_process_cr_candidate(cr_candidate, {}, external_dependencies, major_version=2) - self.assertEqual(1, actual_version) + self.assertEqual(Version(2, 1), actual_version) @patch("imap_l3_processing.glows.l3bc.glows_l3bc_initializer.read_cdf_parents") def test_should_process_cr_existing_cr_should_process(self, mock_read_cdf_parents): @@ -347,16 +352,18 @@ def test_should_process_cr_existing_cr_should_process(self, mock_read_cdf_parent } test_cases = [ - ("new l3a file version", new_l3a_version), - ("new l3a files arrived", new_l3a_file), - ("new ancillary file arrived", new_ancillary) + ("new l3a file version", new_l3a_version, 2), + ("new l3a files arrived", new_l3a_file, 2), + ("new ancillary file arrived", new_ancillary, 2), + ("new major version", {}, 3) ] - for name, new_server_data in test_cases: + for name, new_server_data, new_major_version in test_cases: with self.subTest(name): mock_read_cdf_parents.reset_mock() - existing_file_version = 2 + existing_file_major_version = 2 + existing_file_minor_version = 3 already_processed_cr = CRToProcess( l3a_file_names={ @@ -392,14 +399,16 @@ def test_should_process_cr_existing_cr_should_process(self, mock_read_cdf_parent "some_zip_file.zip" } - l3bs = {2091: f"imap_glows_l3b_ion-rate-profile_20100101-cr02091_v00{existing_file_version}.cdf"} + existing_file_version = Version(existing_file_major_version, existing_file_minor_version) + l3bs = {2091: f"imap_glows_l3b_ion-rate-profile_20100101-cr02091_{existing_file_version}.cdf"} actual_version = GlowsL3BCInitializer.should_process_cr_candidate(cr_candidate, l3bs, - external_dependencies) + external_dependencies, new_major_version) + expected_version = Version(new_major_version, existing_file_minor_version + 1) mock_read_cdf_parents.assert_called_once_with( - f"imap_glows_l3b_ion-rate-profile_20100101-cr02091_v00{existing_file_version}.cdf") + f"imap_glows_l3b_ion-rate-profile_20100101-cr02091_{existing_file_version}.cdf") - self.assertEqual(existing_file_version + 1, actual_version) + self.assertEqual(expected_version, actual_version) @patch("imap_l3_processing.glows.l3bc.glows_l3bc_initializer.read_cdf_parents") def test_should_process_cr_existing_cr_should_not_reprocess(self, mock_read_cdf_parents): @@ -418,8 +427,8 @@ def test_should_process_cr_existing_cr_should_not_reprocess(self, mock_read_cdf_ ) cr_candidate.buffer_time_has_elapsed_since_cr = Mock(return_value=True) cr_candidate.has_valid_external_dependencies = Mock(return_value=True) - - l3bs = {2091: "imap_glows_l3b_ion-rate-profile_20100101-cr02091_v001.cdf"} + existing_version = Version(2, 1) + l3bs = {2091: f"imap_glows_l3b_ion-rate-profile_20100101-cr02091_{existing_version}.cdf"} external_dependencies = ExternalDependencies( f107_index_file_path=Path("f107_index_file_path"), @@ -437,9 +446,9 @@ def test_should_process_cr_existing_cr_should_not_reprocess(self, mock_read_cdf_ "some_zip_file.zip" } - self.assertIsNone(GlowsL3BCInitializer.should_process_cr_candidate(cr_candidate, l3bs, external_dependencies)) + self.assertIsNone(GlowsL3BCInitializer.should_process_cr_candidate(cr_candidate, l3bs, external_dependencies, existing_version.major)) - mock_read_cdf_parents.assert_called_once_with("imap_glows_l3b_ion-rate-profile_20100101-cr02091_v001.cdf") + mock_read_cdf_parents.assert_called_once_with(f"imap_glows_l3b_ion-rate-profile_20100101-cr02091_{existing_version}.cdf") def test_should_process_cr_with_invalid_dependencies(self): cr_candidate = CRToProcess( @@ -462,7 +471,7 @@ def test_should_process_cr_with_invalid_dependencies(self): omni2_data_path=Path("omni2_data_path"), ) - self.assertIsNone(GlowsL3BCInitializer.should_process_cr_candidate(cr_candidate, {}, external_dependencies)) + self.assertIsNone(GlowsL3BCInitializer.should_process_cr_candidate(cr_candidate, {}, external_dependencies, 1)) cr_candidate.has_valid_external_dependencies.assert_called_once_with(external_dependencies) diff --git a/tests/glows/test_glows_processor.py b/tests/glows/test_glows_processor.py index 5f54bb7e7..fcefd7890 100644 --- a/tests/glows/test_glows_processor.py +++ b/tests/glows/test_glows_processor.py @@ -16,6 +16,7 @@ import imap_data_access import numpy as np from imap_data_access import AncillaryFilePath +from imap_data_access.file_validation import Version from spacepy.pycdf import CDF from imap_l3_processing.constants import TEMP_CDF_FOLDER_PATH @@ -295,25 +296,27 @@ def test_process_l3bc(self, mock_glows_l3bc_initializer, mock_archive_dependenci mock_archive_dependencies.side_effect = [Path("path1.zip"), Path("path2.zip")] l3a_data_1 = {"filename": "l3a_file_1"} + first_dependency_version = Version(1, 1) first_dependency = GlowsL3BCDependencies(l3a_data=[l3a_data_1], external_files=sentinel.external_files_1, ancillary_files={ 'bad_days_list': sentinel.bad_days_list_1, }, carrington_rotation_number=first_cr_number_to_process, - version=1, + version=first_dependency_version, start_date=Mock(), end_date=Mock(), repointing_file_path=sentinel.repointing_file_path) l3a_data_2 = {"filename": "l3a_file_2"} + second_dependency_version = Version(1, 2) second_dependency = GlowsL3BCDependencies(l3a_data=[l3a_data_2], external_files=sentinel.external_files_2, ancillary_files={ 'bad_days_list': sentinel.bad_days_list_2, }, carrington_rotation_number=second_cr_number_to_process, - version=2, + version=second_dependency_version, start_date=Mock(), end_date=Mock(), repointing_file_path=sentinel.repointing_file_path) @@ -333,9 +336,10 @@ def test_process_l3bc(self, mock_glows_l3bc_initializer, mock_archive_dependenci Path("path/to/l3b_file_2.cdf"), Path("path/to/l3c_file_2.cdf")] + input_version = Version(2,1) input_metadata = InputMetadata('glows', "l3b", datetime(2024, 10, 7, 10, 00, 00), datetime(2024, 10, 8, 10, 00, 00), - 'v02') + str(input_version)) mock_glows_l3bc_initializer.get_crs_to_process.return_value = GlowsL3BCInitializerData( external_dependencies=external_deps, @@ -345,7 +349,7 @@ def test_process_l3bc(self, mock_glows_l3bc_initializer, mock_archive_dependenci repoint_file_path=sentinel.repoint_file_path ) - processor = GlowsProcessor(dependencies=Mock(), input_metadata=input_metadata) + processor = GlowsProcessor(dependencies=sentinel.dependencies, input_metadata=input_metadata) products = processor.process() self.assertEqual([Path("path/to/l3b_file_1.cdf"), @@ -366,14 +370,16 @@ def test_process_l3bc(self, mock_glows_l3bc_initializer, mock_archive_dependenci mock_generate_l3bc.assert_has_calls( [call(dependencies_with_filtered_list_1), call(dependencies_with_filtered_list_2)]) + mock_glows_l3bc_initializer.get_crs_to_process.assert_called_once_with(sentinel.dependencies, input_version.major) + expected_l3b_metadata_1 = InputMetadata("glows", "l3b", first_dependency.start_date, - first_dependency.end_date, 'v001', "ion-rate-profile") + first_dependency.end_date, str(first_dependency_version), "ion-rate-profile") expected_l3b_metadata_2 = InputMetadata("glows", "l3b", second_dependency.start_date, - second_dependency.end_date, 'v002', "ion-rate-profile") + second_dependency.end_date, str(second_dependency_version), "ion-rate-profile") expected_l3c_metadata_1 = InputMetadata("glows", "l3c", first_dependency.start_date, - first_dependency.end_date, 'v001', "sw-profile") + first_dependency.end_date, str(first_dependency_version), "sw-profile") expected_l3c_metadata_2 = InputMetadata("glows", "l3c", second_dependency.start_date, - second_dependency.end_date, 'v002', "sw-profile") + second_dependency.end_date, str(second_dependency_version), "sw-profile") mock_l3b_model_class.from_instrument_team_dictionary.assert_has_calls( [call(sentinel.l3b_data_1, expected_l3b_metadata_1), call(sentinel.l3b_data_2, expected_l3b_metadata_2)]) @@ -410,7 +416,7 @@ def test_process_l3bc_catches_no_data_error_and_continues(self, mock_glows_initi mock_archive_dependencies, mock_generate_l3bc, mock_l3c_from_instrument_team_dictionary, mock_l3b_from_instrument_team_dictionary, _): - bc_dependencies_1 = GlowsL3BCDependencies(version=1, + bc_dependencies_1 = GlowsL3BCDependencies(version=Version(1, 1), carrington_rotation_number=2096, start_date=datetime(year=2021, month=1, day=1), end_date=datetime(year=2021, month=1, day=1), @@ -419,7 +425,7 @@ def test_process_l3bc_catches_no_data_error_and_continues(self, mock_glows_initi ancillary_files=defaultdict(Mock), repointing_file_path=sentinel.repointing_file_path ) - bc_dependencies_2 = GlowsL3BCDependencies(version=1, + bc_dependencies_2 = GlowsL3BCDependencies(version=Version(1,1), carrington_rotation_number=2096, start_date=datetime(year=2021, month=1, day=1), end_date=datetime(year=2021, month=1, day=1), @@ -508,7 +514,7 @@ def test_process_l3bc_catches_exceptions_from_science_code_and_continues(self, m 'omni_raw_data': get_test_instrument_team_data_path('glows/omni_2010.dat'), }, carrington_rotation_number=1, - start_date=Mock(), end_date=Mock(), version=1, + start_date=Mock(), end_date=Mock(), version=Version(1,1), repointing_file_path=sentinel.repointing_file_path ) @@ -553,7 +559,7 @@ def test_l3bc_uses_all_l3a_file_names_for_l3b_parents(self, mock_save_data, mock 'omni_raw_data': get_test_instrument_team_data_path('glows/omni2_all_years.dat'), }, carrington_rotation_number=2092, - start_date=Mock(), end_date=Mock(), version=1, + start_date=Mock(), end_date=Mock(), version=Version(1,1), repointing_file_path=sentinel.repointing_file_path, ) @@ -648,7 +654,7 @@ def test_process_l3d(self, mock_read_pipeline_settings, mock_glows_l3d_initializ mock_save_data.return_value = Path("l3d_cdf.cdf") - processor = GlowsProcessor(Mock(), Mock(data_level="l3b")) + processor = GlowsProcessor(Mock(), Mock(data_level="l3b", version=str(Version(1,1)))) products = processor.process() mock_convert_l3b_to_json.assert_has_calls([call(sentinel.l3b_file_1), call(sentinel.l3b_file_2)]) @@ -1546,10 +1552,10 @@ def test_process_l3e_skips_repointing_on_exception(self, mock_process_ultra_sf, @patch("imap_l3_processing.glows.glows_processor.json") @patch("imap_l3_processing.glows.glows_processor.ZipFile") def test_archive_dependencies(self, mock_zip, mock_json): - expected_filepath = TEMP_CDF_FOLDER_PATH / "imap_glows_l3b-archive_20250314_v001.zip" + version_number = Version(1,1) + expected_filepath = TEMP_CDF_FOLDER_PATH / f"imap_glows_l3b-archive_20250314_{version_number}.zip" expected_json_filename = "cr_to_process.json" - version_number = 1 l3bc_dependencies = GlowsL3BCDependencies( version=version_number, carrington_rotation_number=2095, diff --git a/tests/test_imap_l3_data_processor.py b/tests/test_imap_l3_data_processor.py index 88dfd9324..bbcd5a2e6 100644 --- a/tests/test_imap_l3_data_processor.py +++ b/tests/test_imap_l3_data_processor.py @@ -1,3 +1,4 @@ +import json from datetime import datetime from pathlib import Path from unittest import TestCase @@ -58,7 +59,7 @@ def test_invokes_correct_processor(self, mock_argparse, mock_codice_hi, mock_cod mock_argument_parser = mock_argparse.ArgumentParser.return_value mock_argument_parser.parse_args.return_value.instrument = instrument mock_argument_parser.parse_args.return_value.data_level = data_level - mock_argument_parser.parse_args.return_value.dependency = "dependency_string" + mock_argument_parser.parse_args.return_value.dependency = "[]" mock_argument_parser.parse_args.return_value.start_date = "20250101" mock_argument_parser.parse_args.return_value.end_date = None mock_argument_parser.parse_args.return_value.repointing = "repoint00022" @@ -81,13 +82,13 @@ def test_invokes_correct_processor(self, mock_argparse, mock_codice_hi, mock_cod call("--start-date"), call("--end-date", required=False), call("--repointing", required=False), - call("--version"), + call("--version", required=False), call("--dependency"), call("--upload-to-sdc", action="store_true", required=False, help="Upload completed output files to the IMAP SDC.") ]) - mock_processing_input.return_value.deserialize.assert_called_once_with("dependency_string") + mock_processing_input.return_value.deserialize.assert_called_once_with("[]") expected_processor.assert_called_once_with(mock_processing_input.return_value, expected_input_metadata) mock_upload.assert_called_once_with(sentinel.cdf) @@ -234,7 +235,7 @@ def test_get_spice_kernels_based_on_input_collection(self, _, __, mock_processin mock_argument_parser.parse_args.return_value.end_date = "20160630" mock_argument_parser.parse_args.return_value.version = "v101" mock_argument_parser.parse_args.return_value.descriptor = "dont care" - mock_argument_parser.parse_args.return_value.dependency = "also dont care" + mock_argument_parser.parse_args.return_value.dependency = "[]" mock_argument_parser.parse_args.return_value.repointing = None mock_download.side_effect = [ @@ -307,7 +308,7 @@ def test_uses_input_from_processing_input_collection(self, mock_argparse, mock_p call("--start-date"), call("--end-date", required=False), call("--repointing", required=False), - call("--version"), + call("--version", required=False), call("--dependency"), call("--upload-to-sdc", action="store_true", required=False, help="Upload completed output files to the IMAP SDC.") @@ -517,3 +518,124 @@ def test_throws_exception_when_attempting_to_process_non_l3_data_levels(self, mo self.assertEqual(str(exception_manager.exception), "Level l4 data processing has not yet been implemented for swapi") mock_upload.assert_not_called() + + @patch('imap_l3_data_processor.ProcessingInputCollection') + @patch('imap_l3_data_processor.imap_data_access.upload') + @patch('imap_l3_data_processor.SweProcessor') + @patch('imap_l3_data_processor.argparse') + def test_version_comes_from_dependency(self, mock_argparse, mock_processor_class, mock_upload, + mock_processing_input_collection): + cases = [("20170630", datetime(2017, 6, 30)), (None, datetime(2016, 6, 30))] + + instrument_argument = "swe" + data_level_argument = "l3" + start_date_argument = "20160630" + version_argument = "v092" + descriptor_argument = "pitch-angle" + science_input_1 = ScienceInput("imap_swe_l1_sci_20250101_v112.cdf", "imap_swe_l1_sci_20250102_v112.cdf") + science_input_2 = ScienceInput("imap_mag_l1d_norm-dsrf_20250101_v112.cdf") + ancillary_input = AncillaryInput("imap_swe_ancillary_20250101_v112.cdf") + imap_data_access_dependency = ProcessingInputCollection(science_input_1, science_input_2, ancillary_input) + + version_input = { + "sci": {"major_version": 2, "minor_version": 1} + } + combined_input = { + "version": version_input, + "dependency": json.loads(imap_data_access_dependency.serialize()), + } + + mock_processing_input_collection.return_value = imap_data_access_dependency + mock_processing_input_collection.deserialize = Mock() + mock_processing_input_collection.get_science_inputs = Mock(return_value=[]) + + mock_argument_parser = mock_argparse.ArgumentParser.return_value + + mock_argument_parser.parse_args.return_value.instrument = instrument_argument + mock_argument_parser.parse_args.return_value.data_level = data_level_argument + mock_argument_parser.parse_args.return_value.dependency = json.dumps(combined_input) + mock_argument_parser.parse_args.return_value.start_date = start_date_argument + mock_argument_parser.parse_args.return_value.version = version_argument + mock_argument_parser.parse_args.return_value.descriptor = descriptor_argument + mock_argument_parser.parse_args.return_value.repointing = None + + mock_processor = mock_processor_class.return_value + + for input_end_date, expected_end_date in cases: + with self.subTest(input_end_date): + mock_upload.reset_mock() + + mock_argument_parser.parse_args.return_value.end_date = input_end_date + + mock_processor_class.return_value.process.return_value = [Mock()] + + imap_l3_processor() + + parser = mock_argparse.ArgumentParser() + parser.add_argument.assert_has_calls([ + call("--instrument"), + call("--data-level"), + call("--descriptor"), + call("--start-date"), + call("--end-date", required=False), + call("--repointing", required=False), + call("--version", required=False), + call("--dependency"), + call("--upload-to-sdc", action="store_true", required=False, + help="Upload completed output files to the IMAP SDC.") + ]) + + expected_input_metadata = InputMetadata("swe", "l3", datetime(year=2016, month=6, day=30), + expected_end_date, "v002.0001", "pitch-angle") + + mock_processor_class.assert_called_with(imap_data_access_dependency, expected_input_metadata) + + mock_processor.process.assert_called() + mock_upload.assert_called_once_with(mock_processor_class.return_value.process.return_value[0]) + + + @patch('imap_l3_data_processor.ProcessingInputCollection') + @patch('imap_l3_data_processor.imap_data_access.upload') + @patch('imap_l3_data_processor.SweProcessor') + @patch('imap_l3_data_processor.argparse') + def test_raises_error_if_multiple_versions_are_specified(self, mock_argparse, mock_processor_class, mock_upload, + mock_processing_input_collection): + + instrument_argument = "swe" + data_level_argument = "l3" + start_date_argument = "20160630" + version_argument = "v092" + descriptor_argument = "pitch-angle" + science_input_1 = ScienceInput("imap_swe_l1_sci_20250101_v112.cdf", "imap_swe_l1_sci_20250102_v112.cdf") + science_input_2 = ScienceInput("imap_mag_l1d_norm-dsrf_20250101_v112.cdf") + ancillary_input = AncillaryInput("imap_swe_ancillary_20250101_v112.cdf") + imap_data_access_dependency = ProcessingInputCollection(science_input_1, science_input_2, ancillary_input) + + version_input = { + "sci": {"major_version": 2, "minor_version": 1}, + "another_var": {"major_version": 2, "minor_version": 1} + } + combined_input = { + "version": version_input, + "dependency": json.loads(imap_data_access_dependency.serialize()), + } + + mock_processing_input_collection.return_value = imap_data_access_dependency + mock_processing_input_collection.deserialize = Mock() + mock_processing_input_collection.get_science_inputs = Mock(return_value=[]) + + mock_argument_parser = mock_argparse.ArgumentParser.return_value + + mock_argument_parser.parse_args.return_value.instrument = instrument_argument + mock_argument_parser.parse_args.return_value.data_level = data_level_argument + mock_argument_parser.parse_args.return_value.dependency = json.dumps(combined_input) + mock_argument_parser.parse_args.return_value.start_date = start_date_argument + mock_argument_parser.parse_args.return_value.version = version_argument + mock_argument_parser.parse_args.return_value.descriptor = descriptor_argument + mock_argument_parser.parse_args.return_value.repointing = None + mock_argument_parser.parse_args.return_value.end_date = None + + with self.assertRaises(ValueError) as exception: + mock_processor_class.return_value.process.return_value = [Mock()] + imap_l3_processor() + self.assertEqual(str(exception.exception), "Expected only a single version to be specified in the dependency.") diff --git a/uv.lock b/uv.lock index fe264c234..2caa5135f 100644 --- a/uv.lock +++ b/uv.lock @@ -1,5 +1,5 @@ version = 1 -revision = 2 +revision = 3 requires-python = ">=3.10" resolution-markers = [ "python_full_version >= '3.12' and platform_machine == 'ARM64' and sys_platform == 'win32'", @@ -691,14 +691,14 @@ wheels = [ [[package]] name = "imap-data-access" -version = "0.37.0" +version = "0.42.0.dev0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "requests" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/95/1b/39b1cfa538ca15195e7077fc1c953daf99a36aad17b398f4067f503b191c/imap_data_access-0.37.0.tar.gz", hash = "sha256:7887ad43c4404c6465035af73621237aef7a8d88d506439dd4618aef1db8cf87", size = 49971, upload-time = "2025-10-15T19:25:38.795Z" } +sdist = { url = "https://files.pythonhosted.org/packages/1b/9a/9a8efafcbf985fbf0b50dff4ec29593b2468d6d93de95d59531811a60ca4/imap_data_access-0.42.0.dev0.tar.gz", hash = "sha256:d75b5f2df51fb220d58c5fb009fd5d2eec24c22abdeac5035bc32df55ee2c338", size = 59535, upload-time = "2026-06-18T16:02:19.287Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/c1/00/0348ebb2217f2c9c0b0c3cf7e80d150f6e63d7636c80189c46442c500d82/imap_data_access-0.37.0-py3-none-any.whl", hash = "sha256:9801aba311311815866336636fdad9e37b530498f4c3f21abce283d860a7c643", size = 35793, upload-time = "2025-10-15T19:25:37.521Z" }, + { url = "https://files.pythonhosted.org/packages/7c/7b/be7b42051f060016417b1aedf84994a87b506cca1cb86b08d735fb86e8bf/imap_data_access-0.42.0.dev0-py3-none-any.whl", hash = "sha256:ba6618e809033cf8effa1ed0a63add082ff32ce5cde239a59530ec7065237da2", size = 41593, upload-time = "2026-06-18T16:02:17.989Z" }, ] [[package]] @@ -741,7 +741,7 @@ requires-dist = [ { name = "astropy-healpix", specifier = "==1.1.2" }, { name = "bitstring", specifier = "==4.3.0" }, { name = "hatchling" }, - { name = "imap-data-access", specifier = ">=0.37.0" }, + { name = "imap-data-access", specifier = ">=0.42.0.dev0" }, { name = "imap-processing", specifier = ">=1.0.33" }, { name = "lmfit", specifier = "==1.3.3" }, { name = "netcdf4", specifier = ">=1.6" }, From cb7a3b3da77af1a3dd9f3a4db781c0104beb432c Mon Sep 17 00:00:00 2001 From: Harrison Date: Thu, 25 Jun 2026 10:52:23 -0400 Subject: [PATCH 03/11] Harrison 3295 #118: PMCL/KJON - Handle updated version structure in GLOWS L3d/e --- imap_l3_processing/glows/glows_processor.py | 124 ++++---- .../glows/l3d/glows_l3d_initializer.py | 14 +- .../glows/l3e/glows_l3e_initializer.py | 18 +- .../glows/l3e/glows_l3e_utils.py | 26 +- tests/glows/l3d/test_glows_l3d_initializer.py | 165 +++++++---- tests/glows/l3e/test_glows_l3e_initializer.py | 83 +++++- tests/glows/l3e/test_glows_l3e_utils.py | 273 +++++++++++++----- tests/glows/test_glows_processor.py | 70 ++--- 8 files changed, 521 insertions(+), 252 deletions(-) diff --git a/imap_l3_processing/glows/glows_processor.py b/imap_l3_processing/glows/glows_processor.py index e0ce8de82..d25a83cd5 100644 --- a/imap_l3_processing/glows/glows_processor.py +++ b/imap_l3_processing/glows/glows_processor.py @@ -80,7 +80,7 @@ def process(self): l3bs = list({**l3bc_initializer_data.l3bs_by_cr, **glows_l3bc_output_data.l3bs_by_cr}.values()) l3cs = list({**l3bc_initializer_data.l3cs_by_cr, **glows_l3bc_output_data.l3cs_by_cr}.values()) l3d_initializer_result = GlowsL3DInitializer.should_process_l3d(l3bc_initializer_data.external_dependencies, - l3bs, l3cs) + l3bs, l3cs, major_version) if l3d_initializer_result is None: logger.info("No inputs to L3d have changed. Skipping processing of L3d and L3e!") return products_list @@ -102,7 +102,8 @@ def process(self): if l3e_initializer_output is not None: logger.info(f"Processing L3e for repointings: {l3e_initializer_output.repointings.repointing_numbers}") products_list.extend([*process_l3d_result.l3d_text_file_paths, process_l3d_result.l3d_cdf_file_path]) - products_list.extend(process_l3e(l3e_initializer_output)) + l3e_products = process_l3e(l3e_initializer_output) + products_list.extend(l3e_products) else: logger.info(f"There are no changes between: {old_l3d} and the newly produced L3d. Not uploading L3d or processing L3e!") @@ -197,7 +198,7 @@ def process_l3bc(processor, initializer_data: GlowsL3BCInitializerData): ) def process_l3d( - dependencies: GlowsL3DDependencies, version: int + dependencies: GlowsL3DDependencies, version: Version ) -> Optional[GlowsL3DProcessorOutput]: [create_glows_l3b_json_file_from_cdf(l3b) for l3b in dependencies.l3b_file_paths] @@ -232,14 +233,12 @@ def process_l3d( last_processed_cr = int(output.stdout.split('= ')[-1]) if last_processed_cr: - formatted_version = f"v{version:03}" - output_text_files = [] for text_file in os.listdir(PATH_TO_L3D_TOOLKIT / 'data_l3d_txt'): if str(last_processed_cr) in text_file: output_text_files.append(PATH_TO_L3D_TOOLKIT / 'data_l3d_txt' / text_file) - txt_files_with_correct_version = rename_l3d_text_outputs(output_text_files, formatted_version) + txt_files_with_correct_version = rename_l3d_text_outputs(output_text_files, str(Version(None, version.minor))) for txt_file in txt_files_with_correct_version: shutil.copy(txt_file, generate_imap_file_path(txt_file.name).construct_path()) @@ -248,7 +247,7 @@ def process_l3d( start_date = datetime(1947, 3, 3) data_product_metadata = InputMetadata(instrument="glows", data_level="l3d", descriptor=GLOWS_L3D_DESCRIPTOR, - start_date=start_date, end_date=start_date, version=formatted_version) + start_date=start_date, end_date=start_date, version=str(version)) parent_file_names = get_parent_file_names_from_l3d_json(PATH_TO_L3D_TOOLKIT / 'data_l3d') l3d_data_product = convert_json_to_l3d_data_product(PATH_TO_L3D_TOOLKIT / 'data_l3d' / file_name, @@ -258,13 +257,57 @@ def process_l3d( return GlowsL3DProcessorOutput(l3d_data_product_path, txt_files_with_correct_version, last_processed_cr) return None +def process_l3e(initializer_data: GlowsL3EInitializerOutput): + products_list = [] + + lo_pivot_angles = get_lo_pivot_angles(initializer_data.repointings.repointing_numbers) + for repointing in initializer_data.repointings.repointing_numbers: + with SwallowExceptionAndLog(f"Exception encountered when processing L3e for repointing {repointing}"): + start_repointing, end_repointing = get_pointing_date_range(repointing) + epoch_delta: timedelta = (end_repointing - start_repointing) / 2 + glows_flags = compute_glows_flags_for_window(initializer_data.l3d_cdf_path, start_repointing, end_repointing) + + with SwallowExceptionAndLog(f"Exception encountered when processing L3e lo for repointing {repointing}"): + lo_parent_file_names = initializer_data.dependencies.get_lo_parents() + pivot_info = lo_pivot_angles[repointing] + if pivot_info.parent_filename is not None: + lo_parent_file_names = lo_parent_file_names + [pivot_info.parent_filename] + lo_version = initializer_data.repointings.lo_repointings[repointing] + products_list.extend(process_l3e_lo(lo_parent_file_names, repointing, start_repointing, epoch_delta, pivot_info.pivot_angle, lo_version, glows_flags)) + + with SwallowExceptionAndLog(f"Exception encountered when processing L3e hi-90 for repointing {repointing}"): + hi_parent_file_names = initializer_data.dependencies.get_hi_parents() + hi_90_version = initializer_data.repointings.hi_90_repointings[repointing] + products_list.extend(process_l3e_hi(hi_parent_file_names, repointing, start_repointing, epoch_delta, 90, hi_90_version, glows_flags)) + + with SwallowExceptionAndLog(f"Exception encountered when processing L3e hi-45 for repointing {repointing}"): + hi_parent_file_names = initializer_data.dependencies.get_hi_parents() + hi_45_version = initializer_data.repointings.hi_45_repointings[repointing] + products_list.extend(process_l3e_hi(hi_parent_file_names, repointing, start_repointing, epoch_delta, 135, hi_45_version, glows_flags)) + + ul_parent_file_names = initializer_data.dependencies.get_ul_parents() + ul_sf_version = initializer_data.repointings.ultra_sf_repointings[repointing] + ul_hf_version = initializer_data.repointings.ultra_hf_repointings[repointing] + + with SwallowExceptionAndLog( + f"Exception encountered when processing L3e ultra SF for repointing {repointing}"): + products_list.extend( + process_l3e_ul_sf(ul_parent_file_names, repointing, start_repointing, epoch_delta, ul_sf_version, glows_flags)) + + with SwallowExceptionAndLog( + f"Exception encountered when processing L3e ultra HF for repointing {repointing}"): + products_list.extend( + process_l3e_ul_hf(ul_parent_file_names, repointing, start_repointing, epoch_delta, ul_hf_version, glows_flags)) + + return products_list + def process_l3e_lo( parent_file_names: list[str], repointing: int, repointing_start: datetime, epoch_delta: timedelta, elongation_value: float, - version: int, + version: Version, glows_flags: int, ) -> list[Path]: repointing_midpoint = repointing_start + epoch_delta @@ -281,7 +324,7 @@ def process_l3e_lo( descriptor="survival-probability-lo", start_date=repointing_start, end_date=repointing_start + epoch_delta * 2, - version=f"v{version:03}", + version=str(version), repointing=repointing, ) @@ -303,7 +346,7 @@ def process_l3e_lo( instrument="glows", descriptor=f"{cdf_science_file_path.descriptor}-raw", start_time=cdf_science_file_path.start_date, - version=cdf_science_file_path.version, + version=str(Version(None, version.minor)), extension="dat" ).construct_path() @@ -313,7 +356,7 @@ def process_l3e_lo( def process_l3e_ul_sf(parent_file_names: list[str], repointing: int, repointing_start: datetime, epoch_delta: timedelta, - version: int, glows_flags: int) -> list[Path]: + version: Version, glows_flags: int) -> list[Path]: repointing_midpoint = repointing_start + epoch_delta call_args_object = determine_call_args_for_l3e_executable(repointing_start, repointing_midpoint, 30) call_args = call_args_object.to_argument_list() @@ -328,7 +371,7 @@ def process_l3e_ul_sf(parent_file_names: list[str], repointing: int, repointing_ descriptor=GLOWS_L3E_ULTRA_SF_DESCRIPTOR, start_date=repointing_start, end_date=repointing_start + epoch_delta * 2, - version=f"v{version:03}", + version=str(version), repointing=repointing, ) @@ -347,7 +390,7 @@ def process_l3e_ul_sf(parent_file_names: list[str], repointing: int, repointing_ instrument="glows", descriptor=f"{cdf_science_file_path.descriptor}-raw", start_time=cdf_science_file_path.start_date, - version=cdf_science_file_path.version, + version=str(Version(None, version.minor)), extension="dat" ).construct_path() @@ -357,7 +400,7 @@ def process_l3e_ul_sf(parent_file_names: list[str], repointing: int, repointing_ def process_l3e_ul_hf(parent_file_names: list[str], repointing: int, repointing_start: datetime, epoch_delta: timedelta, - version: int, glows_flags: int) -> list[Path]: + version: Version, glows_flags: int) -> list[Path]: repointing_midpoint = repointing_start + epoch_delta call_args_object = determine_call_args_for_l3e_executable(repointing_start, repointing_midpoint, 30) @@ -377,7 +420,7 @@ def process_l3e_ul_hf(parent_file_names: list[str], repointing: int, repointing_ descriptor=GLOWS_L3E_ULTRA_HF_DESCRIPTOR, start_date=repointing_start, end_date=repointing_start + epoch_delta * 2, - version=f"v{version:03}", + version=str(version), repointing=repointing, ) @@ -396,7 +439,7 @@ def process_l3e_ul_hf(parent_file_names: list[str], repointing: int, repointing_ instrument="glows", descriptor=f"{cdf_science_file_path.descriptor}-raw", start_time=cdf_science_file_path.start_date, - version=cdf_science_file_path.version, + version=str(Version(None, version.minor)), extension="dat" ).construct_path() @@ -404,7 +447,7 @@ def process_l3e_ul_hf(parent_file_names: list[str], repointing: int, repointing_ return [ul_cdf, new_dat_path] -def process_l3e_hi(parent_file_names: list[str], repointing: int, repointing_start: datetime, epoch_delta: timedelta, elongation: int, version: int, glows_flags: int) -> list[Path]: +def process_l3e_hi(parent_file_names: list[str], repointing: int, repointing_start: datetime, epoch_delta: timedelta, elongation: int, version: Version, glows_flags: int) -> list[Path]: repointing_midpoint = repointing_start + epoch_delta l3e_hi_args = determine_call_args_for_l3e_executable(repointing_start, repointing_midpoint, elongation) call_args = l3e_hi_args.to_argument_list() @@ -414,7 +457,7 @@ def process_l3e_hi(parent_file_names: list[str], repointing: int, repointing_sta run(["./survProbHi"] + call_args) input_metadata = InputMetadata(instrument='glows', descriptor=f'survival-probability-hi-{180-elongation}', - version=f'v{version:03}', start_date=repointing_start, end_date=repointing_start + epoch_delta * 2, + version=str(version), start_date=repointing_start, end_date=repointing_start + epoch_delta * 2, repointing=repointing, data_level='l3e') output_path = Path(f'probSur.Imap.Hi_{call_args[0]}_{call_args[1][:8]}_{call_args[-1][:5]}.dat') @@ -435,7 +478,7 @@ def process_l3e_hi(parent_file_names: list[str], repointing: int, repointing_sta instrument="glows", descriptor=f"{cdf_science_file_path.descriptor}-raw", start_time=cdf_science_file_path.start_date, - version=cdf_science_file_path.version, + version=str(Version(None, version.minor)), extension="dat" ).construct_path() @@ -443,49 +486,6 @@ def process_l3e_hi(parent_file_names: list[str], repointing: int, repointing_sta return [hi_cdf, new_dat_path] -def process_l3e(initializer_data: GlowsL3EInitializerOutput): - products_list = [] - - lo_pivot_angles = get_lo_pivot_angles(initializer_data.repointings.repointing_numbers) - for repointing in initializer_data.repointings.repointing_numbers: - with SwallowExceptionAndLog(f"Exception encountered when processing L3e for repointing {repointing}"): - start_repointing, end_repointing = get_pointing_date_range(repointing) - epoch_delta: timedelta = (end_repointing - start_repointing) / 2 - glows_flags = compute_glows_flags_for_window(initializer_data.l3d_cdf_path, start_repointing, end_repointing) - - with SwallowExceptionAndLog(f"Exception encountered when processing L3e lo for repointing {repointing}"): - lo_parent_file_names = initializer_data.dependencies.get_lo_parents() - pivot_info = lo_pivot_angles[repointing] - if pivot_info.parent_filename is not None: - lo_parent_file_names = lo_parent_file_names + [pivot_info.parent_filename] - lo_version = initializer_data.repointings.lo_repointings[repointing] - products_list.extend(process_l3e_lo(lo_parent_file_names, repointing, start_repointing, epoch_delta, pivot_info.pivot_angle, lo_version, glows_flags)) - - with SwallowExceptionAndLog(f"Exception encountered when processing L3e hi-90 for repointing {repointing}"): - hi_parent_file_names = initializer_data.dependencies.get_hi_parents() - hi_90_version = initializer_data.repointings.hi_90_repointings[repointing] - products_list.extend(process_l3e_hi(hi_parent_file_names, repointing, start_repointing, epoch_delta, 90, hi_90_version, glows_flags)) - - with SwallowExceptionAndLog(f"Exception encountered when processing L3e hi-45 for repointing {repointing}"): - hi_parent_file_names = initializer_data.dependencies.get_hi_parents() - hi_45_version = initializer_data.repointings.hi_45_repointings[repointing] - products_list.extend(process_l3e_hi(hi_parent_file_names, repointing, start_repointing, epoch_delta, 135, hi_45_version, glows_flags)) - - ul_parent_file_names = initializer_data.dependencies.get_ul_parents() - ul_sf_version = initializer_data.repointings.ultra_sf_repointings[repointing] - ul_hf_version = initializer_data.repointings.ultra_hf_repointings[repointing] - - with SwallowExceptionAndLog( - f"Exception encountered when processing L3e ultra SF for repointing {repointing}"): - products_list.extend( - process_l3e_ul_sf(ul_parent_file_names, repointing, start_repointing, epoch_delta, ul_sf_version, glows_flags)) - - with SwallowExceptionAndLog( - f"Exception encountered when processing L3e ultra HF for repointing {repointing}"): - products_list.extend( - process_l3e_ul_hf(ul_parent_file_names, repointing, start_repointing, epoch_delta, ul_hf_version, glows_flags)) - - return products_list class SwallowExceptionAndLog: def __init__(self, message: str): diff --git a/imap_l3_processing/glows/l3d/glows_l3d_initializer.py b/imap_l3_processing/glows/l3d/glows_l3d_initializer.py index f5dbf7990..0388d5550 100644 --- a/imap_l3_processing/glows/l3d/glows_l3d_initializer.py +++ b/imap_l3_processing/glows/l3d/glows_l3d_initializer.py @@ -4,6 +4,7 @@ import imap_data_access from imap_data_access import ProcessingInputCollection, ScienceInput, AncillaryInput, ScienceFilePath +from imap_data_access.file_validation import Version from imap_l3_processing.glows.descriptors import PLASMA_SPEED_DESCRIPTOR, PROTON_DENSITY_DESCRIPTOR, \ UV_ANISOTROPY_DESCRIPTOR, PHOTOION_DESCRIPTOR, LYA_DESCRIPTOR, ELECTRON_DENSITY_DESCRIPTOR, \ @@ -19,8 +20,8 @@ class GlowsL3DInitializer: @staticmethod - def should_process_l3d(external_deps: ExternalDependencies, l3bs: list[str], l3cs: list[str]) -> Optional[ - tuple[int, GlowsL3DDependencies, Optional[Path]]]: + def should_process_l3d(external_deps: ExternalDependencies, l3bs: list[str], l3cs: list[str], major_version: int|None) -> Optional[ + tuple[Version, GlowsL3DDependencies, Optional[str]]]: if len(l3bs) == 0 and len(l3cs) == 0: logger.info("Found no L3b and L3c files!") return None @@ -106,12 +107,15 @@ def should_process_l3d(external_deps: ExternalDependencies, l3bs: list[str], l3c old_l3d = Path(most_recent_l3d["file_path"]).name logger.info(f"Old L3d parents: {l3d_parents}, new L3d deps: {updated_input_files}") - if updated_input_files.issubset(l3d_parents): + most_recent_l3d_version = Version.from_version(most_recent_l3d['version']) + same_major_version = most_recent_l3d_version.major == major_version + if same_major_version and updated_input_files.issubset(l3d_parents): return None - version_to_generate = int(most_recent_l3d['version'][1:]) + 1 + minor_version_to_generate = most_recent_l3d_version.minor + 1 else: old_l3d = None - version_to_generate = 1 + minor_version_to_generate = 1 + version_to_generate = Version(major_version, minor_version_to_generate) return (version_to_generate, GlowsL3DDependencies.fetch_dependencies(processing_input_collection, external_deps), diff --git a/imap_l3_processing/glows/l3e/glows_l3e_initializer.py b/imap_l3_processing/glows/l3e/glows_l3e_initializer.py index 1b0a7b7a8..ca22854c6 100644 --- a/imap_l3_processing/glows/l3e/glows_l3e_initializer.py +++ b/imap_l3_processing/glows/l3e/glows_l3e_initializer.py @@ -6,6 +6,7 @@ import imap_data_access from imap_data_access import ProcessingInputCollection, AncillaryInput, ScienceInput, ScienceFilePath, SPICEInput, \ RepointInput +from imap_data_access.file_validation import Version from imap_l3_processing.glows.l3bc.utils import get_pointing_date_range from imap_l3_processing.glows.l3d.models import GlowsL3DProcessorOutput @@ -25,12 +26,19 @@ class GlowsL3EInitializerOutput: class GlowsL3EInitializer: @staticmethod - def get_repointings_to_process(l3d_output: GlowsL3DProcessorOutput, previous_l3d: Optional[str], repointing_file_path: Path) -> Optional[GlowsL3EInitializerOutput]: + def get_repointings_to_process( + l3d_output: GlowsL3DProcessorOutput, + previous_l3d: Optional[str], + repointing_file_path: Path, + ) -> Optional[GlowsL3EInitializerOutput]: latest_l3d_cr = None + major_version_from_output = Version.from_version(ScienceFilePath(l3d_output.l3d_cdf_file_path).version).major if previous_l3d is not None: - latest_l3d_cr = find_first_updated_cr(l3d_output.l3d_cdf_file_path, previous_l3d) - if not latest_l3d_cr: - return None + major_version_from_previous = Version.from_version(ScienceFilePath(previous_l3d).version).major + if major_version_from_output == major_version_from_previous: + latest_l3d_cr = find_first_updated_cr(l3d_output.l3d_cdf_file_path, previous_l3d) + if not latest_l3d_cr: + return None pipeline_settings_l3bcde = get_most_recently_uploaded_ancillary(imap_data_access.query(table='ancillary', instrument='glows', descriptor='pipeline-settings-l3bcde')) energy_grid_lo = get_most_recently_uploaded_ancillary(imap_data_access.query(table='ancillary', instrument='glows', descriptor='energy-grid-lo')) @@ -59,7 +67,7 @@ def get_repointings_to_process(l3d_output: GlowsL3DProcessorOutput, previous_l3d first_cr = max(first_cr, latest_l3d_cr - 1) last_cr = ScienceFilePath(l3d_output.l3d_cdf_file_path).cr - glows_repointings = determine_l3e_files_to_produce(first_cr, last_cr, repointing_file_path) + glows_repointings = determine_l3e_files_to_produce(first_cr, last_cr, repointing_file_path, major_version_from_output) if len(glows_repointings.repointing_numbers) > 0: earliest_repointing_start, _ = get_pointing_date_range(min(glows_repointings.repointing_numbers)) diff --git a/imap_l3_processing/glows/l3e/glows_l3e_utils.py b/imap_l3_processing/glows/l3e/glows_l3e_utils.py index f719205df..3fb701d2e 100644 --- a/imap_l3_processing/glows/l3e/glows_l3e_utils.py +++ b/imap_l3_processing/glows/l3e/glows_l3e_utils.py @@ -7,6 +7,7 @@ import numpy as np import spiceypy from astropy.time import Time +from imap_data_access.file_validation import Version from imap_processing.spice.repoint import set_global_repoint_table_paths, get_repoint_data from spacepy.pycdf import CDF @@ -59,15 +60,15 @@ def _decimal_time(t: datetime) -> str: @dataclass class GlowsL3eRepointings: repointing_numbers: list[int] - hi_90_repointings: dict[int, int] - hi_45_repointings: dict[int, int] - lo_repointings: dict[int, int] - ultra_sf_repointings: dict[int, int] - ultra_hf_repointings: dict[int, int] + hi_90_repointings: dict[int, Version] + hi_45_repointings: dict[int, Version] + lo_repointings: dict[int, Version] + ultra_sf_repointings: dict[int, Version] + ultra_hf_repointings: dict[int, Version] def determine_l3e_files_to_produce(first_cr_processed: int, last_processed_cr: int, - repointing_path: Path) -> GlowsL3eRepointings: + repointing_path: Path, major_version: int|None) -> GlowsL3eRepointings: descriptors = [ GLOWS_L3E_HI_90_DESCRIPTOR, GLOWS_L3E_HI_45_DESCRIPTOR, @@ -98,14 +99,15 @@ def determine_l3e_files_to_produce(first_cr_processed: int, last_processed_cr: i updated_pointings_per_instruments = [] for descriptor in descriptors: l3e_files = imap_data_access.query(instrument='glows', data_level='l3e', version="latest", descriptor=descriptor) - updated_pointing = {int(l3e['repointing']): int(l3e['version'][1:]) for l3e in l3e_files} - + existing_file_versions = {int(l3e['repointing']): Version.from_version(l3e['version']) for l3e in l3e_files} + new_file_versions = {} for pointing_number in pointing_numbers: - if pointing_number in updated_pointing: - updated_pointing[pointing_number] = updated_pointing[pointing_number] + 1 + if pointing_number in existing_file_versions: + previous_version = existing_file_versions[pointing_number] + new_file_versions[pointing_number] = Version(major_version, previous_version.minor + 1) else: - updated_pointing[pointing_number] = 1 - updated_pointings_per_instruments.append(updated_pointing) + new_file_versions[pointing_number] = Version(major_version, 1) + updated_pointings_per_instruments.append(new_file_versions) return GlowsL3eRepointings(pointing_numbers, *updated_pointings_per_instruments) diff --git a/tests/glows/l3d/test_glows_l3d_initializer.py b/tests/glows/l3d/test_glows_l3d_initializer.py index 8c380db1a..c63093f8c 100644 --- a/tests/glows/l3d/test_glows_l3d_initializer.py +++ b/tests/glows/l3d/test_glows_l3d_initializer.py @@ -1,6 +1,8 @@ import unittest from unittest.mock import patch, call, Mock +from imap_data_access.file_validation import Version + from imap_l3_processing.glows.l3d.glows_l3d_initializer import GlowsL3DInitializer from tests.test_helpers import create_mock_query_results @@ -29,14 +31,6 @@ def test_l3d_initializer(self, mock_fetch_l3d_dependencies, mock_download, mock_ external_dependencies = Mock() - mock_query.side_effect = self._create_ancillary_query_results([ - 'imap_glows_l3d_solar-hist_19470301-cr00001_v000.cdf', - 'imap_glows_l3d_solar-hist_19470301-cr00001_v001.cdf', - 'imap_glows_l3d_solar-hist_19470301-cr00001_v002.cdf', - 'imap_glows_l3d_solar-hist_19470301-cr00002_v000.cdf', - 'imap_glows_l3d_solar-hist_19470301-cr00002_v001.cdf' - ]) - mock_read_cdf_parents.return_value = { 'imap_glows_l3b_ion-rate-profile_20200101-cr00001_v000.cdf', 'imap_glows_l3b_ion-rate-profile_20200201-cr00002_v000.cdf', @@ -44,62 +38,133 @@ def test_l3d_initializer(self, mock_fetch_l3d_dependencies, mock_download, mock_ 'imap_glows_l3c_sw-profile_20200101-cr00001_v000.cdf', 'imap_glows_l3c_sw-profile_20200201-cr00002_v000.cdf', 'imap_glows_l3c_sw-profile_20200301-cr00003_v000.cdf', - 'imap_glows_plasma-speed-2026a_19470301_v001.dat', - 'imap_glows_proton-density-2026a_19470301_v000.dat', - 'imap_glows_uv-anisotropy-2026a_19470301_v000.dat', - 'imap_glows_photoion-2026a_19470301_v000.dat', - 'imap_glows_lya-2026a_19470301_v000.dat', - 'imap_glows_electron-density-2026a_19470301_v000.dat', + 'imap_glows_plasma-speed-2026d_19470301_v001.dat', + 'imap_glows_proton-density-2026d_19470301_v000.dat', + 'imap_glows_lya-2026d_19470301_v000.dat', + 'imap_glows_uv-anisotropy-2026d_19470301_v000.dat', + 'imap_glows_photoion-2026d_19470301_v000.dat', + 'imap_glows_electron-density-2026d_19470301_v000.dat' 'lyman-alpha-composite.nc' } mock_read_pipeline_settings.return_value = {"start_cr": 1} - actual_version, actual_l3d_deps, actual_old_l3d = GlowsL3DInitializer.should_process_l3d(external_dependencies, - l3bs, l3cs) - - most_recent_l3d = 'imap_glows_l3d_solar-hist_19470301-cr00002_v001.cdf' - mock_read_cdf_parents.assert_called_once_with(most_recent_l3d) - - mock_query.assert_has_calls([ - call(instrument='glows', data_level="l3d", descriptor="solar-hist"), - call(table='ancillary', instrument='glows', descriptor='plasma-speed-2026d', version='latest'), - call(table='ancillary', instrument='glows', descriptor='proton-density-2026d', version='latest'), - call(table='ancillary', instrument='glows', descriptor='uv-anisotropy-2026d', version='latest'), - call(table='ancillary', instrument='glows', descriptor='photoion-2026d', version='latest'), - call(table='ancillary', instrument='glows', descriptor='lya-2026d', version='latest'), - call(table='ancillary', instrument='glows', descriptor='electron-density-2026d', version='latest'), - call(table='ancillary', instrument='glows', descriptor='pipeline-settings-l3bcde', version='latest'), - ]) - - mock_download.assert_called_once_with("imap_glows_pipeline-settings-l3bcde_19470301_v000.json") - mock_read_pipeline_settings.assert_called_once_with(mock_download.return_value) + test_cases = { + 'matching versions with legacy': (None, None), + 'matching versions with new version format': (1, 1), + 'upgraded major version': (1, 2), + } + for name, (existing_major_version, requested_major_version) in test_cases.items(): + with self.subTest(name=name): + most_recent_l3d = f'imap_glows_l3d_solar-hist_19470301-cr00002_{Version(existing_major_version, 1)}.cdf' + mock_query.side_effect = self._create_ancillary_query_results([ + 'imap_glows_l3d_solar-hist_19470301-cr00001_v000.cdf', + 'imap_glows_l3d_solar-hist_19470301-cr00001_v001.cdf', + 'imap_glows_l3d_solar-hist_19470301-cr00001_v002.cdf', + 'imap_glows_l3d_solar-hist_19470301-cr00002_v000.cdf', + most_recent_l3d, + ]) + mock_read_cdf_parents.reset_mock() + mock_download.reset_mock() + mock_read_pipeline_settings.reset_mock() + mock_fetch_l3d_dependencies.reset_mock() + actual_version, actual_l3d_deps, actual_old_l3d = GlowsL3DInitializer.should_process_l3d(external_dependencies, + l3bs, l3cs, requested_major_version) + + mock_read_cdf_parents.assert_called_once_with(most_recent_l3d) + + mock_query.assert_has_calls([ + call(instrument='glows', data_level="l3d", descriptor="solar-hist"), + call(table='ancillary', instrument='glows', descriptor='plasma-speed-2026d', version='latest'), + call(table='ancillary', instrument='glows', descriptor='proton-density-2026d', version='latest'), + call(table='ancillary', instrument='glows', descriptor='uv-anisotropy-2026d', version='latest'), + call(table='ancillary', instrument='glows', descriptor='photoion-2026d', version='latest'), + call(table='ancillary', instrument='glows', descriptor='lya-2026d', version='latest'), + call(table='ancillary', instrument='glows', descriptor='electron-density-2026d', version='latest'), + call(table='ancillary', instrument='glows', descriptor='pipeline-settings-l3bcde', version='latest'), + ]) + + mock_download.assert_called_once_with("imap_glows_pipeline-settings-l3bcde_19470301_v000.json") + mock_read_pipeline_settings.assert_called_once_with(mock_download.return_value) + + [fetch_dependencies_call] = mock_fetch_l3d_dependencies.call_args_list + + [actual_l3d_inputs, actual_external_deps] = fetch_dependencies_call.args + + pipeline_l3d_input_paths = actual_l3d_inputs.get_file_paths(source="glows") + pipeline_l3d_input_filenames = [p.name for p in pipeline_l3d_input_paths] + self.assertEqual([ + 'imap_glows_l3b_ion-rate-profile_20200101-cr00001_v000.cdf', + 'imap_glows_l3b_ion-rate-profile_20200201-cr00002_v001.cdf', + 'imap_glows_l3b_ion-rate-profile_20200301-cr00003_v000.cdf', + 'imap_glows_l3c_sw-profile_20200101-cr00001_v000.cdf', + 'imap_glows_l3c_sw-profile_20200201-cr00002_v001.cdf', + 'imap_glows_l3c_sw-profile_20200301-cr00003_v000.cdf', + 'imap_glows_plasma-speed-2026d_19470301_v001.dat', + 'imap_glows_proton-density-2026d_19470301_v000.dat', + 'imap_glows_uv-anisotropy-2026d_19470301_v000.dat', + 'imap_glows_photoion-2026d_19470301_v000.dat', + 'imap_glows_lya-2026d_19470301_v000.dat', + 'imap_glows_electron-density-2026d_19470301_v000.dat', + 'imap_glows_pipeline-settings-l3bcde_19470301_v000.json' + ], pipeline_l3d_input_filenames) + + self.assertEqual(external_dependencies, actual_external_deps) + + self.assertEqual(Version(requested_major_version, 2), actual_version) + self.assertEqual(mock_fetch_l3d_dependencies.return_value, actual_l3d_deps) + self.assertEqual(most_recent_l3d, actual_old_l3d) - [fetch_dependencies_call] = mock_fetch_l3d_dependencies.call_args_list - [actual_l3d_inputs, actual_external_deps] = fetch_dependencies_call.args + @patch('imap_l3_processing.glows.l3d.glows_l3d_initializer.read_cdf_parents') + @patch('imap_l3_processing.glows.l3d.glows_l3d_initializer.read_pipeline_settings') + @patch('imap_l3_processing.glows.l3d.glows_l3d_initializer.imap_data_access.download') + @patch('imap_l3_processing.glows.l3d.glows_l3d_initializer.imap_data_access.query') + @patch('imap_l3_processing.glows.l3d.glows_l3d_initializer.GlowsL3DDependencies.fetch_dependencies') + def test_l3d_initializer_should_process_l3d_when_major_version_changes(self, mock_fetch_l3d_dependencies, mock_query, _, mock_read_pipeline_settings, mock_read_cdf_parents): + most_recent_l3d = 'imap_glows_l3d_solar-hist_19470301-cr00002_v001.cdf' + external_dependencies = Mock() + l3bs = [ + 'imap_glows_l3b_ion-rate-profile_20200101-cr00000_v000.cdf', + 'imap_glows_l3b_ion-rate-profile_20200101-cr00001_v000.cdf', + 'imap_glows_l3b_ion-rate-profile_20200201-cr00002_v000.cdf', + 'imap_glows_l3b_ion-rate-profile_20200301-cr00003_v000.cdf', + ] + l3cs = [ + 'imap_glows_l3c_sw-profile_20200101-cr00000_v000.cdf', + 'imap_glows_l3c_sw-profile_20200101-cr00001_v000.cdf', + 'imap_glows_l3c_sw-profile_20200201-cr00002_v000.cdf', + 'imap_glows_l3c_sw-profile_20200301-cr00003_v000.cdf', + ] - pipeline_l3d_input_paths = actual_l3d_inputs.get_file_paths(source="glows") - pipeline_l3d_input_filenames = [p.name for p in pipeline_l3d_input_paths] - self.assertEqual([ + mock_read_cdf_parents.return_value = { 'imap_glows_l3b_ion-rate-profile_20200101-cr00001_v000.cdf', - 'imap_glows_l3b_ion-rate-profile_20200201-cr00002_v001.cdf', + 'imap_glows_l3b_ion-rate-profile_20200201-cr00002_v000.cdf', 'imap_glows_l3b_ion-rate-profile_20200301-cr00003_v000.cdf', 'imap_glows_l3c_sw-profile_20200101-cr00001_v000.cdf', - 'imap_glows_l3c_sw-profile_20200201-cr00002_v001.cdf', + 'imap_glows_l3c_sw-profile_20200201-cr00002_v000.cdf', 'imap_glows_l3c_sw-profile_20200301-cr00003_v000.cdf', + 'lyman-alpha-composite.nc', 'imap_glows_plasma-speed-2026d_19470301_v001.dat', 'imap_glows_proton-density-2026d_19470301_v000.dat', + 'imap_glows_lya-2026d_19470301_v000.dat', 'imap_glows_uv-anisotropy-2026d_19470301_v000.dat', 'imap_glows_photoion-2026d_19470301_v000.dat', - 'imap_glows_lya-2026d_19470301_v000.dat', - 'imap_glows_electron-density-2026d_19470301_v000.dat', - 'imap_glows_pipeline-settings-l3bcde_19470301_v000.json' - ], pipeline_l3d_input_filenames) + 'imap_glows_electron-density-2026d_19470301_v000.dat' + + } + + mock_read_pipeline_settings.return_value = {"start_cr": 1} - self.assertEqual(external_dependencies, actual_external_deps) + mock_query.side_effect = self._create_ancillary_query_results([ + most_recent_l3d + ]) + new_major_version = 1 + actual_version, actual_l3d_deps, actual_old_l3d = GlowsL3DInitializer.should_process_l3d(external_dependencies, + l3bs, l3cs, + new_major_version) - self.assertEqual(2, actual_version) + self.assertEqual(Version(new_major_version, 2), actual_version) self.assertEqual(mock_fetch_l3d_dependencies.return_value, actual_l3d_deps) self.assertEqual(most_recent_l3d, actual_old_l3d) @@ -117,7 +182,7 @@ def test_l3d_initializer_returns_no_old_cdf_if_none_found(self, mock_fetch_l3d_d mock_query.side_effect = self._create_ancillary_query_results([]) - _, __, old_l3d = GlowsL3DInitializer.should_process_l3d(external_deps, l3bs, l3cs) + _, __, old_l3d = GlowsL3DInitializer.should_process_l3d(external_deps, l3bs, l3cs, None) mock_read_cdf_parents.assert_not_called() mock_fetch_l3d_deps.assert_called_once() self.assertIsNone(old_l3d) @@ -157,7 +222,7 @@ def test_l3d_initializer_should_not_process(self, mock_fetch_l3d_deps, mock_quer mock_read_pipeline_settings.return_value = {"start_cr": 1} - actual_l3d_deps = GlowsL3DInitializer.should_process_l3d(external_dependencies, l3bs, l3cs) + actual_l3d_deps = GlowsL3DInitializer.should_process_l3d(external_dependencies, l3bs, l3cs, None) mock_fetch_l3d_deps.assert_not_called() self.assertIsNone(actual_l3d_deps) @@ -168,7 +233,7 @@ def test_l3d_initializer_should_not_process_when_no_l3bs(self, mock_fetch_l3d_de external_dependencies = Mock() - actual_l3d_deps = GlowsL3DInitializer.should_process_l3d(external_dependencies, l3bs, l3cs) + actual_l3d_deps = GlowsL3DInitializer.should_process_l3d(external_dependencies, l3bs, l3cs, None) mock_fetch_l3d_deps.assert_not_called() self.assertIsNone(actual_l3d_deps) @@ -187,7 +252,7 @@ def test_l3d_initializer_does_not_process_when_no_l3bs_after_start_cr(self, mock mock_read_pipeline_settings.return_value = {"start_cr": 1} - actual_l3d_deps = GlowsL3DInitializer.should_process_l3d(external_dependencies, l3bs, l3cs) + actual_l3d_deps = GlowsL3DInitializer.should_process_l3d(external_dependencies, l3bs, l3cs, None) mock_fetch_l3d_deps.assert_not_called() self.assertIsNone(actual_l3d_deps) diff --git a/tests/glows/l3e/test_glows_l3e_initializer.py b/tests/glows/l3e/test_glows_l3e_initializer.py index 38f07c2d5..74f500972 100644 --- a/tests/glows/l3e/test_glows_l3e_initializer.py +++ b/tests/glows/l3e/test_glows_l3e_initializer.py @@ -4,6 +4,7 @@ from unittest.mock import patch, call from imap_data_access import RepointInput +from imap_data_access.file_validation import Version from imap_l3_processing.glows.l3d.models import GlowsL3DProcessorOutput from imap_l3_processing.glows.l3e.glows_l3e_initializer import GlowsL3EInitializer, GlowsL3EInitializerOutput @@ -42,7 +43,7 @@ def test_get_repointings_to_process(self, mock_query, mock_get_most_recently_upl updated_l3d = Path('path/to/imap_glows_l3d_solar-hist_19470303-cr02091_v000.cdf') updated_l3d_text_file_path = Path("imap_glows_e-dens_19470303_20100101_v000.dat") glows_l3d_processor_output = GlowsL3DProcessorOutput(updated_l3d, [updated_l3d_text_file_path], 2091) - previous_l3d = 'previous_l3d' + previous_l3d = 'imap_glows_l3d_solar-hist_19470303-cr02090_v000.cdf' mock_find_first_updated_cr.return_value = 2091 @@ -50,10 +51,10 @@ def test_get_repointings_to_process(self, mock_query, mock_get_most_recently_upl mock_l3e_dependencies.pipeline_settings = {"start_cr": 2089} mock_l3e_dependencies.repointing_file = Path('path/to/repointing_file') - expected_hi_45 = {1234: 1, 2468: 1} - expected_hi_90 = {1234: 2, 2468: 2} - expected_lo = {1234: 3, 2468: 3} - expected_ultra = {1234: 4, 2468: 4} + expected_hi_45 = {1234: Version(None, 1), 2468: Version(None, 1)} + expected_hi_90 = {1234: Version(None, 2), 2468: Version(None, 2)} + expected_lo = {1234: Version(None, 3), 2468: Version(None, 3)} + expected_ultra = {1234: Version(None, 4), 2468: Version(None, 4)} expected_repointings = GlowsL3eRepointings( repointing_numbers=[2468, 1234], @@ -84,7 +85,7 @@ def test_get_repointings_to_process(self, mock_query, mock_get_most_recently_upl mock_find_first_updated_cr.assert_called_once_with(updated_l3d, previous_l3d) - mock_determine_l3e_files_to_produce.assert_called_once_with(2090, 2091, repointing_file_path) + mock_determine_l3e_files_to_produce.assert_called_once_with(2090, 2091, repointing_file_path, None) mock_query.assert_has_calls([ call(table="ancillary", instrument='glows', descriptor='pipeline-settings-l3bcde'), @@ -136,7 +137,7 @@ def test_get_repointings_to_process_with_identical_l3d_files(self, mock_find_fir updated_l3d = Path('path/to/imap_glows_l3d_solar-hist_19470303-cr02091_v000.cdf') updated_l3d_text_file_path = Path("imap_glows_e-dens_19470303_20100101_v000.dat") glows_l3d_processor_output = GlowsL3DProcessorOutput(updated_l3d, [updated_l3d_text_file_path], 2091) - previous_l3d = 'previous_l3d' + previous_l3d = 'imap_glows_l3d_solar-hist_19470303-cr02091_v000.cdf' mock_find_first_updated_cr.return_value = None @@ -147,6 +148,70 @@ def test_get_repointings_to_process_with_identical_l3d_files(self, mock_find_fir mock_find_first_updated_cr.assert_called_once_with(glows_l3d_processor_output.l3d_cdf_file_path, previous_l3d) self.assertIsNone(actual_initializer_output) + @patch('imap_l3_processing.glows.l3e.glows_l3e_initializer.get_pointing_date_range') + @patch('imap_l3_processing.glows.l3e.glows_l3e_initializer.GlowsL3EDependencies.fetch_dependencies') + @patch('imap_l3_processing.glows.l3e.glows_l3e_initializer.determine_l3e_files_to_produce') + @patch('imap_l3_processing.glows.l3e.glows_l3e_initializer.find_first_updated_cr') + @patch('imap_l3_processing.glows.l3e.glows_l3e_initializer.get_most_recently_uploaded_ancillary') + @patch('imap_l3_processing.glows.l3e.glows_l3e_initializer.imap_data_access.query') + def test_get_repointings_to_process_with_identical_l3d_files_but_different_major_version(self, mock_query, mock_get_most_recently_uploaded_ancillary, + mock_find_first_updated_cr, mock_determine_l3e_files_to_produce, + mock_fetch_dependencies, mock_get_pointing_date_range): + + repointing_file_path = Path("imap_2026_105_01.repoint.csv") + mock_l3e_dependencies = mock_fetch_dependencies.return_value + mock_l3e_dependencies.pipeline_settings = {"start_cr": 2089} + updated_l3d = Path('path/to/imap_glows_l3d_solar-hist_19470303-cr02091_v009.0000.cdf') + updated_l3d_text_file_path = Path("imap_glows_e-dens_19470303_20100101_v000.dat") + glows_l3d_processor_output = GlowsL3DProcessorOutput(updated_l3d, [updated_l3d_text_file_path], 2091) + previous_l3d = 'imap_glows_l3d_solar-hist_19470303-cr02091_v000.cdf' + + mock_query.side_effect = create_mock_query_results([ + 'imap_glows_pipeline-settings-l3bcde_20200101_v000.cdf', + 'imap_glows_energy-grid-lo_20200101_v000.cdf', + 'imap_glows_tess-xyz-8_20200101_v000.cdf', + 'imap_glows_energy-grid-hi_20200101_v000.cdf', + 'imap_glows_energy-grid-ultra_20200101_v000.cdf', + 'imap_glows_tess-ang-16_20200101_v000.cdf', + ]) + + mock_get_most_recently_uploaded_ancillary.side_effect = [ + create_mock_query_results(['imap_glows_pipeline-settings-l3bcde_20200101_v000.cdf'])[0], + create_mock_query_results(['imap_glows_energy-grid-lo_20200101_v000.cdf'])[0], + create_mock_query_results(['imap_glows_tess-xyz-8_20200101_v000.cdf'])[0], + create_mock_query_results(['imap_glows_energy-grid-hi_20200101_v000.cdf'])[0], + create_mock_query_results(['imap_glows_energy-grid-ultra_20200101_v000.cdf'])[0], + create_mock_query_results(['imap_glows_tess-ang-16_20200101_v000.cdf'])[0], + ] + + mock_determine_l3e_files_to_produce.return_value = GlowsL3eRepointings( + repointing_numbers=list(range(100, 160)), + ultra_sf_repointings={}, + ultra_hf_repointings={}, + lo_repointings={}, + hi_45_repointings={}, + hi_90_repointings={} + ) + mock_get_pointing_date_range.side_effect = [ + (datetime(2010, 1, 1), datetime(2010, 1, 2)), + (datetime(2010, 4, 1), datetime(2010, 4, 2)), + ] + + actual_initializer_output = GlowsL3EInitializer.get_repointings_to_process(glows_l3d_processor_output, + previous_l3d, + repointing_file_path) + + expected_output = GlowsL3EInitializerOutput( + dependencies=mock_fetch_dependencies.return_value, + repointings=mock_determine_l3e_files_to_produce.return_value, + l3d_cdf_path=updated_l3d, + ) + self.assertIsNotNone(actual_initializer_output) + mock_determine_l3e_files_to_produce.assert_called_once_with(2089, 2091, repointing_file_path, 9) + + mock_find_first_updated_cr.assert_not_called() + self.assertEqual(expected_output, actual_initializer_output) + @patch('imap_l3_processing.glows.l3e.glows_l3e_initializer.imap_data_access.query') @patch('imap_l3_processing.glows.l3e.glows_l3e_initializer.get_most_recently_uploaded_ancillary') @patch('imap_l3_processing.glows.l3e.glows_l3e_initializer.GlowsL3EDependencies.fetch_dependencies') @@ -174,7 +239,7 @@ def test_get_repointings_to_process_handles_no_previous_l3d(self, mock_find_firs hi_90_repointings={} ) - updated_l3d = Path('path/to/imap_glows_l3d_solar-hist_19470303-cr02091_v000.cdf') + updated_l3d = Path('path/to/imap_glows_l3d_solar-hist_19470303-cr02091_v012.0001.cdf') glows_l3d_processor_output = GlowsL3DProcessorOutput(updated_l3d, [], 2091) previous_l3d = None @@ -185,4 +250,4 @@ def test_get_repointings_to_process_handles_no_previous_l3d(self, mock_find_firs repointing_file_path) mock_find_first_updated_cr.assert_not_called() - mock_determine_l3e_files_to_produce.assert_called_once_with(2089, 2091, repointing_file_path) + mock_determine_l3e_files_to_produce.assert_called_once_with(2089, 2091, repointing_file_path, 12) diff --git a/tests/glows/l3e/test_glows_l3e_utils.py b/tests/glows/l3e/test_glows_l3e_utils.py index 6143ade9e..b3f0ab50f 100644 --- a/tests/glows/l3e/test_glows_l3e_utils.py +++ b/tests/glows/l3e/test_glows_l3e_utils.py @@ -5,6 +5,7 @@ from unittest.mock import patch, Mock, call, sentinel import numpy as np +from imap_data_access.file_validation import Version from spacepy.pycdf import CDF, const from imap_l3_processing.glows.l3e.glows_l3e_call_arguments import GlowsL3eCallArguments @@ -64,83 +65,205 @@ def test_determine_call_args_for_l3e_executable(self, mock_pxform: Mock, mock_re @patch("imap_l3_processing.glows.l3e.glows_l3e_utils.imap_data_access.query") def test_determine_l3e_files_to_produce(self, mock_query): - last_processed_cr = 2094 - first_cr_processed = 2093 - repoint_pathing = get_test_data_path("fake_1_day_repointing_file.csv") - - expected_repointings = [i for i in range(3682, 3736)] - - expected_hi_90_repointing_to_version = {i: 1 for i in expected_repointings} - expected_hi_45_repointing_to_version = {i: 1 for i in expected_repointings} - expected_lo_repointing_to_version = {i: 1 for i in expected_repointings} - expected_ultra_repointing_to_version = {i: 1 for i in expected_repointings} - - expected_hi_90_repointing_to_version.update({ - 3682: 2, 3683: 3, 3684: 4, 3685: 5, 3686: 6 - }) - expected_hi_45_repointing_to_version.update({ - 3683: 3, 3684: 4, 3685: 5, 3686: 6, 3687: 7 - }) - expected_lo_repointing_to_version.update({ - 3684: 4, 3685: 5, 3686: 6, 3687: 7, 3688: 8 - }) - expected_ultra_repointing_to_version.update({ - 3685: 5, 3686: 6, 3687: 7, 3688: 8, 3689: 9 - }) - - mock_query.side_effect = [ - create_mock_query_results([ - 'imap_glows_l3e_survival-probability-hi-90_20250101-repoint03682_v001.cdf', - 'imap_glows_l3e_survival-probability-hi-90_20250101-repoint03683_v002.cdf', - 'imap_glows_l3e_survival-probability-hi-90_20250101-repoint03684_v003.cdf', - 'imap_glows_l3e_survival-probability-hi-90_20250101-repoint03685_v004.cdf', - 'imap_glows_l3e_survival-probability-hi-90_20250101-repoint03686_v005.cdf', - ]), - create_mock_query_results([ - 'imap_glows_l3e_survival-probability-hi-45_20250101-repoint03683_v002.cdf', - 'imap_glows_l3e_survival-probability-hi-45_20250101-repoint03684_v003.cdf', - 'imap_glows_l3e_survival-probability-hi-45_20250101-repoint03685_v004.cdf', - 'imap_glows_l3e_survival-probability-hi-45_20250101-repoint03686_v005.cdf', - 'imap_glows_l3e_survival-probability-hi-45_20250101-repoint03687_v006.cdf', - ]), - create_mock_query_results([ - 'imap_glows_l3e_survival-probability-lo_20250101-repoint03684_v003.cdf', - 'imap_glows_l3e_survival-probability-lo_20250101-repoint03685_v004.cdf', - 'imap_glows_l3e_survival-probability-lo_20250101-repoint03686_v005.cdf', - 'imap_glows_l3e_survival-probability-lo_20250101-repoint03687_v006.cdf', - 'imap_glows_l3e_survival-probability-lo_20250101-repoint03688_v007.cdf', - ]), - create_mock_query_results([ - 'imap_glows_l3e_survival-probability-ul-sf_20250101-repoint03685_v004.cdf', - 'imap_glows_l3e_survival-probability-ul-sf_20250101-repoint03686_v005.cdf', - 'imap_glows_l3e_survival-probability-ul-sf_20250101-repoint03687_v006.cdf', - 'imap_glows_l3e_survival-probability-ul-sf_20250101-repoint03688_v007.cdf', - 'imap_glows_l3e_survival-probability-ul-sf_20250101-repoint03689_v008.cdf', - ]), - create_mock_query_results([ - 'imap_glows_l3e_survival-probability-ul-hf_20250101-repoint03685_v004.cdf', - 'imap_glows_l3e_survival-probability-ul-hf_20250101-repoint03686_v005.cdf', - 'imap_glows_l3e_survival-probability-ul-hf_20250101-repoint03687_v006.cdf', - 'imap_glows_l3e_survival-probability-ul-hf_20250101-repoint03688_v007.cdf', - 'imap_glows_l3e_survival-probability-ul-hf_20250101-repoint03689_v008.cdf', - ]), - ] + for major_version in (None, 3): + with self.subTest(major_version=major_version): + last_processed_cr = 2094 + first_cr_processed = 2093 + repoint_pathing = get_test_data_path("fake_1_day_repointing_file.csv") + + expected_repointings = [i for i in range(3682, 3736)] + + expected_hi_90_repointing_to_version = {i: Version(major_version, 1) for i in expected_repointings} + expected_hi_45_repointing_to_version = {i: Version(major_version, 1) for i in expected_repointings} + expected_lo_repointing_to_version = {i: Version(major_version, 1) for i in expected_repointings} + expected_ultra_repointing_to_version = {i: Version(major_version, 1) for i in expected_repointings} + + expected_hi_90_repointing_to_version.update({ + 3682: Version(major_version, 2), + 3683: Version(major_version, 3), + 3684: Version(major_version, 4), + 3685: Version(major_version, 5), + 3686: Version(major_version, 6) + }) + expected_hi_45_repointing_to_version.update({ + 3683: Version(major_version, 3), + 3684: Version(major_version, 4), + 3685: Version(major_version, 5), + 3686: Version(major_version, 6), + 3687: Version(major_version, 7) + }) + expected_lo_repointing_to_version.update({ + 3684: Version(major_version, 4), + 3685: Version(major_version, 5), + 3686: Version(major_version, 6), + 3687: Version(major_version, 7), + 3688: Version(major_version, 8) + }) + expected_ultra_repointing_to_version.update({ + 3685: Version(major_version, 5), + 3686: Version(major_version, 6), + 3687: Version(major_version, 7), + 3688: Version(major_version, 8), + 3689: Version(major_version, 9) + }) + + mock_query.side_effect = [ + create_mock_query_results([ + 'imap_glows_l3e_survival-probability-hi-90_20250101-repoint03682_v001.cdf', + 'imap_glows_l3e_survival-probability-hi-90_20250101-repoint03683_v002.cdf', + 'imap_glows_l3e_survival-probability-hi-90_20250101-repoint03684_v003.cdf', + 'imap_glows_l3e_survival-probability-hi-90_20250101-repoint03685_v004.cdf', + 'imap_glows_l3e_survival-probability-hi-90_20250101-repoint03686_v005.cdf', + ]), + create_mock_query_results([ + 'imap_glows_l3e_survival-probability-hi-45_20250101-repoint03683_v002.cdf', + 'imap_glows_l3e_survival-probability-hi-45_20250101-repoint03684_v003.cdf', + 'imap_glows_l3e_survival-probability-hi-45_20250101-repoint03685_v004.cdf', + 'imap_glows_l3e_survival-probability-hi-45_20250101-repoint03686_v005.cdf', + 'imap_glows_l3e_survival-probability-hi-45_20250101-repoint03687_v006.cdf', + ]), + create_mock_query_results([ + 'imap_glows_l3e_survival-probability-lo_20250101-repoint03684_v003.cdf', + 'imap_glows_l3e_survival-probability-lo_20250101-repoint03685_v004.cdf', + 'imap_glows_l3e_survival-probability-lo_20250101-repoint03686_v005.cdf', + 'imap_glows_l3e_survival-probability-lo_20250101-repoint03687_v006.cdf', + 'imap_glows_l3e_survival-probability-lo_20250101-repoint03688_v007.cdf', + ]), + create_mock_query_results([ + 'imap_glows_l3e_survival-probability-ul-sf_20250101-repoint03685_v004.cdf', + 'imap_glows_l3e_survival-probability-ul-sf_20250101-repoint03686_v005.cdf', + 'imap_glows_l3e_survival-probability-ul-sf_20250101-repoint03687_v006.cdf', + 'imap_glows_l3e_survival-probability-ul-sf_20250101-repoint03688_v007.cdf', + 'imap_glows_l3e_survival-probability-ul-sf_20250101-repoint03689_v008.cdf', + ]), + create_mock_query_results([ + 'imap_glows_l3e_survival-probability-ul-hf_20250101-repoint03685_v004.cdf', + 'imap_glows_l3e_survival-probability-ul-hf_20250101-repoint03686_v005.cdf', + 'imap_glows_l3e_survival-probability-ul-hf_20250101-repoint03687_v006.cdf', + 'imap_glows_l3e_survival-probability-ul-hf_20250101-repoint03688_v007.cdf', + 'imap_glows_l3e_survival-probability-ul-hf_20250101-repoint03689_v008.cdf', + ]), + ] + + repointings = determine_l3e_files_to_produce(first_cr_processed, last_processed_cr, repoint_pathing, major_version) + mock_query.assert_has_calls([ + call(instrument="glows", data_level="l3e", version='latest', descriptor='survival-probability-hi-90'), + call(instrument="glows", data_level="l3e", version='latest', descriptor='survival-probability-hi-45'), + call(instrument="glows", data_level="l3e", version='latest', descriptor='survival-probability-lo'), + call(instrument="glows", data_level="l3e", version='latest', descriptor='survival-probability-ul-sf'), + call(instrument="glows", data_level="l3e", version='latest', descriptor='survival-probability-ul-hf'), + ]) + + self.assertEqual(expected_hi_90_repointing_to_version, repointings.hi_90_repointings) + self.assertEqual(expected_hi_45_repointing_to_version, repointings.hi_45_repointings) + self.assertEqual(expected_lo_repointing_to_version, repointings.lo_repointings) + self.assertEqual(expected_ultra_repointing_to_version, repointings.ultra_sf_repointings) + self.assertEqual(expected_ultra_repointing_to_version, repointings.ultra_hf_repointings) + self.assertEqual(expected_repointings, repointings.repointing_numbers) - repointings = determine_l3e_files_to_produce(first_cr_processed, last_processed_cr, repoint_pathing) - mock_query.assert_has_calls([ - call(instrument="glows", data_level="l3e", version='latest', descriptor='survival-probability-hi-90'), - call(instrument="glows", data_level="l3e", version='latest', descriptor='survival-probability-hi-45'), - call(instrument="glows", data_level="l3e", version='latest', descriptor='survival-probability-lo'), - call(instrument="glows", data_level="l3e", version='latest', descriptor='survival-probability-ul-sf'), - call(instrument="glows", data_level="l3e", version='latest', descriptor='survival-probability-ul-hf'), - ]) + @patch("imap_l3_processing.glows.l3e.glows_l3e_utils.imap_data_access.query") + def test_determine_l3e_files_to_produce_reprocesses_when_major_version_updated(self, mock_query): + old_new_major_versions = ( + (None, 1), + (1, 2) + ) - self.assertEqual(expected_hi_90_repointing_to_version, repointings.hi_90_repointings) - self.assertEqual(expected_hi_45_repointing_to_version, repointings.hi_45_repointings) - self.assertEqual(expected_lo_repointing_to_version, repointings.lo_repointings) - self.assertEqual(expected_ultra_repointing_to_version, repointings.ultra_sf_repointings) - self.assertEqual(expected_ultra_repointing_to_version, repointings.ultra_hf_repointings) - self.assertEqual(expected_repointings, repointings.repointing_numbers) + for old_major_version, new_major_version in old_new_major_versions: + with self.subTest(old_major_version=old_major_version, new_major_version=new_major_version): + last_processed_cr = 2094 + first_cr_processed = 2093 + repoint_pathing = get_test_data_path("fake_1_day_repointing_file.csv") + + expected_repointings = [i for i in range(3682, 3736)] + + expected_hi_90_repointing_to_version = {i: Version(new_major_version, 1) for i in expected_repointings} + expected_hi_45_repointing_to_version = {i: Version(new_major_version, 1) for i in expected_repointings} + expected_lo_repointing_to_version = {i: Version(new_major_version, 1) for i in expected_repointings} + expected_ultra_repointing_to_version = {i: Version(new_major_version, 1) for i in expected_repointings} + + expected_hi_90_repointing_to_version.update({ + 3682: Version(new_major_version, 2), + 3683: Version(new_major_version, 3), + 3684: Version(new_major_version, 4), + 3685: Version(new_major_version, 5), + 3686: Version(new_major_version, 6) + }) + expected_hi_45_repointing_to_version.update({ + 3683: Version(new_major_version, 3), + 3684: Version(new_major_version, 4), + 3685: Version(new_major_version, 5), + 3686: Version(new_major_version, 6), + 3687: Version(new_major_version, 7) + }) + expected_lo_repointing_to_version.update({ + 3684: Version(new_major_version, 4), + 3685: Version(new_major_version, 5), + 3686: Version(new_major_version, 6), + 3687: Version(new_major_version, 7), + 3688: Version(new_major_version, 8) + }) + expected_ultra_repointing_to_version.update({ + 3685: Version(new_major_version, 5), + 3686: Version(new_major_version, 6), + 3687: Version(new_major_version, 7), + 3688: Version(new_major_version, 8), + 3689: Version(new_major_version, 9) + }) + + + mock_query.side_effect = [ + create_mock_query_results([ + f'imap_glows_l3e_survival-probability-hi-90_20250101-repoint03682_{Version(old_major_version,1)}.cdf', + f'imap_glows_l3e_survival-probability-hi-90_20250101-repoint03683_{Version(old_major_version,2)}.cdf', + f'imap_glows_l3e_survival-probability-hi-90_20250101-repoint03684_{Version(old_major_version,3)}.cdf', + f'imap_glows_l3e_survival-probability-hi-90_20250101-repoint03685_{Version(old_major_version,4)}.cdf', + f'imap_glows_l3e_survival-probability-hi-90_20250101-repoint03686_{Version(old_major_version,5)}.cdf', + ]), + create_mock_query_results([ + f'imap_glows_l3e_survival-probability-hi-45_20250101-repoint03683_{Version(old_major_version,2)}.cdf', + f'imap_glows_l3e_survival-probability-hi-45_20250101-repoint03684_{Version(old_major_version,3)}.cdf', + f'imap_glows_l3e_survival-probability-hi-45_20250101-repoint03685_{Version(old_major_version,4)}.cdf', + f'imap_glows_l3e_survival-probability-hi-45_20250101-repoint03686_{Version(old_major_version,5)}.cdf', + f'imap_glows_l3e_survival-probability-hi-45_20250101-repoint03687_{Version(old_major_version,6)}.cdf', + ]), + create_mock_query_results([ + f'imap_glows_l3e_survival-probability-lo_20250101-repoint03684_{Version(old_major_version,3)}.cdf', + f'imap_glows_l3e_survival-probability-lo_20250101-repoint03685_{Version(old_major_version,4)}.cdf', + f'imap_glows_l3e_survival-probability-lo_20250101-repoint03686_{Version(old_major_version,5)}.cdf', + f'imap_glows_l3e_survival-probability-lo_20250101-repoint03687_{Version(old_major_version,6)}.cdf', + f'imap_glows_l3e_survival-probability-lo_20250101-repoint03688_{Version(old_major_version,7)}.cdf', + ]), + create_mock_query_results([ + f'imap_glows_l3e_survival-probability-ul-sf_20250101-repoint03685_{Version(old_major_version,4)}.cdf', + f'imap_glows_l3e_survival-probability-ul-sf_20250101-repoint03686_{Version(old_major_version,5)}.cdf', + f'imap_glows_l3e_survival-probability-ul-sf_20250101-repoint03687_{Version(old_major_version,6)}.cdf', + f'imap_glows_l3e_survival-probability-ul-sf_20250101-repoint03688_{Version(old_major_version,7)}.cdf', + f'imap_glows_l3e_survival-probability-ul-sf_20250101-repoint03689_{Version(old_major_version,8)}.cdf', + ]), + create_mock_query_results([ + f'imap_glows_l3e_survival-probability-ul-hf_20250101-repoint03685_{Version(old_major_version,4)}.cdf', + f'imap_glows_l3e_survival-probability-ul-hf_20250101-repoint03686_{Version(old_major_version,5)}.cdf', + f'imap_glows_l3e_survival-probability-ul-hf_20250101-repoint03687_{Version(old_major_version,6)}.cdf', + f'imap_glows_l3e_survival-probability-ul-hf_20250101-repoint03688_{Version(old_major_version,7)}.cdf', + f'imap_glows_l3e_survival-probability-ul-hf_20250101-repoint03689_{Version(old_major_version,8)}.cdf', + ]), + ] + + repointings = determine_l3e_files_to_produce(first_cr_processed, last_processed_cr, repoint_pathing, new_major_version) + mock_query.assert_has_calls([ + call(instrument="glows", data_level="l3e", version='latest', descriptor='survival-probability-hi-90'), + call(instrument="glows", data_level="l3e", version='latest', descriptor='survival-probability-hi-45'), + call(instrument="glows", data_level="l3e", version='latest', descriptor='survival-probability-lo'), + call(instrument="glows", data_level="l3e", version='latest', descriptor='survival-probability-ul-sf'), + call(instrument="glows", data_level="l3e", version='latest', descriptor='survival-probability-ul-hf'), + ]) + + self.assertEqual(expected_hi_90_repointing_to_version, repointings.hi_90_repointings) + self.assertEqual(expected_hi_45_repointing_to_version, repointings.hi_45_repointings) + self.assertEqual(expected_lo_repointing_to_version, repointings.lo_repointings) + self.assertEqual(expected_ultra_repointing_to_version, repointings.ultra_sf_repointings) + self.assertEqual(expected_ultra_repointing_to_version, repointings.ultra_hf_repointings) + self.assertEqual(expected_repointings, repointings.repointing_numbers) @patch('imap_l3_processing.glows.l3e.glows_l3e_utils.imap_data_access.download') @patch('imap_l3_processing.glows.l3e.glows_l3e_utils.CDF') diff --git a/tests/glows/test_glows_processor.py b/tests/glows/test_glows_processor.py index fcefd7890..f19cc553b 100644 --- a/tests/glows/test_glows_processor.py +++ b/tests/glows/test_glows_processor.py @@ -632,7 +632,7 @@ def test_process_l3d(self, mock_read_pipeline_settings, mock_glows_l3d_initializ ) old_l3d = Path('imap_glows_l3d_solar-hist_19470303-cr02090_v001.cdf') - l3d_output_version = 5 + l3d_output_version = Version(12, 5) mock_glows_l3d_initializer.should_process_l3d.return_value = ( l3d_output_version, glows_l3d_dependencies, old_l3d) @@ -659,7 +659,8 @@ def test_process_l3d(self, mock_read_pipeline_settings, mock_glows_l3d_initializ mock_convert_l3b_to_json.assert_has_calls([call(sentinel.l3b_file_1), call(sentinel.l3b_file_2)]) mock_convert_l3c_to_json.assert_has_calls([call(sentinel.l3c_file_1), call(sentinel.l3c_file_2)]) - + mock_glows_l3d_initializer.should_process_l3d.assert_called_with( + self.mock_external_deps, [], [], 1) self.assertEqual([ Path("imap_glows_e-dens_19470303_20100101_v000.dat"), Path("imap_glows_lya_19470303_20100101_v000.dat"), @@ -715,14 +716,14 @@ def test_process_l3d(self, mock_read_pipeline_settings, mock_glows_l3d_initializ Path(PATH_TO_L3D_TOOLKIT / 'data_l3d_txt' / f'{expected_end_cr}_txt_file_1', ), Path(PATH_TO_L3D_TOOLKIT / 'data_l3d_txt' / f'{expected_end_cr}_txt_file_2', ), ] - mock_rename_l3d.assert_has_calls([call(expected_l3d_txt_paths, f'v00{l3d_output_version}')]) + mock_rename_l3d.assert_has_calls([call(expected_l3d_txt_paths, str(Version(None, l3d_output_version.minor)))]) mock_get_parent_file_names_from_l3d_json.assert_called_once_with(expected_working_directory / 'data_l3d') expected_data_product_metadata = InputMetadata(instrument="glows", data_level="l3d", descriptor="solar-hist", start_date=datetime(1947, 3, 3), end_date=datetime(1947, 3, 3), - version=f"v00{l3d_output_version}") + version=f"{l3d_output_version}") mock_convert_json_to_l3d_data_product.assert_called_once_with( expected_working_directory / 'data_l3d' / f'imap_glows_l3d_solar-params-history_19470303-cr0{expected_end_cr}_v00.json', @@ -778,7 +779,7 @@ def test_process_l3d_adds_parent_file_names_to_output(self, mock_spicepy, mock_r l3d_dependencies = GlowsL3DDependencies(l3b_file_paths=l3b_file_paths, l3c_file_paths=l3c_file_paths, ancillary_files=ancillary_inputs, external_files=external_inputs, end_cr=cr_number) - glows_l3d_output = process_l3d(l3d_dependencies, 1) + glows_l3d_output = process_l3d(l3d_dependencies, Version(1,1)) [save_data_call_args] = mock_save_data.call_args_list actual_data_product = save_data_call_args.args[0] @@ -934,8 +935,7 @@ def test_process_glows_l3d_drift(self, mock_save_data, _): end_cr=expected_cr, ) - version = 4 - glows_l3d_output = process_l3d(l3d_dependencies, version) + glows_l3d_output = process_l3d(l3d_dependencies, Version(1, 4)) expected_txt_filenames = ["imap_glows_e-dens_19470303_20100629_v004.dat", "imap_glows_lya_19470303_20100629_v004.dat", @@ -1002,7 +1002,7 @@ def test_process_glows_l3d_drift(self, mock_save_data, _): [save_data_call_args] = mock_save_data.call_args_list l3d_data_product = save_data_call_args.args[0] - expected_cdf_filename = "imap_glows_l3d_solar-hist_19470303-cr02098_v004.cdf" + expected_cdf_filename = "imap_glows_l3d_solar-hist_19470303-cr02098_v001.0004.cdf" with tempfile.TemporaryDirectory() as tmp_dir: tmp_dir = Path(tmp_dir) save_data(l3d_data_product, cr_number=expected_cr, folder_path=tmp_dir) @@ -1063,11 +1063,11 @@ def test_process_l3e(self, mock_process_ultra, mock_process_ultra_hf, mock_proce dependencies=mock_dependencies, repointings=GlowsL3eRepointings( repointing_numbers=[25], - hi_90_repointings={25: 1}, - hi_45_repointings={25: 2}, - lo_repointings={25: 3}, - ultra_sf_repointings={25: 4}, - ultra_hf_repointings={25: 4}, + hi_90_repointings={25: Version(None,1)}, + hi_45_repointings={25: Version(None,2)}, + lo_repointings={25: Version(None,3)}, + ultra_sf_repointings={25: Version(None,4)}, + ultra_hf_repointings={25: Version(None,4)}, ), l3d_cdf_path=l3d_cdf_path, ) @@ -1077,13 +1077,15 @@ def test_process_l3e(self, mock_process_ultra, mock_process_ultra_hf, mock_proce mock_compute_glows_flags_for_window.assert_called_once_with(l3d_cdf_path, start_epoch, end_epoch) mock_process_hi.assert_has_calls([ - call(["hi_ancillary.dat"], 25, start_epoch, epoch_delta, 90, 1, 4), - call(["hi_ancillary.dat"], 25, start_epoch, epoch_delta, 135, 2, 4) + call(["hi_ancillary.dat"], 25, start_epoch, epoch_delta, 90, Version(None,1), 4), + call(["hi_ancillary.dat"], 25, start_epoch, epoch_delta, 135, Version(None,2), 4) ]) mock_process_lo.assert_called_once_with(["lo_ancillary.dat", "l1b_nhk.cdf"], 25, start_epoch, epoch_delta, 75, - 3, 4) - mock_process_ultra.assert_called_once_with(["ul_ancillary.dat"], 25, start_epoch, epoch_delta, 4, 4) - mock_process_ultra_hf.assert_called_once_with(["ul_ancillary.dat"], 25, start_epoch, epoch_delta, 4, 4) + Version(None, 3), 4) + mock_process_ultra.assert_called_once_with(["ul_ancillary.dat"], 25, start_epoch, epoch_delta, + Version(None, 4), 4) + mock_process_ultra_hf.assert_called_once_with(["ul_ancillary.dat"], 25, start_epoch, epoch_delta, + Version(None, 4), 4) self.assertEqual(expected_l3e_products, actual_l3e_products) mock_get_lo_pivot_angles.assert_called_once_with([25]) @@ -1094,7 +1096,7 @@ def test_process_l3e(self, mock_process_ultra, mock_process_ultra_hf, mock_proce @patch("imap_l3_processing.glows.glows_processor.run") @patch("imap_l3_processing.glows.glows_processor.determine_call_args_for_l3e_executable") @patch("imap_l3_processing.glows.glows_processor.shutil") - def test_process_l3e_ultra(self, mock_shutil, mock_determine_call_args, mock_run, + def test_process_l3e_ultra_sf(self, mock_shutil, mock_determine_call_args, mock_run, mock_convert_dat_to_glows_l3e_ul_product, mock_save_data, mock_get_parent_file_names): mock_get_parent_file_names.return_value = ["l3d_file", "ancillary_1", "ancillary_2", "ancillary_3"] @@ -1103,10 +1105,10 @@ def test_process_l3e_ultra(self, mock_shutil, mock_determine_call_args, mock_run epoch_end_date = datetime(year=2024, month=10, day=7, hour=23) epoch_delta = (epoch_end_date - epoch_start_date) / 2 repointing = 20 - version = 12 + version = Version(1,12) expected_input_metadata = InputMetadata('glows', "l3e", start_date=epoch_start_date, end_date=epoch_end_date, - version='v012', descriptor=GLOWS_L3E_ULTRA_SF_DESCRIPTOR, + version='v001.0012', descriptor=GLOWS_L3E_ULTRA_SF_DESCRIPTOR, repointing=repointing) ultra_args = ["20241007_000000", "date.001", "vx", "vy", "vz", "30.000"] @@ -1117,7 +1119,7 @@ def test_process_l3e_ultra(self, mock_shutil, mock_determine_call_args, mock_run mock_convert_dat_to_glows_l3e_ul_product.return_value = sentinel.ultra_data_1 - mock_save_data.return_value = "imap_glows_l3e_survival-probability-ul-sf_20241007-repoint00020_v012.cdf" + mock_save_data.return_value = "imap_glows_l3e_survival-probability-ul-sf_20241007-repoint00020_v001.0012.cdf" parent_file_names = ["l3d_file", "ancillary_1", "ancillary_2", "ancillary_3"] glows_flags = 4 @@ -1144,7 +1146,7 @@ def test_process_l3e_ultra(self, mock_shutil, mock_determine_call_args, mock_run survival_data_product.parent_file_names) np.testing.assert_array_equal(survival_data_product.glows_flags, np.array([glows_flags], dtype=np.uint16)) - self.assertEqual(products, ["imap_glows_l3e_survival-probability-ul-sf_20241007-repoint00020_v012.cdf", + self.assertEqual(products, ["imap_glows_l3e_survival-probability-ul-sf_20241007-repoint00020_v001.0012.cdf", expected_first_data_path]) @patch('imap_l3_processing.glows.glows_processor.Processor.get_parent_file_names') @@ -1162,10 +1164,10 @@ def test_process_l3e_ultra_hf(self, mock_shutil, mock_determine_call_args, mock_ epoch_end_date = datetime(year=2024, month=10, day=7, hour=23) epoch_delta = (epoch_end_date - epoch_start_date) / 2 repointing = 20 - version = 12 + version = Version(1, 12) input_metadata = InputMetadata('glows', "l3e", start_date=epoch_start_date, end_date=epoch_end_date, - version='v012', descriptor=GLOWS_L3E_ULTRA_HF_DESCRIPTOR, repointing=repointing) + version='v001.0012', descriptor=GLOWS_L3E_ULTRA_HF_DESCRIPTOR, repointing=repointing) mock_determine_call_args.return_value = GlowsL3eCallArguments( formatted_date="20241007_000000", @@ -1197,7 +1199,7 @@ def test_process_l3e_ultra_hf(self, mock_shutil, mock_determine_call_args, mock_ mock_convert_dat_to_glows_l3e_ul_product.return_value = sentinel.ultra_data_hf - mock_save_data.return_value = "imap_glows_l3e_survival-probability-ul-hf_20241007-repoint00020_v012.cdf" + mock_save_data.return_value = "imap_glows_l3e_survival-probability-ul-hf_20241007-repoint00020_v001.0012.cdf" parent_file_names = ["l3d_file", "ancillary_1", "ancillary_2", "ancillary_3"] glows_flags = 8 @@ -1224,7 +1226,7 @@ def test_process_l3e_ultra_hf(self, mock_shutil, mock_determine_call_args, mock_ survival_data_product.parent_file_names) np.testing.assert_array_equal(survival_data_product.glows_flags, np.array([glows_flags], dtype=np.uint16)) - self.assertEqual(products, ["imap_glows_l3e_survival-probability-ul-hf_20241007-repoint00020_v012.cdf", + self.assertEqual(products, ["imap_glows_l3e_survival-probability-ul-hf_20241007-repoint00020_v001.0012.cdf", expected_first_data_path]) @patch('imap_l3_processing.glows.glows_processor.save_data') @@ -1252,11 +1254,11 @@ def test_process_l3e_hi(self, mock_shutil, mock_determine_call_args, epoch_end_date = datetime(year=2024, month=10, day=7, hour=23) epoch_delta = (epoch_end_date - epoch_start_date) / 2 repointing = 20 - version = 12 + version = Version(1, 12) expected_input_metadata = InputMetadata( instrument='glows', data_level="l3e", start_date=epoch_start_date, end_date=epoch_end_date, - version='v012', descriptor=f'survival-probability-hi-{descriptor_elongation}', + version=str(version), descriptor=f'survival-probability-hi-{descriptor_elongation}', repointing=repointing) l3e_dependencies = MagicMock(spec=GlowsL3EDependencies) @@ -1271,7 +1273,7 @@ def test_process_l3e_hi(self, mock_shutil, mock_determine_call_args, mock_convert_dat_to_glows_l3e_hi_product.return_value = sentinel.hi_data_1 saved_cdf_path = Path( - f"imap_glows_l3e_survival-probability-hi-{descriptor_elongation}_20241007-repoint00020_v001.cdf") + f"imap_glows_l3e_survival-probability-hi-{descriptor_elongation}_20241007-repoint00020_v001.0012.cdf") mock_save_data.return_value = saved_cdf_path @@ -1303,7 +1305,7 @@ def test_process_l3e_hi(self, mock_shutil, mock_determine_call_args, np.testing.assert_array_equal(survival_data_product.glows_flags, np.array([glows_flags], dtype=np.uint16)) expected_output_data_path = AncillaryFilePath( - f"imap_glows_survival-probability-hi-{descriptor_elongation}-raw_20241007_v001.dat" + f"imap_glows_survival-probability-hi-{descriptor_elongation}-raw_20241007_v012.dat" ).construct_path() mock_shutil.move.assert_called_once_with(first_output_data_path, expected_output_data_path) @@ -1330,11 +1332,11 @@ def test_process_l3e_lo(self, mock_shutil, epoch_end_date = datetime(year=2024, month=10, day=7, hour=23) epoch_delta = (epoch_end_date - epoch_start_date) / 2 repointing = 20 - version = 12 + version = Version(1,12) expected_input_metadata = InputMetadata( instrument='glows', data_level="l3e", start_date=epoch_start_date, end_date=epoch_end_date, - version='v012', descriptor=f'survival-probability-lo', repointing=repointing) + version=str(version), descriptor=f'survival-probability-lo', repointing=repointing) lo_call_args = ["20241007_000000", "date.100", "vx", "vy", "vz", f"{elongation:.3f}"] @@ -1357,7 +1359,7 @@ def test_process_l3e_lo(self, mock_shutil, lo_data_1 = Mock() mock_convert_dat_to_glows_l3e_lo_product.return_value = lo_data_1 - output_cdf_path = Path("imap_glows_l3e_survival-probability-lo_20241007-repoint00020_v012.cdf") + output_cdf_path = Path("imap_glows_l3e_survival-probability-lo_20241007-repoint00020_v001.0012.cdf") mock_save_data.return_value = output_cdf_path parent_file_names = ["l3d_file", "ancillary_1", "ancillary_2", "ancillary_3"] From 486d6db22359d8cf43a8e5dc6a7ae9aeb1e854c4 Mon Sep 17 00:00:00 2001 From: Harrison Date: Thu, 25 Jun 2026 16:11:37 -0400 Subject: [PATCH 04/11] Harrison 3295 #118: PMCL/KJON - Handle new versioning format in L3 maps --- imap_l3_data_processor.py | 4 +- .../hi/hi_combined_initializer.py | 5 +- imap_l3_processing/maps/map_initializer.py | 15 +- imap_l3_processing/maps/sp_map_initializer.py | 5 +- .../ultra/ultra_combined_nsp_initializer.py | 5 +- tests/hi/test_hi_combined_initializer.py | 72 ++++- tests/hi/test_hi_sp_initializer.py | 305 +++++++++++------- .../test_map_processor_integration.py | 15 +- tests/lo/l3/test_lo_sp_initializer.py | 13 +- tests/periodically_run_tests.json | 20 +- tests/test_imap_l3_data_processor.py | 44 ++- .../test_ultra_combined_nsp_initializer.py | 253 +++++++++------ tests/ultra/test_ultra_sp_initializer.py | 13 +- 13 files changed, 492 insertions(+), 277 deletions(-) diff --git a/imap_l3_data_processor.py b/imap_l3_data_processor.py index 7e9d08c11..523bf1a3d 100644 --- a/imap_l3_data_processor.py +++ b/imap_l3_data_processor.py @@ -75,6 +75,7 @@ def imap_l3_processor(): version = args.version processing_input_collection.deserialize(args.dependency) else: + processing_input_collection.deserialize(json.dumps(parsed_dependency["dependency"])) version_numbers = list(parsed_dependency["version"].values()) match version_numbers: case [{"major_version": major, "minor_version": minor}]: @@ -110,8 +111,9 @@ def imap_l3_processor(): initializer = initializer_class() paths = [] maps_to_produce = [] + major_version_from_dependency = Version.from_version(version).major for map_descriptor in descriptors: - maps_to_produce.extend(initializer.get_maps_that_should_be_produced(map_descriptor)) + maps_to_produce.extend(initializer.get_maps_that_should_be_produced(map_descriptor, major_version_from_dependency)) logger.info(f"maps to produce {[m.input_metadata.descriptor for m in maps_to_produce]}") if len(maps_to_produce) == 0: diff --git a/imap_l3_processing/hi/hi_combined_initializer.py b/imap_l3_processing/hi/hi_combined_initializer.py index 59d6f03f9..b360b69f4 100644 --- a/imap_l3_processing/hi/hi_combined_initializer.py +++ b/imap_l3_processing/hi/hi_combined_initializer.py @@ -3,6 +3,7 @@ import imap_data_access from dateutil.relativedelta import relativedelta +from imap_data_access.file_validation import Version from imap_l3_processing.maps.map_descriptors import MapDescriptorParts, parse_map_descriptor, Sensor, \ map_descriptor_parts_to_string, SpinPhase @@ -34,7 +35,7 @@ def __init__(self, canonical_map_start_date: datetime = datetime(2026, 1, 1)): def furnish_spice_dependencies(self, map_to_produce: PossibleMapToProduce): pass - def get_maps_that_can_be_produced(self, input_descriptor: str) -> list[PossibleMapToProduce]: + def get_maps_that_can_be_produced(self, input_descriptor: str, major_version: int|None) -> list[PossibleMapToProduce]: possible_maps_to_produce = [] map_descriptor: MapDescriptorParts = parse_map_descriptor(input_descriptor) @@ -73,7 +74,7 @@ def get_maps_that_can_be_produced(self, input_descriptor: str) -> list[PossibleM data_level='l3', start_date=start_date, end_date=start_date + relativedelta(months=+12), - version='v001', + version=str(Version(major_version, 1)), descriptor=input_descriptor, repointing=None ) diff --git a/imap_l3_processing/maps/map_initializer.py b/imap_l3_processing/maps/map_initializer.py index 68415b5e8..4492e75ff 100644 --- a/imap_l3_processing/maps/map_initializer.py +++ b/imap_l3_processing/maps/map_initializer.py @@ -6,6 +6,7 @@ import imap_data_access from imap_data_access import ScienceFilePath, ImapFilePath, ProcessingInputCollection +from imap_data_access.file_validation import Version from imap_data_access.processing_input import generate_imap_input from imap_l3_processing.models import InputMetadata @@ -35,11 +36,11 @@ def furnish_spice_dependencies(self, map_to_produce: PossibleMapToProduce): raise NotImplementedError() @abc.abstractmethod - def get_maps_that_can_be_produced(self, l3_descriptor: str): + def get_maps_that_can_be_produced(self, l3_descriptor: str, major_version: int|None): raise NotImplementedError() - def get_maps_that_should_be_produced(self, descriptor: str) -> list[PossibleMapToProduce]: - possible_maps = self.get_maps_that_can_be_produced(descriptor) + def get_maps_that_should_be_produced(self, descriptor: str, major_version: int|None) -> list[PossibleMapToProduce]: + possible_maps = self.get_maps_that_can_be_produced(descriptor, major_version) maps_to_make = [] for possible_map in possible_maps: @@ -47,10 +48,12 @@ def get_maps_that_should_be_produced(self, descriptor: str) -> list[PossibleMapT if start_dates_for_l3_descriptor := self.existing_l3_maps.get(descriptor): if l3_result := start_dates_for_l3_descriptor.get(start_time): existing_parents = read_cdf_parents(l3_result) - if possible_map.input_files.issubset(existing_parents): + existing_major_version = Version.from_version(ScienceFilePath(l3_result).version).major + if possible_map.input_files.issubset(existing_parents) and major_version == existing_major_version: continue - new_version = int(ScienceFilePath(l3_result).version[1:]) + 1 - possible_map.input_metadata.version = f'v{new_version:03}' + new_minor_version = Version.from_version(ScienceFilePath(l3_result).version).minor + 1 + new_version = Version(major_version, new_minor_version) + possible_map.input_metadata.version = str(new_version) maps_to_make.append(possible_map) return maps_to_make diff --git a/imap_l3_processing/maps/sp_map_initializer.py b/imap_l3_processing/maps/sp_map_initializer.py index 503c660fa..c35971b1b 100644 --- a/imap_l3_processing/maps/sp_map_initializer.py +++ b/imap_l3_processing/maps/sp_map_initializer.py @@ -3,6 +3,7 @@ from datetime import datetime from imap_data_access import ScienceFilePath +from imap_data_access.file_validation import Version from imap_l3_processing.maps.map_descriptors import MapDescriptorParts, parse_map_descriptor, \ get_duration_from_map_descriptor, map_descriptor_parts_to_string, Sensor @@ -29,7 +30,7 @@ def _get_ancillary_files(self) -> list[str]: def _get_l2_dependencies(self, descriptor: MapDescriptorParts) -> list[MapDescriptorParts]: raise NotImplementedError() - def get_maps_that_can_be_produced(self, l3_descriptor: str) -> list[PossibleMapToProduce]: + def get_maps_that_can_be_produced(self, l3_descriptor: str, major_version: int|None) -> list[PossibleMapToProduce]: l3_descriptor_parts = parse_map_descriptor(l3_descriptor) map_duration = get_duration_from_map_descriptor(l3_descriptor_parts) @@ -80,7 +81,7 @@ def get_maps_that_can_be_produced(self, l3_descriptor: str) -> list[PossibleMapT repoint in glows_file_by_repointing] input_metadata = InputMetadata(instrument=self.instrument, data_level='l3', start_date=start_date, - end_date=start_date + map_duration, version='v001', + end_date=start_date + map_duration, version=str(Version(major_version, 1)), descriptor=l3_descriptor) possible_map_to_produce = PossibleMapToProduce( diff --git a/imap_l3_processing/ultra/ultra_combined_nsp_initializer.py b/imap_l3_processing/ultra/ultra_combined_nsp_initializer.py index 5e9531c7c..626be309f 100644 --- a/imap_l3_processing/ultra/ultra_combined_nsp_initializer.py +++ b/imap_l3_processing/ultra/ultra_combined_nsp_initializer.py @@ -3,6 +3,7 @@ from pathlib import Path import imap_data_access +from imap_data_access.file_validation import Version from imap_l3_processing.maps.map_descriptors import Sensor, map_descriptor_parts_to_string, \ parse_map_descriptor, get_duration_from_map_descriptor @@ -45,7 +46,7 @@ def furnish_spice_dependencies(self, map_to_produce: PossibleMapToProduce): kernel_types=ULTRA_SP_SPICE_KERNELS ) - def get_maps_that_can_be_produced(self, descriptor: str) -> list[PossibleMapToProduce]: + def get_maps_that_can_be_produced(self, descriptor: str, major_version: int|None) -> list[PossibleMapToProduce]: descriptor_parts = parse_map_descriptor(descriptor) if descriptor_parts is None: raise ValueError(f"Invalid map descriptor: {descriptor}") @@ -79,7 +80,7 @@ def get_maps_that_can_be_produced(self, descriptor: str) -> list[PossibleMapToPr data_level='l3', start_date=start_date, end_date=start_date + get_duration_from_map_descriptor(descriptor_parts), - version='v001', + version=str(Version(major_version, 1)), descriptor=descriptor, ) )) diff --git a/tests/hi/test_hi_combined_initializer.py b/tests/hi/test_hi_combined_initializer.py index 341584efc..0a63ef2b1 100644 --- a/tests/hi/test_hi_combined_initializer.py +++ b/tests/hi/test_hi_combined_initializer.py @@ -1,3 +1,4 @@ +import itertools import unittest from datetime import datetime from unittest.mock import patch, Mock, call @@ -32,8 +33,9 @@ def teardown(self): self.hi_combined_initializer_mock_query.stop() def test_get_maps_that_should_be_produced_no_existing_combined_sp(self): - for deg in ["6deg", "4deg"]: - with self.subTest(msg=f"'hic-ena-h-hf-sp-full-hae-{deg}-1yr'"): + + for deg, (major_version, expected_version) in itertools.product(["6deg", "4deg"], [(None, 'v001'), (1, 'v001.0001')]): + with self.subTest(deg=deg,major_version=major_version): self.hi_combined_initializer_mock_query.side_effect = [ create_mock_query_results([ f'imap_hi_l3_h45-ena-h-hf-sp-ram-hae-{deg}-1yr_20250101_v001.cdf', @@ -59,7 +61,7 @@ def test_get_maps_that_should_be_produced_no_existing_combined_sp(self): data_level='l3', start_date=datetime(2025, 1, 1), end_date=datetime(2026, 1, 1), - version='v001', + version=expected_version, descriptor=f'hic-ena-h-hf-sp-full-hae-{deg}-1yr' ) ) @@ -67,7 +69,7 @@ def test_get_maps_that_should_be_produced_no_existing_combined_sp(self): initializer = HiCombinedInitializer() actual_maps_to_produce = initializer.get_maps_that_should_be_produced( - f'hic-ena-h-hf-sp-full-hae-{deg}-1yr') + f'hic-ena-h-hf-sp-full-hae-{deg}-1yr', major_version) self.hi_combined_initializer_mock_query.assert_has_calls([ call(instrument='hi', data_level='l3'), @@ -76,6 +78,56 @@ def test_get_maps_that_should_be_produced_no_existing_combined_sp(self): self.assertEqual(expected_maps_to_produce, actual_maps_to_produce) + @patch('imap_l3_processing.maps.map_initializer.read_cdf_parents') + def test_get_maps_that_should_be_produced_new_major_version(self, mock_read_cdf_parents: Mock): + mock_read_cdf_parents.return_value = { + *l3_sp_input_maps + } + + mock_read_cdf_parents.reset_mock() + self.hi_combined_initializer_mock_query.side_effect = [ + create_mock_query_results([ + f'imap_hi_l3_h45-ena-h-hf-sp-ram-hae-6deg-1yr_20250101_v001.cdf', + f'imap_hi_l3_h45-ena-h-hf-sp-anti-hae-6deg-1yr_20250101_v001.cdf', + f'imap_hi_l3_h90-ena-h-hf-sp-ram-hae-6deg-1yr_20250101_v001.cdf', + f'imap_hi_l3_h90-ena-h-hf-sp-anti-hae-6deg-1yr_20250101_v001.cdf', + f'imap_hi_l3_hic-ena-h-hf-sp-full-hae-6deg-1yr_20250101_v001.cdf', + ]), + create_mock_query_results([]) + ] + + expected_maps_to_produce = [ + PossibleMapToProduce( + input_files={ + f'imap_hi_l3_h45-ena-h-hf-sp-ram-hae-6deg-1yr_20250101_v001.cdf', + f'imap_hi_l3_h45-ena-h-hf-sp-anti-hae-6deg-1yr_20250101_v001.cdf', + + f'imap_hi_l3_h90-ena-h-hf-sp-ram-hae-6deg-1yr_20250101_v001.cdf', + f'imap_hi_l3_h90-ena-h-hf-sp-anti-hae-6deg-1yr_20250101_v001.cdf', + }, + input_metadata=InputMetadata( + instrument='hi', + data_level='l3', + start_date=datetime(2025, 1, 1), + end_date=datetime(2026, 1, 1), + version='v002.0002', + descriptor=f'hic-ena-h-hf-sp-full-hae-6deg-1yr' + ) + ) + ] + + initializer = HiCombinedInitializer() + actual_maps_to_produce = initializer.get_maps_that_should_be_produced( + f'hic-ena-h-hf-sp-full-hae-6deg-1yr', 2) + + mock_read_cdf_parents.assert_called_once_with(f'imap_hi_l3_hic-ena-h-hf-sp-full-hae-6deg-1yr_20250101_v001.cdf') + self.hi_combined_initializer_mock_query.assert_has_calls([ + call(instrument='hi', data_level='l3'), + call(instrument='hi', data_level='l2') + ]) + + self.assertEqual(expected_maps_to_produce, actual_maps_to_produce) + def test_get_maps_that_should_be_produced_no_existing_combined_nsp(self): self.maxDiff = 2000 for deg in ["6deg", "4deg"]: @@ -113,7 +165,7 @@ def test_get_maps_that_should_be_produced_no_existing_combined_nsp(self): initializer = HiCombinedInitializer() actual_maps_to_produce = initializer.get_maps_that_should_be_produced( - f'hic-ena-h-hf-nsp-full-hae-{deg}-1yr') + f'hic-ena-h-hf-nsp-full-hae-{deg}-1yr', None) self.hi_combined_initializer_mock_query.assert_has_calls([ call(instrument='hi', data_level='l3'), @@ -144,7 +196,7 @@ def test_get_maps_that_should_be_produced_waits_for_full_set_of_new_inputs(self, expected_maps_to_produce = [] initializer = HiCombinedInitializer() - actual_maps_to_produce = initializer.get_maps_that_should_be_produced('hic-ena-h-hf-sp-full-hae-6deg-1yr') + actual_maps_to_produce = initializer.get_maps_that_should_be_produced('hic-ena-h-hf-sp-full-hae-6deg-1yr', None) self.hi_combined_initializer_mock_query.assert_has_calls([ call(instrument='hi', data_level='l3'), @@ -166,7 +218,8 @@ def test_get_maps_that_should_be_produced_filters_based_on_input_descriptor(self expected_maps_to_produce = [] initializer = HiCombinedInitializer() - actual_maps_to_produce = initializer.get_maps_that_should_be_produced('hic-ena-h-hf-nsp-full-hae-6deg-1yr') + actual_maps_to_produce = initializer.get_maps_that_should_be_produced('hic-ena-h-hf-nsp-full-hae-6deg-1yr', + None) self.hi_combined_initializer_mock_query.assert_has_calls([ call(instrument='hi', data_level='l3'), @@ -213,7 +266,7 @@ def test_get_maps_that_should_be_produced_increments_versions_correctly(self, mo ] initializer = HiCombinedInitializer() - actual_maps_to_produce = initializer.get_maps_that_should_be_produced('hic-ena-h-hf-sp-full-hae-6deg-1yr') + actual_maps_to_produce = initializer.get_maps_that_should_be_produced('hic-ena-h-hf-sp-full-hae-6deg-1yr', None) self.hi_combined_initializer_mock_query.assert_has_calls([ call(instrument='hi', data_level='l3'), @@ -274,7 +327,8 @@ def test_get_maps_that_should_be_produced_can_produce_multiple_maps(self, mock_r ] initializer = HiCombinedInitializer() - actual_maps_to_produce = initializer.get_maps_that_should_be_produced(f'hic-ena-h-hf-nsp-full-hae-6deg-1yr') + actual_maps_to_produce = initializer.get_maps_that_should_be_produced(f'hic-ena-h-hf-nsp-full-hae-6deg-1yr', + None) mock_read_cdf_parents.assert_not_called() diff --git a/tests/hi/test_hi_sp_initializer.py b/tests/hi/test_hi_sp_initializer.py index b96c16bfa..20b8ef184 100644 --- a/tests/hi/test_hi_sp_initializer.py +++ b/tests/hi/test_hi_sp_initializer.py @@ -21,123 +21,131 @@ def tearDown(self): @patch('imap_l3_processing.maps.map_initializer.read_cdf_parents') def test_get_maps_that_can_be_produced(self, mock_read_cdf_parents): - self.mock_query.side_effect = [ - create_mock_query_results([ - 'imap_glows_l3e_survival-probability-hi-45_20100101-repoint00001_v001.cdf', - 'imap_glows_l3e_survival-probability-hi-45_20100102-repoint00002_v001.cdf', - 'imap_glows_l3e_survival-probability-hi-45_20100103-repoint00003_v001.cdf', - - 'imap_glows_l3e_survival-probability-hi-45_20100401-repoint00101_v001.cdf', - 'imap_glows_l3e_survival-probability-hi-45_20100402-repoint00102_v001.cdf', - 'imap_glows_l3e_survival-probability-hi-45_20100403-repoint00103_v001.cdf', - - 'imap_glows_l3e_survival-probability-hi-45_20100702-repoint00202_v001.cdf', - ]), - create_mock_query_results([]), - create_mock_query_results([ - 'imap_hi_l2_h45-ena-h-sf-nsp-anti-hae-4deg-3mo_20100101_v001.cdf', - 'imap_hi_l2_h45-ena-h-sf-nsp-anti-hae-4deg-3mo_20100401_v001.cdf', - 'imap_hi_l2_h45-ena-h-sf-nsp-anti-hae-4deg-3mo_20100701_v001.cdf', - 'imap_hi_l2_h45-ena-h-sf-nsp-anti-hae-4deg-3mo_20101001_v001.cdf' - ]), - create_mock_query_results([]) + cases = [ + (None, "v001"), + (4, "v004.0001"), ] + for major_version, expected_version in cases: + with self.subTest(major_version=major_version): + + self.mock_query.side_effect = [ + create_mock_query_results([ + 'imap_glows_l3e_survival-probability-hi-45_20100101-repoint00001_v001.cdf', + 'imap_glows_l3e_survival-probability-hi-45_20100102-repoint00002_v001.cdf', + 'imap_glows_l3e_survival-probability-hi-45_20100103-repoint00003_v001.cdf', + + 'imap_glows_l3e_survival-probability-hi-45_20100401-repoint00101_v001.cdf', + 'imap_glows_l3e_survival-probability-hi-45_20100402-repoint00102_v001.cdf', + 'imap_glows_l3e_survival-probability-hi-45_20100403-repoint00103_v001.cdf', + + 'imap_glows_l3e_survival-probability-hi-45_20100702-repoint00202_v001.cdf', + ]), + create_mock_query_results([]), + create_mock_query_results([ + 'imap_hi_l2_h45-ena-h-sf-nsp-anti-hae-4deg-3mo_20100101_v001.cdf', + 'imap_hi_l2_h45-ena-h-sf-nsp-anti-hae-4deg-3mo_20100401_v001.cdf', + 'imap_hi_l2_h45-ena-h-sf-nsp-anti-hae-4deg-3mo_20100701_v001.cdf', + 'imap_hi_l2_h45-ena-h-sf-nsp-anti-hae-4deg-3mo_20101001_v001.cdf' + ]), + create_mock_query_results([]) + ] + + mock_read_cdf_parents.side_effect = self.create_fake_read_cdf_parents("45") + + expected_possible_maps = [ + PossibleMapToProduce( + input_files={ + 'imap_hi_l2_h45-ena-h-sf-nsp-anti-hae-4deg-3mo_20100101_v001.cdf', + 'imap_glows_l3e_survival-probability-hi-45_20100101-repoint00001_v001.cdf', + 'imap_glows_l3e_survival-probability-hi-45_20100102-repoint00002_v001.cdf', + 'imap_glows_l3e_survival-probability-hi-45_20100103-repoint00003_v001.cdf', + 'imap_hi_l1c_45sensor-pset_20100101-repoint00001_v001.cdf', + 'imap_hi_l1c_45sensor-pset_20100102-repoint00002_v001.cdf', + 'imap_hi_l1c_45sensor-pset_20100103-repoint00003_v001.cdf', + }, + input_metadata=InputMetadata( + instrument="hi", + data_level="l3", + start_date=datetime(2010, 1, 1), + end_date=datetime(2010, 4, 2, 7, 30), + version=expected_version, + descriptor='h45-ena-h-sf-sp-anti-hae-4deg-3mo', + ) + ), + PossibleMapToProduce( + input_files={ + 'imap_hi_l2_h45-ena-h-sf-nsp-anti-hae-4deg-3mo_20100401_v001.cdf', + 'imap_glows_l3e_survival-probability-hi-45_20100401-repoint00101_v001.cdf', + 'imap_glows_l3e_survival-probability-hi-45_20100402-repoint00102_v001.cdf', + 'imap_glows_l3e_survival-probability-hi-45_20100403-repoint00103_v001.cdf', + 'imap_hi_l1c_45sensor-pset_20100401-repoint00101_v001.cdf', + 'imap_hi_l1c_45sensor-pset_20100402-repoint00102_v001.cdf', + 'imap_hi_l1c_45sensor-pset_20100403-repoint00103_v001.cdf', + }, + input_metadata=InputMetadata( + instrument="hi", + data_level="l3", + start_date=datetime(2010, 4, 1), + end_date=datetime(2010, 7, 1, 7, 30), + version=expected_version, + descriptor='h45-ena-h-sf-sp-anti-hae-4deg-3mo', + ) + ), + PossibleMapToProduce( + input_files={ + 'imap_hi_l2_h45-ena-h-sf-nsp-anti-hae-4deg-3mo_20100701_v001.cdf', + 'imap_glows_l3e_survival-probability-hi-45_20100702-repoint00202_v001.cdf', + 'imap_hi_l1c_45sensor-pset_20100401-repoint00201_v001.cdf', + 'imap_hi_l1c_45sensor-pset_20100402-repoint00202_v001.cdf', + 'imap_hi_l1c_45sensor-pset_20100403-repoint00203_v001.cdf', + }, + input_metadata=InputMetadata( + instrument="hi", + data_level="l3", + start_date=datetime(2010, 7, 1), + end_date=datetime(2010, 9, 30, 7, 30), + version=expected_version, + descriptor='h45-ena-h-sf-sp-anti-hae-4deg-3mo', + ) + ), + PossibleMapToProduce( + input_files={ + 'imap_hi_l2_h45-ena-h-sf-nsp-anti-hae-4deg-3mo_20101001_v001.cdf', + 'imap_hi_l1c_45sensor-pset_20101001-repoint00301_v001.cdf', + 'imap_hi_l1c_45sensor-pset_20101002-repoint00302_v001.cdf', + 'imap_hi_l1c_45sensor-pset_20101003-repoint00303_v001.cdf', + }, + input_metadata=InputMetadata( + instrument="hi", + data_level="l3", + start_date=datetime(2010, 10, 1), + end_date=datetime(2010, 12, 31, 7, 30), + version=expected_version, + descriptor='h45-ena-h-sf-sp-anti-hae-4deg-3mo', + ) + ), + ] - mock_read_cdf_parents.side_effect = self.create_fake_read_cdf_parents("45") + initializer = HiSPInitializer() - expected_possible_maps = [ - PossibleMapToProduce( - input_files={ - 'imap_hi_l2_h45-ena-h-sf-nsp-anti-hae-4deg-3mo_20100101_v001.cdf', - 'imap_glows_l3e_survival-probability-hi-45_20100101-repoint00001_v001.cdf', - 'imap_glows_l3e_survival-probability-hi-45_20100102-repoint00002_v001.cdf', - 'imap_glows_l3e_survival-probability-hi-45_20100103-repoint00003_v001.cdf', - 'imap_hi_l1c_45sensor-pset_20100101-repoint00001_v001.cdf', - 'imap_hi_l1c_45sensor-pset_20100102-repoint00002_v001.cdf', - 'imap_hi_l1c_45sensor-pset_20100103-repoint00003_v001.cdf', - }, - input_metadata=InputMetadata( - instrument="hi", - data_level="l3", - start_date=datetime(2010, 1, 1), - end_date=datetime(2010, 4, 2, 7, 30), - version="v001", - descriptor='h45-ena-h-sf-sp-anti-hae-4deg-3mo', - ) - ), - PossibleMapToProduce( - input_files={ - 'imap_hi_l2_h45-ena-h-sf-nsp-anti-hae-4deg-3mo_20100401_v001.cdf', - 'imap_glows_l3e_survival-probability-hi-45_20100401-repoint00101_v001.cdf', - 'imap_glows_l3e_survival-probability-hi-45_20100402-repoint00102_v001.cdf', - 'imap_glows_l3e_survival-probability-hi-45_20100403-repoint00103_v001.cdf', - 'imap_hi_l1c_45sensor-pset_20100401-repoint00101_v001.cdf', - 'imap_hi_l1c_45sensor-pset_20100402-repoint00102_v001.cdf', - 'imap_hi_l1c_45sensor-pset_20100403-repoint00103_v001.cdf', - }, - input_metadata=InputMetadata( - instrument="hi", - data_level="l3", - start_date=datetime(2010, 4, 1), - end_date=datetime(2010, 7, 1, 7, 30), - version="v001", - descriptor='h45-ena-h-sf-sp-anti-hae-4deg-3mo', - ) - ), - PossibleMapToProduce( - input_files={ - 'imap_hi_l2_h45-ena-h-sf-nsp-anti-hae-4deg-3mo_20100701_v001.cdf', - 'imap_glows_l3e_survival-probability-hi-45_20100702-repoint00202_v001.cdf', - 'imap_hi_l1c_45sensor-pset_20100401-repoint00201_v001.cdf', - 'imap_hi_l1c_45sensor-pset_20100402-repoint00202_v001.cdf', - 'imap_hi_l1c_45sensor-pset_20100403-repoint00203_v001.cdf', - }, - input_metadata=InputMetadata( - instrument="hi", - data_level="l3", - start_date=datetime(2010, 7, 1), - end_date=datetime(2010, 9, 30, 7, 30), - version="v001", - descriptor='h45-ena-h-sf-sp-anti-hae-4deg-3mo', - ) - ), - PossibleMapToProduce( - input_files={ - 'imap_hi_l2_h45-ena-h-sf-nsp-anti-hae-4deg-3mo_20101001_v001.cdf', - 'imap_hi_l1c_45sensor-pset_20101001-repoint00301_v001.cdf', - 'imap_hi_l1c_45sensor-pset_20101002-repoint00302_v001.cdf', - 'imap_hi_l1c_45sensor-pset_20101003-repoint00303_v001.cdf', - }, - input_metadata=InputMetadata( - instrument="hi", - data_level="l3", - start_date=datetime(2010, 10, 1), - end_date=datetime(2010, 12, 31, 7, 30), - version="v001", - descriptor='h45-ena-h-sf-sp-anti-hae-4deg-3mo', - ) - ), - ] + self.mock_query.assert_has_calls([ + call(instrument='glows', data_level='l3e', descriptor='survival-probability-hi-45', version="latest"), + call(instrument='glows', data_level='l3e', descriptor='survival-probability-hi-90', version="latest"), + call(instrument='hi', data_level='l2'), + call(instrument='hi', data_level='l3'), + ]) - initializer = HiSPInitializer() + actual_possible_maps = initializer.get_maps_that_can_be_produced('h45-ena-h-sf-sp-anti-hae-4deg-3mo', + major_version) - self.mock_query.assert_has_calls([ - call(instrument='glows', data_level='l3e', descriptor='survival-probability-hi-45', version="latest"), - call(instrument='glows', data_level='l3e', descriptor='survival-probability-hi-90', version="latest"), - call(instrument='hi', data_level='l2'), - call(instrument='hi', data_level='l3'), - ]) + mock_read_cdf_parents.assert_has_calls([ + call('imap_hi_l2_h45-ena-h-sf-nsp-anti-hae-4deg-3mo_20100101_v001.cdf'), + call('imap_hi_l2_h45-ena-h-sf-nsp-anti-hae-4deg-3mo_20100401_v001.cdf'), + call('imap_hi_l2_h45-ena-h-sf-nsp-anti-hae-4deg-3mo_20100701_v001.cdf'), + call('imap_hi_l2_h45-ena-h-sf-nsp-anti-hae-4deg-3mo_20101001_v001.cdf'), + ], any_order=True) - actual_possible_maps = initializer.get_maps_that_can_be_produced('h45-ena-h-sf-sp-anti-hae-4deg-3mo') - - mock_read_cdf_parents.assert_has_calls([ - call('imap_hi_l2_h45-ena-h-sf-nsp-anti-hae-4deg-3mo_20100101_v001.cdf'), - call('imap_hi_l2_h45-ena-h-sf-nsp-anti-hae-4deg-3mo_20100401_v001.cdf'), - call('imap_hi_l2_h45-ena-h-sf-nsp-anti-hae-4deg-3mo_20100701_v001.cdf'), - call('imap_hi_l2_h45-ena-h-sf-nsp-anti-hae-4deg-3mo_20101001_v001.cdf'), - ], any_order=True) - - self.assertEqual(expected_possible_maps, actual_possible_maps) + self.assertEqual(expected_possible_maps, actual_possible_maps) @patch('imap_l3_processing.maps.map_initializer.read_cdf_parents') def test_get_maps_that_can_be_produced_with_no_glows_data(self, mock_read_cdf_parents): @@ -155,7 +163,8 @@ def test_get_maps_that_can_be_produced_with_no_glows_data(self, mock_read_cdf_pa initializer = HiSPInitializer() - actual_possible_maps = initializer.get_maps_that_can_be_produced('h45-ena-h-sf-sp-anti-hae-4deg-3mo') + actual_possible_maps = initializer.get_maps_that_can_be_produced('h45-ena-h-sf-sp-anti-hae-4deg-3mo', + None) self.assertEqual(2, len(actual_possible_maps)) for possible_map in actual_possible_maps: @@ -199,7 +208,8 @@ def test_get_maps_that_can_be_produced_full_spin_descriptor(self, mock_read_cdf_ ] initializer = HiSPInitializer() - actual_possible_maps = initializer.get_maps_that_can_be_produced('h90-ena-h-sf-sp-full-hae-4deg-6mo') + actual_possible_maps = initializer.get_maps_that_can_be_produced('h90-ena-h-sf-sp-full-hae-4deg-6mo', + None) self.mock_query.assert_has_calls([ call(instrument='glows', data_level='l3e', descriptor='survival-probability-hi-45', version="latest"), @@ -265,7 +275,8 @@ def test_get_maps_that_can_be_produced_full_spin_with_mismatched_parents(self, m initializer = HiSPInitializer() with self.assertLogs(logger=logger, level='WARNING') as log_context: - actual_possible_maps = initializer.get_maps_that_can_be_produced('h90-ena-h-sf-sp-full-hae-4deg-6mo') + actual_possible_maps = initializer.get_maps_that_can_be_produced('h90-ena-h-sf-sp-full-hae-4deg-6mo', + None) expected_message = ('Expected all input maps to be created from the same pointing sets! l2_file_paths: ' 'imap_hi_l2_h90-ena-h-sf-nsp-anti-hae-4deg-6mo_20100101_v001.cdf, ' 'imap_hi_l2_h90-ena-h-sf-nsp-ram-hae-4deg-6mo_20100101_v001.cdf') @@ -382,7 +393,7 @@ def test_get_maps_that_should_be_produced(self, mock_read_cdf_parents): call(instrument='hi', data_level='l3'), ]) - actual_possible_maps = initializer.get_maps_that_should_be_produced('h90-ena-h-sf-sp-anti-hae-4deg-3mo') + actual_possible_maps = initializer.get_maps_that_should_be_produced('h90-ena-h-sf-sp-anti-hae-4deg-3mo', None) mock_read_cdf_parents.assert_has_calls([ call(f'imap_hi_l2_h90-ena-h-sf-nsp-anti-hae-4deg-3mo_20100101_v001.cdf'), @@ -396,6 +407,70 @@ def test_get_maps_that_should_be_produced(self, mock_read_cdf_parents): self.assertEqual(expected_possible_maps, actual_possible_maps) + + + @patch('imap_l3_processing.maps.map_initializer.read_cdf_parents') + def test_get_maps_that_should_be_produced_remakes_when_major_version_changes(self, mock_read_cdf_parents): + self.mock_query.side_effect = [ + create_mock_query_results([]), + create_mock_query_results([ + 'imap_glows_l3e_survival-probability-hi-90_20100101-repoint00001_v001.cdf', + 'imap_glows_l3e_survival-probability-hi-90_20100102-repoint00002_v001.cdf', + 'imap_glows_l3e_survival-probability-hi-90_20100103-repoint00003_v001.cdf', + ]), + create_mock_query_results([ + 'imap_hi_l2_h90-ena-h-sf-nsp-anti-hae-4deg-3mo_20100101_v001.cdf', + 'imap_hi_l2_h90-ena-h-sf-nsp-anti-hae-4deg-3mo_20100101_v000.cdf', + ]), + create_mock_query_results([ + 'imap_hi_l3_h90-ena-h-sf-sp-anti-hae-4deg-3mo_20100101_v001.cdf', + 'imap_hi_l3_h90-ena-h-sf-sp-anti-hae-4deg-3mo_20100101_v000.cdf', + ]), + ] + + mock_read_cdf_parents.side_effect = self.create_fake_read_cdf_parents("90") + + expected_possible_maps = [ + PossibleMapToProduce( + input_files={ + 'imap_hi_l2_h90-ena-h-sf-nsp-anti-hae-4deg-3mo_20100101_v001.cdf', + 'imap_glows_l3e_survival-probability-hi-90_20100101-repoint00001_v001.cdf', + 'imap_glows_l3e_survival-probability-hi-90_20100102-repoint00002_v001.cdf', + 'imap_glows_l3e_survival-probability-hi-90_20100103-repoint00003_v001.cdf', + 'imap_hi_l1c_90sensor-pset_20100101-repoint00001_v001.cdf', + 'imap_hi_l1c_90sensor-pset_20100102-repoint00002_v001.cdf', + 'imap_hi_l1c_90sensor-pset_20100103-repoint00003_v001.cdf', + }, + input_metadata=InputMetadata( + instrument="hi", + data_level="l3", + start_date=datetime(2010, 1, 1), + end_date=datetime(2010, 4, 2, 7, 30), + version="v001.0002", + descriptor='h90-ena-h-sf-sp-anti-hae-4deg-3mo', + ) + ), + ] + + initializer = HiSPInitializer() + + self.mock_query.assert_has_calls([ + call(instrument='glows', data_level='l3e', descriptor='survival-probability-hi-45', version="latest"), + call(instrument='glows', data_level='l3e', descriptor='survival-probability-hi-90', version="latest"), + call(instrument='hi', data_level='l2'), + call(instrument='hi', data_level='l3'), + ]) + + actual_possible_maps = initializer.get_maps_that_should_be_produced('h90-ena-h-sf-sp-anti-hae-4deg-3mo', 1) + + mock_read_cdf_parents.assert_has_calls([ + call(f'imap_hi_l2_h90-ena-h-sf-nsp-anti-hae-4deg-3mo_20100101_v001.cdf'), + + call(f'imap_hi_l3_h90-ena-h-sf-sp-anti-hae-4deg-3mo_20100101_v001.cdf'), + ]) + + self.assertEqual(expected_possible_maps, actual_possible_maps) + def test_get_dependencies(self): cases = [ ("h90-ena-h-sf-sp-anti-hae-4deg-3mo", ["h90-ena-h-sf-nsp-anti-hae-4deg-3mo"]), diff --git a/tests/integration/test_map_processor_integration.py b/tests/integration/test_map_processor_integration.py index 93a2253b1..0c59209fc 100644 --- a/tests/integration/test_map_processor_integration.py +++ b/tests/integration/test_map_processor_integration.py @@ -219,7 +219,16 @@ def test_hi_combined_sensor(self, mock_parse_cli_arguments): mock_arguments.end_date = None mock_arguments.repointing = None mock_arguments.version = "v001" - mock_arguments.dependency = "[]" + mock_arguments.dependency = """ + { "version": { + "hic-maps": { + "major_version": 4, + "minor_version": 7 + } + }, + "dependency": [] + } + """ mock_arguments.upload_to_sdc = False mock_parse_cli_arguments.return_value = mock_arguments @@ -227,14 +236,14 @@ def test_hi_combined_sensor(self, mock_parse_cli_arguments): imap_l3_data_processor.imap_l3_processor() expected_map_path = ScienceFilePath( - 'imap_hi_l3_hic-ena-h-hf-sp-full-hae-6deg-1yr_20250415_v001.cdf').construct_path() + 'imap_hi_l3_hic-ena-h-hf-sp-full-hae-6deg-1yr_20250415_v004.0001.cdf').construct_path() self.assertTrue(expected_map_path.exists(), f"Expected file {expected_map_path.name} not found") with CDF(str(expected_map_path)) as cdf: self.assertEqual(set(sp_files), set(cdf.attrs["Parents"])) expected_map_path = ScienceFilePath( - 'imap_hi_l3_hic-ena-h-hf-nsp-full-hae-6deg-1yr_20250415_v001.cdf').construct_path() + 'imap_hi_l3_hic-ena-h-hf-nsp-full-hae-6deg-1yr_20250415_v004.0001.cdf').construct_path() self.assertTrue(expected_map_path.exists(), f"Expected file {expected_map_path.name} not found") with CDF(str(expected_map_path)) as cdf: diff --git a/tests/lo/l3/test_lo_sp_initializer.py b/tests/lo/l3/test_lo_sp_initializer.py index 1cfc66cd7..6f6104dd9 100644 --- a/tests/lo/l3/test_lo_sp_initializer.py +++ b/tests/lo/l3/test_lo_sp_initializer.py @@ -40,8 +40,8 @@ def test_get_maps_that_should_be_produced(self, mock_read_cdf_parents): 'imap_lo_l2_l090-ena-h-sf-nsp-ram-hae-6deg-12mo_20100701_v001.cdf', 'imap_lo_l2_l090-ena-h-sf-nsp-ram-hae-6deg-12mo_20101001_v001.cdf', - 'imap_lo_l3_l090-ena-h-sf-sp-ram-hae-6deg-12mo_20100101_v001.cdf', - 'imap_lo_l3_l090-ena-h-sf-sp-ram-hae-6deg-12mo_20100401_v001.cdf' + 'imap_lo_l3_l090-ena-h-sf-sp-ram-hae-6deg-12mo_20100101_v003.0001.cdf', + 'imap_lo_l3_l090-ena-h-sf-sp-ram-hae-6deg-12mo_20100401_v003.0001.cdf' ]) mock_read_cdf_parents.side_effect = [ @@ -86,7 +86,8 @@ def test_get_maps_that_should_be_produced(self, mock_read_cdf_parents): ] initializer = LoSPInitializer() - actual_maps_to_produce = initializer.get_maps_that_should_be_produced("l090-ena-h-sf-sp-ram-hae-6deg-12mo") + actual_maps_to_produce = initializer.get_maps_that_should_be_produced("l090-ena-h-sf-sp-ram-hae-6deg-12mo", + 3) self.mock_query.assert_has_calls([ call(instrument='glows', data_level='l3e', descriptor='survival-probability-lo', version="latest"), @@ -100,8 +101,8 @@ def test_get_maps_that_should_be_produced(self, mock_read_cdf_parents): call('imap_lo_l2_l090-ena-h-sf-nsp-ram-hae-6deg-12mo_20100701_v001.cdf'), call('imap_lo_l2_l090-ena-h-sf-nsp-ram-hae-6deg-12mo_20101001_v001.cdf'), - call('imap_lo_l3_l090-ena-h-sf-sp-ram-hae-6deg-12mo_20100101_v001.cdf'), - call('imap_lo_l3_l090-ena-h-sf-sp-ram-hae-6deg-12mo_20100401_v001.cdf') + call('imap_lo_l3_l090-ena-h-sf-sp-ram-hae-6deg-12mo_20100101_v003.0001.cdf'), + call('imap_lo_l3_l090-ena-h-sf-sp-ram-hae-6deg-12mo_20100401_v003.0001.cdf') ]) expected_possible_map_to_produce = PossibleMapToProduce( @@ -110,7 +111,7 @@ def test_get_maps_that_should_be_produced(self, mock_read_cdf_parents): data_level="l3", start_date=datetime(2010, 4, 1), end_date=datetime(2011, 4, 1, 6, 0), - version="v002", + version="v003.0002", descriptor="l090-ena-h-sf-sp-ram-hae-6deg-12mo" ), input_files={ diff --git a/tests/periodically_run_tests.json b/tests/periodically_run_tests.json index c79344c3b..69b8e3111 100644 --- a/tests/periodically_run_tests.json +++ b/tests/periodically_run_tests.json @@ -1,14 +1,14 @@ { - "test_l3bcde_first_time_processing": "2026-06-03T15:44:15.445151", - "test_spectral_fit_against_validation_data": "2026-06-10T15:45:51.002557", + "test_l3bcde_first_time_processing": "2026-06-23T13:32:02.351597", + "test_spectral_fit_against_validation_data": "2026-06-23T13:30:47.289531", "test_calculate_pickup_ions_with_minimize": "2026-05-20T10:52:46.704900", - "test_l3bcde_reprocessing": "2026-06-10T15:49:11.328315", - "test_glows_l3abcde_from_l2": "2026-06-10T15:47:58.463278", + "test_l3bcde_reprocessing": "2026-06-25T09:12:12.228439", + "test_glows_l3abcde_from_l2": "2026-06-25T09:12:11.727186", "test_codice_lo_3d_distributions": "2026-05-28T09:43:22.990231", - "test_alpha_sigma_matches_empirical_std": "2026-06-10T15:52:57.807349", - "test_proton_sigma_matches_empirical_std": "2026-06-10T15:53:35.510111", - "test_swe_processor_with_local_data": "2026-06-10T15:51:38.513730", - "test_ultra_all_sp_maps": "2026-06-10T15:50:02.825494", - "test_ultra_combined_nsp_maps": "2026-06-10T15:50:23.153090", - "test_ultra_combined_sp_maps": "2026-06-10T15:50:52.562588" + "test_alpha_sigma_matches_empirical_std": null, + "test_proton_sigma_matches_empirical_std": null, + "test_swe_processor_with_local_data": "2026-06-23T13:36:38.979488", + "test_ultra_all_sp_maps": "2026-06-23T13:33:14.635122", + "test_ultra_combined_nsp_maps": "2026-06-23T13:33:55.733624", + "test_ultra_combined_sp_maps": "2026-06-23T13:35:07.874145" } \ No newline at end of file diff --git a/tests/test_imap_l3_data_processor.py b/tests/test_imap_l3_data_processor.py index bbcd5a2e6..0b68d58ae 100644 --- a/tests/test_imap_l3_data_processor.py +++ b/tests/test_imap_l3_data_processor.py @@ -1,3 +1,4 @@ +import itertools import json from datetime import datetime from pathlib import Path @@ -109,7 +110,7 @@ def test_scheduled_map_jobs_invoke_correct_initializer(self, mock_argparse, mock mock_ultra_processor_class, mock_ultra_initializer_class, mock_ultra_combined_nsp_initializer_class, mock_hi_combined_initializer_class): - test_cases = [ + instrument_cases = [ ("hi", "sp-maps", mock_hi_initializer_class, mock_hi_processor_class, HI_SP_MAP_DESCRIPTORS), ("hi", "hic-maps", mock_hi_combined_initializer_class, mock_hi_processor_class, HI_COMBINED_DESCRIPTORS), ("lo", "all-maps", mock_lo_initializer_class, mock_lo_processor_class, LO_SP_MAP_DESCRIPTORS), @@ -122,27 +123,45 @@ def test_scheduled_map_jobs_invoke_correct_initializer(self, mock_argparse, mock ULTRA_COMBINED_NSP_DESCRIPTORS), ] - for instrument, descriptor, mock_initializer_class, mock_processor_class, expected_descriptors in test_cases: + dependency_with_version_info = """ + { + "version": { + "descriptor": { + "major_version": 123, + "minor_version": 0 + } + }, + "dependency": [] + } + """ + dependency_cases = [ + ("old style", "[]", 'v001', None, 'v001'), + ("new style", dependency_with_version_info, None, 123, 'v123.0000'), + ] + + for instrument_case, dependency_case in itertools.product(instrument_cases, dependency_cases): + instrument, descriptor, mock_initializer_class, mock_processor_class, expected_descriptors = instrument_case + dependency_name, dependency_contents, version_argument, expected_major_version, expected_version_str = dependency_case mock_upload.reset_mock() mock_argparse.reset_mock() mock_initializer_class.reset_mock() mock_processor_class.reset_mock() - with self.subTest(instrument=instrument): + with self.subTest(instrument=instrument, descriptor=descriptor, dependency=dependency_name): data_level = "l3" mock_argument_parser = mock_argparse.ArgumentParser.return_value mock_argument_parser.parse_args.return_value.instrument = instrument mock_argument_parser.parse_args.return_value.data_level = data_level - mock_argument_parser.parse_args.return_value.dependency = "[]" + mock_argument_parser.parse_args.return_value.dependency = dependency_contents mock_argument_parser.parse_args.return_value.start_date = "20250101" mock_argument_parser.parse_args.return_value.end_date = None mock_argument_parser.parse_args.return_value.repointing = "repoint00022" - mock_argument_parser.parse_args.return_value.version = sentinel.version + mock_argument_parser.parse_args.return_value.version = version_argument mock_argument_parser.parse_args.return_value.descriptor = descriptor expected_input_metadata = InputMetadata(instrument, data_level, datetime(2025, 1, 1), datetime(2025, 1, 1), - sentinel.version, descriptor=descriptor, + expected_version_str, descriptor=descriptor, ) mock_initializer = mock_initializer_class.return_value @@ -156,7 +175,7 @@ def test_scheduled_map_jobs_invoke_correct_initializer(self, mock_argparse, mock imap_l3_processor() mock_initializer.get_maps_that_should_be_produced.assert_has_calls([ - call(descriptor) for descriptor in expected_descriptors + call(descriptor, expected_major_version) for descriptor in expected_descriptors ]) self.assertEqual(len(expected_descriptors), mock_processor_class.call_count) @@ -189,7 +208,7 @@ def test_failing_to_produce_an_sp_map_continues(self, mock_argparse, mock_hi_pro mock_argument_parser.parse_args.return_value.start_date = "20250101" mock_argument_parser.parse_args.return_value.end_date = None mock_argument_parser.parse_args.return_value.repointing = "repoint00022" - mock_argument_parser.parse_args.return_value.version = sentinel.version + mock_argument_parser.parse_args.return_value.version = "v001" mock_argument_parser.parse_args.return_value.descriptor = descriptor mock_hi_initializer.get_maps_that_should_be_produced.side_effect = [ @@ -540,14 +559,15 @@ def test_version_comes_from_dependency(self, mock_argparse, mock_processor_class version_input = { "sci": {"major_version": 2, "minor_version": 1} } + serialized_dependency_paths = imap_data_access_dependency.serialize() combined_input = { "version": version_input, - "dependency": json.loads(imap_data_access_dependency.serialize()), + "dependency": json.loads(serialized_dependency_paths), } mock_processing_input_collection.return_value = imap_data_access_dependency - mock_processing_input_collection.deserialize = Mock() - mock_processing_input_collection.get_science_inputs = Mock(return_value=[]) + imap_data_access_dependency.deserialize = Mock() + imap_data_access_dependency.get_science_inputs = Mock(return_value=[]) mock_argument_parser = mock_argparse.ArgumentParser.return_value @@ -564,6 +584,7 @@ def test_version_comes_from_dependency(self, mock_argparse, mock_processor_class for input_end_date, expected_end_date in cases: with self.subTest(input_end_date): mock_upload.reset_mock() + imap_data_access_dependency.deserialize.reset_mock() mock_argument_parser.parse_args.return_value.end_date = input_end_date @@ -590,6 +611,7 @@ def test_version_comes_from_dependency(self, mock_argparse, mock_processor_class mock_processor_class.assert_called_with(imap_data_access_dependency, expected_input_metadata) + imap_data_access_dependency.deserialize.assert_called_once_with(serialized_dependency_paths) mock_processor.process.assert_called() mock_upload.assert_called_once_with(mock_processor_class.return_value.process.return_value[0]) diff --git a/tests/ultra/test_ultra_combined_nsp_initializer.py b/tests/ultra/test_ultra_combined_nsp_initializer.py index 8a1b1069e..b77b64699 100644 --- a/tests/ultra/test_ultra_combined_nsp_initializer.py +++ b/tests/ultra/test_ultra_combined_nsp_initializer.py @@ -2,6 +2,8 @@ from datetime import datetime from unittest.mock import patch, call +from imap_data_access.file_validation import Version + from imap_l3_processing.maps.map_initializer import MapInitializer, PossibleMapToProduce from imap_l3_processing.models import InputMetadata from imap_l3_processing.ultra.ultra_combined_nsp_initializer import UltraCombinedNSPInitializer @@ -22,111 +24,156 @@ def test_is_map_initializer(self): @patch('imap_l3_processing.maps.map_initializer.read_cdf_parents') def test_get_maps_that_should_be_produced(self, mock_read_cdf_parents): - self.mock_query.side_effect = ImapQueryPatcher([ - f'imap_ultra_l2_u45-ena-h-sf-nsp-full-hae-4deg-3mo_20100101_v001.cdf', - f'imap_ultra_l2_u90-ena-h-sf-nsp-full-hae-4deg-3mo_20100101_v001.cdf', - - f'imap_ultra_l2_u45-ena-h-sf-nsp-full-hae-4deg-3mo_20100401_v001.cdf', - f'imap_ultra_l2_u90-ena-h-sf-nsp-full-hae-4deg-3mo_20100401_v001.cdf', - f'imap_ultra_l2_u90-ena-h-sf-nsp-full-hae-4deg-3mo_20100401_v002.cdf', - - f'imap_ultra_l2_u45-ena-h-sf-nsp-full-hae-4deg-3mo_20101001_v001.cdf', - - f'imap_ultra_l3_ulc-ena-h-sf-nsp-full-hae-4deg-3mo_20100101_v001.cdf', - f'imap_ultra_l3_ulc-ena-h-sf-nsp-full-hae-4deg-3mo_20100401_v001.cdf', - 'imap_ultra_l2-energy-bin-group-sizes_20250101_v000.csv' - ]) - - mock_read_cdf_parents.side_effect = [ - [ - f"imap_ultra_l1c_45sensor-spacecraftpset_20100101-repoint00001_v001.cdf", - f"imap_ultra_l1c_45sensor-spacecraftpset_20100102-repoint00002_v001.cdf", - f"imap_ultra_l1c_45sensor-spacecraftpset_20100103-repoint00003_v001.cdf", - - ], - [ - f"imap_ultra_l1c_90sensor-spacecraftpset_20100101-repoint00001_v001.cdf", - f"imap_ultra_l1c_90sensor-spacecraftpset_20100102-repoint00002_v001.cdf", - f"imap_ultra_l1c_90sensor-spacecraftpset_20100103-repoint00003_v001.cdf" - ], - [ - f'imap_ultra_l1c_45sensor-spacecraftpset_20100401-repoint00101_v001.cdf', - f'imap_ultra_l1c_45sensor-spacecraftpset_20100402-repoint00102_v001.cdf', - f'imap_ultra_l1c_45sensor-spacecraftpset_20100403-repoint00103_v001.cdf' - ], [ - f'imap_ultra_l1c_90sensor-spacecraftpset_20100401-repoint00101_v002.cdf', - f'imap_ultra_l1c_90sensor-spacecraftpset_20100402-repoint00102_v002.cdf', - f'imap_ultra_l1c_90sensor-spacecraftpset_20100403-repoint00103_v002.cdf' - ], - [ - f"imap_ultra_l1c_45sensor-spacecraftpset_20100101-repoint00001_v001.cdf", - f"imap_ultra_l1c_45sensor-spacecraftpset_20100102-repoint00002_v001.cdf", - f"imap_ultra_l1c_45sensor-spacecraftpset_20100103-repoint00003_v001.cdf", - f"imap_ultra_l1c_90sensor-spacecraftpset_20100101-repoint00001_v001.cdf", - f"imap_ultra_l1c_90sensor-spacecraftpset_20100102-repoint00002_v001.cdf", - f"imap_ultra_l1c_90sensor-spacecraftpset_20100103-repoint00003_v001.cdf", - f'imap_ultra_l2_u45-ena-h-sf-nsp-full-hae-4deg-3mo_20100101_v001.cdf', - f'imap_ultra_l2_u90-ena-h-sf-nsp-full-hae-4deg-3mo_20100101_v001.cdf', - 'imap_ultra_l2-energy-bin-group-sizes_20250101_v000.csv', - ], - [ - f'imap_ultra_l1c_45sensor-spacecraftpset_20100401-repoint00101_v001.cdf', - f'imap_ultra_l1c_45sensor-spacecraftpset_20100402-repoint00102_v001.cdf', - f'imap_ultra_l1c_45sensor-spacecraftpset_20100403-repoint00103_v001.cdf', - f'imap_ultra_l1c_90sensor-spacecraftpset_20100401-repoint00101_v001.cdf', - f'imap_ultra_l1c_90sensor-spacecraftpset_20100402-repoint00102_v001.cdf', - f'imap_ultra_l1c_90sensor-spacecraftpset_20100403-repoint00103_v001.cdf', - f'imap_ultra_l2_u45-ena-h-sf-nsp-full-hae-4deg-3mo_20100401_v001.cdf', - f'imap_ultra_l2_u90-ena-h-sf-nsp-full-hae-4deg-3mo_20100401_v001.cdf', - 'imap_ultra_l2-energy-bin-group-sizes_20250101_v000.csv', - ] + self.maxDiff = None + cases = [ + (None, "v002"), + (4, "v004.0002"), ] - - initializer = UltraCombinedNSPInitializer() - actual_maps_to_produce = initializer.get_maps_that_should_be_produced(f"ulc-ena-h-sf-nsp-full-hae-4deg-3mo") - - self.mock_query.assert_has_calls([ - call(instrument='ultra', data_level='l2'), - call(instrument="ultra", table='ancillary', descriptor='l2-energy-bin-group-sizes', - version='latest'), - call(instrument='ultra', data_level='l3') - ]) - - mock_read_cdf_parents.assert_has_calls([ - call(f'imap_ultra_l2_u45-ena-h-sf-nsp-full-hae-4deg-3mo_20100101_v001.cdf'), - call(f'imap_ultra_l2_u90-ena-h-sf-nsp-full-hae-4deg-3mo_20100101_v001.cdf'), - call(f'imap_ultra_l2_u45-ena-h-sf-nsp-full-hae-4deg-3mo_20100401_v001.cdf'), - call(f'imap_ultra_l2_u90-ena-h-sf-nsp-full-hae-4deg-3mo_20100401_v002.cdf'), - - call(f'imap_ultra_l3_ulc-ena-h-sf-nsp-full-hae-4deg-3mo_20100101_v001.cdf'), - call(f'imap_ultra_l3_ulc-ena-h-sf-nsp-full-hae-4deg-3mo_20100401_v001.cdf'), - ]) - - expected_possible_map_to_produce = PossibleMapToProduce( - input_metadata=InputMetadata( - instrument="ultra", - data_level="l3", - start_date=datetime(2010, 4, 1), - end_date=datetime(2010, 7, 1, 7, 30), - version="v002", - descriptor=f"ulc-ena-h-sf-nsp-full-hae-4deg-3mo" - ), - input_files={ - f'imap_ultra_l1c_45sensor-spacecraftpset_20100401-repoint00101_v001.cdf', - f'imap_ultra_l1c_45sensor-spacecraftpset_20100402-repoint00102_v001.cdf', - f'imap_ultra_l1c_45sensor-spacecraftpset_20100403-repoint00103_v001.cdf', - f'imap_ultra_l1c_90sensor-spacecraftpset_20100401-repoint00101_v002.cdf', - f'imap_ultra_l1c_90sensor-spacecraftpset_20100402-repoint00102_v002.cdf', - f'imap_ultra_l1c_90sensor-spacecraftpset_20100403-repoint00103_v002.cdf', - f'imap_ultra_l2_u45-ena-h-sf-nsp-full-hae-4deg-3mo_20100401_v001.cdf', - f'imap_ultra_l2_u90-ena-h-sf-nsp-full-hae-4deg-3mo_20100401_v002.cdf', - f'imap_ultra_l2-energy-bin-group-sizes_20250101_v000.csv' - } - ) - self.assertEqual([expected_possible_map_to_produce], actual_maps_to_produce) + for major_version, expected_version in cases: + with self.subTest(major_version=major_version): + self.mock_query.side_effect = ImapQueryPatcher([ + f'imap_ultra_l2_u45-ena-h-sf-nsp-full-hae-4deg-3mo_20100101_v001.cdf', + f'imap_ultra_l2_u90-ena-h-sf-nsp-full-hae-4deg-3mo_20100101_v001.cdf', + + f'imap_ultra_l2_u45-ena-h-sf-nsp-full-hae-4deg-3mo_20100401_v001.cdf', + f'imap_ultra_l2_u90-ena-h-sf-nsp-full-hae-4deg-3mo_20100401_v001.cdf', + f'imap_ultra_l2_u90-ena-h-sf-nsp-full-hae-4deg-3mo_20100401_v002.cdf', + + f'imap_ultra_l2_u45-ena-h-sf-nsp-full-hae-4deg-3mo_20100701_v001.cdf', + f'imap_ultra_l2_u90-ena-h-sf-nsp-full-hae-4deg-3mo_20100701_v001.cdf', + + f'imap_ultra_l2_u45-ena-h-sf-nsp-full-hae-4deg-3mo_20101001_v001.cdf', + + f'imap_ultra_l3_ulc-ena-h-sf-nsp-full-hae-4deg-3mo_20100101_{Version(major_version, 1)}.cdf', + f'imap_ultra_l3_ulc-ena-h-sf-nsp-full-hae-4deg-3mo_20100401_{Version(major_version, 1)}.cdf', + 'imap_ultra_l2-energy-bin-group-sizes_20250101_v000.csv' + ]) + + mock_read_cdf_parents.side_effect = [ + [ + f"imap_ultra_l1c_45sensor-spacecraftpset_20100101-repoint00001_v001.cdf", + f"imap_ultra_l1c_45sensor-spacecraftpset_20100102-repoint00002_v001.cdf", + f"imap_ultra_l1c_45sensor-spacecraftpset_20100103-repoint00003_v001.cdf", + + ], + [ + f"imap_ultra_l1c_90sensor-spacecraftpset_20100101-repoint00001_v001.cdf", + f"imap_ultra_l1c_90sensor-spacecraftpset_20100102-repoint00002_v001.cdf", + f"imap_ultra_l1c_90sensor-spacecraftpset_20100103-repoint00003_v001.cdf" + ], + [ + f'imap_ultra_l1c_45sensor-spacecraftpset_20100401-repoint00101_v001.cdf', + f'imap_ultra_l1c_45sensor-spacecraftpset_20100402-repoint00102_v001.cdf', + f'imap_ultra_l1c_45sensor-spacecraftpset_20100403-repoint00103_v001.cdf' + ], [ + f'imap_ultra_l1c_90sensor-spacecraftpset_20100401-repoint00101_v002.cdf', + f'imap_ultra_l1c_90sensor-spacecraftpset_20100402-repoint00102_v002.cdf', + f'imap_ultra_l1c_90sensor-spacecraftpset_20100403-repoint00103_v002.cdf' + ], + [ + f'imap_ultra_l1c_45sensor-spacecraftpset_20100701-repoint00201_v001.cdf', + f'imap_ultra_l1c_45sensor-spacecraftpset_20100702-repoint00202_v001.cdf', + f'imap_ultra_l1c_45sensor-spacecraftpset_20100703-repoint00203_v001.cdf' + ], + [ + f'imap_ultra_l1c_90sensor-spacecraftpset_20100701-repoint00201_v001.cdf', + f'imap_ultra_l1c_90sensor-spacecraftpset_20100702-repoint00202_v001.cdf', + f'imap_ultra_l1c_90sensor-spacecraftpset_20100703-repoint00203_v001.cdf' + ], + [ + f"imap_ultra_l1c_45sensor-spacecraftpset_20100101-repoint00001_v001.cdf", + f"imap_ultra_l1c_45sensor-spacecraftpset_20100102-repoint00002_v001.cdf", + f"imap_ultra_l1c_45sensor-spacecraftpset_20100103-repoint00003_v001.cdf", + f"imap_ultra_l1c_90sensor-spacecraftpset_20100101-repoint00001_v001.cdf", + f"imap_ultra_l1c_90sensor-spacecraftpset_20100102-repoint00002_v001.cdf", + f"imap_ultra_l1c_90sensor-spacecraftpset_20100103-repoint00003_v001.cdf", + f'imap_ultra_l2_u45-ena-h-sf-nsp-full-hae-4deg-3mo_20100101_v001.cdf', + f'imap_ultra_l2_u90-ena-h-sf-nsp-full-hae-4deg-3mo_20100101_v001.cdf', + 'imap_ultra_l2-energy-bin-group-sizes_20250101_v000.csv', + ], + [ + f'imap_ultra_l1c_45sensor-spacecraftpset_20100401-repoint00101_v001.cdf', + f'imap_ultra_l1c_45sensor-spacecraftpset_20100402-repoint00102_v001.cdf', + f'imap_ultra_l1c_45sensor-spacecraftpset_20100403-repoint00103_v001.cdf', + f'imap_ultra_l1c_90sensor-spacecraftpset_20100401-repoint00101_v001.cdf', + f'imap_ultra_l1c_90sensor-spacecraftpset_20100402-repoint00102_v001.cdf', + f'imap_ultra_l1c_90sensor-spacecraftpset_20100403-repoint00103_v001.cdf', + f'imap_ultra_l2_u45-ena-h-sf-nsp-full-hae-4deg-3mo_20100401_v001.cdf', + f'imap_ultra_l2_u90-ena-h-sf-nsp-full-hae-4deg-3mo_20100401_v001.cdf', + 'imap_ultra_l2-energy-bin-group-sizes_20250101_v000.csv', + ] + ] + initializer = UltraCombinedNSPInitializer() + actual_maps_to_produce = initializer.get_maps_that_should_be_produced(f"ulc-ena-h-sf-nsp-full-hae-4deg-3mo", + major_version) + + self.mock_query.assert_has_calls([ + call(instrument='ultra', data_level='l2'), + call(instrument="ultra", table='ancillary', descriptor='l2-energy-bin-group-sizes', + version='latest'), + call(instrument='ultra', data_level='l3') + ]) + + mock_read_cdf_parents.assert_has_calls([ + call(f'imap_ultra_l2_u45-ena-h-sf-nsp-full-hae-4deg-3mo_20100101_v001.cdf'), + call(f'imap_ultra_l2_u90-ena-h-sf-nsp-full-hae-4deg-3mo_20100101_v001.cdf'), + call(f'imap_ultra_l2_u45-ena-h-sf-nsp-full-hae-4deg-3mo_20100401_v001.cdf'), + call(f'imap_ultra_l2_u90-ena-h-sf-nsp-full-hae-4deg-3mo_20100401_v002.cdf'), + call(f'imap_ultra_l2_u45-ena-h-sf-nsp-full-hae-4deg-3mo_20100701_v001.cdf'), + call(f'imap_ultra_l2_u90-ena-h-sf-nsp-full-hae-4deg-3mo_20100701_v001.cdf'), + + call(f'imap_ultra_l3_ulc-ena-h-sf-nsp-full-hae-4deg-3mo_20100101_{Version(major_version, 1)}.cdf'), + call(f'imap_ultra_l3_ulc-ena-h-sf-nsp-full-hae-4deg-3mo_20100401_{Version(major_version, 1)}.cdf'), + ]) + + expected_possible_map_to_produce_with_previous_version = PossibleMapToProduce( + input_metadata=InputMetadata( + instrument="ultra", + data_level="l3", + start_date=datetime(2010, 4, 1), + end_date=datetime(2010, 7, 1, 7, 30), + version=expected_version, + descriptor=f"ulc-ena-h-sf-nsp-full-hae-4deg-3mo" + ), + input_files={ + f'imap_ultra_l1c_45sensor-spacecraftpset_20100401-repoint00101_v001.cdf', + f'imap_ultra_l1c_45sensor-spacecraftpset_20100402-repoint00102_v001.cdf', + f'imap_ultra_l1c_45sensor-spacecraftpset_20100403-repoint00103_v001.cdf', + f'imap_ultra_l1c_90sensor-spacecraftpset_20100401-repoint00101_v002.cdf', + f'imap_ultra_l1c_90sensor-spacecraftpset_20100402-repoint00102_v002.cdf', + f'imap_ultra_l1c_90sensor-spacecraftpset_20100403-repoint00103_v002.cdf', + f'imap_ultra_l2_u45-ena-h-sf-nsp-full-hae-4deg-3mo_20100401_v001.cdf', + f'imap_ultra_l2_u90-ena-h-sf-nsp-full-hae-4deg-3mo_20100401_v002.cdf', + f'imap_ultra_l2-energy-bin-group-sizes_20250101_v000.csv' + } + ) + expected_possible_map_to_produce_with_no_previous_version = PossibleMapToProduce( + input_metadata=InputMetadata( + instrument="ultra", + data_level="l3", + start_date=datetime(2010, 7, 1), + end_date=datetime(2010, 9, 30, 7, 30), + version=str(Version(major_version, 1)), + descriptor=f"ulc-ena-h-sf-nsp-full-hae-4deg-3mo" + ), + input_files={ + f'imap_ultra_l1c_45sensor-spacecraftpset_20100701-repoint00201_v001.cdf', + f'imap_ultra_l1c_45sensor-spacecraftpset_20100702-repoint00202_v001.cdf', + f'imap_ultra_l1c_45sensor-spacecraftpset_20100703-repoint00203_v001.cdf', + f'imap_ultra_l1c_90sensor-spacecraftpset_20100701-repoint00201_v001.cdf', + f'imap_ultra_l1c_90sensor-spacecraftpset_20100702-repoint00202_v001.cdf', + f'imap_ultra_l1c_90sensor-spacecraftpset_20100703-repoint00203_v001.cdf', + f'imap_ultra_l2_u45-ena-h-sf-nsp-full-hae-4deg-3mo_20100701_v001.cdf', + f'imap_ultra_l2_u90-ena-h-sf-nsp-full-hae-4deg-3mo_20100701_v001.cdf', + f'imap_ultra_l2-energy-bin-group-sizes_20250101_v000.csv' + } + ) + expected_maps_to_produce = [expected_possible_map_to_produce_with_previous_version, + expected_possible_map_to_produce_with_no_previous_version] + self.assertEqual(expected_maps_to_produce, actual_maps_to_produce) def test_initializer_raises_when_given_invalid_map_descriptor(self): initializer = UltraCombinedNSPInitializer() with self.assertRaises(ValueError) as e: - initializer.get_maps_that_should_be_produced(f"ulc-invalid") - self.assertEqual(e.exception.args, ("Invalid map descriptor: ulc-invalid",)) + initializer.get_maps_that_should_be_produced(f"ulc-invalid", None) + self.assertEqual(e.exception.args, ("Invalid map descriptor: ulc-invalid",)) \ No newline at end of file diff --git a/tests/ultra/test_ultra_sp_initializer.py b/tests/ultra/test_ultra_sp_initializer.py index b6b72f3ae..bcc874d5c 100644 --- a/tests/ultra/test_ultra_sp_initializer.py +++ b/tests/ultra/test_ultra_sp_initializer.py @@ -99,11 +99,10 @@ def test_get_maps_that_should_be_produced(self, mock_read_cdf_parents): initializer = UltraSPInitializer() actual_maps_to_produce = initializer.get_maps_that_should_be_produced( - f"u{sensor}-ena-h-sf-sp-full-hae-4deg-3mo") + f"u{sensor}-ena-h-sf-sp-full-hae-4deg-3mo", None) with self.assertRaises(NotImplementedError) as e: - initializer.get_maps_that_should_be_produced( - f"u{sensor}-ena-h-hk-sp-full-hae-4deg-3mo") + initializer.get_maps_that_should_be_produced(f"u{sensor}-ena-h-hk-sp-full-hae-4deg-3mo", None) self.assertEqual(e.exception.args, ("Reference frame should be either Spacecraft or Heliospheric",)) self.mock_query.assert_has_calls([ @@ -253,8 +252,8 @@ def test_get_maps_that_should_be_produced_lists_combined_maps(self, mock_read_cd ] initializer = UltraSPInitializer() - actual_maps_to_produce = initializer.get_maps_that_should_be_produced( - f"ulc-ena-h-sf-sp-full-hae-4deg-3mo") + actual_maps_to_produce = initializer.get_maps_that_should_be_produced(f"ulc-ena-h-sf-sp-full-hae-4deg-3mo", + None) self.mock_query.assert_has_calls([ call(instrument='glows', data_level='l3e', descriptor=GLOWS_L3E_ULTRA_SF_DESCRIPTOR, version="latest"), @@ -409,8 +408,8 @@ def test_get_maps_that_should_be_produced_lists_cg_corrected_maps(self, mock_rea ] initializer = UltraSPInitializer() - actual_maps_to_produce = initializer.get_maps_that_should_be_produced( - f"ulc-ena-h-hf-sp-full-hae-4deg-3mo") + actual_maps_to_produce = initializer.get_maps_that_should_be_produced(f"ulc-ena-h-hf-sp-full-hae-4deg-3mo", + None) self.mock_query.assert_has_calls([ call(instrument='glows', data_level='l3e', descriptor=GLOWS_L3E_ULTRA_SF_DESCRIPTOR, version="latest"), From eba1c1b65df1ac01f8304f0276161426daaf53fc Mon Sep 17 00:00:00 2001 From: Harrison Date: Thu, 25 Jun 2026 16:58:37 -0400 Subject: [PATCH 05/11] Harrison 3295 #118: Version for zipped archive ancillary from L3b uses minor version only for compatibility --- imap_l3_processing/glows/glows_processor.py | 3 ++- tests/glows/test_glows_processor.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/imap_l3_processing/glows/glows_processor.py b/imap_l3_processing/glows/glows_processor.py index d25a83cd5..b85d60d08 100644 --- a/imap_l3_processing/glows/glows_processor.py +++ b/imap_l3_processing/glows/glows_processor.py @@ -134,7 +134,8 @@ def add_spin_angle_delta(data: dict, ancillary_files: dict) -> dict: @staticmethod def archive_dependencies(l3bc_deps: GlowsL3BCDependencies, external_dependencies: ExternalDependencies) -> Path: start_date = l3bc_deps.start_date.strftime("%Y%m%d") - zip_path = TEMP_CDF_FOLDER_PATH / f"imap_glows_l3b-archive_{start_date}_{l3bc_deps.version}.zip" + minor_version_str = str(Version(None,l3bc_deps.version.minor)) + zip_path = TEMP_CDF_FOLDER_PATH / f"imap_glows_l3b-archive_{start_date}_{minor_version_str}.zip" json_filename = "cr_to_process.json" with ZipFile(zip_path, "w", ZIP_DEFLATED) as file: file.write(external_dependencies.lyman_alpha_path, "lyman_alpha_composite.nc") diff --git a/tests/glows/test_glows_processor.py b/tests/glows/test_glows_processor.py index f19cc553b..a83489797 100644 --- a/tests/glows/test_glows_processor.py +++ b/tests/glows/test_glows_processor.py @@ -1555,7 +1555,7 @@ def test_process_l3e_skips_repointing_on_exception(self, mock_process_ultra_sf, @patch("imap_l3_processing.glows.glows_processor.ZipFile") def test_archive_dependencies(self, mock_zip, mock_json): version_number = Version(1,1) - expected_filepath = TEMP_CDF_FOLDER_PATH / f"imap_glows_l3b-archive_20250314_{version_number}.zip" + expected_filepath = TEMP_CDF_FOLDER_PATH / f"imap_glows_l3b-archive_20250314_v001.zip" expected_json_filename = "cr_to_process.json" l3bc_dependencies = GlowsL3BCDependencies( From 3a321a2e15719cf6aa052466467839674a472e38 Mon Sep 17 00:00:00 2001 From: Harrison Date: Wed, 1 Jul 2026 15:21:48 -0400 Subject: [PATCH 06/11] Harrison 3295 #118 - AFAH/KJON - WIP implementing new versioning scheme to use VersionMap with a version per output descriptor. --- imap_l3_data_processor.py | 15 +- .../hi/hi_combined_initializer.py | 6 +- imap_l3_processing/maps/map_initializer.py | 22 +- imap_l3_processing/maps/sp_map_initializer.py | 15 +- imap_l3_processing/models.py | 15 +- .../ultra/ultra_combined_nsp_initializer.py | 6 +- imap_l3_processing/utils.py | 5 +- tests/cdf/test_cdf_utils.py | 5 +- tests/codice/l3/lo/test_models.py | 5 +- tests/hi/test_hi_combined_initializer.py | 283 +++++++++++------- tests/hi/test_hi_sp_initializer.py | 158 +++++----- .../test_map_processor_integration.py | 15 +- tests/lo/l3/test_lo_sp_initializer.py | 11 +- tests/periodically_run_tests.json | 10 +- tests/swapi/test_swapi_processor.py | 44 +-- tests/test_helpers.py | 5 +- tests/test_imap_l3_data_processor.py | 188 +++++------- tests/test_models.py | 63 +++- tests/test_utils.py | 34 ++- .../test_ultra_combined_nsp_initializer.py | 66 +++- tests/ultra/test_ultra_sp_initializer.py | 25 +- 21 files changed, 579 insertions(+), 417 deletions(-) diff --git a/imap_l3_data_processor.py b/imap_l3_data_processor.py index 523bf1a3d..244127ebd 100644 --- a/imap_l3_data_processor.py +++ b/imap_l3_data_processor.py @@ -20,7 +20,7 @@ from imap_l3_processing.lo.l3.lo_sp_initializer import LoSPInitializer, LO_SP_MAP_DESCRIPTORS from imap_l3_processing.lo.lo_processor import LoProcessor from imap_l3_processing.maps.map_descriptors import parse_map_descriptor -from imap_l3_processing.models import InputMetadata +from imap_l3_processing.models import InputMetadata, VersionMap from imap_l3_processing.swapi.swapi_processor import SwapiProcessor from imap_l3_processing.swe.swe_processor import SweProcessor from imap_l3_processing.ultra.ultra_combined_nsp_initializer import UltraCombinedNSPInitializer, \ @@ -72,16 +72,11 @@ def imap_l3_processor(): parsed_dependency = json.loads(args.dependency) if isinstance(parsed_dependency, list): - version = args.version processing_input_collection.deserialize(args.dependency) + version_map = VersionMap({}, Version(None, args.version)) else: processing_input_collection.deserialize(json.dumps(parsed_dependency["dependency"])) - version_numbers = list(parsed_dependency["version"].values()) - match version_numbers: - case [{"major_version": major, "minor_version": minor}]: - version = str(Version(major, minor)) - case _: - raise ValueError("Expected only a single version to be specified in the dependency.") + version_map = VersionMap({k: Version(v["major_version"], v["minor_version"]) for k, v in parsed_dependency["version"].items()}) repointing_number = None if args.repointing is not None: @@ -95,7 +90,7 @@ def imap_l3_processor(): args.data_level, _convert_to_datetime(args.start_date), _convert_to_datetime(args.end_date or args.start_date), - version, descriptor=args.descriptor, repointing=repointing_number) + version_map, descriptor=args.descriptor, repointing=repointing_number) if args.instrument in ["hi", "lo", "ultra"] and args.data_level == 'l3' and not parse_map_descriptor( args.descriptor): initializer_class, processor_class, descriptors = { @@ -111,8 +106,8 @@ def imap_l3_processor(): initializer = initializer_class() paths = [] maps_to_produce = [] - major_version_from_dependency = Version.from_version(version).major for map_descriptor in descriptors: + major_version_from_dependency = version_map.lookup(map_descriptor).major maps_to_produce.extend(initializer.get_maps_that_should_be_produced(map_descriptor, major_version_from_dependency)) logger.info(f"maps to produce {[m.input_metadata.descriptor for m in maps_to_produce]}") diff --git a/imap_l3_processing/hi/hi_combined_initializer.py b/imap_l3_processing/hi/hi_combined_initializer.py index b360b69f4..4c88bbd0c 100644 --- a/imap_l3_processing/hi/hi_combined_initializer.py +++ b/imap_l3_processing/hi/hi_combined_initializer.py @@ -8,7 +8,7 @@ from imap_l3_processing.maps.map_descriptors import MapDescriptorParts, parse_map_descriptor, Sensor, \ map_descriptor_parts_to_string, SpinPhase from imap_l3_processing.maps.map_initializer import PossibleMapToProduce, MapInitializer -from imap_l3_processing.models import InputMetadata +from imap_l3_processing.models import InputMetadata, VersionMap HI_COMBINED_DESCRIPTORS = [ "hic-ena-h-hf-nsp-full-hae-6deg-1yr", @@ -35,7 +35,7 @@ def __init__(self, canonical_map_start_date: datetime = datetime(2026, 1, 1)): def furnish_spice_dependencies(self, map_to_produce: PossibleMapToProduce): pass - def get_maps_that_can_be_produced(self, input_descriptor: str, major_version: int|None) -> list[PossibleMapToProduce]: + def get_maps_that_can_be_produced(self, input_descriptor: str) -> list[PossibleMapToProduce]: possible_maps_to_produce = [] map_descriptor: MapDescriptorParts = parse_map_descriptor(input_descriptor) @@ -74,7 +74,7 @@ def get_maps_that_can_be_produced(self, input_descriptor: str, major_version: in data_level='l3', start_date=start_date, end_date=start_date + relativedelta(months=+12), - version=str(Version(major_version, 1)), + version=VersionMap({},Version(None,1)), descriptor=input_descriptor, repointing=None ) diff --git a/imap_l3_processing/maps/map_initializer.py b/imap_l3_processing/maps/map_initializer.py index 4492e75ff..0c8c990c2 100644 --- a/imap_l3_processing/maps/map_initializer.py +++ b/imap_l3_processing/maps/map_initializer.py @@ -9,7 +9,7 @@ from imap_data_access.file_validation import Version from imap_data_access.processing_input import generate_imap_input -from imap_l3_processing.models import InputMetadata +from imap_l3_processing.models import InputMetadata, VersionMap from imap_l3_processing.utils import read_cdf_parents logger = logging.getLogger(__name__) @@ -36,24 +36,26 @@ def furnish_spice_dependencies(self, map_to_produce: PossibleMapToProduce): raise NotImplementedError() @abc.abstractmethod - def get_maps_that_can_be_produced(self, l3_descriptor: str, major_version: int|None): + def get_maps_that_can_be_produced(self, descriptor: str): raise NotImplementedError() def get_maps_that_should_be_produced(self, descriptor: str, major_version: int|None) -> list[PossibleMapToProduce]: - possible_maps = self.get_maps_that_can_be_produced(descriptor, major_version) + possible_maps = self.get_maps_that_can_be_produced(descriptor) maps_to_make = [] for possible_map in possible_maps: start_time = possible_map.input_metadata.start_date.strftime("%Y%m%d") - if start_dates_for_l3_descriptor := self.existing_l3_maps.get(descriptor): - if l3_result := start_dates_for_l3_descriptor.get(start_time): - existing_parents = read_cdf_parents(l3_result) - existing_major_version = Version.from_version(ScienceFilePath(l3_result).version).major + possible_map.input_metadata.version = VersionMap({descriptor: Version(major_version, 1)}) + + if highest_version_path_by_start_date := self.existing_l3_maps.get(descriptor): + if path_with_highest_version := highest_version_path_by_start_date.get(start_time): + existing_parents = read_cdf_parents(path_with_highest_version) + existing_major_version = Version.from_version(ScienceFilePath(path_with_highest_version).version).major if possible_map.input_files.issubset(existing_parents) and major_version == existing_major_version: continue - new_minor_version = Version.from_version(ScienceFilePath(l3_result).version).minor + 1 - new_version = Version(major_version, new_minor_version) - possible_map.input_metadata.version = str(new_version) + existing_highest_version = Version.from_version(ScienceFilePath(path_with_highest_version).version) + new_version = Version(major_version, existing_highest_version.minor + 1) + possible_map.input_metadata.version = VersionMap({descriptor: new_version}) maps_to_make.append(possible_map) return maps_to_make diff --git a/imap_l3_processing/maps/sp_map_initializer.py b/imap_l3_processing/maps/sp_map_initializer.py index c35971b1b..9d270c797 100644 --- a/imap_l3_processing/maps/sp_map_initializer.py +++ b/imap_l3_processing/maps/sp_map_initializer.py @@ -8,7 +8,7 @@ from imap_l3_processing.maps.map_descriptors import MapDescriptorParts, parse_map_descriptor, \ get_duration_from_map_descriptor, map_descriptor_parts_to_string, Sensor from imap_l3_processing.maps.map_initializer import PossibleMapToProduce, MapInitializer -from imap_l3_processing.models import InputMetadata +from imap_l3_processing.models import InputMetadata, VersionMap logger = logging.getLogger(__name__) @@ -30,19 +30,19 @@ def _get_ancillary_files(self) -> list[str]: def _get_l2_dependencies(self, descriptor: MapDescriptorParts) -> list[MapDescriptorParts]: raise NotImplementedError() - def get_maps_that_can_be_produced(self, l3_descriptor: str, major_version: int|None) -> list[PossibleMapToProduce]: - l3_descriptor_parts = parse_map_descriptor(l3_descriptor) + def get_maps_that_can_be_produced(self, descriptor: str) -> list[PossibleMapToProduce]: + l3_descriptor_parts = parse_map_descriptor(descriptor) map_duration = get_duration_from_map_descriptor(l3_descriptor_parts) glows_file_by_repointing = self._collect_glows_psets_by_repoint(l3_descriptor_parts) if len(glows_file_by_repointing) == 0: - logger.info(f"No GLOWS data available for descriptor {l3_descriptor}. " + logger.info(f"No GLOWS data available for descriptor {descriptor}. " f"Processing will use default survival probability of 1.0.") l2_descriptors = self._get_l2_dependencies(l3_descriptor_parts) l2_descriptor_strs = [map_descriptor_parts_to_string(parts) for parts in l2_descriptors] - assert l2_descriptor_strs, f"Expected at least one L2 dependency for l3 map: {l3_descriptor}" + assert l2_descriptor_strs, f"Expected at least one L2 dependency for l3 map: {descriptor}" possible_start_dates = set(self.input_maps_by_descriptor[l2_descriptor_strs[0]].keys()) for l2_descriptor in l2_descriptor_strs[1:]: @@ -81,9 +81,8 @@ def get_maps_that_can_be_produced(self, l3_descriptor: str, major_version: int|N repoint in glows_file_by_repointing] input_metadata = InputMetadata(instrument=self.instrument, data_level='l3', start_date=start_date, - end_date=start_date + map_duration, version=str(Version(major_version, 1)), - descriptor=l3_descriptor) - + end_date=start_date + map_duration, version=VersionMap({}, Version(None, 1)), + descriptor=descriptor) possible_map_to_produce = PossibleMapToProduce( input_files=set(l2_file_paths + glows_files + l1c_names + self._get_ancillary_files()), input_metadata=input_metadata diff --git a/imap_l3_processing/models.py b/imap_l3_processing/models.py index 8a1bf7d3a..6b1b1b47d 100644 --- a/imap_l3_processing/models.py +++ b/imap_l3_processing/models.py @@ -7,17 +7,30 @@ from typing import Union, Optional, TypeVar, Generic import numpy as np +from imap_data_access.file_validation import Version from imap_l3_processing.data_utils import rebin +@dataclass +class VersionMap: + mapping: dict[str, Version] + fallback: Version | None = None + + def lookup(self, descriptor: str) -> Version: + result = self.mapping.get(descriptor, self.fallback) + if result is None: + raise KeyError(f"No version found for descriptor {descriptor}") + return result + + @dataclass class InputMetadata: instrument: str data_level: str start_date: datetime end_date: Optional[datetime] - version: str + version: VersionMap descriptor: str = "" repointing: Optional[int] = None diff --git a/imap_l3_processing/ultra/ultra_combined_nsp_initializer.py b/imap_l3_processing/ultra/ultra_combined_nsp_initializer.py index 626be309f..59bed3a3b 100644 --- a/imap_l3_processing/ultra/ultra_combined_nsp_initializer.py +++ b/imap_l3_processing/ultra/ultra_combined_nsp_initializer.py @@ -8,7 +8,7 @@ from imap_l3_processing.maps.map_descriptors import Sensor, map_descriptor_parts_to_string, \ parse_map_descriptor, get_duration_from_map_descriptor from imap_l3_processing.maps.map_initializer import MapInitializer, PossibleMapToProduce -from imap_l3_processing.models import InputMetadata +from imap_l3_processing.models import InputMetadata, VersionMap from imap_l3_processing.ultra.ultra_sp_initializer import ULTRA_SP_SPICE_KERNELS from imap_l3_processing.utils import furnish_spice_metakernel @@ -46,7 +46,7 @@ def furnish_spice_dependencies(self, map_to_produce: PossibleMapToProduce): kernel_types=ULTRA_SP_SPICE_KERNELS ) - def get_maps_that_can_be_produced(self, descriptor: str, major_version: int|None) -> list[PossibleMapToProduce]: + def get_maps_that_can_be_produced(self, descriptor: str) -> list[PossibleMapToProduce]: descriptor_parts = parse_map_descriptor(descriptor) if descriptor_parts is None: raise ValueError(f"Invalid map descriptor: {descriptor}") @@ -80,7 +80,7 @@ def get_maps_that_can_be_produced(self, descriptor: str, major_version: int|None data_level='l3', start_date=start_date, end_date=start_date + get_duration_from_map_descriptor(descriptor_parts), - version=str(Version(major_version, 1)), + version=VersionMap({},Version(None,1)), descriptor=descriptor, ) )) diff --git a/imap_l3_processing/utils.py b/imap_l3_processing/utils.py index ec3573d7f..34f4afd83 100644 --- a/imap_l3_processing/utils.py +++ b/imap_l3_processing/utils.py @@ -57,7 +57,7 @@ def save_data(data: DataProduct, delete_if_present: bool = False, folder_path: P start_time=formatted_start_date, repointing=data.input_metadata.repointing, cr=cr_number, - version=data.input_metadata.version, + version=str(data.input_metadata.version.lookup(data.input_metadata.descriptor)) ) file_path = science_file_path.construct_path() @@ -72,7 +72,8 @@ def save_data(data: DataProduct, delete_if_present: bool = False, folder_path: P file_path.unlink(missing_ok=True) attribute_manager = ImapAttributeManager() - attribute_manager.add_global_attribute("Data_version", data.input_metadata.version.replace('v', '')) + version = str(data.input_metadata.version.lookup(data.input_metadata.descriptor)).replace('v','') + attribute_manager.add_global_attribute("Data_version", version) attribute_manager.add_instrument_attrs(data.input_metadata.instrument, data.input_metadata.data_level, data.input_metadata.descriptor) attribute_manager.add_global_attribute("Generation_date", date.today().strftime("%Y%m%d")) diff --git a/tests/cdf/test_cdf_utils.py b/tests/cdf/test_cdf_utils.py index 889ae1034..ae8ed73ed 100644 --- a/tests/cdf/test_cdf_utils.py +++ b/tests/cdf/test_cdf_utils.py @@ -3,11 +3,12 @@ from unittest.mock import Mock, call, patch, sentinel import numpy as np +from imap_data_access.file_validation import Version from spacepy import pycdf from imap_l3_processing.cdf.cdf_utils import write_cdf, read_variable_and_mask_fill_values, read_numeric_variable from imap_l3_processing.cdf.imap_attribute_manager import ImapAttributeManager -from imap_l3_processing.models import DataProduct, DataProductVariable, InputMetadata +from imap_l3_processing.models import DataProduct, DataProductVariable, InputMetadata, VersionMap from tests.temp_file_test_case import TempFileTestCase @@ -337,7 +338,7 @@ class TestDataProduct(DataProduct): def __init__(self): super().__init__( input_metadata=InputMetadata("glows", "l3", datetime(year=2025, month=5, day=10), - datetime(year=2025, month=5, day=12), "v003", "descriptor"), + datetime(year=2025, month=5, day=12), VersionMap({'descriptor': Version(None, 3)}), "descriptor"), parent_file_names=[] ) diff --git a/tests/codice/l3/lo/test_models.py b/tests/codice/l3/lo/test_models.py index a5b87f1f3..83d862adf 100644 --- a/tests/codice/l3/lo/test_models.py +++ b/tests/codice/l3/lo/test_models.py @@ -5,6 +5,7 @@ from unittest.mock import Mock, sentinel import numpy as np +from imap_data_access.file_validation import Version from spacepy.pycdf import CDF from imap_l3_processing.codice.l3.lo.models import ( @@ -31,7 +32,7 @@ ELEVATION_ANGLE_LABEL_VAR_NAME, ENERGY_LABEL_VAR_NAME, SPIN_ANGLE_DELTA_VAR_NAME, SPIN_ANGLE_VAR_NAME, SPIN_ANGLE_LABEL_VAR_NAME, ) -from imap_l3_processing.models import InputMetadata +from imap_l3_processing.models import InputMetadata, VersionMap from imap_l3_processing.utils import save_data from tests.swapi.cdf_model_test_case import CdfModelTestCase from tests.test_helpers import get_test_data_path @@ -268,7 +269,7 @@ def test_codice_lo_l3a_partial_density_read_from_cdf(self): fe_loq_partial_density=rng.random(len(epoch), dtype='float32'), fe_hiq_partial_density=rng.random(len(epoch), dtype='float32'), ) - input_metadata = InputMetadata("codice", "l3a", datetime.now(), datetime.now(), "v000", "lo-partial-densities") + input_metadata = InputMetadata("codice", "l3a", datetime.now(), datetime.now(), VersionMap({"lo-partial-densities": Version(None, 1)}), "lo-partial-densities") codice_lo_l3a_partial_densities = CodiceLoL3aPartialDensityDataProduct(input_metadata, density_data) diff --git a/tests/hi/test_hi_combined_initializer.py b/tests/hi/test_hi_combined_initializer.py index 0a63ef2b1..970f5d0bd 100644 --- a/tests/hi/test_hi_combined_initializer.py +++ b/tests/hi/test_hi_combined_initializer.py @@ -1,11 +1,12 @@ -import itertools import unittest from datetime import datetime from unittest.mock import patch, Mock, call +from imap_data_access.file_validation import Version + from imap_l3_processing.hi.hi_combined_initializer import HiCombinedInitializer from imap_l3_processing.maps.map_initializer import PossibleMapToProduce -from imap_l3_processing.models import InputMetadata +from imap_l3_processing.models import InputMetadata, VersionMap from tests.test_helpers import create_mock_query_results l3_sp_input_maps = [ @@ -33,44 +34,105 @@ def teardown(self): self.hi_combined_initializer_mock_query.stop() def test_get_maps_that_should_be_produced_no_existing_combined_sp(self): + for deg in ["6deg", "4deg"]: + descriptor = f'hic-ena-h-hf-sp-full-hae-{deg}-1yr' + + for input_major_version in [None, 2]: + expected_version_map = VersionMap({descriptor: Version(input_major_version, 1)}) + + with self.subTest(deg=deg, major_version=input_major_version): + self.hi_combined_initializer_mock_query.side_effect = [ + create_mock_query_results([ + f'imap_hi_l3_h45-ena-h-hf-sp-ram-hae-{deg}-1yr_20250101_v001.cdf', + f'imap_hi_l3_h45-ena-h-hf-sp-anti-hae-{deg}-1yr_20250101_v001.cdf', + + f'imap_hi_l3_h90-ena-h-hf-sp-ram-hae-{deg}-1yr_20250101_v001.cdf', + f'imap_hi_l3_h90-ena-h-hf-sp-anti-hae-{deg}-1yr_20250101_v001.cdf', + ]), + create_mock_query_results([]) + ] + + expected_maps_to_produce = [ + PossibleMapToProduce( + input_files={ + f'imap_hi_l3_h45-ena-h-hf-sp-ram-hae-{deg}-1yr_20250101_v001.cdf', + f'imap_hi_l3_h45-ena-h-hf-sp-anti-hae-{deg}-1yr_20250101_v001.cdf', + + f'imap_hi_l3_h90-ena-h-hf-sp-ram-hae-{deg}-1yr_20250101_v001.cdf', + f'imap_hi_l3_h90-ena-h-hf-sp-anti-hae-{deg}-1yr_20250101_v001.cdf', + }, + input_metadata=InputMetadata( + instrument='hi', + data_level='l3', + start_date=datetime(2025, 1, 1), + end_date=datetime(2026, 1, 1), + version=expected_version_map, + descriptor=descriptor + ) + ) + ] + + initializer = HiCombinedInitializer() + actual_maps_to_produce = initializer.get_maps_that_should_be_produced( + descriptor, input_major_version) + + self.hi_combined_initializer_mock_query.assert_has_calls([ + call(instrument='hi', data_level='l3'), + call(instrument='hi', data_level='l2') + ]) + + self.assertEqual(expected_maps_to_produce, actual_maps_to_produce) + + @patch('imap_l3_processing.maps.map_initializer.read_cdf_parents') + def test_get_maps_that_should_be_produced_new_major_version(self, mock_read_cdf_parents: Mock): + mock_read_cdf_parents.return_value = { + *l3_sp_input_maps + } + + existing_file_without_major_number = f'imap_hi_l3_hic-ena-h-hf-sp-full-hae-6deg-1yr_20250101_v001.cdf' + existing_file_with_lower_major_number = f'imap_hi_l3_hic-ena-h-hf-sp-full-hae-6deg-1yr_20250101_v001.0001.cdf' + for existing_file in [existing_file_without_major_number, existing_file_with_lower_major_number]: + with self.subTest(existing_file=existing_file): + mock_read_cdf_parents.reset_mock() + self.hi_combined_initializer_mock_query.reset_mock() - for deg, (major_version, expected_version) in itertools.product(["6deg", "4deg"], [(None, 'v001'), (1, 'v001.0001')]): - with self.subTest(deg=deg,major_version=major_version): self.hi_combined_initializer_mock_query.side_effect = [ create_mock_query_results([ - f'imap_hi_l3_h45-ena-h-hf-sp-ram-hae-{deg}-1yr_20250101_v001.cdf', - f'imap_hi_l3_h45-ena-h-hf-sp-anti-hae-{deg}-1yr_20250101_v001.cdf', - - f'imap_hi_l3_h90-ena-h-hf-sp-ram-hae-{deg}-1yr_20250101_v001.cdf', - f'imap_hi_l3_h90-ena-h-hf-sp-anti-hae-{deg}-1yr_20250101_v001.cdf', + f'imap_hi_l3_h45-ena-h-hf-sp-ram-hae-6deg-1yr_20250101_v001.cdf', + f'imap_hi_l3_h45-ena-h-hf-sp-anti-hae-6deg-1yr_20250101_v001.cdf', + f'imap_hi_l3_h90-ena-h-hf-sp-ram-hae-6deg-1yr_20250101_v001.cdf', + f'imap_hi_l3_h90-ena-h-hf-sp-anti-hae-6deg-1yr_20250101_v001.cdf', + existing_file, ]), create_mock_query_results([]) ] + descriptor = f'hic-ena-h-hf-sp-full-hae-6deg-1yr' expected_maps_to_produce = [ PossibleMapToProduce( input_files={ - f'imap_hi_l3_h45-ena-h-hf-sp-ram-hae-{deg}-1yr_20250101_v001.cdf', - f'imap_hi_l3_h45-ena-h-hf-sp-anti-hae-{deg}-1yr_20250101_v001.cdf', + f'imap_hi_l3_h45-ena-h-hf-sp-ram-hae-6deg-1yr_20250101_v001.cdf', + f'imap_hi_l3_h45-ena-h-hf-sp-anti-hae-6deg-1yr_20250101_v001.cdf', - f'imap_hi_l3_h90-ena-h-hf-sp-ram-hae-{deg}-1yr_20250101_v001.cdf', - f'imap_hi_l3_h90-ena-h-hf-sp-anti-hae-{deg}-1yr_20250101_v001.cdf', + f'imap_hi_l3_h90-ena-h-hf-sp-ram-hae-6deg-1yr_20250101_v001.cdf', + f'imap_hi_l3_h90-ena-h-hf-sp-anti-hae-6deg-1yr_20250101_v001.cdf', }, input_metadata=InputMetadata( instrument='hi', data_level='l3', start_date=datetime(2025, 1, 1), end_date=datetime(2026, 1, 1), - version=expected_version, - descriptor=f'hic-ena-h-hf-sp-full-hae-{deg}-1yr' + version=VersionMap({descriptor: Version(2, 2)}), + descriptor=descriptor ) ) ] initializer = HiCombinedInitializer() actual_maps_to_produce = initializer.get_maps_that_should_be_produced( - f'hic-ena-h-hf-sp-full-hae-{deg}-1yr', major_version) + f'hic-ena-h-hf-sp-full-hae-6deg-1yr', 2) + mock_read_cdf_parents.assert_called_once_with(existing_file) self.hi_combined_initializer_mock_query.assert_has_calls([ call(instrument='hi', data_level='l3'), call(instrument='hi', data_level='l2') @@ -78,60 +140,10 @@ def test_get_maps_that_should_be_produced_no_existing_combined_sp(self): self.assertEqual(expected_maps_to_produce, actual_maps_to_produce) - @patch('imap_l3_processing.maps.map_initializer.read_cdf_parents') - def test_get_maps_that_should_be_produced_new_major_version(self, mock_read_cdf_parents: Mock): - mock_read_cdf_parents.return_value = { - *l3_sp_input_maps - } - - mock_read_cdf_parents.reset_mock() - self.hi_combined_initializer_mock_query.side_effect = [ - create_mock_query_results([ - f'imap_hi_l3_h45-ena-h-hf-sp-ram-hae-6deg-1yr_20250101_v001.cdf', - f'imap_hi_l3_h45-ena-h-hf-sp-anti-hae-6deg-1yr_20250101_v001.cdf', - f'imap_hi_l3_h90-ena-h-hf-sp-ram-hae-6deg-1yr_20250101_v001.cdf', - f'imap_hi_l3_h90-ena-h-hf-sp-anti-hae-6deg-1yr_20250101_v001.cdf', - f'imap_hi_l3_hic-ena-h-hf-sp-full-hae-6deg-1yr_20250101_v001.cdf', - ]), - create_mock_query_results([]) - ] - - expected_maps_to_produce = [ - PossibleMapToProduce( - input_files={ - f'imap_hi_l3_h45-ena-h-hf-sp-ram-hae-6deg-1yr_20250101_v001.cdf', - f'imap_hi_l3_h45-ena-h-hf-sp-anti-hae-6deg-1yr_20250101_v001.cdf', - - f'imap_hi_l3_h90-ena-h-hf-sp-ram-hae-6deg-1yr_20250101_v001.cdf', - f'imap_hi_l3_h90-ena-h-hf-sp-anti-hae-6deg-1yr_20250101_v001.cdf', - }, - input_metadata=InputMetadata( - instrument='hi', - data_level='l3', - start_date=datetime(2025, 1, 1), - end_date=datetime(2026, 1, 1), - version='v002.0002', - descriptor=f'hic-ena-h-hf-sp-full-hae-6deg-1yr' - ) - ) - ] - - initializer = HiCombinedInitializer() - actual_maps_to_produce = initializer.get_maps_that_should_be_produced( - f'hic-ena-h-hf-sp-full-hae-6deg-1yr', 2) - - mock_read_cdf_parents.assert_called_once_with(f'imap_hi_l3_hic-ena-h-hf-sp-full-hae-6deg-1yr_20250101_v001.cdf') - self.hi_combined_initializer_mock_query.assert_has_calls([ - call(instrument='hi', data_level='l3'), - call(instrument='hi', data_level='l2') - ]) - - self.assertEqual(expected_maps_to_produce, actual_maps_to_produce) - def test_get_maps_that_should_be_produced_no_existing_combined_nsp(self): - self.maxDiff = 2000 for deg in ["6deg", "4deg"]: - with self.subTest(msg=f"'hic-ena-h-hf-sp-full-hae-{deg}-1yr'"): + descriptor = f'hic-ena-h-hf-nsp-full-hae-{deg}-1yr' + with self.subTest(msg=descriptor): self.hi_combined_initializer_mock_query.side_effect = [ create_mock_query_results([]), create_mock_query_results([ @@ -157,15 +169,15 @@ def test_get_maps_that_should_be_produced_no_existing_combined_nsp(self): data_level='l3', start_date=datetime(2025, 1, 1), end_date=datetime(2026, 1, 1), - version='v001', - descriptor=f'hic-ena-h-hf-nsp-full-hae-{deg}-1yr' + version=VersionMap({descriptor: Version(None, 1)}), + descriptor=descriptor ) ) ] initializer = HiCombinedInitializer() actual_maps_to_produce = initializer.get_maps_that_should_be_produced( - f'hic-ena-h-hf-nsp-full-hae-{deg}-1yr', None) + descriptor, None) self.hi_combined_initializer_mock_query.assert_has_calls([ call(instrument='hi', data_level='l3'), @@ -230,50 +242,61 @@ def test_get_maps_that_should_be_produced_filters_based_on_input_descriptor(self @patch('imap_l3_processing.maps.map_initializer.read_cdf_parents') def test_get_maps_that_should_be_produced_increments_versions_correctly(self, mock_read_cdf_parents): - self.hi_combined_initializer_mock_query.side_effect = [ - create_mock_query_results([ - 'imap_hi_l3_hic-ena-h-hf-sp-full-hae-6deg-1yr_20250101_v001.cdf', + cases = [(None, "v001", 2), (2, "v002.0001", 2), (2, "v002.0002", 3)] - 'imap_hi_l3_h45-ena-h-hf-sp-anti-hae-6deg-1yr_20250101_v002.cdf', - *l3_sp_input_maps, - ]), - create_mock_query_results([]) - ] + for input_major_version, existing_file_version, expected_minor_version in cases: + with self.subTest(input_major_version=input_major_version, existing_file_version=existing_file_version, expected_minor_version=expected_minor_version): + self.hi_combined_initializer_mock_query.reset_mock() + mock_read_cdf_parents.reset_mock() + self.hi_combined_initializer_mock_query.reset_mock() + mock_read_cdf_parents.reset_mock() - expected_maps_to_produce = [ - PossibleMapToProduce( - input_files={ - 'imap_hi_l3_h45-ena-h-hf-sp-ram-hae-6deg-1yr_20250101_v001.cdf', - 'imap_hi_l3_h90-ena-h-hf-sp-ram-hae-6deg-1yr_20250101_v001.cdf', - 'imap_hi_l3_h90-ena-h-hf-sp-anti-hae-6deg-1yr_20250101_v001.cdf', + self.hi_combined_initializer_mock_query.side_effect = [ + create_mock_query_results([ + f'imap_hi_l3_hic-ena-h-hf-sp-full-hae-6deg-1yr_20250101_{existing_file_version}.cdf', - 'imap_hi_l3_h45-ena-h-hf-sp-anti-hae-6deg-1yr_20250101_v002.cdf', + 'imap_hi_l3_h45-ena-h-hf-sp-anti-hae-6deg-1yr_20250101_v002.cdf', + *l3_sp_input_maps, + ]), + create_mock_query_results([]) + ] - }, - input_metadata=InputMetadata( - instrument='hi', - data_level='l3', - start_date=datetime(2025, 1, 1), - end_date=datetime(2026, 1, 1), - version='v002', - descriptor='hic-ena-h-hf-sp-full-hae-6deg-1yr' - ) - ) - ] + descriptor = 'hic-ena-h-hf-sp-full-hae-6deg-1yr' + expected_maps_to_produce = [ + PossibleMapToProduce( + input_files={ + 'imap_hi_l3_h45-ena-h-hf-sp-ram-hae-6deg-1yr_20250101_v001.cdf', + 'imap_hi_l3_h90-ena-h-hf-sp-ram-hae-6deg-1yr_20250101_v001.cdf', + 'imap_hi_l3_h90-ena-h-hf-sp-anti-hae-6deg-1yr_20250101_v001.cdf', - mock_read_cdf_parents.return_value = [ - *l3_sp_input_maps - ] + 'imap_hi_l3_h45-ena-h-hf-sp-anti-hae-6deg-1yr_20250101_v002.cdf', - initializer = HiCombinedInitializer() - actual_maps_to_produce = initializer.get_maps_that_should_be_produced('hic-ena-h-hf-sp-full-hae-6deg-1yr', None) + }, + input_metadata=InputMetadata( + instrument='hi', + data_level='l3', + start_date=datetime(2025, 1, 1), + end_date=datetime(2026, 1, 1), + version=VersionMap({descriptor: Version(input_major_version, expected_minor_version)}), + descriptor=descriptor + ) + ) + ] - self.hi_combined_initializer_mock_query.assert_has_calls([ - call(instrument='hi', data_level='l3'), - call(instrument='hi', data_level='l2') - ]) + mock_read_cdf_parents.return_value = [ + *l3_sp_input_maps + ] - self.assertEqual(expected_maps_to_produce, actual_maps_to_produce) + initializer = HiCombinedInitializer() + actual_maps_to_produce = initializer.get_maps_that_should_be_produced(descriptor, input_major_version) + mock_read_cdf_parents.assert_called_once_with(f'imap_hi_l3_hic-ena-h-hf-sp-full-hae-6deg-1yr_20250101_{existing_file_version}.cdf') + + self.hi_combined_initializer_mock_query.assert_has_calls([ + call(instrument='hi', data_level='l3'), + call(instrument='hi', data_level='l2') + ]) + + self.assertEqual(expected_maps_to_produce, actual_maps_to_produce) @patch('imap_l3_processing.maps.map_initializer.read_cdf_parents') def test_get_maps_that_should_be_produced_can_produce_multiple_maps(self, mock_read_cdf_parents): @@ -289,6 +312,7 @@ def test_get_maps_that_should_be_produced_can_produce_multiple_maps(self, mock_r ]), ] + descriptor = 'hic-ena-h-hf-nsp-full-hae-6deg-1yr' expected_maps_to_produce = [ PossibleMapToProduce( input_files={ @@ -303,8 +327,8 @@ def test_get_maps_that_should_be_produced_can_produce_multiple_maps(self, mock_r data_level='l3', start_date=datetime(2025, 1, 1), end_date=datetime(2026, 1, 1), - version='v001', - descriptor=f'hic-ena-h-hf-nsp-full-hae-6deg-1yr' + version=VersionMap({descriptor: Version(None, 1)}), + descriptor=descriptor ) ), PossibleMapToProduce( @@ -320,8 +344,8 @@ def test_get_maps_that_should_be_produced_can_produce_multiple_maps(self, mock_r data_level='l3', start_date=datetime(2026, 1, 1), end_date=datetime(2027, 1, 1), - version='v001', - descriptor=f'hic-ena-h-hf-nsp-full-hae-6deg-1yr' + version=VersionMap({descriptor: Version(None, 1)}), + descriptor=descriptor ) ), ] @@ -338,3 +362,40 @@ def test_get_maps_that_should_be_produced_can_produce_multiple_maps(self, mock_r ]) self.assertEqual(expected_maps_to_produce, actual_maps_to_produce) + + @patch('imap_l3_processing.maps.map_initializer.read_cdf_parents') + def test_get_maps_that_can_be_produced(self, mock_read_cdf_parents): + query_results_for_existing_l3 = create_mock_query_results([]) + query_results_for_available_inputs = create_mock_query_results(['imap_hi_l2_h45-ena-h-hf-nsp-ram-hae-6deg-1yr_20260101_v001.cdf', + 'imap_hi_l2_h45-ena-h-hf-nsp-anti-hae-6deg-1yr_20260101_v001.cdf', + 'imap_hi_l2_h90-ena-h-hf-nsp-ram-hae-6deg-1yr_20260101_v001.cdf', + 'imap_hi_l2_h90-ena-h-hf-nsp-anti-hae-6deg-1yr_20260101_v001.cdf', ]) + self.hi_combined_initializer_mock_query.side_effect = [ + query_results_for_existing_l3, query_results_for_available_inputs, + ] + + initializer = HiCombinedInitializer() + descriptor = 'hic-ena-h-hf-nsp-anti-hae-6deg-1yr' + + actual_possible_maps = initializer.get_maps_that_can_be_produced(descriptor) + + expected_possible_maps = [ + PossibleMapToProduce( + input_files={ + 'imap_hi_l2_h45-ena-h-hf-nsp-ram-hae-6deg-1yr_20260101_v001.cdf', + 'imap_hi_l2_h45-ena-h-hf-nsp-anti-hae-6deg-1yr_20260101_v001.cdf', + 'imap_hi_l2_h90-ena-h-hf-nsp-ram-hae-6deg-1yr_20260101_v001.cdf', + 'imap_hi_l2_h90-ena-h-hf-nsp-anti-hae-6deg-1yr_20260101_v001.cdf', + }, + input_metadata=InputMetadata( + instrument='hi', + data_level='l3', + start_date=datetime(2026,1,1), + end_date=datetime(2027,1,1), + version=VersionMap({}, Version(None, 1)), + descriptor=descriptor, + ) + ) + ] + + self.assertEqual(expected_possible_maps, actual_possible_maps) \ No newline at end of file diff --git a/tests/hi/test_hi_sp_initializer.py b/tests/hi/test_hi_sp_initializer.py index 20b8ef184..9eb790f8b 100644 --- a/tests/hi/test_hi_sp_initializer.py +++ b/tests/hi/test_hi_sp_initializer.py @@ -3,11 +3,13 @@ from typing import Callable from unittest.mock import patch, call, Mock, sentinel +from imap_data_access.file_validation import Version + from imap_l3_processing.hi.hi_sp_initializer import HiSPInitializer, HI_SP_SPICE_KERNELS from imap_l3_processing.maps.map_descriptors import parse_map_descriptor, map_descriptor_parts_to_string from imap_l3_processing.maps.map_initializer import PossibleMapToProduce from imap_l3_processing.maps.sp_map_initializer import logger -from imap_l3_processing.models import InputMetadata +from imap_l3_processing.models import InputMetadata, VersionMap from tests.test_helpers import create_mock_query_results @@ -20,10 +22,11 @@ def tearDown(self): self.query_patcher.stop() @patch('imap_l3_processing.maps.map_initializer.read_cdf_parents') - def test_get_maps_that_can_be_produced(self, mock_read_cdf_parents): + def test_get_maps_that_can_be_produced_always_returns_possible_maps_with_default_version_map(self, mock_read_cdf_parents): + descriptor = 'h45-ena-h-sf-sp-anti-hae-4deg-3mo' cases = [ - (None, "v001"), - (4, "v004.0001"), + (None, VersionMap({}, Version(None, 1))), + (4, VersionMap({}, Version(None, 1))), ] for major_version, expected_version in cases: with self.subTest(major_version=major_version): @@ -69,7 +72,7 @@ def test_get_maps_that_can_be_produced(self, mock_read_cdf_parents): start_date=datetime(2010, 1, 1), end_date=datetime(2010, 4, 2, 7, 30), version=expected_version, - descriptor='h45-ena-h-sf-sp-anti-hae-4deg-3mo', + descriptor=descriptor, ) ), PossibleMapToProduce( @@ -88,7 +91,7 @@ def test_get_maps_that_can_be_produced(self, mock_read_cdf_parents): start_date=datetime(2010, 4, 1), end_date=datetime(2010, 7, 1, 7, 30), version=expected_version, - descriptor='h45-ena-h-sf-sp-anti-hae-4deg-3mo', + descriptor=descriptor, ) ), PossibleMapToProduce( @@ -105,7 +108,7 @@ def test_get_maps_that_can_be_produced(self, mock_read_cdf_parents): start_date=datetime(2010, 7, 1), end_date=datetime(2010, 9, 30, 7, 30), version=expected_version, - descriptor='h45-ena-h-sf-sp-anti-hae-4deg-3mo', + descriptor=descriptor, ) ), PossibleMapToProduce( @@ -121,7 +124,7 @@ def test_get_maps_that_can_be_produced(self, mock_read_cdf_parents): start_date=datetime(2010, 10, 1), end_date=datetime(2010, 12, 31, 7, 30), version=expected_version, - descriptor='h45-ena-h-sf-sp-anti-hae-4deg-3mo', + descriptor=descriptor, ) ), ] @@ -135,8 +138,7 @@ def test_get_maps_that_can_be_produced(self, mock_read_cdf_parents): call(instrument='hi', data_level='l3'), ]) - actual_possible_maps = initializer.get_maps_that_can_be_produced('h45-ena-h-sf-sp-anti-hae-4deg-3mo', - major_version) + actual_possible_maps = initializer.get_maps_that_can_be_produced('h45-ena-h-sf-sp-anti-hae-4deg-3mo') mock_read_cdf_parents.assert_has_calls([ call('imap_hi_l2_h45-ena-h-sf-nsp-anti-hae-4deg-3mo_20100101_v001.cdf'), @@ -163,8 +165,7 @@ def test_get_maps_that_can_be_produced_with_no_glows_data(self, mock_read_cdf_pa initializer = HiSPInitializer() - actual_possible_maps = initializer.get_maps_that_can_be_produced('h45-ena-h-sf-sp-anti-hae-4deg-3mo', - None) + actual_possible_maps = initializer.get_maps_that_can_be_produced('h45-ena-h-sf-sp-anti-hae-4deg-3mo') self.assertEqual(2, len(actual_possible_maps)) for possible_map in actual_possible_maps: @@ -208,8 +209,8 @@ def test_get_maps_that_can_be_produced_full_spin_descriptor(self, mock_read_cdf_ ] initializer = HiSPInitializer() - actual_possible_maps = initializer.get_maps_that_can_be_produced('h90-ena-h-sf-sp-full-hae-4deg-6mo', - None) + descriptor = 'h90-ena-h-sf-sp-full-hae-4deg-6mo' + actual_possible_maps = initializer.get_maps_that_can_be_produced(descriptor) self.mock_query.assert_has_calls([ call(instrument='glows', data_level='l3e', descriptor='survival-probability-hi-45', version="latest"), @@ -237,8 +238,8 @@ def test_get_maps_that_can_be_produced_full_spin_descriptor(self, mock_read_cdf_ data_level="l3", start_date=datetime(2010, 1, 1), end_date=datetime(2010, 7, 2, 15), - version="v001", - descriptor='h90-ena-h-sf-sp-full-hae-4deg-6mo', + version=VersionMap({},Version(None,1)), + descriptor=descriptor, ) ) @@ -275,8 +276,7 @@ def test_get_maps_that_can_be_produced_full_spin_with_mismatched_parents(self, m initializer = HiSPInitializer() with self.assertLogs(logger=logger, level='WARNING') as log_context: - actual_possible_maps = initializer.get_maps_that_can_be_produced('h90-ena-h-sf-sp-full-hae-4deg-6mo', - None) + actual_possible_maps = initializer.get_maps_that_can_be_produced('h90-ena-h-sf-sp-full-hae-4deg-6mo') expected_message = ('Expected all input maps to be created from the same pointing sets! l2_file_paths: ' 'imap_hi_l2_h90-ena-h-sf-nsp-anti-hae-4deg-6mo_20100101_v001.cdf, ' 'imap_hi_l2_h90-ena-h-sf-nsp-ram-hae-4deg-6mo_20100101_v001.cdf') @@ -329,6 +329,7 @@ def test_get_maps_that_should_be_produced(self, mock_read_cdf_parents): mock_read_cdf_parents.side_effect = self.create_fake_read_cdf_parents("90") + descriptor = 'h90-ena-h-sf-sp-anti-hae-4deg-3mo' expected_possible_maps = [ PossibleMapToProduce( input_files={ @@ -345,8 +346,8 @@ def test_get_maps_that_should_be_produced(self, mock_read_cdf_parents): data_level="l3", start_date=datetime(2010, 4, 1), end_date=datetime(2010, 7, 1, 7, 30), - version="v002", - descriptor='h90-ena-h-sf-sp-anti-hae-4deg-3mo', + version=VersionMap({descriptor:Version(None,2)}), + descriptor=descriptor, ) ), PossibleMapToProduce( @@ -362,8 +363,8 @@ def test_get_maps_that_should_be_produced(self, mock_read_cdf_parents): data_level="l3", start_date=datetime(2010, 7, 1), end_date=datetime(2010, 9, 30, 7, 30), - version="v001", - descriptor='h90-ena-h-sf-sp-anti-hae-4deg-3mo', + version=VersionMap({descriptor:Version(None,1)}), + descriptor=descriptor, ) ), PossibleMapToProduce( @@ -378,8 +379,8 @@ def test_get_maps_that_should_be_produced(self, mock_read_cdf_parents): data_level="l3", start_date=datetime(2010, 10, 1), end_date=datetime(2010, 12, 31, 7, 30), - version="v001", - descriptor='h90-ena-h-sf-sp-anti-hae-4deg-3mo', + version=VersionMap({descriptor:Version(None,1)}), + descriptor=descriptor, ) ), ] @@ -393,7 +394,7 @@ def test_get_maps_that_should_be_produced(self, mock_read_cdf_parents): call(instrument='hi', data_level='l3'), ]) - actual_possible_maps = initializer.get_maps_that_should_be_produced('h90-ena-h-sf-sp-anti-hae-4deg-3mo', None) + actual_possible_maps = initializer.get_maps_that_should_be_produced(descriptor, None) mock_read_cdf_parents.assert_has_calls([ call(f'imap_hi_l2_h90-ena-h-sf-nsp-anti-hae-4deg-3mo_20100101_v001.cdf'), @@ -411,65 +412,70 @@ def test_get_maps_that_should_be_produced(self, mock_read_cdf_parents): @patch('imap_l3_processing.maps.map_initializer.read_cdf_parents') def test_get_maps_that_should_be_produced_remakes_when_major_version_changes(self, mock_read_cdf_parents): - self.mock_query.side_effect = [ - create_mock_query_results([]), - create_mock_query_results([ - 'imap_glows_l3e_survival-probability-hi-90_20100101-repoint00001_v001.cdf', - 'imap_glows_l3e_survival-probability-hi-90_20100102-repoint00002_v001.cdf', - 'imap_glows_l3e_survival-probability-hi-90_20100103-repoint00003_v001.cdf', - ]), - create_mock_query_results([ - 'imap_hi_l2_h90-ena-h-sf-nsp-anti-hae-4deg-3mo_20100101_v001.cdf', - 'imap_hi_l2_h90-ena-h-sf-nsp-anti-hae-4deg-3mo_20100101_v000.cdf', - ]), - create_mock_query_results([ - 'imap_hi_l3_h90-ena-h-sf-sp-anti-hae-4deg-3mo_20100101_v001.cdf', - 'imap_hi_l3_h90-ena-h-sf-sp-anti-hae-4deg-3mo_20100101_v000.cdf', - ]), - ] + for major_version in [2,3]: + with self.subTest(major_version=major_version): + self.mock_query.reset_mock() + mock_read_cdf_parents.reset_mock() + self.mock_query.side_effect = [ + create_mock_query_results([]), + create_mock_query_results([ + 'imap_glows_l3e_survival-probability-hi-90_20100101-repoint00001_v001.cdf', + 'imap_glows_l3e_survival-probability-hi-90_20100102-repoint00002_v001.cdf', + 'imap_glows_l3e_survival-probability-hi-90_20100103-repoint00003_v001.cdf', + ]), + create_mock_query_results([ + 'imap_hi_l2_h90-ena-h-sf-nsp-anti-hae-4deg-3mo_20100101_v001.cdf', + 'imap_hi_l2_h90-ena-h-sf-nsp-anti-hae-4deg-3mo_20100101_v000.cdf', + ]), + create_mock_query_results([ + 'imap_hi_l3_h90-ena-h-sf-sp-anti-hae-4deg-3mo_20100101_v001.cdf', + 'imap_hi_l3_h90-ena-h-sf-sp-anti-hae-4deg-3mo_20100101_v000.cdf', + ]), + ] - mock_read_cdf_parents.side_effect = self.create_fake_read_cdf_parents("90") + mock_read_cdf_parents.side_effect = self.create_fake_read_cdf_parents("90") - expected_possible_maps = [ - PossibleMapToProduce( - input_files={ - 'imap_hi_l2_h90-ena-h-sf-nsp-anti-hae-4deg-3mo_20100101_v001.cdf', - 'imap_glows_l3e_survival-probability-hi-90_20100101-repoint00001_v001.cdf', - 'imap_glows_l3e_survival-probability-hi-90_20100102-repoint00002_v001.cdf', - 'imap_glows_l3e_survival-probability-hi-90_20100103-repoint00003_v001.cdf', - 'imap_hi_l1c_90sensor-pset_20100101-repoint00001_v001.cdf', - 'imap_hi_l1c_90sensor-pset_20100102-repoint00002_v001.cdf', - 'imap_hi_l1c_90sensor-pset_20100103-repoint00003_v001.cdf', - }, - input_metadata=InputMetadata( - instrument="hi", - data_level="l3", - start_date=datetime(2010, 1, 1), - end_date=datetime(2010, 4, 2, 7, 30), - version="v001.0002", - descriptor='h90-ena-h-sf-sp-anti-hae-4deg-3mo', - ) - ), - ] + descriptor = 'h90-ena-h-sf-sp-anti-hae-4deg-3mo' + expected_possible_maps = [ + PossibleMapToProduce( + input_files={ + 'imap_hi_l2_h90-ena-h-sf-nsp-anti-hae-4deg-3mo_20100101_v001.cdf', + 'imap_glows_l3e_survival-probability-hi-90_20100101-repoint00001_v001.cdf', + 'imap_glows_l3e_survival-probability-hi-90_20100102-repoint00002_v001.cdf', + 'imap_glows_l3e_survival-probability-hi-90_20100103-repoint00003_v001.cdf', + 'imap_hi_l1c_90sensor-pset_20100101-repoint00001_v001.cdf', + 'imap_hi_l1c_90sensor-pset_20100102-repoint00002_v001.cdf', + 'imap_hi_l1c_90sensor-pset_20100103-repoint00003_v001.cdf', + }, + input_metadata=InputMetadata( + instrument="hi", + data_level="l3", + start_date=datetime(2010, 1, 1), + end_date=datetime(2010, 4, 2, 7, 30), + version=VersionMap({descriptor:Version(major_version,2)}), + descriptor=descriptor, + ) + ), + ] - initializer = HiSPInitializer() + initializer = HiSPInitializer() - self.mock_query.assert_has_calls([ - call(instrument='glows', data_level='l3e', descriptor='survival-probability-hi-45', version="latest"), - call(instrument='glows', data_level='l3e', descriptor='survival-probability-hi-90', version="latest"), - call(instrument='hi', data_level='l2'), - call(instrument='hi', data_level='l3'), - ]) + self.mock_query.assert_has_calls([ + call(instrument='glows', data_level='l3e', descriptor='survival-probability-hi-45', version="latest"), + call(instrument='glows', data_level='l3e', descriptor='survival-probability-hi-90', version="latest"), + call(instrument='hi', data_level='l2'), + call(instrument='hi', data_level='l3'), + ]) - actual_possible_maps = initializer.get_maps_that_should_be_produced('h90-ena-h-sf-sp-anti-hae-4deg-3mo', 1) + actual_possible_maps = initializer.get_maps_that_should_be_produced(descriptor, major_version) - mock_read_cdf_parents.assert_has_calls([ - call(f'imap_hi_l2_h90-ena-h-sf-nsp-anti-hae-4deg-3mo_20100101_v001.cdf'), + mock_read_cdf_parents.assert_has_calls([ + call(f'imap_hi_l2_h90-ena-h-sf-nsp-anti-hae-4deg-3mo_20100101_v001.cdf'), - call(f'imap_hi_l3_h90-ena-h-sf-sp-anti-hae-4deg-3mo_20100101_v001.cdf'), - ]) + call(f'imap_hi_l3_h90-ena-h-sf-sp-anti-hae-4deg-3mo_20100101_v001.cdf'), + ]) - self.assertEqual(expected_possible_maps, actual_possible_maps) + self.assertEqual(expected_possible_maps, actual_possible_maps) def test_get_dependencies(self): cases = [ diff --git a/tests/integration/test_map_processor_integration.py b/tests/integration/test_map_processor_integration.py index 0c59209fc..4cb7bd669 100644 --- a/tests/integration/test_map_processor_integration.py +++ b/tests/integration/test_map_processor_integration.py @@ -1,3 +1,4 @@ +import json import logging import unittest from datetime import timedelta @@ -10,6 +11,7 @@ from spiceypy import spiceypy import imap_l3_data_processor +from imap_l3_processing.hi.hi_combined_initializer import HI_COMBINED_DESCRIPTORS from tests.integration.integration_test_helpers import mock_imap_data_access from tests.test_helpers import get_run_local_data_path, run_periodically, get_test_data_path @@ -219,16 +221,13 @@ def test_hi_combined_sensor(self, mock_parse_cli_arguments): mock_arguments.end_date = None mock_arguments.repointing = None mock_arguments.version = "v001" - mock_arguments.dependency = """ - { "version": { - "hic-maps": { - "major_version": 4, - "minor_version": 7 - } - }, + + version_descriptors = {descriptor: {"major_version": 4, "minor_version": 7} for descriptor in HI_COMBINED_DESCRIPTORS} + dependency_with_version_info = { + "version": version_descriptors, "dependency": [] } - """ + mock_arguments.dependency = json.dumps(dependency_with_version_info) mock_arguments.upload_to_sdc = False mock_parse_cli_arguments.return_value = mock_arguments diff --git a/tests/lo/l3/test_lo_sp_initializer.py b/tests/lo/l3/test_lo_sp_initializer.py index 6f6104dd9..a314d0472 100644 --- a/tests/lo/l3/test_lo_sp_initializer.py +++ b/tests/lo/l3/test_lo_sp_initializer.py @@ -2,9 +2,11 @@ from datetime import datetime from unittest.mock import patch, call +from imap_data_access.file_validation import Version + from imap_l3_processing.lo.l3.lo_sp_initializer import LoSPInitializer, LO_SP_MAP_KERNELS from imap_l3_processing.maps.map_initializer import PossibleMapToProduce, MapInitializer -from imap_l3_processing.models import InputMetadata +from imap_l3_processing.models import InputMetadata, VersionMap from tests.integration.integration_test_helpers import ImapQueryPatcher @@ -86,7 +88,8 @@ def test_get_maps_that_should_be_produced(self, mock_read_cdf_parents): ] initializer = LoSPInitializer() - actual_maps_to_produce = initializer.get_maps_that_should_be_produced("l090-ena-h-sf-sp-ram-hae-6deg-12mo", + descriptor = "l090-ena-h-sf-sp-ram-hae-6deg-12mo" + actual_maps_to_produce = initializer.get_maps_that_should_be_produced(descriptor, 3) self.mock_query.assert_has_calls([ @@ -111,8 +114,8 @@ def test_get_maps_that_should_be_produced(self, mock_read_cdf_parents): data_level="l3", start_date=datetime(2010, 4, 1), end_date=datetime(2011, 4, 1, 6, 0), - version="v003.0002", - descriptor="l090-ena-h-sf-sp-ram-hae-6deg-12mo" + version=VersionMap({descriptor: Version(3, 2)}), + descriptor=descriptor ), input_files={ 'imap_glows_l3e_survival-probability-lo_20100401-repoint00101_v002.cdf', diff --git a/tests/periodically_run_tests.json b/tests/periodically_run_tests.json index 69b8e3111..362b75b74 100644 --- a/tests/periodically_run_tests.json +++ b/tests/periodically_run_tests.json @@ -1,14 +1,14 @@ { "test_l3bcde_first_time_processing": "2026-06-23T13:32:02.351597", - "test_spectral_fit_against_validation_data": "2026-06-23T13:30:47.289531", + "test_spectral_fit_against_validation_data": "2026-07-01T08:47:35.621214", "test_calculate_pickup_ions_with_minimize": "2026-05-20T10:52:46.704900", "test_l3bcde_reprocessing": "2026-06-25T09:12:12.228439", "test_glows_l3abcde_from_l2": "2026-06-25T09:12:11.727186", "test_codice_lo_3d_distributions": "2026-05-28T09:43:22.990231", "test_alpha_sigma_matches_empirical_std": null, "test_proton_sigma_matches_empirical_std": null, - "test_swe_processor_with_local_data": "2026-06-23T13:36:38.979488", - "test_ultra_all_sp_maps": "2026-06-23T13:33:14.635122", - "test_ultra_combined_nsp_maps": "2026-06-23T13:33:55.733624", - "test_ultra_combined_sp_maps": "2026-06-23T13:35:07.874145" + "test_swe_processor_with_local_data": "2026-07-01T09:13:44.538970", + "test_ultra_all_sp_maps": "2026-07-01T09:11:19.247396", + "test_ultra_combined_nsp_maps": "2026-07-01T09:49:45.053794", + "test_ultra_combined_sp_maps": "2026-07-01T09:12:32.440571" } \ No newline at end of file diff --git a/tests/swapi/test_swapi_processor.py b/tests/swapi/test_swapi_processor.py index 0e275b922..8ae4398ab 100644 --- a/tests/swapi/test_swapi_processor.py +++ b/tests/swapi/test_swapi_processor.py @@ -5,12 +5,13 @@ import numpy as np from imap_data_access import config +from imap_data_access.file_validation import Version from imap_data_access.processing_input import ProcessingInputCollection, ScienceInput, AncillaryInput from uncertainties.unumpy import uarray, nominal_values, std_devs from imap_l3_processing.constants import THIRTY_SECONDS_IN_NANOSECONDS, \ FIVE_MINUTES_IN_NANOSECONDS -from imap_l3_processing.models import InputMetadata +from imap_l3_processing.models import InputMetadata, VersionMap from imap_l3_processing.swapi.descriptors import DENSITY_OF_NEUTRAL_HELIUM_DESCRIPTOR, \ EFFICIENCY_LOOKUP_TABLE_DESCRIPTOR, \ GEOMETRIC_FACTOR_SW_LOOKUP_TABLE_DESCRIPTOR, \ @@ -22,6 +23,7 @@ from imap_l3_processing.swapi.l3b.science.calculate_solar_wind_vdf import DeltaMinusPlus from imap_l3_processing.swapi.quality_flags import SwapiL3Flags from imap_l3_processing.swapi.swapi_processor import SwapiProcessor +from tests.test_helpers import create_mock_version_map class TestSwapiProcessor(TestCase): @@ -46,7 +48,8 @@ def test_process_l3a_pui(self, mock_spicepy, end_date = datetime(2025, 9, 26) outgoing_data_level = "l3a" start_date = datetime(2025, 9, 25) - input_version = "v123" + descriptor = "pui-he" + input_version = VersionMap({descriptor: Version(None, 123)}) outgoing_version = "123" start_date_as_str = datetime.strftime(start_date, "%Y%m%d") @@ -110,13 +113,13 @@ def test_process_l3a_pui(self, mock_spicepy, input_metadata = InputMetadata(instrument, outgoing_data_level, start_date, end_date, input_version) pickup_ion_data = mock_pickup_ion_data_constructor.return_value - expected_pickup_ion_metadata = replace(input_metadata, descriptor="pui-he") + expected_pickup_ion_metadata = replace(input_metadata, descriptor=descriptor) pickup_ion_data.input_metadata = expected_pickup_ion_metadata - input_metadata.descriptor = "pui-he" + input_metadata.descriptor = descriptor expected_cdf_path = (config["DATA_DIR"] / "imap" / "swapi" / "l3a" / "2025" / "09" / - f"imap_swapi_l3a_pui-he_{start_date_as_str}_{input_version}.cdf") + f"imap_swapi_l3a_pui-he_{start_date_as_str}_v123.cdf") mock_chunk_l2_data.side_effect = [ [chunk_of_five], @@ -175,7 +178,7 @@ def test_process_l3a_pui(self, mock_spicepy, call("Logical_source", f"imap_swapi_l3a_pui-he"), call("Logical_file_id", - f"imap_swapi_l3a_pui-he_{start_date_as_str}_{input_version}"), + f"imap_swapi_l3a_pui-he_{start_date_as_str}_v123"), ]) actual_positional = mock_pickup_ion_data_constructor.call_args.args @@ -184,7 +187,7 @@ def test_process_l3a_pui(self, mock_spicepy, for key, expected_val in pui_runner_result.items(): np.testing.assert_array_equal(expected_val, actual_kwargs[key]) - mock_manager.add_instrument_attrs.assert_called_once_with("swapi", "l3a", "pui-he") + mock_manager.add_instrument_attrs.assert_called_once_with("swapi", "l3a", descriptor) self.assertEqual(input_file_names, pickup_ion_data.parent_file_names) mock_write_cdf.assert_called_once_with(str(expected_cdf_path), pickup_ion_data, mock_manager) @@ -211,7 +214,8 @@ def test_process_l3a_proton(self, mock_spicepy, end_date = datetime(2025, 6, 13) outgoing_data_level = "l3a" start_date = datetime(2025, 6, 12) - input_version = "v123" + descriptor = "proton-sw" + input_version = VersionMap({descriptor: Version(None, 123)}) outgoing_version = "123" start_date_as_str = datetime.strftime(start_date, "%Y%m%d") @@ -273,13 +277,14 @@ def test_process_l3a_proton(self, mock_spicepy, input_metadata = InputMetadata(instrument, outgoing_data_level, start_date, end_date, input_version) proton_solar_wind_data = mock_proton_solar_wind_data_constructor.return_value - expected_proton_metadata = replace(input_metadata, descriptor="proton-sw") + + expected_proton_metadata = replace(input_metadata, descriptor=descriptor) proton_solar_wind_data.input_metadata = expected_proton_metadata - input_metadata.descriptor = "proton-sw" + input_metadata.descriptor = descriptor expected_cdf_path = (config["DATA_DIR"] / "imap" / "swapi" / "l3a" / "2025" / "06" / - f"imap_swapi_l3a_proton-sw_{start_date_as_str}_{input_version}.cdf") + f"imap_swapi_l3a_proton-sw_{start_date_as_str}_v123.cdf") mock_chunk_l2_data.return_value = [chunk_of_five] @@ -316,10 +321,10 @@ def test_process_l3a_proton(self, mock_spicepy, call("Logical_source", f"imap_swapi_l3a_proton-sw"), call("Logical_file_id", - f"imap_swapi_l3a_proton-sw_{start_date_as_str}_{input_version}"), + f"imap_swapi_l3a_proton-sw_{start_date_as_str}_v123"), ]) - mock_manager.add_instrument_attrs.assert_called_once_with("swapi", "l3a", "proton-sw") + mock_manager.add_instrument_attrs.assert_called_once_with("swapi", "l3a", descriptor) self.assertEqual(input_file_names, proton_solar_wind_data.parent_file_names) mock_write_cdf.assert_called_once_with(str(expected_cdf_path), proton_solar_wind_data, mock_manager) @@ -346,7 +351,8 @@ def test_process_l3a_alpha(self, mock_spicepy, end_date = datetime(2025, 8, 29) outgoing_data_level = "l3a" start_date = datetime(2025, 8, 28) - input_version = "v123" + l3a_descriptor = "alpha-sw" + input_version = create_mock_version_map(descriptor=l3a_descriptor, minor_version=123) outgoing_version = "123" start_date_as_str = datetime.strftime(start_date, "%Y%m%d") @@ -397,13 +403,13 @@ def test_process_l3a_alpha(self, mock_spicepy, input_metadata = InputMetadata(instrument, outgoing_data_level, start_date, end_date, input_version) alpha_solar_wind_data = mock_alpha_solar_wind_data_constructor.return_value - expected_alpha_metadata = replace(input_metadata, descriptor="alpha-sw") + expected_alpha_metadata = replace(input_metadata, descriptor=l3a_descriptor) alpha_solar_wind_data.input_metadata = expected_alpha_metadata - input_metadata.descriptor = "alpha-sw" + input_metadata.descriptor = l3a_descriptor expected_cdf_path = (config["DATA_DIR"] / "imap" / "swapi" / "l3a" / "2025" / "08" / - f"imap_swapi_l3a_alpha-sw_{start_date_as_str}_{input_version}.cdf") + f"imap_swapi_l3a_alpha-sw_{start_date_as_str}_v123.cdf") mock_chunk_l2_data.return_value = [chunk_of_five] @@ -440,9 +446,9 @@ def test_process_l3a_alpha(self, mock_spicepy, call("Data_version", outgoing_version), call("Generation_date", date.today().strftime("%Y%m%d")), call("Logical_source", "imap_swapi_l3a_alpha-sw"), - call("Logical_file_id", f"imap_swapi_l3a_alpha-sw_{start_date_as_str}_{input_version}"), + call("Logical_file_id", f"imap_swapi_l3a_alpha-sw_{start_date_as_str}_v123"), ]) - mock_manager.add_instrument_attrs.assert_called_once_with("swapi", "l3a", "alpha-sw") + mock_manager.add_instrument_attrs.assert_called_once_with("swapi", "l3a", l3a_descriptor) self.assertEqual(input_file_names, alpha_solar_wind_data.parent_file_names) mock_write_cdf.assert_called_once_with(str(expected_cdf_path), alpha_solar_wind_data, mock_manager) diff --git a/tests/test_helpers.py b/tests/test_helpers.py index e82ccdb2d..97bffbad8 100644 --- a/tests/test_helpers.py +++ b/tests/test_helpers.py @@ -12,10 +12,11 @@ import imap_data_access import numpy as np from imap_data_access import ScienceFilePath, AncillaryFilePath, SPICEFilePath -from imap_data_access.file_validation import generate_imap_file_path +from imap_data_access.file_validation import generate_imap_file_path, Version import imap_l3_processing import tests +from imap_l3_processing.models import VersionMap from imap_l3_processing.swe.l3.models import SweConfiguration, SweL3MomentData from imap_l3_processing.swe.l3.science.moment_calculations import Moments, MomentFitResults @@ -255,6 +256,8 @@ def create_mock_query_results(file_names: list[Path | str], ingestion_dates: Opt raise NotImplementedError(f"Unexpected file path type {imap_file_path}") return file_paths +def create_mock_version_map(descriptor: Optional[str] = 'descriptor', major_version: Optional[int] = None, minor_version: Optional[int] = 1): + return VersionMap({descriptor:Version(major_version, minor_version)}) @dataclass class PeriodicallyRunTest: diff --git a/tests/test_imap_l3_data_processor.py b/tests/test_imap_l3_data_processor.py index 0b68d58ae..4b8f08704 100644 --- a/tests/test_imap_l3_data_processor.py +++ b/tests/test_imap_l3_data_processor.py @@ -5,6 +5,7 @@ from unittest import TestCase from unittest.mock import patch, call, Mock, sentinel +from imap_data_access.file_validation import Version from imap_data_access.processing_input import ScienceInput, ProcessingInputCollection, \ AncillaryInput, SPICEInput @@ -13,7 +14,7 @@ from imap_l3_processing.hi.hi_sp_initializer import HI_SP_MAP_DESCRIPTORS from imap_l3_processing.lo.l3.lo_sp_initializer import LO_SP_MAP_DESCRIPTORS from imap_l3_processing.maps.map_initializer import PossibleMapToProduce -from imap_l3_processing.models import InputMetadata +from imap_l3_processing.models import InputMetadata, VersionMap from imap_l3_processing.ultra.ultra_combined_nsp_initializer import ULTRA_COMBINED_NSP_DESCRIPTORS from imap_l3_processing.ultra.ultra_sp_initializer import ULTRA_45_DESCRIPTORS, ULTRA_90_DESCRIPTORS, \ ULTRA_COMBINED_SP_DESCRIPTORS @@ -64,12 +65,12 @@ def test_invokes_correct_processor(self, mock_argparse, mock_codice_hi, mock_cod mock_argument_parser.parse_args.return_value.start_date = "20250101" mock_argument_parser.parse_args.return_value.end_date = None mock_argument_parser.parse_args.return_value.repointing = "repoint00022" - mock_argument_parser.parse_args.return_value.version = sentinel.version + mock_argument_parser.parse_args.return_value.version = "v099" mock_argument_parser.parse_args.return_value.descriptor = descriptor expected_input_metadata = InputMetadata(instrument, data_level, datetime(2025, 1, 1), datetime(2025, 1, 1), - sentinel.version, descriptor=descriptor, + VersionMap({},Version(None, 99)), descriptor=descriptor, repointing=22) expected_processor.return_value.process.return_value = [sentinel.cdf] @@ -123,72 +124,67 @@ def test_scheduled_map_jobs_invoke_correct_initializer(self, mock_argparse, mock ULTRA_COMBINED_NSP_DESCRIPTORS), ] - dependency_with_version_info = """ - { - "version": { - "descriptor": { - "major_version": 123, - "minor_version": 0 - } - }, - "dependency": [] - } - """ - dependency_cases = [ - ("old style", "[]", 'v001', None, 'v001'), - ("new style", dependency_with_version_info, None, 123, 'v123.0000'), - ] + for instrument, descriptor, mock_initializer_class, mock_processor_class, expected_descriptors in instrument_cases: + version_descriptors = {descriptor: { "major_version": 123,"minor_version": 0} for descriptor in expected_descriptors} + dependency_with_version_info = { + "version": version_descriptors, + "dependency": [] + } + dependency_with_version_info_as_string = json.dumps(dependency_with_version_info) - for instrument_case, dependency_case in itertools.product(instrument_cases, dependency_cases): - instrument, descriptor, mock_initializer_class, mock_processor_class, expected_descriptors = instrument_case - dependency_name, dependency_contents, version_argument, expected_major_version, expected_version_str = dependency_case - mock_upload.reset_mock() - mock_argparse.reset_mock() - mock_initializer_class.reset_mock() - mock_processor_class.reset_mock() + dependency_cases = [ + ("old style", "[]", None, VersionMap({descriptor: Version(None, 1)})), + ("new style", dependency_with_version_info_as_string, 123, VersionMap({descriptor: Version(123, 1)})), + ] + for source_of_version_info, dependency_contents, expected_major_version, expected_version_map in dependency_cases: - with self.subTest(instrument=instrument, descriptor=descriptor, dependency=dependency_name): - data_level = "l3" - mock_argument_parser = mock_argparse.ArgumentParser.return_value - mock_argument_parser.parse_args.return_value.instrument = instrument - mock_argument_parser.parse_args.return_value.data_level = data_level - mock_argument_parser.parse_args.return_value.dependency = dependency_contents - mock_argument_parser.parse_args.return_value.start_date = "20250101" - mock_argument_parser.parse_args.return_value.end_date = None - mock_argument_parser.parse_args.return_value.repointing = "repoint00022" - mock_argument_parser.parse_args.return_value.version = version_argument - mock_argument_parser.parse_args.return_value.descriptor = descriptor + mock_upload.reset_mock() + mock_argparse.reset_mock() + mock_initializer_class.reset_mock() + mock_processor_class.reset_mock() - expected_input_metadata = InputMetadata(instrument, data_level, datetime(2025, 1, 1), - datetime(2025, 1, 1), - expected_version_str, descriptor=descriptor, - ) + with self.subTest(instrument=instrument, descriptor=descriptor, dependency=source_of_version_info): + data_level = "l3" + mock_argument_parser = mock_argparse.ArgumentParser.return_value + mock_argument_parser.parse_args.return_value.instrument = instrument + mock_argument_parser.parse_args.return_value.data_level = data_level + mock_argument_parser.parse_args.return_value.dependency = dependency_contents + mock_argument_parser.parse_args.return_value.start_date = "20250101" + mock_argument_parser.parse_args.return_value.end_date = None + mock_argument_parser.parse_args.return_value.repointing = "repoint00022" + mock_argument_parser.parse_args.return_value.version = "v001" + mock_argument_parser.parse_args.return_value.descriptor = descriptor - mock_initializer = mock_initializer_class.return_value - mock_processor = mock_processor_class.return_value + expected_input_metadata = InputMetadata(instrument, data_level, datetime(2025, 1, 1), + datetime(2025, 1, 1), + expected_version_map, descriptor=descriptor, + ) - possible_map_to_produce = PossibleMapToProduce(set(), expected_input_metadata) - mock_initializer.get_maps_that_should_be_produced.return_value = [possible_map_to_produce] + mock_initializer = mock_initializer_class.return_value + mock_processor = mock_processor_class.return_value - mock_processor.process.return_value = [sentinel.cdf] + possible_map_to_produce = PossibleMapToProduce(set(), expected_input_metadata) + mock_initializer.get_maps_that_should_be_produced.return_value = [possible_map_to_produce] - imap_l3_processor() + mock_processor.process.return_value = [sentinel.cdf] - mock_initializer.get_maps_that_should_be_produced.assert_has_calls([ - call(descriptor, expected_major_version) for descriptor in expected_descriptors - ]) + imap_l3_processor() - self.assertEqual(len(expected_descriptors), mock_processor_class.call_count) + mock_initializer.get_maps_that_should_be_produced.assert_has_calls([ + call(descriptor, expected_major_version) for descriptor in expected_descriptors + ]) - self.assertEqual(len(expected_descriptors), mock_initializer.furnish_spice_dependencies.call_count) - mock_initializer.furnish_spice_dependencies.assert_called_with(possible_map_to_produce) + self.assertEqual(len(expected_descriptors), mock_processor_class.call_count) - mock_processor_class.assert_called_with(possible_map_to_produce.processing_input_collection, - expected_input_metadata) - self.assertEqual(len(expected_descriptors), mock_processor.process.call_count) + self.assertEqual(len(expected_descriptors), mock_initializer.furnish_spice_dependencies.call_count) + mock_initializer.furnish_spice_dependencies.assert_called_with(possible_map_to_produce) - self.assertEqual(len(expected_descriptors), mock_upload.call_count) - mock_upload.assert_called_with(sentinel.cdf) + mock_processor_class.assert_called_with(possible_map_to_produce.processing_input_collection, + expected_input_metadata) + self.assertEqual(len(expected_descriptors), mock_processor.process.call_count) + + self.assertEqual(len(expected_descriptors), mock_upload.call_count) + mock_upload.assert_called_with(sentinel.cdf) @patch('imap_l3_data_processor.imap_data_access.upload') @patch('imap_l3_data_processor.HiSPInitializer') @@ -279,7 +275,7 @@ def test_get_spice_kernels_based_on_input_collection(self, _, __, mock_processin @patch('imap_l3_data_processor.imap_data_access.upload') @patch('imap_l3_data_processor.SweProcessor') @patch('imap_l3_data_processor.argparse') - def test_uses_input_from_processing_input_collection(self, mock_argparse, mock_processor_class, mock_upload, + def test_sets_end_date_to_start_date_if_not_specified(self, mock_argparse, mock_processor_class, mock_upload, mock_processing_input_collection): cases = [("20170630", datetime(2017, 6, 30)), (None, datetime(2016, 6, 30))] @@ -334,7 +330,7 @@ def test_uses_input_from_processing_input_collection(self, mock_argparse, mock_p ]) expected_input_metadata = InputMetadata("swe", "l3", datetime(year=2016, month=6, day=30), - expected_end_date, "v092", "pitch-angle") + expected_end_date, VersionMap({}, Version(None, 92)), "pitch-angle") mock_processor_class.assert_called_with(imap_data_access_dependency, expected_input_metadata) @@ -544,8 +540,6 @@ def test_throws_exception_when_attempting_to_process_non_l3_data_levels(self, mo @patch('imap_l3_data_processor.argparse') def test_version_comes_from_dependency(self, mock_argparse, mock_processor_class, mock_upload, mock_processing_input_collection): - cases = [("20170630", datetime(2017, 6, 30)), (None, datetime(2016, 6, 30))] - instrument_argument = "swe" data_level_argument = "l3" start_date_argument = "20160630" @@ -556,13 +550,14 @@ def test_version_comes_from_dependency(self, mock_argparse, mock_processor_class ancillary_input = AncillaryInput("imap_swe_ancillary_20250101_v112.cdf") imap_data_access_dependency = ProcessingInputCollection(science_input_1, science_input_2, ancillary_input) - version_input = { - "sci": {"major_version": 2, "minor_version": 1} - } serialized_dependency_paths = imap_data_access_dependency.serialize() - combined_input = { - "version": version_input, - "dependency": json.loads(serialized_dependency_paths), + dependency_without_version_info = json.loads(serialized_dependency_paths) + dependency_with_version_info = { + "version": { + "sci": {"major_version": 2, "minor_version": 1}, + "var": {"major_version": 3, "minor_version": 2} + }, + "dependency": dependency_without_version_info, } mock_processing_input_collection.return_value = imap_data_access_dependency @@ -573,16 +568,18 @@ def test_version_comes_from_dependency(self, mock_argparse, mock_processor_class mock_argument_parser.parse_args.return_value.instrument = instrument_argument mock_argument_parser.parse_args.return_value.data_level = data_level_argument - mock_argument_parser.parse_args.return_value.dependency = json.dumps(combined_input) mock_argument_parser.parse_args.return_value.start_date = start_date_argument mock_argument_parser.parse_args.return_value.version = version_argument mock_argument_parser.parse_args.return_value.descriptor = descriptor_argument mock_argument_parser.parse_args.return_value.repointing = None mock_processor = mock_processor_class.return_value - - for input_end_date, expected_end_date in cases: - with self.subTest(input_end_date): + cases = [("version in dependency", dependency_with_version_info, VersionMap({'sci':Version(2,1), 'var':Version(3,2)})), + ("version on command line", dependency_without_version_info, VersionMap({}, Version(None,92)))] + input_end_date, expected_end_date = "20170630", datetime(2017, 6, 30) + for name, dependency, expected_version_map in cases: + with self.subTest(name): + mock_argument_parser.parse_args.return_value.dependency = json.dumps(dependency) mock_upload.reset_mock() imap_data_access_dependency.deserialize.reset_mock() @@ -607,57 +604,10 @@ def test_version_comes_from_dependency(self, mock_argparse, mock_processor_class ]) expected_input_metadata = InputMetadata("swe", "l3", datetime(year=2016, month=6, day=30), - expected_end_date, "v002.0001", "pitch-angle") + expected_end_date, expected_version_map, "pitch-angle") mock_processor_class.assert_called_with(imap_data_access_dependency, expected_input_metadata) imap_data_access_dependency.deserialize.assert_called_once_with(serialized_dependency_paths) mock_processor.process.assert_called() mock_upload.assert_called_once_with(mock_processor_class.return_value.process.return_value[0]) - - - @patch('imap_l3_data_processor.ProcessingInputCollection') - @patch('imap_l3_data_processor.imap_data_access.upload') - @patch('imap_l3_data_processor.SweProcessor') - @patch('imap_l3_data_processor.argparse') - def test_raises_error_if_multiple_versions_are_specified(self, mock_argparse, mock_processor_class, mock_upload, - mock_processing_input_collection): - - instrument_argument = "swe" - data_level_argument = "l3" - start_date_argument = "20160630" - version_argument = "v092" - descriptor_argument = "pitch-angle" - science_input_1 = ScienceInput("imap_swe_l1_sci_20250101_v112.cdf", "imap_swe_l1_sci_20250102_v112.cdf") - science_input_2 = ScienceInput("imap_mag_l1d_norm-dsrf_20250101_v112.cdf") - ancillary_input = AncillaryInput("imap_swe_ancillary_20250101_v112.cdf") - imap_data_access_dependency = ProcessingInputCollection(science_input_1, science_input_2, ancillary_input) - - version_input = { - "sci": {"major_version": 2, "minor_version": 1}, - "another_var": {"major_version": 2, "minor_version": 1} - } - combined_input = { - "version": version_input, - "dependency": json.loads(imap_data_access_dependency.serialize()), - } - - mock_processing_input_collection.return_value = imap_data_access_dependency - mock_processing_input_collection.deserialize = Mock() - mock_processing_input_collection.get_science_inputs = Mock(return_value=[]) - - mock_argument_parser = mock_argparse.ArgumentParser.return_value - - mock_argument_parser.parse_args.return_value.instrument = instrument_argument - mock_argument_parser.parse_args.return_value.data_level = data_level_argument - mock_argument_parser.parse_args.return_value.dependency = json.dumps(combined_input) - mock_argument_parser.parse_args.return_value.start_date = start_date_argument - mock_argument_parser.parse_args.return_value.version = version_argument - mock_argument_parser.parse_args.return_value.descriptor = descriptor_argument - mock_argument_parser.parse_args.return_value.repointing = None - mock_argument_parser.parse_args.return_value.end_date = None - - with self.assertRaises(ValueError) as exception: - mock_processor_class.return_value.process.return_value = [Mock()] - imap_l3_processor() - self.assertEqual(str(exception.exception), "Expected only a single version to be specified in the dependency.") diff --git a/tests/test_models.py b/tests/test_models.py index cfd8d8895..216687320 100644 --- a/tests/test_models.py +++ b/tests/test_models.py @@ -2,8 +2,9 @@ from datetime import datetime, timedelta import numpy as np +from imap_data_access.file_validation import Version -from imap_l3_processing.models import MagData +from imap_l3_processing.models import MagData, VersionMap class TestModels(unittest.TestCase): @@ -49,6 +50,66 @@ def test_calculate_average_mag_data_handles_missing_mag_data(self): epoch_delta=hit_data_delta) np.testing.assert_array_equal(actual_average, expected_average) + def test_version_map(self): + version_map = VersionMap( + { + "sci": Version(1, 5), + "other": Version(2, 90), + }, + fallback=Version(1, 99), + ) + self.assertEqual(version_map.lookup("sci"), Version(1, 5)) + self.assertEqual(version_map.lookup("other"), Version(2, 90)) + self.assertEqual(version_map.lookup("unknown"), Version(1, 99)) + + fallback_version_map = VersionMap( + {}, + fallback=Version(1, 1), + ) + self.assertEqual(fallback_version_map.lookup("sci"), Version(1, 1)) + + def test_version_map_throws_error_if_no_fallback(self): + version_map = VersionMap( + { + "sci": Version(1, 5) + }, + ) + + with self.assertRaises(KeyError): + version_map.lookup("other") + + + def test_version_map_equality(self): + version_map = VersionMap( + { + "sci": Version(1, 5), + "other": Version(2, 90), + }, + fallback=Version(1, 99), + ) + equal_version_map = VersionMap( + { + "sci": Version(1, 5), + "other": Version(2, 90), + }, + fallback=Version(1, 99), + ) + different_map = VersionMap( + { + "sci": Version(1, 3), + }, + fallback=Version(1, 99), + ) + different_fallback = VersionMap( + { + "sci": Version(1, 5), + "other": Version(2, 90), + }, + fallback=None, + ) + self.assertEqual(version_map, equal_version_map) + self.assertNotEqual(version_map, different_map) + self.assertNotEqual(version_map, different_fallback) if __name__ == '__main__': unittest.main() diff --git a/tests/test_utils.py b/tests/test_utils.py index 16f476c9b..8cba99c83 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -8,6 +8,7 @@ import imap_data_access import numpy as np from imap_data_access import config +from imap_data_access.file_validation import Version from imap_data_access.processing_input import ScienceInput, ProcessingInputCollection from imap_processing.spice.geometry import SpiceFrame from requests import RequestException @@ -16,7 +17,7 @@ from imap_l3_processing.constants import TEMP_CDF_FOLDER_PATH, TT2000_EPOCH from imap_l3_processing.maps.map_models import GlowsL3eRectangularMapInputData, InputRectangularPointingSet, \ RectangularSpectralIndexDataProduct, RectangularIntensityDataProduct -from imap_l3_processing.models import InputMetadata +from imap_l3_processing.models import InputMetadata, VersionMap from imap_l3_processing.swapi.l3a.models import SwapiL3AlphaSolarWindData from imap_l3_processing.swapi.quality_flags import SwapiL3Flags from imap_l3_processing.utils import format_time, read_mag_data, save_data, \ @@ -26,7 +27,7 @@ from imap_l3_processing.version import VERSION from tests.cdf.test_cdf_utils import TestDataProduct from tests.maps.test_builders import create_rectangular_spectral_index_map_data, create_rectangular_intensity_map_data -from tests.test_helpers import get_spice_data_path, with_tempdir, create_dataclass_mock +from tests.test_helpers import get_spice_data_path, with_tempdir, create_dataclass_mock, create_mock_version_map class TestUtils(TestCase): @@ -45,7 +46,7 @@ def tearDown(self) -> None: def test_save_data(self, mock_science_file_path_class, mock_write_cdf, mock_today, mock_attribute_manager): mock_today.today.return_value = date(2024, 9, 16) - input_metadata = InputMetadata("swapi", "l2", datetime(2024, 9, 17), datetime(2024, 9, 18), "v002", + input_metadata = InputMetadata("swapi", "l2", datetime(2024, 9, 17), datetime(2024, 9, 18), VersionMap({"descriptor":Version(1,2)}), "descriptor", repointing=None) epoch = np.array([1, 2, 3]) @@ -77,7 +78,7 @@ def test_save_data(self, mock_science_file_path_class, mock_write_cdf, mock_toda data_level="l2", descriptor="descriptor", start_time="20240917", - version="v002", + version="v001.0002", repointing=None, cr=sentinel.cr ) @@ -88,7 +89,7 @@ def test_save_data(self, mock_science_file_path_class, mock_write_cdf, mock_toda mock_attribute_manager.return_value) mock_attribute_manager.return_value.add_global_attribute.assert_has_calls([ - call("Data_version", "002"), + call("Data_version", "001.0002"), call("Generation_date", "20240916"), call("Logical_source", "imap_swapi_l2_descriptor"), call("Logical_file_id", expected_file_path.stem), @@ -196,8 +197,9 @@ def test_save_throws_exception_if_given_cr_and_repointing(self, _, mock_today, _ def test_save_data_does_not_add_parent_attribute_if_empty(self, mock_write_cdf, mock_today, _): mock_today.today.return_value = date(2024, 9, 16) - input_metadata = InputMetadata("swapi", "l2", datetime(2024, 9, 17), datetime(2024, 9, 18), "v002", - "descriptor") + descriptor = "descriptor" + input_metadata = InputMetadata("swapi", "l2", datetime(2024, 9, 17), datetime(2024, 9, 18), create_mock_version_map(descriptor=descriptor, minor_version=2), + descriptor) epoch = np.array([1, 2, 3]) quality_flags = np.repeat([SwapiL3Flags.NONE], 3) @@ -235,8 +237,9 @@ def test_save_data_does_not_add_parent_attribute_if_empty(self, mock_write_cdf, def test_save_data_custom_path(self, mock_write_cdf, mock_today, _): mock_today.today.return_value = date(2024, 9, 16) - input_metadata = InputMetadata("swapi", "l2", datetime(2024, 9, 17), datetime(2024, 9, 18), "v002", - "descriptor") + descriptor = "descriptor" + input_metadata = InputMetadata("swapi", "l2", datetime(2024, 9, 17), datetime(2024, 9, 18), create_mock_version_map(descriptor=descriptor, minor_version=2), + descriptor) epoch = np.array([1, 2, 3]) quality_flags = np.repeat([SwapiL3Flags.NONE], 3) @@ -270,7 +273,9 @@ def test_save_data_custom_path(self, mock_write_cdf, mock_today, _): @patch("imap_l3_processing.utils.write_cdf") @patch("imap_l3_processing.utils.ImapAttributeManager.add_global_attribute") def test_save_data_procedurally_generates_map_global_metadata(self, mock_add_global_attr, _): - input_metadata = InputMetadata("ultra", "l3", datetime(2025, 1, 1), datetime(2025, 1, 1), "v001", "descriptor") + descriptor = "descriptor" + input_metadata = InputMetadata("ultra", "l3", datetime(2025, 1, 1), datetime(2025, 1, 1), create_mock_version_map(descriptor=descriptor, minor_version=1), + descriptor) epoch_as_datetime = np.array([datetime(2025, 1, 1)]) epoch_as_int = np.array([(datetime(2025, 1, 1) - TT2000_EPOCH).total_seconds() * 1e9]) @@ -302,8 +307,9 @@ def test_save_data_procedurally_generates_map_global_metadata(self, mock_add_glo @patch("imap_l3_processing.utils.write_cdf") def test_save_data_procedurally_generates_all_map_global_metadata_if_absent(self, mock_write_cdf, mock_add_instrument_attrs): - non_map_input_metadata = InputMetadata("swapi", "l3", datetime(2024, 9, 17), datetime(2024, 9, 18), "v002", - "descriptor") + descriptor = "descriptor" + non_map_input_metadata = InputMetadata("swapi", "l3", datetime(2024, 9, 17), datetime(2024, 9, 18), create_mock_version_map(descriptor=descriptor, minor_version=2) , + descriptor) def add_swapi_attrs_from_file(attr_manager, instrument, level, descriptor): for logical_source in ["imap_swapi_l3_descriptor", "imap_ultra_l3_ena-map-descriptor"]: @@ -315,11 +321,11 @@ def add_swapi_attrs_from_file(attr_manager, instrument, level, descriptor): mock_add_instrument_attrs.side_effect = add_swapi_attrs_from_file - map_input_metadata = InputMetadata("hi", "l3b", datetime(2024, 9, 17), datetime(2024, 9, 18), "v002", + map_input_metadata = InputMetadata("hi", "l3b", datetime(2024, 9, 17), datetime(2024, 9, 18), create_mock_version_map(descriptor="spx-map-descriptor", minor_version=2), "spx-map-descriptor") map_input_metadata_with_existing_global_attrs = InputMetadata("ultra", "l3", datetime(2024, 9, 17), - datetime(2024, 9, 18), "v002", + datetime(2024, 9, 18), create_mock_version_map(descriptor="ena-map-descriptor", minor_version=2), "ena-map-descriptor") cases = [ diff --git a/tests/ultra/test_ultra_combined_nsp_initializer.py b/tests/ultra/test_ultra_combined_nsp_initializer.py index b77b64699..729fa0c70 100644 --- a/tests/ultra/test_ultra_combined_nsp_initializer.py +++ b/tests/ultra/test_ultra_combined_nsp_initializer.py @@ -5,7 +5,7 @@ from imap_data_access.file_validation import Version from imap_l3_processing.maps.map_initializer import MapInitializer, PossibleMapToProduce -from imap_l3_processing.models import InputMetadata +from imap_l3_processing.models import InputMetadata, VersionMap from imap_l3_processing.ultra.ultra_combined_nsp_initializer import UltraCombinedNSPInitializer from tests.integration.integration_test_helpers import ImapQueryPatcher @@ -22,12 +22,60 @@ def tearDown(self): def test_is_map_initializer(self): self.assertIsInstance(UltraCombinedNSPInitializer(), MapInitializer) + @patch('imap_l3_processing.maps.map_initializer.read_cdf_parents') + def test_get_maps_that_can_be_produced(self, mock_read_cdf_parents): + self.mock_query.side_effect = ImapQueryPatcher([ + 'imap_ultra_l2_u45-ena-h-sf-nsp-full-hae-4deg-3mo_20100101_v001.cdf', + 'imap_ultra_l2_u90-ena-h-sf-nsp-full-hae-4deg-3mo_20100101_v001.cdf', + ]) + + initializer = UltraCombinedNSPInitializer() + descriptor = 'ulc-ena-h-sf-nsp-full-hae-4deg-3mo' + mock_read_cdf_parents.side_effect = [ + [ + "imap_ultra_l1c_45sensor-spacecraftpset_20100101-repoint00001_v001.cdf", + "imap_ultra_l1c_45sensor-spacecraftpset_20100102-repoint00002_v001.cdf", + "imap_ultra_l1c_45sensor-spacecraftpset_20100103-repoint00003_v001.cdf", + + ], + [ + "imap_ultra_l1c_90sensor-spacecraftpset_20100101-repoint00001_v001.cdf", + "imap_ultra_l1c_90sensor-spacecraftpset_20100102-repoint00002_v001.cdf", + "imap_ultra_l1c_90sensor-spacecraftpset_20100103-repoint00003_v001.cdf" + ], + ] + actual_possible_maps = initializer.get_maps_that_can_be_produced(descriptor) + + expected_possible_maps = [ + PossibleMapToProduce( + input_files={ + 'imap_ultra_l2_u45-ena-h-sf-nsp-full-hae-4deg-3mo_20100101_v001.cdf', + 'imap_ultra_l2_u90-ena-h-sf-nsp-full-hae-4deg-3mo_20100101_v001.cdf', + "imap_ultra_l1c_45sensor-spacecraftpset_20100101-repoint00001_v001.cdf", + "imap_ultra_l1c_45sensor-spacecraftpset_20100102-repoint00002_v001.cdf", + "imap_ultra_l1c_45sensor-spacecraftpset_20100103-repoint00003_v001.cdf", + "imap_ultra_l1c_90sensor-spacecraftpset_20100101-repoint00001_v001.cdf", + "imap_ultra_l1c_90sensor-spacecraftpset_20100102-repoint00002_v001.cdf", + "imap_ultra_l1c_90sensor-spacecraftpset_20100103-repoint00003_v001.cdf", + }, + input_metadata=InputMetadata( + instrument='ultra', + data_level='l3', + start_date=datetime(2010, 1, 1), + end_date=datetime(2010, 4, 2, 7, 30), + version=VersionMap({},Version(None,1)), + descriptor=descriptor, + ) + ) + ] + + self.assertEqual(expected_possible_maps, actual_possible_maps) + @patch('imap_l3_processing.maps.map_initializer.read_cdf_parents') def test_get_maps_that_should_be_produced(self, mock_read_cdf_parents): - self.maxDiff = None cases = [ - (None, "v002"), - (4, "v004.0002"), + (None, Version(None,2)), + (4, Version(4,2)), ] for major_version, expected_version in cases: with self.subTest(major_version=major_version): @@ -126,14 +174,16 @@ def test_get_maps_that_should_be_produced(self, mock_read_cdf_parents): call(f'imap_ultra_l3_ulc-ena-h-sf-nsp-full-hae-4deg-3mo_20100401_{Version(major_version, 1)}.cdf'), ]) + descriptor = "ulc-ena-h-sf-nsp-full-hae-4deg-3mo" + expected_possible_map_to_produce_with_previous_version = PossibleMapToProduce( input_metadata=InputMetadata( instrument="ultra", data_level="l3", start_date=datetime(2010, 4, 1), end_date=datetime(2010, 7, 1, 7, 30), - version=expected_version, - descriptor=f"ulc-ena-h-sf-nsp-full-hae-4deg-3mo" + version=VersionMap({descriptor:expected_version}), + descriptor=descriptor ), input_files={ f'imap_ultra_l1c_45sensor-spacecraftpset_20100401-repoint00101_v001.cdf', @@ -153,8 +203,8 @@ def test_get_maps_that_should_be_produced(self, mock_read_cdf_parents): data_level="l3", start_date=datetime(2010, 7, 1), end_date=datetime(2010, 9, 30, 7, 30), - version=str(Version(major_version, 1)), - descriptor=f"ulc-ena-h-sf-nsp-full-hae-4deg-3mo" + version=VersionMap({descriptor: Version(major_version, 1)}), + descriptor=descriptor ), input_files={ f'imap_ultra_l1c_45sensor-spacecraftpset_20100701-repoint00201_v001.cdf', diff --git a/tests/ultra/test_ultra_sp_initializer.py b/tests/ultra/test_ultra_sp_initializer.py index bcc874d5c..09f938116 100644 --- a/tests/ultra/test_ultra_sp_initializer.py +++ b/tests/ultra/test_ultra_sp_initializer.py @@ -2,10 +2,12 @@ from datetime import datetime from unittest.mock import patch, call +from imap_data_access.file_validation import Version + from imap_l3_processing.glows.descriptors import GLOWS_L3E_ULTRA_SF_DESCRIPTOR, GLOWS_L3E_ULTRA_HF_DESCRIPTOR from imap_l3_processing.maps.map_initializer import PossibleMapToProduce from imap_l3_processing.maps.sp_map_initializer import SPMapInitializer -from imap_l3_processing.models import InputMetadata +from imap_l3_processing.models import InputMetadata, VersionMap from imap_l3_processing.ultra.ultra_sp_initializer import UltraSPInitializer, ULTRA_SP_SPICE_KERNELS from tests.integration.integration_test_helpers import ImapQueryPatcher @@ -125,15 +127,15 @@ def test_get_maps_that_should_be_produced(self, mock_read_cdf_parents): call(f'imap_ultra_l3_u{sensor}-ena-h-sf-sp-full-hae-4deg-3mo_20100101_v001.cdf'), call(f'imap_ultra_l3_u{sensor}-ena-h-sf-sp-full-hae-4deg-3mo_20100401_v001.cdf'), ]) - + descriptor = f"u{sensor}-ena-h-sf-sp-full-hae-4deg-3mo" expected_possible_map_to_produce = PossibleMapToProduce( input_metadata=InputMetadata( instrument="ultra", data_level="l3", start_date=datetime(2010, 4, 1), end_date=datetime(2010, 7, 1, 7, 30), - version="v002", - descriptor=f"u{sensor}-ena-h-sf-sp-full-hae-4deg-3mo" + version=VersionMap({descriptor: Version(None, 2)}), + descriptor=descriptor ), input_files={ 'imap_glows_l3e_survival-probability-ul-sf_20100401-repoint00101_v002.cdf', @@ -277,6 +279,7 @@ def test_get_maps_that_should_be_produced_lists_combined_maps(self, mock_read_cd call(f'imap_ultra_l3_ulc-ena-h-sf-sp-full-hae-4deg-3mo_20100101_v001.cdf'), call(f'imap_ultra_l3_ulc-ena-h-sf-sp-full-hae-4deg-3mo_20100401_v001.cdf'), ]) + descriptor = f"ulc-ena-h-sf-sp-full-hae-4deg-3mo" expected_possible_map_to_produce = PossibleMapToProduce( input_metadata=InputMetadata( @@ -284,8 +287,8 @@ def test_get_maps_that_should_be_produced_lists_combined_maps(self, mock_read_cd data_level="l3", start_date=datetime(2010, 4, 1), end_date=datetime(2010, 7, 1, 7, 30), - version="v002", - descriptor=f"ulc-ena-h-sf-sp-full-hae-4deg-3mo" + version=VersionMap({descriptor: Version(None, 2)}), + descriptor=descriptor ), input_files={ 'imap_glows_l3e_survival-probability-ul-sf_20100401-repoint00101_v002.cdf', @@ -434,14 +437,15 @@ def test_get_maps_that_should_be_produced_lists_cg_corrected_maps(self, mock_rea call(f'imap_ultra_l3_ulc-ena-h-hf-sp-full-hae-4deg-3mo_20100401_v001.cdf'), ]) + descriptor = f"ulc-ena-h-hf-sp-full-hae-4deg-3mo" expected_possible_map_to_produce = PossibleMapToProduce( input_metadata=InputMetadata( instrument="ultra", data_level="l3", start_date=datetime(2010, 4, 1), end_date=datetime(2010, 7, 1, 7, 30), - version="v002", - descriptor=f"ulc-ena-h-hf-sp-full-hae-4deg-3mo" + version=VersionMap({descriptor: Version(None, 2)}), + descriptor=descriptor, ), input_files={ 'imap_glows_l3e_survival-probability-ul-hf_20100401-repoint00101_v002.cdf', @@ -466,13 +470,14 @@ def test_furnish_spice_dependencies(self, mock_furnish_metakernel): start_date = datetime(2025, 4, 15) end_date = datetime(2025, 7, 15) + descriptor = "u90-ena-h-sf-sp-full-hae-4deg-3mo" input_metadata = InputMetadata( instrument="ultra", data_level="l2", start_date=start_date, end_date=end_date, - version="v000", - descriptor="u90-ena-h-sf-sp-full-hae-4deg-3mo", + version=VersionMap({descriptor: Version(None, 0)}), + descriptor=descriptor ) map_to_produce = PossibleMapToProduce(set(), input_metadata) From 429e2fee2f403239538f06ecbe2736dfd88ccbe6 Mon Sep 17 00:00:00 2001 From: Harrison Date: Thu, 2 Jul 2026 10:45:25 -0400 Subject: [PATCH 07/11] Harrison 3295 #118 - AFAH/KJON - WIP implementing new versioning scheme to use VersionMap for GLOWS processing L3b, L3c, and L3d --- imap_l3_processing/glows/glows_processor.py | 18 +-- tests/glows/test_glows_processor.py | 136 ++++++++++++------ .../test_glows_processor_integration.py | 7 +- 3 files changed, 106 insertions(+), 55 deletions(-) diff --git a/imap_l3_processing/glows/glows_processor.py b/imap_l3_processing/glows/glows_processor.py index b85d60d08..9100aa3df 100644 --- a/imap_l3_processing/glows/glows_processor.py +++ b/imap_l3_processing/glows/glows_processor.py @@ -40,7 +40,7 @@ from imap_l3_processing.glows.l3e.glows_l3e_ultra_model import GlowsL3EUltraData from imap_l3_processing.glows.l3e.glows_l3e_utils import determine_call_args_for_l3e_executable, get_lo_pivot_angles, \ compute_glows_flags_for_window -from imap_l3_processing.models import InputMetadata +from imap_l3_processing.models import InputMetadata, VersionMap from imap_l3_processing.processor import Processor from imap_l3_processing.utils import save_data @@ -63,8 +63,8 @@ def process(self): return products elif self.input_metadata.data_level == "l3b": products_list = [] - major_version = Version.from_version(self.input_metadata.version).major - l3bc_initializer_data: GlowsL3BCInitializerData = GlowsL3BCInitializer.get_crs_to_process(self.dependencies, major_version) + l3bc_major_version = self.input_metadata.version.lookup(GLOWS_L3B_DESCRIPTOR).major + l3bc_initializer_data: GlowsL3BCInitializerData = GlowsL3BCInitializer.get_crs_to_process(self.dependencies, l3bc_major_version) if len(l3bc_initializer_data.l3bc_dependencies) > 0: logger.info("Found CRs to Process L3BC:") @@ -77,10 +77,11 @@ def process(self): glows_l3bc_output_data = process_l3bc(self, l3bc_initializer_data) products_list.extend(glows_l3bc_output_data.data_products) + l3d_major_version = self.input_metadata.version.lookup(GLOWS_L3D_DESCRIPTOR).major l3bs = list({**l3bc_initializer_data.l3bs_by_cr, **glows_l3bc_output_data.l3bs_by_cr}.values()) l3cs = list({**l3bc_initializer_data.l3cs_by_cr, **glows_l3bc_output_data.l3cs_by_cr}.values()) l3d_initializer_result = GlowsL3DInitializer.should_process_l3d(l3bc_initializer_data.external_dependencies, - l3bs, l3cs, major_version) + l3bs, l3cs, l3d_major_version) if l3d_initializer_result is None: logger.info("No inputs to L3d have changed. Skipping processing of L3d and L3e!") return products_list @@ -168,14 +169,15 @@ def process_l3bc(processor, initializer_data: GlowsL3BCInitializerData): filtered_days = filter_l3a_files(dependency.l3a_data, dependency.ancillary_files['bad_days_list'], dependency.carrington_rotation_number) with SwallowExceptionAndLog(f"Exception caught in L3BC science code, skipping CR {dependency.carrington_rotation_number}") as manager: - l3b_data, l3c_data = generate_l3bc(replace(dependency, l3a_data=filtered_days)) + dependency_filtering_out_bad_days = replace(dependency, l3a_data=filtered_days) + l3b_data, l3c_data = generate_l3bc(dependency_filtering_out_bad_days) if manager.exception_caught: continue l3b_metadata = InputMetadata("glows", "l3b", dependency.start_date, dependency.end_date, - str(dependency.version), GLOWS_L3B_DESCRIPTOR) + VersionMap({GLOWS_L3B_DESCRIPTOR: dependency.version}), GLOWS_L3B_DESCRIPTOR) l3c_metadata = InputMetadata("glows", "l3c", dependency.start_date, dependency.end_date, - str(dependency.version), GLOWS_L3C_DESCRIPTOR) + VersionMap({GLOWS_L3C_DESCRIPTOR: dependency.version}), GLOWS_L3C_DESCRIPTOR) l3b_data_product = GlowsL3BIonizationRate.from_instrument_team_dictionary(l3b_data, l3b_metadata) l3c_data_product = GlowsL3CSolarWind.from_instrument_team_dictionary(l3c_data, l3c_metadata) @@ -248,7 +250,7 @@ def process_l3d( start_date = datetime(1947, 3, 3) data_product_metadata = InputMetadata(instrument="glows", data_level="l3d", descriptor=GLOWS_L3D_DESCRIPTOR, - start_date=start_date, end_date=start_date, version=str(version)) + start_date=start_date, end_date=start_date, version=VersionMap({GLOWS_L3D_DESCRIPTOR: version})) parent_file_names = get_parent_file_names_from_l3d_json(PATH_TO_L3D_TOOLKIT / 'data_l3d') l3d_data_product = convert_json_to_l3d_data_product(PATH_TO_L3D_TOOLKIT / 'data_l3d' / file_name, diff --git a/tests/glows/test_glows_processor.py b/tests/glows/test_glows_processor.py index a83489797..9ce9faccc 100644 --- a/tests/glows/test_glows_processor.py +++ b/tests/glows/test_glows_processor.py @@ -22,7 +22,8 @@ from imap_l3_processing.constants import TEMP_CDF_FOLDER_PATH from imap_l3_processing.glows import l3d from imap_l3_processing.glows.descriptors import GLOWS_L3A_DESCRIPTOR, GLOWS_L3E_ULTRA_SF_DESCRIPTOR, \ - GLOWS_L3E_ULTRA_HF_DESCRIPTOR + GLOWS_L3E_ULTRA_HF_DESCRIPTOR, GLOWS_L3B_DESCRIPTOR, GLOWS_L3C_DESCRIPTOR, GLOWS_L3D_DESCRIPTOR, \ + GLOWS_L3E_HI_45_DESCRIPTOR, GLOWS_L3E_HI_90_DESCRIPTOR, GLOWS_L3E_LO_DESCRIPTOR from imap_l3_processing.glows.glows_processor import GlowsProcessor, process_l3d, process_l3e, process_l3bc, \ process_l3e_ul_sf, process_l3e_hi, process_l3e_lo, process_l3e_ul_hf from imap_l3_processing.glows.l3a.utils import create_glows_l3a_dictionary_from_cdf, create_glows_l3a_from_dictionary @@ -39,7 +40,7 @@ from imap_l3_processing.glows.l3e.glows_l3e_lo_model import GlowsL3ELoData from imap_l3_processing.glows.l3e.glows_l3e_ultra_model import GlowsL3EUltraData from imap_l3_processing.glows.l3e.glows_l3e_utils import GlowsL3eRepointings, LoPivotAngle -from imap_l3_processing.models import InputMetadata +from imap_l3_processing.models import InputMetadata, VersionMap from imap_l3_processing.utils import save_data from tests.test_helpers import get_test_instrument_team_data_path, get_test_data_path, get_test_data_folder, \ assert_dataclass_fields @@ -174,7 +175,6 @@ def test_does_not_save_cdf_if_process_l3a_returns_none(self, mock_save_data, _): mock_save_data.assert_not_called() self.assertEqual([], products) - @patch('imap_l3_processing.glows.glows_processor.create_glows_l3a_from_dictionary') @patch('imap_l3_processing.glows.glows_processor.L3aData') def test_process_l3a(self, l3a_data_constructor, create_glows_l3a_from_dictionary): @@ -208,7 +208,8 @@ def test_process_l3a(self, l3a_data_constructor, create_glows_l3a_from_dictionar @patch('imap_l3_processing.glows.glows_processor.create_glows_l3a_from_dictionary') @patch('imap_l3_processing.glows.glows_processor.L3aData') - def test_process_l3a_returns_none_when_no_input_histrogram_bins_are_valid(self, l3a_data_constructor, create_glows_l3a_from_dictionary): + def test_process_l3a_returns_none_when_no_input_histrogram_bins_are_valid(self, l3a_data_constructor, + create_glows_l3a_from_dictionary): input_metadata = InputMetadata('glows', "l3a", datetime(2024, 10, 7, 10, 00, 00), datetime(2024, 10, 8, 10, 00, 00), 'v02') @@ -335,11 +336,22 @@ def test_process_l3bc(self, mock_glows_l3bc_initializer, mock_archive_dependenci Path("path/to/l3c_file_1.cdf"), Path("path/to/l3b_file_2.cdf"), Path("path/to/l3c_file_2.cdf")] - - input_version = Version(2,1) + input_major_version = 2 + input_version = VersionMap( + { + GLOWS_L3B_DESCRIPTOR: Version(input_major_version, 1), + GLOWS_L3C_DESCRIPTOR: Version(input_major_version, 1), + GLOWS_L3D_DESCRIPTOR: Version(5, 1), + GLOWS_L3E_HI_45_DESCRIPTOR: Version(5, 1), + GLOWS_L3E_HI_90_DESCRIPTOR: Version(5, 1), + GLOWS_L3E_LO_DESCRIPTOR: Version(5, 1), + GLOWS_L3E_ULTRA_SF_DESCRIPTOR: Version(5, 1), + GLOWS_L3E_ULTRA_HF_DESCRIPTOR: Version(5, 1), + } + ) input_metadata = InputMetadata('glows', "l3b", datetime(2024, 10, 7, 10, 00, 00), datetime(2024, 10, 8, 10, 00, 00), - str(input_version)) + version=input_version) mock_glows_l3bc_initializer.get_crs_to_process.return_value = GlowsL3BCInitializerData( external_dependencies=external_deps, @@ -370,16 +382,25 @@ def test_process_l3bc(self, mock_glows_l3bc_initializer, mock_archive_dependenci mock_generate_l3bc.assert_has_calls( [call(dependencies_with_filtered_list_1), call(dependencies_with_filtered_list_2)]) - mock_glows_l3bc_initializer.get_crs_to_process.assert_called_once_with(sentinel.dependencies, input_version.major) + mock_glows_l3bc_initializer.get_crs_to_process.assert_called_once_with(sentinel.dependencies, + input_major_version) expected_l3b_metadata_1 = InputMetadata("glows", "l3b", first_dependency.start_date, - first_dependency.end_date, str(first_dependency_version), "ion-rate-profile") + first_dependency.end_date, + VersionMap({GLOWS_L3B_DESCRIPTOR: first_dependency_version}), + "ion-rate-profile") expected_l3b_metadata_2 = InputMetadata("glows", "l3b", second_dependency.start_date, - second_dependency.end_date, str(second_dependency_version), "ion-rate-profile") + second_dependency.end_date, + VersionMap({GLOWS_L3B_DESCRIPTOR: second_dependency_version}), + "ion-rate-profile") expected_l3c_metadata_1 = InputMetadata("glows", "l3c", first_dependency.start_date, - first_dependency.end_date, str(first_dependency_version), "sw-profile") + first_dependency.end_date, + VersionMap({GLOWS_L3C_DESCRIPTOR: first_dependency_version}), + "sw-profile") expected_l3c_metadata_2 = InputMetadata("glows", "l3c", second_dependency.start_date, - second_dependency.end_date, str(second_dependency_version), "sw-profile") + second_dependency.end_date, + VersionMap({GLOWS_L3C_DESCRIPTOR: second_dependency_version}), + "sw-profile") mock_l3b_model_class.from_instrument_team_dictionary.assert_has_calls( [call(sentinel.l3b_data_1, expected_l3b_metadata_1), call(sentinel.l3b_data_2, expected_l3b_metadata_2)]) @@ -425,7 +446,7 @@ def test_process_l3bc_catches_no_data_error_and_continues(self, mock_glows_initi ancillary_files=defaultdict(Mock), repointing_file_path=sentinel.repointing_file_path ) - bc_dependencies_2 = GlowsL3BCDependencies(version=Version(1,1), + bc_dependencies_2 = GlowsL3BCDependencies(version=Version(1, 1), carrington_rotation_number=2096, start_date=datetime(year=2021, month=1, day=1), end_date=datetime(year=2021, month=1, day=1), @@ -452,9 +473,20 @@ def test_process_l3bc_catches_no_data_error_and_continues(self, mock_glows_initi l3c_path = Path("path_to/l3c") mock_save_data.side_effect = [l3b_path, l3c_path] + version_map = VersionMap({ + GLOWS_L3B_DESCRIPTOR: Version(1, 1), + GLOWS_L3C_DESCRIPTOR: Version(1, 1), + GLOWS_L3D_DESCRIPTOR: Version(5, 1), + GLOWS_L3E_HI_45_DESCRIPTOR: Version(5, 1), + GLOWS_L3E_HI_90_DESCRIPTOR: Version(5, 1), + GLOWS_L3E_LO_DESCRIPTOR: Version(5, 1), + GLOWS_L3E_ULTRA_SF_DESCRIPTOR: Version(5, 1), + GLOWS_L3E_ULTRA_HF_DESCRIPTOR: Version(5, 1), + }) + input_metadata = InputMetadata('glows', "l3b", datetime(2024, 10, 7, 10, 00, 00), datetime(2024, 10, 8, 10, 00, 00), - 'v001') + version_map) processor = GlowsProcessor(Mock(), input_metadata) @@ -514,7 +546,7 @@ def test_process_l3bc_catches_exceptions_from_science_code_and_continues(self, m 'omni_raw_data': get_test_instrument_team_data_path('glows/omni_2010.dat'), }, carrington_rotation_number=1, - start_date=Mock(), end_date=Mock(), version=Version(1,1), + start_date=Mock(), end_date=Mock(), version=Version(1, 1), repointing_file_path=sentinel.repointing_file_path ) @@ -559,7 +591,7 @@ def test_l3bc_uses_all_l3a_file_names_for_l3b_parents(self, mock_save_data, mock 'omni_raw_data': get_test_instrument_team_data_path('glows/omni2_all_years.dat'), }, carrington_rotation_number=2092, - start_date=Mock(), end_date=Mock(), version=Version(1,1), + start_date=Mock(), end_date=Mock(), version=Version(1, 1), repointing_file_path=sentinel.repointing_file_path, ) @@ -632,11 +664,12 @@ def test_process_l3d(self, mock_read_pipeline_settings, mock_glows_l3d_initializ ) old_l3d = Path('imap_glows_l3d_solar-hist_19470303-cr02090_v001.cdf') - l3d_output_version = Version(12, 5) + input_major_version = 12 + l3d_output_version = Version(input_major_version, 5) mock_glows_l3d_initializer.should_process_l3d.return_value = ( l3d_output_version, glows_l3d_dependencies, old_l3d) - mock_run.return_value = CompletedProcess(args=[], returncode=0, stdout=f'Processed CR= {expected_end_cr}') + mock_run.return_value = CompletedProcess(args=[], returncode=0, stdout=f'Processed CR= {expected_end_cr}') mock_os.listdir.return_value = [ f'{cr_number}_txt_file_1', @@ -654,13 +687,25 @@ def test_process_l3d(self, mock_read_pipeline_settings, mock_glows_l3d_initializ mock_save_data.return_value = Path("l3d_cdf.cdf") - processor = GlowsProcessor(Mock(), Mock(data_level="l3b", version=str(Version(1,1)))) + input_version_map = VersionMap({ + GLOWS_L3B_DESCRIPTOR: Version(2, 1), + GLOWS_L3C_DESCRIPTOR: Version(2, 1), + GLOWS_L3D_DESCRIPTOR: Version(input_major_version, 1), + GLOWS_L3E_HI_45_DESCRIPTOR: Version(2, 1), + GLOWS_L3E_HI_90_DESCRIPTOR: Version(2, 1), + GLOWS_L3E_LO_DESCRIPTOR: Version(2, 1), + GLOWS_L3E_ULTRA_SF_DESCRIPTOR: Version(2, 1), + GLOWS_L3E_ULTRA_HF_DESCRIPTOR: Version(2, 1), + }) + input_metadata = InputMetadata('glows', "l3b", datetime(2024, 10, 7), None, version=input_version_map) + + processor = GlowsProcessor(Mock(), input_metadata) products = processor.process() mock_convert_l3b_to_json.assert_has_calls([call(sentinel.l3b_file_1), call(sentinel.l3b_file_2)]) mock_convert_l3c_to_json.assert_has_calls([call(sentinel.l3c_file_1), call(sentinel.l3c_file_2)]) mock_glows_l3d_initializer.should_process_l3d.assert_called_with( - self.mock_external_deps, [], [], 1) + self.mock_external_deps, [], [], input_major_version) self.assertEqual([ Path("imap_glows_e-dens_19470303_20100101_v000.dat"), Path("imap_glows_lya_19470303_20100101_v000.dat"), @@ -723,7 +768,7 @@ def test_process_l3d(self, mock_read_pipeline_settings, mock_glows_l3d_initializ expected_data_product_metadata = InputMetadata(instrument="glows", data_level="l3d", descriptor="solar-hist", start_date=datetime(1947, 3, 3), end_date=datetime(1947, 3, 3), - version=f"{l3d_output_version}") + version=VersionMap({GLOWS_L3D_DESCRIPTOR: l3d_output_version})) mock_convert_json_to_l3d_data_product.assert_called_once_with( expected_working_directory / 'data_l3d' / f'imap_glows_l3d_solar-params-history_19470303-cr0{expected_end_cr}_v00.json', @@ -777,9 +822,10 @@ def test_process_l3d_adds_parent_file_names_to_output(self, mock_spicepy, mock_r mock_run.return_value = CompletedProcess(args=[], returncode=0, stdout=f'Processed CR= {cr_number}') l3d_dependencies = GlowsL3DDependencies(l3b_file_paths=l3b_file_paths, l3c_file_paths=l3c_file_paths, - ancillary_files=ancillary_inputs, external_files=external_inputs, end_cr=cr_number) + ancillary_files=ancillary_inputs, external_files=external_inputs, + end_cr=cr_number) - glows_l3d_output = process_l3d(l3d_dependencies, Version(1,1)) + _ = process_l3d(l3d_dependencies, Version(1, 1)) [save_data_call_args] = mock_save_data.call_args_list actual_data_product = save_data_call_args.args[0] @@ -827,7 +873,6 @@ def test_process_l3d_adds_parent_file_names_to_output(self, mock_spicepy, mock_r ) self.assertEqual("v001", mock_rename_l3d_text_output.call_args[0][1]) - @patch('imap_l3_processing.glows.glows_processor.rename_l3d_text_outputs') @patch('imap_l3_processing.glows.glows_processor.get_parent_file_names_from_l3d_json') @patch('imap_l3_processing.glows.glows_processor.os') @@ -870,7 +915,6 @@ def test_process_l3d_handles_unexpected_exception_from_science(self, mock_read_p '2096_txt_file_6' ] - unexpected_exception = r"""Traceback (most recent call last): File "...\imap-L3-processing\imap_l3_processing\glows\l3d\science\generate_l3d.py", line 46, in solar_param_hist.update_solar_params_hist(EXT_DEPENDENCIES,data_l3b,data_l3c) @@ -985,7 +1029,7 @@ def test_process_glows_l3d_drift(self, mock_save_data, _): "plasma_speed", 849, [1.94716799e+03, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - 1.25050000e+03, -1], + 1.25050000e+03, -1], [2010.49252260274, 397.0, 399.0, 406.0, 374.0, 316.0, 268.0, 228.0, 209.0, 202.0, 203.0, 198.0, 198.0, 235.0, 283.0, 332.0, 398.0, 436.0, 439.0, 439.0, 2098.5, 88888.0], ), @@ -994,7 +1038,7 @@ def test_process_glows_l3d_drift(self, mock_save_data, _): "uv_anisotropy", 849, [1.94716799e+03, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - 1.25050000e+03, -1], + 1.25050000e+03, -1], [2010.49252260274, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2098.5, 88888.0], ), ] @@ -1037,7 +1081,7 @@ def test_process_l3e(self, mock_process_ultra, mock_process_ultra_hf, mock_proce mock_process_lo.return_value = [Path('path/to/lo_l3e')] mock_process_ultra.return_value = [Path('path/to/ultra_l3e')] mock_process_ultra_hf.return_value = [Path('path/to/ultra_l3e_hf')] - mock_get_lo_pivot_angles.return_value = {25: LoPivotAngle("l1b_nhk.cdf",75)} + mock_get_lo_pivot_angles.return_value = {25: LoPivotAngle("l1b_nhk.cdf", 75)} mock_compute_glows_flags_for_window.return_value = 4 expected_l3e_products = [ @@ -1063,11 +1107,11 @@ def test_process_l3e(self, mock_process_ultra, mock_process_ultra_hf, mock_proce dependencies=mock_dependencies, repointings=GlowsL3eRepointings( repointing_numbers=[25], - hi_90_repointings={25: Version(None,1)}, - hi_45_repointings={25: Version(None,2)}, - lo_repointings={25: Version(None,3)}, - ultra_sf_repointings={25: Version(None,4)}, - ultra_hf_repointings={25: Version(None,4)}, + hi_90_repointings={25: Version(None, 1)}, + hi_45_repointings={25: Version(None, 2)}, + lo_repointings={25: Version(None, 3)}, + ultra_sf_repointings={25: Version(None, 4)}, + ultra_hf_repointings={25: Version(None, 4)}, ), l3d_cdf_path=l3d_cdf_path, ) @@ -1077,8 +1121,8 @@ def test_process_l3e(self, mock_process_ultra, mock_process_ultra_hf, mock_proce mock_compute_glows_flags_for_window.assert_called_once_with(l3d_cdf_path, start_epoch, end_epoch) mock_process_hi.assert_has_calls([ - call(["hi_ancillary.dat"], 25, start_epoch, epoch_delta, 90, Version(None,1), 4), - call(["hi_ancillary.dat"], 25, start_epoch, epoch_delta, 135, Version(None,2), 4) + call(["hi_ancillary.dat"], 25, start_epoch, epoch_delta, 90, Version(None, 1), 4), + call(["hi_ancillary.dat"], 25, start_epoch, epoch_delta, 135, Version(None, 2), 4) ]) mock_process_lo.assert_called_once_with(["lo_ancillary.dat", "l1b_nhk.cdf"], 25, start_epoch, epoch_delta, 75, Version(None, 3), 4) @@ -1097,15 +1141,15 @@ def test_process_l3e(self, mock_process_ultra, mock_process_ultra_hf, mock_proce @patch("imap_l3_processing.glows.glows_processor.determine_call_args_for_l3e_executable") @patch("imap_l3_processing.glows.glows_processor.shutil") def test_process_l3e_ultra_sf(self, mock_shutil, mock_determine_call_args, mock_run, - mock_convert_dat_to_glows_l3e_ul_product, - mock_save_data, mock_get_parent_file_names): + mock_convert_dat_to_glows_l3e_ul_product, + mock_save_data, mock_get_parent_file_names): mock_get_parent_file_names.return_value = ["l3d_file", "ancillary_1", "ancillary_2", "ancillary_3"] epoch_start_date = datetime(year=2024, month=10, day=7) epoch_end_date = datetime(year=2024, month=10, day=7, hour=23) epoch_delta = (epoch_end_date - epoch_start_date) / 2 repointing = 20 - version = Version(1,12) + version = Version(1, 12) expected_input_metadata = InputMetadata('glows', "l3e", start_date=epoch_start_date, end_date=epoch_end_date, version='v001.0012', descriptor=GLOWS_L3E_ULTRA_SF_DESCRIPTOR, @@ -1167,7 +1211,8 @@ def test_process_l3e_ultra_hf(self, mock_shutil, mock_determine_call_args, mock_ version = Version(1, 12) input_metadata = InputMetadata('glows', "l3e", start_date=epoch_start_date, end_date=epoch_end_date, - version='v001.0012', descriptor=GLOWS_L3E_ULTRA_HF_DESCRIPTOR, repointing=repointing) + version='v001.0012', descriptor=GLOWS_L3E_ULTRA_HF_DESCRIPTOR, + repointing=repointing) mock_determine_call_args.return_value = GlowsL3eCallArguments( formatted_date="20241007_000000", @@ -1302,7 +1347,8 @@ def test_process_l3e_hi(self, mock_shutil, mock_determine_call_args, survival_data_product: GlowsL3EHiData = mock_save_data.call_args_list[0].args[0] self.assertEqual(parent_file_names, survival_data_product.parent_file_names) - np.testing.assert_array_equal(survival_data_product.glows_flags, np.array([glows_flags], dtype=np.uint16)) + np.testing.assert_array_equal(survival_data_product.glows_flags, + np.array([glows_flags], dtype=np.uint16)) expected_output_data_path = AncillaryFilePath( f"imap_glows_survival-probability-hi-{descriptor_elongation}-raw_20241007_v012.dat" @@ -1332,7 +1378,7 @@ def test_process_l3e_lo(self, mock_shutil, epoch_end_date = datetime(year=2024, month=10, day=7, hour=23) epoch_delta = (epoch_end_date - epoch_start_date) / 2 repointing = 20 - version = Version(1,12) + version = Version(1, 12) expected_input_metadata = InputMetadata( instrument='glows', data_level="l3e", start_date=epoch_start_date, end_date=epoch_end_date, @@ -1391,7 +1437,8 @@ def test_process_l3e_lo(self, mock_shutil, survival_data_product: GlowsL3ELoData = mock_save_data.call_args_list[0].args[0] self.assertEqual(parent_file_names, survival_data_product.parent_file_names) - np.testing.assert_array_equal(survival_data_product.glows_flags, np.array([glows_flags], dtype=np.uint16)) + np.testing.assert_array_equal(survival_data_product.glows_flags, + np.array([glows_flags], dtype=np.uint16)) self.assertEqual([output_cdf_path, expected_first_output_file_path], products) @@ -1409,7 +1456,8 @@ def test_process_l3e_lo(self, mock_shutil, @patch('imap_l3_processing.glows.glows_processor.process_l3e_lo') @patch('imap_l3_processing.glows.glows_processor.process_l3e_ul_hf') @patch('imap_l3_processing.glows.glows_processor.process_l3e_ul_sf') - def test_process_l3e_skips_repointing_on_exception(self, mock_process_ultra_sf, mock_process_ultra_hf, mock_process_lo, + def test_process_l3e_skips_repointing_on_exception(self, mock_process_ultra_sf, mock_process_ultra_hf, + mock_process_lo, mock_process_hi, mock_get_pointing_date_range, mock_get_lo_pivot_angles, mock_compute_glows_flags_for_window): @@ -1554,7 +1602,7 @@ def test_process_l3e_skips_repointing_on_exception(self, mock_process_ultra_sf, @patch("imap_l3_processing.glows.glows_processor.json") @patch("imap_l3_processing.glows.glows_processor.ZipFile") def test_archive_dependencies(self, mock_zip, mock_json): - version_number = Version(1,1) + version_number = Version(1, 1) expected_filepath = TEMP_CDF_FOLDER_PATH / f"imap_glows_l3b-archive_20250314_v001.zip" expected_json_filename = "cr_to_process.json" diff --git a/tests/integration/test_glows_processor_integration.py b/tests/integration/test_glows_processor_integration.py index ae29622f2..539e30e64 100644 --- a/tests/integration/test_glows_processor_integration.py +++ b/tests/integration/test_glows_processor_integration.py @@ -29,7 +29,7 @@ from imap_l3_processing.utils import save_data from tests.integration.integration_test_helpers import mock_imap_data_access, run_istp_compliance_check from tests.test_helpers import get_test_data_path, get_test_instrument_team_data_path, \ - with_tempdir, get_run_local_data_path, run_periodically + with_tempdir, get_run_local_data_path, run_periodically, create_mock_version_map GLOWS_INTEGRATION_DATA_DIR = get_run_local_data_path( "glows_l3bcde_integration_data_dir" @@ -88,13 +88,14 @@ def test_glows_l3a(self, tmp_dir): date_in_path = l2_science_file_path.start_date start_date = datetime.strptime(date_in_path, "%Y%m%d") end_date = start_date + timedelta(days=1) + l3a_descriptor = 'hist' input_metadata = InputMetadata( instrument='glows', data_level='l3a', - descriptor='hist', + descriptor=l3a_descriptor, start_date=start_date, end_date=end_date, - version='v001', + version=create_mock_version_map(descriptor=l3a_descriptor,minor_version=1), repointing=l2_science_file_path.repointing ) From 92df852d032bc144322eeed3f1d65e3e3d5e07e9 Mon Sep 17 00:00:00 2001 From: Harrison Date: Thu, 9 Jul 2026 15:20:39 -0400 Subject: [PATCH 08/11] Harrison #118 - KJON/AFAH: Added support for individual major versions per product descriptor for GLOWS L3bcde --- imap_l3_processing/glows/descriptors.py | 19 + imap_l3_processing/glows/glows_processor.py | 59 +- .../glows/l3e/glows_l3e_initializer.py | 28 +- .../glows/l3e/glows_l3e_utils.py | 87 ++- tests/glows/l3e/test_glows_l3e_initializer.py | 99 +-- tests/glows/l3e/test_glows_l3e_utils.py | 611 ++++++++++++------ tests/glows/test_glows_processor.py | 172 +++-- .../test_glows_processor_integration.py | 19 +- 8 files changed, 688 insertions(+), 406 deletions(-) diff --git a/imap_l3_processing/glows/descriptors.py b/imap_l3_processing/glows/descriptors.py index 8caefbf27..81824e7f6 100644 --- a/imap_l3_processing/glows/descriptors.py +++ b/imap_l3_processing/glows/descriptors.py @@ -26,3 +26,22 @@ GLOWS_L3E_LO_DESCRIPTOR = 'survival-probability-lo' GLOWS_L3E_ULTRA_SF_DESCRIPTOR = 'survival-probability-ul-sf' GLOWS_L3E_ULTRA_HF_DESCRIPTOR = 'survival-probability-ul-hf' + +GLOWS_L3BCDE_DESCRIPTORS = [ + GLOWS_L3B_DESCRIPTOR, + GLOWS_L3C_DESCRIPTOR, + GLOWS_L3D_DESCRIPTOR, + GLOWS_L3E_HI_45_DESCRIPTOR, + GLOWS_L3E_HI_90_DESCRIPTOR, + GLOWS_L3E_LO_DESCRIPTOR, + GLOWS_L3E_ULTRA_SF_DESCRIPTOR, + GLOWS_L3E_ULTRA_HF_DESCRIPTOR +] + +GLOWS_L3E_DESCRIPTORS = [ + GLOWS_L3E_HI_45_DESCRIPTOR, + GLOWS_L3E_HI_90_DESCRIPTOR, + GLOWS_L3E_LO_DESCRIPTOR, + GLOWS_L3E_ULTRA_SF_DESCRIPTOR, + GLOWS_L3E_ULTRA_HF_DESCRIPTOR, +] \ No newline at end of file diff --git a/imap_l3_processing/glows/glows_processor.py b/imap_l3_processing/glows/glows_processor.py index 9100aa3df..6130d671f 100644 --- a/imap_l3_processing/glows/glows_processor.py +++ b/imap_l3_processing/glows/glows_processor.py @@ -17,7 +17,8 @@ from imap_l3_processing.constants import TEMP_CDF_FOLDER_PATH from imap_l3_processing.glows.descriptors import GLOWS_L3A_DESCRIPTOR, GLOWS_L3E_ULTRA_SF_DESCRIPTOR, \ - GLOWS_L3E_ULTRA_HF_DESCRIPTOR, GLOWS_L3D_DESCRIPTOR, GLOWS_L3B_DESCRIPTOR, GLOWS_L3C_DESCRIPTOR + GLOWS_L3E_ULTRA_HF_DESCRIPTOR, GLOWS_L3D_DESCRIPTOR, GLOWS_L3B_DESCRIPTOR, GLOWS_L3C_DESCRIPTOR, \ + GLOWS_L3E_LO_DESCRIPTOR from imap_l3_processing.glows.l3a.glows_l3a_dependencies import GlowsL3ADependencies from imap_l3_processing.glows.l3a.glows_toolkit.l3a_data import L3aData from imap_l3_processing.glows.l3a.models import GlowsL3LightCurve @@ -99,7 +100,7 @@ def process(self): for txt_file in process_l3d_result.l3d_text_file_paths: logger.info(f"Saved L3d text file output to: {txt_file}") - l3e_initializer_output = GlowsL3EInitializer.get_repointings_to_process(process_l3d_result, old_l3d, l3bc_initializer_data.repoint_file_path) + l3e_initializer_output = GlowsL3EInitializer.get_repointings_to_process(process_l3d_result, old_l3d, l3bc_initializer_data.repoint_file_path, self.input_metadata.version) if l3e_initializer_output is not None: logger.info(f"Processing L3e for repointings: {l3e_initializer_output.repointings.repointing_numbers}") products_list.extend([*process_l3d_result.l3d_text_file_paths, process_l3d_result.l3d_cdf_file_path]) @@ -271,36 +272,41 @@ def process_l3e(initializer_data: GlowsL3EInitializerOutput): glows_flags = compute_glows_flags_for_window(initializer_data.l3d_cdf_path, start_repointing, end_repointing) with SwallowExceptionAndLog(f"Exception encountered when processing L3e lo for repointing {repointing}"): - lo_parent_file_names = initializer_data.dependencies.get_lo_parents() - pivot_info = lo_pivot_angles[repointing] - if pivot_info.parent_filename is not None: - lo_parent_file_names = lo_parent_file_names + [pivot_info.parent_filename] - lo_version = initializer_data.repointings.lo_repointings[repointing] - products_list.extend(process_l3e_lo(lo_parent_file_names, repointing, start_repointing, epoch_delta, pivot_info.pivot_angle, lo_version, glows_flags)) + if repointing in initializer_data.repointings.lo_repointings: + lo_parent_file_names = initializer_data.dependencies.get_lo_parents() + pivot_info = lo_pivot_angles[repointing] + if pivot_info.parent_filename is not None: + lo_parent_file_names = lo_parent_file_names + [pivot_info.parent_filename] + lo_version = initializer_data.repointings.lo_repointings[repointing] + products_list.extend(process_l3e_lo(lo_parent_file_names, repointing, start_repointing, epoch_delta, pivot_info.pivot_angle, lo_version, glows_flags)) with SwallowExceptionAndLog(f"Exception encountered when processing L3e hi-90 for repointing {repointing}"): - hi_parent_file_names = initializer_data.dependencies.get_hi_parents() - hi_90_version = initializer_data.repointings.hi_90_repointings[repointing] - products_list.extend(process_l3e_hi(hi_parent_file_names, repointing, start_repointing, epoch_delta, 90, hi_90_version, glows_flags)) + if repointing in initializer_data.repointings.hi_90_repointings: + hi_parent_file_names = initializer_data.dependencies.get_hi_parents() + hi_90_version = initializer_data.repointings.hi_90_repointings[repointing] + products_list.extend(process_l3e_hi(hi_parent_file_names, repointing, start_repointing, epoch_delta, 90, hi_90_version, glows_flags)) with SwallowExceptionAndLog(f"Exception encountered when processing L3e hi-45 for repointing {repointing}"): - hi_parent_file_names = initializer_data.dependencies.get_hi_parents() - hi_45_version = initializer_data.repointings.hi_45_repointings[repointing] - products_list.extend(process_l3e_hi(hi_parent_file_names, repointing, start_repointing, epoch_delta, 135, hi_45_version, glows_flags)) + if repointing in initializer_data.repointings.hi_45_repointings: + hi_parent_file_names = initializer_data.dependencies.get_hi_parents() + hi_45_version = initializer_data.repointings.hi_45_repointings[repointing] + products_list.extend(process_l3e_hi(hi_parent_file_names, repointing, start_repointing, epoch_delta, 135, hi_45_version, glows_flags)) ul_parent_file_names = initializer_data.dependencies.get_ul_parents() - ul_sf_version = initializer_data.repointings.ultra_sf_repointings[repointing] - ul_hf_version = initializer_data.repointings.ultra_hf_repointings[repointing] with SwallowExceptionAndLog( f"Exception encountered when processing L3e ultra SF for repointing {repointing}"): - products_list.extend( - process_l3e_ul_sf(ul_parent_file_names, repointing, start_repointing, epoch_delta, ul_sf_version, glows_flags)) + if repointing in initializer_data.repointings.ultra_sf_repointings: + ul_sf_version = initializer_data.repointings.ultra_sf_repointings[repointing] + products_list.extend( + process_l3e_ul_sf(ul_parent_file_names, repointing, start_repointing, epoch_delta, ul_sf_version, glows_flags)) with SwallowExceptionAndLog( f"Exception encountered when processing L3e ultra HF for repointing {repointing}"): - products_list.extend( - process_l3e_ul_hf(ul_parent_file_names, repointing, start_repointing, epoch_delta, ul_hf_version, glows_flags)) + if repointing in initializer_data.repointings.ultra_hf_repointings: + ul_hf_version = initializer_data.repointings.ultra_hf_repointings[repointing] + products_list.extend( + process_l3e_ul_hf(ul_parent_file_names, repointing, start_repointing, epoch_delta, ul_hf_version, glows_flags)) return products_list @@ -324,10 +330,10 @@ def process_l3e_lo( input_metadata = InputMetadata( instrument="glows", data_level="l3e", - descriptor="survival-probability-lo", + descriptor=GLOWS_L3E_LO_DESCRIPTOR, start_date=repointing_start, end_date=repointing_start + epoch_delta * 2, - version=str(version), + version=VersionMap({GLOWS_L3E_LO_DESCRIPTOR: version}), repointing=repointing, ) @@ -374,7 +380,7 @@ def process_l3e_ul_sf(parent_file_names: list[str], repointing: int, repointing_ descriptor=GLOWS_L3E_ULTRA_SF_DESCRIPTOR, start_date=repointing_start, end_date=repointing_start + epoch_delta * 2, - version=str(version), + version=VersionMap({GLOWS_L3E_ULTRA_SF_DESCRIPTOR: version}), repointing=repointing, ) @@ -423,7 +429,7 @@ def process_l3e_ul_hf(parent_file_names: list[str], repointing: int, repointing_ descriptor=GLOWS_L3E_ULTRA_HF_DESCRIPTOR, start_date=repointing_start, end_date=repointing_start + epoch_delta * 2, - version=str(version), + version=VersionMap({GLOWS_L3E_ULTRA_HF_DESCRIPTOR: version}), repointing=repointing, ) @@ -459,8 +465,9 @@ def process_l3e_hi(parent_file_names: list[str], repointing: int, repointing_sta run(["./survProbHi"] + call_args) - input_metadata = InputMetadata(instrument='glows', descriptor=f'survival-probability-hi-{180-elongation}', - version=str(version), start_date=repointing_start, end_date=repointing_start + epoch_delta * 2, + descriptor = f'survival-probability-hi-{180-elongation}' + input_metadata = InputMetadata(instrument='glows', descriptor=descriptor, + version=VersionMap({descriptor: version}), start_date=repointing_start, end_date=repointing_start + epoch_delta * 2, repointing=repointing, data_level='l3e') output_path = Path(f'probSur.Imap.Hi_{call_args[0]}_{call_args[1][:8]}_{call_args[-1][:5]}.dat') diff --git a/imap_l3_processing/glows/l3e/glows_l3e_initializer.py b/imap_l3_processing/glows/l3e/glows_l3e_initializer.py index ca22854c6..b4a42b1ae 100644 --- a/imap_l3_processing/glows/l3e/glows_l3e_initializer.py +++ b/imap_l3_processing/glows/l3e/glows_l3e_initializer.py @@ -12,15 +12,16 @@ from imap_l3_processing.glows.l3d.models import GlowsL3DProcessorOutput from imap_l3_processing.glows.l3d.utils import get_most_recently_uploaded_ancillary from imap_l3_processing.glows.l3e.glows_l3e_dependencies import GlowsL3EDependencies -from imap_l3_processing.glows.l3e.glows_l3e_utils import find_first_updated_cr, determine_l3e_files_to_produce, \ - GlowsL3eRepointings +from imap_l3_processing.glows.l3e.glows_l3e_utils import find_first_updated_cr, identify_versions_for_l3e_output_files, \ + GlowsL3eVersionsForRepointings +from imap_l3_processing.models import VersionMap logger = logging.getLogger(__name__) @dataclass class GlowsL3EInitializerOutput: dependencies: GlowsL3EDependencies - repointings: GlowsL3eRepointings + repointings: GlowsL3eVersionsForRepointings l3d_cdf_path: Path @@ -30,15 +31,13 @@ def get_repointings_to_process( l3d_output: GlowsL3DProcessorOutput, previous_l3d: Optional[str], repointing_file_path: Path, + version_map: VersionMap, ) -> Optional[GlowsL3EInitializerOutput]: - latest_l3d_cr = None - major_version_from_output = Version.from_version(ScienceFilePath(l3d_output.l3d_cdf_file_path).version).major + first_updated_cr = None if previous_l3d is not None: - major_version_from_previous = Version.from_version(ScienceFilePath(previous_l3d).version).major - if major_version_from_output == major_version_from_previous: - latest_l3d_cr = find_first_updated_cr(l3d_output.l3d_cdf_file_path, previous_l3d) - if not latest_l3d_cr: - return None + first_updated_cr = find_first_updated_cr(l3d_output.l3d_cdf_file_path, previous_l3d) + if first_updated_cr is None: + return None pipeline_settings_l3bcde = get_most_recently_uploaded_ancillary(imap_data_access.query(table='ancillary', instrument='glows', descriptor='pipeline-settings-l3bcde')) energy_grid_lo = get_most_recently_uploaded_ancillary(imap_data_access.query(table='ancillary', instrument='glows', descriptor='energy-grid-lo')) @@ -63,11 +62,14 @@ def get_repointings_to_process( l3e_deps.copy_dependencies() first_cr = l3e_deps.pipeline_settings["start_cr"] - if latest_l3d_cr is not None: - first_cr = max(first_cr, latest_l3d_cr - 1) + + if first_updated_cr is None: + first_updated_cr = first_cr + else: + first_updated_cr -= 1 last_cr = ScienceFilePath(l3d_output.l3d_cdf_file_path).cr - glows_repointings = determine_l3e_files_to_produce(first_cr, last_cr, repointing_file_path, major_version_from_output) + glows_repointings = identify_versions_for_l3e_output_files(first_cr, last_cr, first_updated_cr, repointing_file_path, version_map) if len(glows_repointings.repointing_numbers) > 0: earliest_repointing_start, _ = get_pointing_date_range(min(glows_repointings.repointing_numbers)) diff --git a/imap_l3_processing/glows/l3e/glows_l3e_utils.py b/imap_l3_processing/glows/l3e/glows_l3e_utils.py index 3fb701d2e..880079bda 100644 --- a/imap_l3_processing/glows/l3e/glows_l3e_utils.py +++ b/imap_l3_processing/glows/l3e/glows_l3e_utils.py @@ -15,9 +15,10 @@ from imap_l3_processing.constants import ONE_AU_IN_KM, TT2000_EPOCH, ONE_SECOND_IN_NANOSECONDS from imap_l3_processing.glows.descriptors import GLOWS_L3E_ULTRA_HF_DESCRIPTOR, GLOWS_L3E_ULTRA_SF_DESCRIPTOR, \ - GLOWS_L3E_LO_DESCRIPTOR, GLOWS_L3E_HI_45_DESCRIPTOR, GLOWS_L3E_HI_90_DESCRIPTOR + GLOWS_L3E_LO_DESCRIPTOR, GLOWS_L3E_HI_45_DESCRIPTOR, GLOWS_L3E_HI_90_DESCRIPTOR, GLOWS_L3E_DESCRIPTORS from imap_l3_processing.glows.l3bc.l3bc_toolkit.funcs import jd_fm_Carrington from imap_l3_processing.glows.l3e.glows_l3e_call_arguments import GlowsL3eCallArguments +from imap_l3_processing.models import VersionMap def determine_call_args_for_l3e_executable(start_date: datetime, repointing_midpoint: datetime, @@ -58,7 +59,7 @@ def _decimal_time(t: datetime) -> str: @dataclass -class GlowsL3eRepointings: +class GlowsL3eVersionsForRepointings: repointing_numbers: list[int] hi_90_repointings: dict[int, Version] hi_45_repointings: dict[int, Version] @@ -67,49 +68,51 @@ class GlowsL3eRepointings: ultra_hf_repointings: dict[int, Version] -def determine_l3e_files_to_produce(first_cr_processed: int, last_processed_cr: int, - repointing_path: Path, major_version: int|None) -> GlowsL3eRepointings: - descriptors = [ - GLOWS_L3E_HI_90_DESCRIPTOR, - GLOWS_L3E_HI_45_DESCRIPTOR, - GLOWS_L3E_LO_DESCRIPTOR, - GLOWS_L3E_ULTRA_SF_DESCRIPTOR, - GLOWS_L3E_ULTRA_HF_DESCRIPTOR, - ] +def identify_versions_for_l3e_output_files(start_cr_of_mission: int, end_cr_of_mission: int, first_updated_cr_from_l3d: Optional[int], + repointing_path: Path, version_map: VersionMap) -> GlowsL3eVersionsForRepointings: set_global_repoint_table_paths([repointing_path]) repointing_data = get_repoint_data() - first_carrington_start_date = Time(jd_fm_Carrington(float(first_cr_processed)), format='jd') - last_cr_end_date = Time(jd_fm_Carrington(float(last_processed_cr + 1)), format='jd') + all_pointing_numbers = get_repoint_numbers_within_cr_window(start_cr_of_mission, end_cr_of_mission, repointing_data) + pointing_numbers_updated_by_l3d = get_repoint_numbers_within_cr_window(first_updated_cr_from_l3d, end_cr_of_mission, repointing_data) - start_ns = (first_carrington_start_date.to_datetime() - TT2000_EPOCH).total_seconds() * ONE_SECOND_IN_NANOSECONDS - end_ns = (last_cr_end_date.to_datetime() - TT2000_EPOCH).total_seconds() * ONE_SECOND_IN_NANOSECONDS - - vectorized_date_conv = np.vectorize(lambda d: (Time(d, format="iso").to_datetime( - leap_second_strict='silent') - TT2000_EPOCH).total_seconds() * ONE_SECOND_IN_NANOSECONDS) - repoint_starts = vectorized_date_conv(repointing_data["repoint_start_utc"]) - repoint_ids = repointing_data["repoint_id"] + updated_pointings_per_instruments = {} + updated_pointing_numbers = {} - pointing_numbers = [] - for i in range(len(repoint_ids)): - if i + 1 < len(repoint_ids) and start_ns < repoint_starts[i + 1] < end_ns: - pointing_numbers.append(int(repoint_ids[i])) - - updated_pointings_per_instruments = [] - for descriptor in descriptors: + for descriptor in GLOWS_L3E_DESCRIPTORS: l3e_files = imap_data_access.query(instrument='glows', data_level='l3e', version="latest", descriptor=descriptor) existing_file_versions = {int(l3e['repointing']): Version.from_version(l3e['version']) for l3e in l3e_files} new_file_versions = {} - for pointing_number in pointing_numbers: + for pointing_number in all_pointing_numbers: + new_major_version = version_map.lookup(descriptor).major if pointing_number in existing_file_versions: previous_version = existing_file_versions[pointing_number] - new_file_versions[pointing_number] = Version(major_version, previous_version.minor + 1) + + if previous_version.major is None: + if new_major_version or pointing_number in pointing_numbers_updated_by_l3d: + new_file_versions[pointing_number] = Version(new_major_version, previous_version.minor + 1) + + elif new_major_version is not None: + higher_major_version_given = new_major_version > previous_version.major + same_major_but_updated_by_l3d = new_major_version == previous_version.major and pointing_number in pointing_numbers_updated_by_l3d + if higher_major_version_given or same_major_but_updated_by_l3d: + new_file_versions[pointing_number] = Version(new_major_version, previous_version.minor + 1) + else: - new_file_versions[pointing_number] = Version(major_version, 1) - updated_pointings_per_instruments.append(new_file_versions) + new_file_versions[pointing_number] = Version(new_major_version, 1) + + updated_pointings_per_instruments[descriptor] = new_file_versions + updated_pointing_numbers = updated_pointing_numbers | new_file_versions.keys() + - return GlowsL3eRepointings(pointing_numbers, *updated_pointings_per_instruments) + return GlowsL3eVersionsForRepointings(list(updated_pointing_numbers), + updated_pointings_per_instruments[GLOWS_L3E_HI_90_DESCRIPTOR], + updated_pointings_per_instruments[GLOWS_L3E_HI_45_DESCRIPTOR], + updated_pointings_per_instruments[GLOWS_L3E_LO_DESCRIPTOR], + updated_pointings_per_instruments[GLOWS_L3E_ULTRA_SF_DESCRIPTOR], + updated_pointings_per_instruments[GLOWS_L3E_ULTRA_HF_DESCRIPTOR], + ) def compute_glows_flags_for_window(l3d_cdf_path: Path, window_start: datetime, window_end: datetime) -> int: @@ -186,3 +189,23 @@ def get_lo_pivot_angles(repointings: list[int]) -> dict[int, LoPivotAngle]: else: result[repointing] = LoPivotAngle(parent_filename=None, pivot_angle=90.0) return result + +def get_repoint_numbers_within_cr_window(start_cr_number: int, end_cr_number: int, repointing_data) -> list[int]: + + first_carrington_start_date = Time(jd_fm_Carrington(float(start_cr_number)), format='jd') + last_cr_end_date = Time(jd_fm_Carrington(float(end_cr_number + 1)), format='jd') + + start_ns = (first_carrington_start_date.to_datetime() - TT2000_EPOCH).total_seconds() * ONE_SECOND_IN_NANOSECONDS + end_ns = (last_cr_end_date.to_datetime() - TT2000_EPOCH).total_seconds() * ONE_SECOND_IN_NANOSECONDS + + vectorized_date_conv = np.vectorize(lambda d: (Time(d, format="iso").to_datetime( + leap_second_strict='silent') - TT2000_EPOCH).total_seconds() * ONE_SECOND_IN_NANOSECONDS) + repoint_starts = vectorized_date_conv(repointing_data["repoint_start_utc"]) + repoint_ids = repointing_data["repoint_id"] + + repoint_numbers = [] + for i in range(len(repoint_ids)): + if i + 1 < len(repoint_ids) and start_ns < repoint_starts[i + 1] < end_ns: + repoint_numbers.append(int(repoint_ids[i])) + + return repoint_numbers \ No newline at end of file diff --git a/tests/glows/l3e/test_glows_l3e_initializer.py b/tests/glows/l3e/test_glows_l3e_initializer.py index 74f500972..1e4963799 100644 --- a/tests/glows/l3e/test_glows_l3e_initializer.py +++ b/tests/glows/l3e/test_glows_l3e_initializer.py @@ -1,26 +1,28 @@ import unittest from datetime import datetime from pathlib import Path -from unittest.mock import patch, call +from unittest.mock import patch, call, sentinel from imap_data_access import RepointInput from imap_data_access.file_validation import Version +from imap_l3_processing.glows.descriptors import GLOWS_L3BCDE_DESCRIPTORS from imap_l3_processing.glows.l3d.models import GlowsL3DProcessorOutput from imap_l3_processing.glows.l3e.glows_l3e_initializer import GlowsL3EInitializer, GlowsL3EInitializerOutput -from imap_l3_processing.glows.l3e.glows_l3e_utils import GlowsL3eRepointings +from imap_l3_processing.glows.l3e.glows_l3e_utils import GlowsL3eVersionsForRepointings +from imap_l3_processing.models import VersionMap from tests.test_helpers import create_mock_query_results class TestGlowsL3EInitializer(unittest.TestCase): @patch('imap_l3_processing.glows.l3e.glows_l3e_initializer.get_pointing_date_range') @patch('imap_l3_processing.glows.l3e.glows_l3e_initializer.GlowsL3EDependencies.fetch_dependencies') - @patch('imap_l3_processing.glows.l3e.glows_l3e_initializer.determine_l3e_files_to_produce') + @patch('imap_l3_processing.glows.l3e.glows_l3e_initializer.identify_versions_for_l3e_output_files') @patch('imap_l3_processing.glows.l3e.glows_l3e_initializer.find_first_updated_cr') @patch('imap_l3_processing.glows.l3e.glows_l3e_initializer.get_most_recently_uploaded_ancillary') @patch('imap_l3_processing.glows.l3e.glows_l3e_initializer.imap_data_access.query') def test_get_repointings_to_process(self, mock_query, mock_get_most_recently_uploaded_ancillary, - mock_find_first_updated_cr, mock_determine_l3e_files_to_produce, + mock_find_first_updated_cr, mock_identify_versions_for_l3e_output_files, mock_fetch_dependencies, mock_get_pointing_date_range): mock_query.side_effect = create_mock_query_results([ 'imap_glows_pipeline-settings-l3bcde_20200101_v000.cdf', @@ -56,7 +58,7 @@ def test_get_repointings_to_process(self, mock_query, mock_get_most_recently_upl expected_lo = {1234: Version(None, 3), 2468: Version(None, 3)} expected_ultra = {1234: Version(None, 4), 2468: Version(None, 4)} - expected_repointings = GlowsL3eRepointings( + expected_repointings = GlowsL3eVersionsForRepointings( repointing_numbers=[2468, 1234], hi_90_repointings=expected_hi_90, hi_45_repointings=expected_hi_45, @@ -71,21 +73,24 @@ def test_get_repointings_to_process(self, mock_query, mock_get_most_recently_upl l3d_cdf_path=updated_l3d, ) - mock_determine_l3e_files_to_produce.return_value = expected_repointings + mock_identify_versions_for_l3e_output_files.return_value = expected_repointings mock_get_pointing_date_range.side_effect = [ (datetime(2010, 1, 1), datetime(2010, 1, 2)), (datetime(2011, 2, 1), datetime(2011, 2, 2)), ] + input_major_version = VersionMap({descriptor: Version(2, 1) for descriptor in GLOWS_L3BCDE_DESCRIPTORS}) + repointing_file_path = Path("imap_2026_105_01.repoint.csv") actual_initializer_output = GlowsL3EInitializer.get_repointings_to_process(glows_l3d_processor_output, previous_l3d, - repointing_file_path) + repointing_file_path, + input_major_version) mock_find_first_updated_cr.assert_called_once_with(updated_l3d, previous_l3d) - mock_determine_l3e_files_to_produce.assert_called_once_with(2090, 2091, repointing_file_path, None) + mock_identify_versions_for_l3e_output_files.assert_called_once_with(2089, 2091, 2090, repointing_file_path, input_major_version) mock_query.assert_has_calls([ call(table="ancillary", instrument='glows', descriptor='pipeline-settings-l3bcde'), @@ -144,81 +149,17 @@ def test_get_repointings_to_process_with_identical_l3d_files(self, mock_find_fir repointing_file_path = Path("imap_2026_105_01.repoint.csv") actual_initializer_output = GlowsL3EInitializer.get_repointings_to_process(glows_l3d_processor_output, previous_l3d, - repointing_file_path) + repointing_file_path, sentinel.version_map) mock_find_first_updated_cr.assert_called_once_with(glows_l3d_processor_output.l3d_cdf_file_path, previous_l3d) self.assertIsNone(actual_initializer_output) - - @patch('imap_l3_processing.glows.l3e.glows_l3e_initializer.get_pointing_date_range') - @patch('imap_l3_processing.glows.l3e.glows_l3e_initializer.GlowsL3EDependencies.fetch_dependencies') - @patch('imap_l3_processing.glows.l3e.glows_l3e_initializer.determine_l3e_files_to_produce') - @patch('imap_l3_processing.glows.l3e.glows_l3e_initializer.find_first_updated_cr') - @patch('imap_l3_processing.glows.l3e.glows_l3e_initializer.get_most_recently_uploaded_ancillary') - @patch('imap_l3_processing.glows.l3e.glows_l3e_initializer.imap_data_access.query') - def test_get_repointings_to_process_with_identical_l3d_files_but_different_major_version(self, mock_query, mock_get_most_recently_uploaded_ancillary, - mock_find_first_updated_cr, mock_determine_l3e_files_to_produce, - mock_fetch_dependencies, mock_get_pointing_date_range): - - repointing_file_path = Path("imap_2026_105_01.repoint.csv") - mock_l3e_dependencies = mock_fetch_dependencies.return_value - mock_l3e_dependencies.pipeline_settings = {"start_cr": 2089} - updated_l3d = Path('path/to/imap_glows_l3d_solar-hist_19470303-cr02091_v009.0000.cdf') - updated_l3d_text_file_path = Path("imap_glows_e-dens_19470303_20100101_v000.dat") - glows_l3d_processor_output = GlowsL3DProcessorOutput(updated_l3d, [updated_l3d_text_file_path], 2091) - previous_l3d = 'imap_glows_l3d_solar-hist_19470303-cr02091_v000.cdf' - - mock_query.side_effect = create_mock_query_results([ - 'imap_glows_pipeline-settings-l3bcde_20200101_v000.cdf', - 'imap_glows_energy-grid-lo_20200101_v000.cdf', - 'imap_glows_tess-xyz-8_20200101_v000.cdf', - 'imap_glows_energy-grid-hi_20200101_v000.cdf', - 'imap_glows_energy-grid-ultra_20200101_v000.cdf', - 'imap_glows_tess-ang-16_20200101_v000.cdf', - ]) - - mock_get_most_recently_uploaded_ancillary.side_effect = [ - create_mock_query_results(['imap_glows_pipeline-settings-l3bcde_20200101_v000.cdf'])[0], - create_mock_query_results(['imap_glows_energy-grid-lo_20200101_v000.cdf'])[0], - create_mock_query_results(['imap_glows_tess-xyz-8_20200101_v000.cdf'])[0], - create_mock_query_results(['imap_glows_energy-grid-hi_20200101_v000.cdf'])[0], - create_mock_query_results(['imap_glows_energy-grid-ultra_20200101_v000.cdf'])[0], - create_mock_query_results(['imap_glows_tess-ang-16_20200101_v000.cdf'])[0], - ] - - mock_determine_l3e_files_to_produce.return_value = GlowsL3eRepointings( - repointing_numbers=list(range(100, 160)), - ultra_sf_repointings={}, - ultra_hf_repointings={}, - lo_repointings={}, - hi_45_repointings={}, - hi_90_repointings={} - ) - mock_get_pointing_date_range.side_effect = [ - (datetime(2010, 1, 1), datetime(2010, 1, 2)), - (datetime(2010, 4, 1), datetime(2010, 4, 2)), - ] - - actual_initializer_output = GlowsL3EInitializer.get_repointings_to_process(glows_l3d_processor_output, - previous_l3d, - repointing_file_path) - - expected_output = GlowsL3EInitializerOutput( - dependencies=mock_fetch_dependencies.return_value, - repointings=mock_determine_l3e_files_to_produce.return_value, - l3d_cdf_path=updated_l3d, - ) - self.assertIsNotNone(actual_initializer_output) - mock_determine_l3e_files_to_produce.assert_called_once_with(2089, 2091, repointing_file_path, 9) - - mock_find_first_updated_cr.assert_not_called() - self.assertEqual(expected_output, actual_initializer_output) - + @patch('imap_l3_processing.glows.l3e.glows_l3e_initializer.imap_data_access.query') @patch('imap_l3_processing.glows.l3e.glows_l3e_initializer.get_most_recently_uploaded_ancillary') @patch('imap_l3_processing.glows.l3e.glows_l3e_initializer.GlowsL3EDependencies.fetch_dependencies') - @patch('imap_l3_processing.glows.l3e.glows_l3e_initializer.determine_l3e_files_to_produce') + @patch('imap_l3_processing.glows.l3e.glows_l3e_initializer.identify_versions_for_l3e_output_files') @patch('imap_l3_processing.glows.l3e.glows_l3e_initializer.find_first_updated_cr') def test_get_repointings_to_process_handles_no_previous_l3d(self, mock_find_first_updated_cr, - mock_determine_l3e_files_to_produce, + mock_identify_versions_for_l3e_output_files, mock_fetch_dependencies, mock_get_most_recently_uploaded_ancillary, _, ): @@ -230,7 +171,7 @@ def test_get_repointings_to_process_handles_no_previous_l3d(self, mock_find_firs create_mock_query_results(['imap_glows_energy-grid-ultra_20200101_v000.cdf'])[0], create_mock_query_results(['imap_glows_tess-ang-16_20200101_v000.cdf'])[0], ] - mock_determine_l3e_files_to_produce.return_value = GlowsL3eRepointings( + mock_identify_versions_for_l3e_output_files.return_value = GlowsL3eVersionsForRepointings( repointing_numbers=[], ultra_sf_repointings={}, ultra_hf_repointings={}, @@ -247,7 +188,7 @@ def test_get_repointings_to_process_handles_no_previous_l3d(self, mock_find_firs repointing_file_path = Path("imap_2026_105_01.repoint.csv") _ = GlowsL3EInitializer.get_repointings_to_process(glows_l3d_processor_output, previous_l3d, - repointing_file_path) + repointing_file_path, sentinel.version_map) mock_find_first_updated_cr.assert_not_called() - mock_determine_l3e_files_to_produce.assert_called_once_with(2089, 2091, repointing_file_path, 12) + mock_identify_versions_for_l3e_output_files.assert_called_once_with(2089, 2091, 2089, repointing_file_path, sentinel.version_map) diff --git a/tests/glows/l3e/test_glows_l3e_utils.py b/tests/glows/l3e/test_glows_l3e_utils.py index b3f0ab50f..08b89f3db 100644 --- a/tests/glows/l3e/test_glows_l3e_utils.py +++ b/tests/glows/l3e/test_glows_l3e_utils.py @@ -6,12 +6,17 @@ import numpy as np from imap_data_access.file_validation import Version +from imap_processing.spice.repoint import get_repoint_data, set_global_repoint_table_paths from spacepy.pycdf import CDF, const +from imap_l3_processing.glows.descriptors import GLOWS_L3E_DESCRIPTORS, GLOWS_L3E_HI_45_DESCRIPTOR, \ + GLOWS_L3E_HI_90_DESCRIPTOR, GLOWS_L3E_LO_DESCRIPTOR, GLOWS_L3E_ULTRA_SF_DESCRIPTOR, GLOWS_L3E_ULTRA_HF_DESCRIPTOR from imap_l3_processing.glows.l3e.glows_l3e_call_arguments import GlowsL3eCallArguments from imap_l3_processing.glows.l3e.glows_l3e_utils import determine_call_args_for_l3e_executable, \ - determine_l3e_files_to_produce, find_first_updated_cr, get_lo_pivot_angles, \ - get_lo_pivot_angle_from_l1b_file, LoPivotAngle, compute_glows_flags_for_window + identify_versions_for_l3e_output_files, find_first_updated_cr, get_lo_pivot_angles, \ + get_lo_pivot_angle_from_l1b_file, LoPivotAngle, compute_glows_flags_for_window, \ + get_repoint_numbers_within_cr_window +from imap_l3_processing.models import VersionMap from tests.test_helpers import get_test_data_path, create_mock_query_results @@ -63,208 +68,6 @@ def test_determine_call_args_for_l3e_executable(self, mock_pxform: Mock, mock_re self.assertEqual(np.rad2deg(spin_axis_lat), call_args.spin_axis_latitude) self.assertEqual(elongation, call_args.elongation) - @patch("imap_l3_processing.glows.l3e.glows_l3e_utils.imap_data_access.query") - def test_determine_l3e_files_to_produce(self, mock_query): - for major_version in (None, 3): - with self.subTest(major_version=major_version): - last_processed_cr = 2094 - first_cr_processed = 2093 - repoint_pathing = get_test_data_path("fake_1_day_repointing_file.csv") - - expected_repointings = [i for i in range(3682, 3736)] - - expected_hi_90_repointing_to_version = {i: Version(major_version, 1) for i in expected_repointings} - expected_hi_45_repointing_to_version = {i: Version(major_version, 1) for i in expected_repointings} - expected_lo_repointing_to_version = {i: Version(major_version, 1) for i in expected_repointings} - expected_ultra_repointing_to_version = {i: Version(major_version, 1) for i in expected_repointings} - - expected_hi_90_repointing_to_version.update({ - 3682: Version(major_version, 2), - 3683: Version(major_version, 3), - 3684: Version(major_version, 4), - 3685: Version(major_version, 5), - 3686: Version(major_version, 6) - }) - expected_hi_45_repointing_to_version.update({ - 3683: Version(major_version, 3), - 3684: Version(major_version, 4), - 3685: Version(major_version, 5), - 3686: Version(major_version, 6), - 3687: Version(major_version, 7) - }) - expected_lo_repointing_to_version.update({ - 3684: Version(major_version, 4), - 3685: Version(major_version, 5), - 3686: Version(major_version, 6), - 3687: Version(major_version, 7), - 3688: Version(major_version, 8) - }) - expected_ultra_repointing_to_version.update({ - 3685: Version(major_version, 5), - 3686: Version(major_version, 6), - 3687: Version(major_version, 7), - 3688: Version(major_version, 8), - 3689: Version(major_version, 9) - }) - - mock_query.side_effect = [ - create_mock_query_results([ - 'imap_glows_l3e_survival-probability-hi-90_20250101-repoint03682_v001.cdf', - 'imap_glows_l3e_survival-probability-hi-90_20250101-repoint03683_v002.cdf', - 'imap_glows_l3e_survival-probability-hi-90_20250101-repoint03684_v003.cdf', - 'imap_glows_l3e_survival-probability-hi-90_20250101-repoint03685_v004.cdf', - 'imap_glows_l3e_survival-probability-hi-90_20250101-repoint03686_v005.cdf', - ]), - create_mock_query_results([ - 'imap_glows_l3e_survival-probability-hi-45_20250101-repoint03683_v002.cdf', - 'imap_glows_l3e_survival-probability-hi-45_20250101-repoint03684_v003.cdf', - 'imap_glows_l3e_survival-probability-hi-45_20250101-repoint03685_v004.cdf', - 'imap_glows_l3e_survival-probability-hi-45_20250101-repoint03686_v005.cdf', - 'imap_glows_l3e_survival-probability-hi-45_20250101-repoint03687_v006.cdf', - ]), - create_mock_query_results([ - 'imap_glows_l3e_survival-probability-lo_20250101-repoint03684_v003.cdf', - 'imap_glows_l3e_survival-probability-lo_20250101-repoint03685_v004.cdf', - 'imap_glows_l3e_survival-probability-lo_20250101-repoint03686_v005.cdf', - 'imap_glows_l3e_survival-probability-lo_20250101-repoint03687_v006.cdf', - 'imap_glows_l3e_survival-probability-lo_20250101-repoint03688_v007.cdf', - ]), - create_mock_query_results([ - 'imap_glows_l3e_survival-probability-ul-sf_20250101-repoint03685_v004.cdf', - 'imap_glows_l3e_survival-probability-ul-sf_20250101-repoint03686_v005.cdf', - 'imap_glows_l3e_survival-probability-ul-sf_20250101-repoint03687_v006.cdf', - 'imap_glows_l3e_survival-probability-ul-sf_20250101-repoint03688_v007.cdf', - 'imap_glows_l3e_survival-probability-ul-sf_20250101-repoint03689_v008.cdf', - ]), - create_mock_query_results([ - 'imap_glows_l3e_survival-probability-ul-hf_20250101-repoint03685_v004.cdf', - 'imap_glows_l3e_survival-probability-ul-hf_20250101-repoint03686_v005.cdf', - 'imap_glows_l3e_survival-probability-ul-hf_20250101-repoint03687_v006.cdf', - 'imap_glows_l3e_survival-probability-ul-hf_20250101-repoint03688_v007.cdf', - 'imap_glows_l3e_survival-probability-ul-hf_20250101-repoint03689_v008.cdf', - ]), - ] - - repointings = determine_l3e_files_to_produce(first_cr_processed, last_processed_cr, repoint_pathing, major_version) - mock_query.assert_has_calls([ - call(instrument="glows", data_level="l3e", version='latest', descriptor='survival-probability-hi-90'), - call(instrument="glows", data_level="l3e", version='latest', descriptor='survival-probability-hi-45'), - call(instrument="glows", data_level="l3e", version='latest', descriptor='survival-probability-lo'), - call(instrument="glows", data_level="l3e", version='latest', descriptor='survival-probability-ul-sf'), - call(instrument="glows", data_level="l3e", version='latest', descriptor='survival-probability-ul-hf'), - ]) - - self.assertEqual(expected_hi_90_repointing_to_version, repointings.hi_90_repointings) - self.assertEqual(expected_hi_45_repointing_to_version, repointings.hi_45_repointings) - self.assertEqual(expected_lo_repointing_to_version, repointings.lo_repointings) - self.assertEqual(expected_ultra_repointing_to_version, repointings.ultra_sf_repointings) - self.assertEqual(expected_ultra_repointing_to_version, repointings.ultra_hf_repointings) - self.assertEqual(expected_repointings, repointings.repointing_numbers) - - @patch("imap_l3_processing.glows.l3e.glows_l3e_utils.imap_data_access.query") - def test_determine_l3e_files_to_produce_reprocesses_when_major_version_updated(self, mock_query): - old_new_major_versions = ( - (None, 1), - (1, 2) - ) - - for old_major_version, new_major_version in old_new_major_versions: - with self.subTest(old_major_version=old_major_version, new_major_version=new_major_version): - last_processed_cr = 2094 - first_cr_processed = 2093 - repoint_pathing = get_test_data_path("fake_1_day_repointing_file.csv") - - expected_repointings = [i for i in range(3682, 3736)] - - expected_hi_90_repointing_to_version = {i: Version(new_major_version, 1) for i in expected_repointings} - expected_hi_45_repointing_to_version = {i: Version(new_major_version, 1) for i in expected_repointings} - expected_lo_repointing_to_version = {i: Version(new_major_version, 1) for i in expected_repointings} - expected_ultra_repointing_to_version = {i: Version(new_major_version, 1) for i in expected_repointings} - - expected_hi_90_repointing_to_version.update({ - 3682: Version(new_major_version, 2), - 3683: Version(new_major_version, 3), - 3684: Version(new_major_version, 4), - 3685: Version(new_major_version, 5), - 3686: Version(new_major_version, 6) - }) - expected_hi_45_repointing_to_version.update({ - 3683: Version(new_major_version, 3), - 3684: Version(new_major_version, 4), - 3685: Version(new_major_version, 5), - 3686: Version(new_major_version, 6), - 3687: Version(new_major_version, 7) - }) - expected_lo_repointing_to_version.update({ - 3684: Version(new_major_version, 4), - 3685: Version(new_major_version, 5), - 3686: Version(new_major_version, 6), - 3687: Version(new_major_version, 7), - 3688: Version(new_major_version, 8) - }) - expected_ultra_repointing_to_version.update({ - 3685: Version(new_major_version, 5), - 3686: Version(new_major_version, 6), - 3687: Version(new_major_version, 7), - 3688: Version(new_major_version, 8), - 3689: Version(new_major_version, 9) - }) - - - mock_query.side_effect = [ - create_mock_query_results([ - f'imap_glows_l3e_survival-probability-hi-90_20250101-repoint03682_{Version(old_major_version,1)}.cdf', - f'imap_glows_l3e_survival-probability-hi-90_20250101-repoint03683_{Version(old_major_version,2)}.cdf', - f'imap_glows_l3e_survival-probability-hi-90_20250101-repoint03684_{Version(old_major_version,3)}.cdf', - f'imap_glows_l3e_survival-probability-hi-90_20250101-repoint03685_{Version(old_major_version,4)}.cdf', - f'imap_glows_l3e_survival-probability-hi-90_20250101-repoint03686_{Version(old_major_version,5)}.cdf', - ]), - create_mock_query_results([ - f'imap_glows_l3e_survival-probability-hi-45_20250101-repoint03683_{Version(old_major_version,2)}.cdf', - f'imap_glows_l3e_survival-probability-hi-45_20250101-repoint03684_{Version(old_major_version,3)}.cdf', - f'imap_glows_l3e_survival-probability-hi-45_20250101-repoint03685_{Version(old_major_version,4)}.cdf', - f'imap_glows_l3e_survival-probability-hi-45_20250101-repoint03686_{Version(old_major_version,5)}.cdf', - f'imap_glows_l3e_survival-probability-hi-45_20250101-repoint03687_{Version(old_major_version,6)}.cdf', - ]), - create_mock_query_results([ - f'imap_glows_l3e_survival-probability-lo_20250101-repoint03684_{Version(old_major_version,3)}.cdf', - f'imap_glows_l3e_survival-probability-lo_20250101-repoint03685_{Version(old_major_version,4)}.cdf', - f'imap_glows_l3e_survival-probability-lo_20250101-repoint03686_{Version(old_major_version,5)}.cdf', - f'imap_glows_l3e_survival-probability-lo_20250101-repoint03687_{Version(old_major_version,6)}.cdf', - f'imap_glows_l3e_survival-probability-lo_20250101-repoint03688_{Version(old_major_version,7)}.cdf', - ]), - create_mock_query_results([ - f'imap_glows_l3e_survival-probability-ul-sf_20250101-repoint03685_{Version(old_major_version,4)}.cdf', - f'imap_glows_l3e_survival-probability-ul-sf_20250101-repoint03686_{Version(old_major_version,5)}.cdf', - f'imap_glows_l3e_survival-probability-ul-sf_20250101-repoint03687_{Version(old_major_version,6)}.cdf', - f'imap_glows_l3e_survival-probability-ul-sf_20250101-repoint03688_{Version(old_major_version,7)}.cdf', - f'imap_glows_l3e_survival-probability-ul-sf_20250101-repoint03689_{Version(old_major_version,8)}.cdf', - ]), - create_mock_query_results([ - f'imap_glows_l3e_survival-probability-ul-hf_20250101-repoint03685_{Version(old_major_version,4)}.cdf', - f'imap_glows_l3e_survival-probability-ul-hf_20250101-repoint03686_{Version(old_major_version,5)}.cdf', - f'imap_glows_l3e_survival-probability-ul-hf_20250101-repoint03687_{Version(old_major_version,6)}.cdf', - f'imap_glows_l3e_survival-probability-ul-hf_20250101-repoint03688_{Version(old_major_version,7)}.cdf', - f'imap_glows_l3e_survival-probability-ul-hf_20250101-repoint03689_{Version(old_major_version,8)}.cdf', - ]), - ] - - repointings = determine_l3e_files_to_produce(first_cr_processed, last_processed_cr, repoint_pathing, new_major_version) - mock_query.assert_has_calls([ - call(instrument="glows", data_level="l3e", version='latest', descriptor='survival-probability-hi-90'), - call(instrument="glows", data_level="l3e", version='latest', descriptor='survival-probability-hi-45'), - call(instrument="glows", data_level="l3e", version='latest', descriptor='survival-probability-lo'), - call(instrument="glows", data_level="l3e", version='latest', descriptor='survival-probability-ul-sf'), - call(instrument="glows", data_level="l3e", version='latest', descriptor='survival-probability-ul-hf'), - ]) - - self.assertEqual(expected_hi_90_repointing_to_version, repointings.hi_90_repointings) - self.assertEqual(expected_hi_45_repointing_to_version, repointings.hi_45_repointings) - self.assertEqual(expected_lo_repointing_to_version, repointings.lo_repointings) - self.assertEqual(expected_ultra_repointing_to_version, repointings.ultra_sf_repointings) - self.assertEqual(expected_ultra_repointing_to_version, repointings.ultra_hf_repointings) - self.assertEqual(expected_repointings, repointings.repointing_numbers) - @patch('imap_l3_processing.glows.l3e.glows_l3e_utils.imap_data_access.download') @patch('imap_l3_processing.glows.l3e.glows_l3e_utils.CDF') def test_find_first_updated_cr(self, mock_CDF, mock_download): @@ -379,15 +182,17 @@ def test_get_lo_pivot_angle_from_l1b_file_scenarios(self): cases = [ ("realistic", epochs, [89.1, 89.9, 89.9, 89.9, 89.9, 89.9, 89.9, 89.9, 89.9], 90), ("basic", epochs, [10, 20, 30, 40, 50, 60, 70, 80, 90], 50), - ("only uses data within 3-15 hours from first point", shifted_epochs, [999, 999, 30, 40, 50, 60, 70, 999, 999], 50), + ("only uses data within 3-15 hours from first point", shifted_epochs, + [999, 999, 30, 40, 50, 60, 70, 999, 999], 50), ("uses median and rounds", epochs, [999, 999, 120.2, 34.4, 86.8, 50.9, 77.7, 999, 999], 78), - ("fallback to 90 if no points in interval", first_thirty_minutes + after_22_hours_thirty_minutes, [10, 10, 10, 10], 90), + ("fallback to 90 if no points in interval", first_thirty_minutes + after_22_hours_thirty_minutes, + [10, 10, 10, 10], 90), ("fallback to 90 if no points at all", [], [], 90), ] for name, epochs, pivot_angles, expected in cases: with self.subTest(name): with tempfile.TemporaryDirectory() as tmp_dir: - cdf_path = Path(tmp_dir,"l1b.cdf") + cdf_path = Path(tmp_dir, "l1b.cdf") with CDF(str(cdf_path), create=True) as cdf: cdf["epoch"] = epochs cdf["pcc_coarse_pot_pri"] = pivot_angles @@ -442,8 +247,10 @@ def test_get_lo_pivot_angles(self, mock_imap_data_access, mock_get_pivot_angle_f Path("local/path/to/file5.cdf"): 72, Path("local/path/to/file6.cdf"): 84, } + def mock_read_from_cdf(path: Path): return pivot_angles_by_file_path[path] + mock_get_pivot_angle_from_file.side_effect = mock_read_from_cdf result = get_lo_pivot_angles([3, 4, 6, 10]) @@ -465,3 +272,393 @@ def mock_read_from_cdf(path: Path): 6: LoPivotAngle(parent_filename="file6.cdf", pivot_angle=84), 10: LoPivotAngle(parent_filename=None, pivot_angle=90), }, result) + + def test_get_repoint_numbers_within_cr_window(self): + start_cr = 2093 + end_cr = 2094 + expected_repoint_numbers = list(range(3682, 3736)) + + repointing_path = get_test_data_path("fake_1_day_repointing_file.csv") + + set_global_repoint_table_paths([repointing_path]) + repointing_data = get_repoint_data() + + actual_repoint_numbers = get_repoint_numbers_within_cr_window(start_cr, end_cr, repointing_data) + + np.testing.assert_array_equal(actual_repoint_numbers, expected_repoint_numbers) + + @patch('imap_l3_processing.glows.l3e.glows_l3e_utils.get_repoint_numbers_within_cr_window') + @patch('imap_l3_processing.glows.l3e.glows_l3e_utils.imap_data_access') + def test_identify_versions_for_l3e_output_files_gives_minor_version_1_for_non_existing_l3e(self, + mock_imap_data_access, + mock_get_repoint_numbers_within_cr_window): + start_cr_of_mission = 2093 + end_cr_of_mission = 2094 + first_cr_updated_in_l3d = None + repointing_path = get_test_data_path("fake_1_day_repointing_file.csv") + version_map = VersionMap({desc: Version(3 + i, 5) for i, desc in enumerate(GLOWS_L3E_DESCRIPTORS)}) + + mock_imap_data_access.query.side_effect = [ + create_mock_query_results([]), + create_mock_query_results([]), + create_mock_query_results([]), + create_mock_query_results([]), + create_mock_query_results([]) + ] + + all_repointing_numbers = list(range(3682, 3736)) + updated_repointing_numbers = list() + mock_get_repoint_numbers_within_cr_window.side_effect = [ + all_repointing_numbers, + updated_repointing_numbers + ] + + result = identify_versions_for_l3e_output_files(start_cr_of_mission, end_cr_of_mission, first_cr_updated_in_l3d, + repointing_path, version_map) + + mock_imap_data_access.query.assert_has_calls([ + call(instrument='glows', data_level='l3e', version="latest", descriptor=GLOWS_L3E_HI_45_DESCRIPTOR), + call(instrument='glows', data_level='l3e', version="latest", descriptor=GLOWS_L3E_HI_90_DESCRIPTOR), + call(instrument='glows', data_level='l3e', version="latest", descriptor=GLOWS_L3E_LO_DESCRIPTOR), + call(instrument='glows', data_level='l3e', version="latest", descriptor=GLOWS_L3E_ULTRA_SF_DESCRIPTOR), + call(instrument='glows', data_level='l3e', version="latest", descriptor=GLOWS_L3E_ULTRA_HF_DESCRIPTOR) + ]) + + expected_versions_for_hi45_repoint_number = {repoint_number: Version(3, 1) for repoint_number in + all_repointing_numbers} + expected_versions_for_hi90_repoint_number = {repoint_number: Version(4, 1) for repoint_number in + all_repointing_numbers} + expected_versions_for_lo_repoint_number = {repoint_number: Version(5, 1) for repoint_number in + all_repointing_numbers} + expected_versions_for_ultra_sf_repoint_number = {repoint_number: Version(6, 1) for repoint_number in + all_repointing_numbers} + expected_versions_for_ultra_hf_repoint_number = {repoint_number: Version(7, 1) for repoint_number in + all_repointing_numbers} + + self.assertCountEqual(all_repointing_numbers, result.repointing_numbers) + self.assertEqual(expected_versions_for_hi90_repoint_number, result.hi_90_repointings) + self.assertEqual(expected_versions_for_hi45_repoint_number, result.hi_45_repointings) + self.assertEqual(expected_versions_for_lo_repoint_number, result.lo_repointings) + self.assertEqual(expected_versions_for_ultra_sf_repoint_number, result.ultra_sf_repointings) + self.assertEqual(expected_versions_for_ultra_hf_repoint_number, result.ultra_hf_repointings) + + + @patch('imap_l3_processing.glows.l3e.glows_l3e_utils.get_repoint_numbers_within_cr_window') + @patch('imap_l3_processing.glows.l3e.glows_l3e_utils.imap_data_access') + def test_identify_versions_for_l3e_output_files_gives_minor_version_1_for_non_existing_l3e(self, + mock_imap_data_access, + mock_get_repoint_numbers_within_cr_window): + start_cr_of_mission = 2093 + end_cr_of_mission = 2094 + first_cr_updated_in_l3d = None + repointing_path = get_test_data_path("fake_1_day_repointing_file.csv") + version_map = VersionMap({}, Version(None, 1)) + + mock_imap_data_access.query.side_effect = [ + create_mock_query_results([]), + create_mock_query_results([]), + create_mock_query_results([]), + create_mock_query_results([]), + create_mock_query_results([]) + ] + + all_repointing_numbers = list(range(3682, 3736)) + updated_repointing_numbers = list() + mock_get_repoint_numbers_within_cr_window.side_effect = [ + all_repointing_numbers, + updated_repointing_numbers + ] + + result = identify_versions_for_l3e_output_files(start_cr_of_mission, end_cr_of_mission, first_cr_updated_in_l3d, + repointing_path, version_map) + + mock_imap_data_access.query.assert_has_calls([ + call(instrument='glows', data_level='l3e', version="latest", descriptor=GLOWS_L3E_HI_45_DESCRIPTOR), + call(instrument='glows', data_level='l3e', version="latest", descriptor=GLOWS_L3E_HI_90_DESCRIPTOR), + call(instrument='glows', data_level='l3e', version="latest", descriptor=GLOWS_L3E_LO_DESCRIPTOR), + call(instrument='glows', data_level='l3e', version="latest", descriptor=GLOWS_L3E_ULTRA_SF_DESCRIPTOR), + call(instrument='glows', data_level='l3e', version="latest", descriptor=GLOWS_L3E_ULTRA_HF_DESCRIPTOR) + ]) + + expected_versions_for_hi45_repoint_number = {repoint_number: Version(None, 1) for repoint_number in + all_repointing_numbers} + expected_versions_for_hi90_repoint_number = {repoint_number: Version(None, 1) for repoint_number in + all_repointing_numbers} + expected_versions_for_lo_repoint_number = {repoint_number: Version(None, 1) for repoint_number in + all_repointing_numbers} + expected_versions_for_ultra_sf_repoint_number = {repoint_number: Version(None, 1) for repoint_number in + all_repointing_numbers} + expected_versions_for_ultra_hf_repoint_number = {repoint_number: Version(None, 1) for repoint_number in + all_repointing_numbers} + + self.assertCountEqual(all_repointing_numbers, result.repointing_numbers) + self.assertEqual(expected_versions_for_hi90_repoint_number, result.hi_90_repointings) + self.assertEqual(expected_versions_for_hi45_repoint_number, result.hi_45_repointings) + self.assertEqual(expected_versions_for_lo_repoint_number, result.lo_repointings) + self.assertEqual(expected_versions_for_ultra_sf_repoint_number, result.ultra_sf_repointings) + self.assertEqual(expected_versions_for_ultra_hf_repoint_number, result.ultra_hf_repointings) + + @patch('imap_l3_processing.glows.l3e.glows_l3e_utils.get_repoint_numbers_within_cr_window') + @patch('imap_l3_processing.glows.l3e.glows_l3e_utils.imap_data_access') + def test_identify_versions_for_l3e_output_files_increments_major_and_minor_when_given_higher_major_version(self, + mock_imap_data_access, + mock_get_repoint_numbers_within_cr_window): + start_cr_of_mission = 2093 + end_cr_of_mission = 2094 + first_cr_updated_in_l3d = None + repointing_path = get_test_data_path("fake_1_day_repointing_file.csv") + version_map = VersionMap({desc: Version(3 + i, 5) for i, desc in enumerate(GLOWS_L3E_DESCRIPTORS)}) + + all_repointing_numbers = list(range(3682, 3736)) + updated_repointing_numbers = list() + + cases = (2, None) + for old_major_version in cases: + with self.subTest(old_major_version): + mock_get_repoint_numbers_within_cr_window.reset_mock() + mock_imap_data_access.reset_mock() + + mock_get_repoint_numbers_within_cr_window.side_effect = [ + all_repointing_numbers, + updated_repointing_numbers + ] + + mock_imap_data_access.query.side_effect = [ + create_mock_query_results([ + f'imap_glows_l3e_survival-probability-hi-90_20250101-repoint03682_{Version(old_major_version, 1)}.cdf', + f'imap_glows_l3e_survival-probability-hi-90_20250101-repoint03683_{Version(3, 1)}.cdf', + f'imap_glows_l3e_survival-probability-hi-90_20250101-repoint03735_{Version(3, 1)}.cdf' + ]), + create_mock_query_results([ + f'imap_glows_l3e_survival-probability-hi-45_20250101-repoint03683_{Version(old_major_version, 2)}.cdf', + f'imap_glows_l3e_survival-probability-hi-45_20250101-repoint03684_{Version(4, 2)}.cdf', + f'imap_glows_l3e_survival-probability-hi-45_20250101-repoint03735_{Version(4, 2)}.cdf' + ]), + create_mock_query_results([ + f'imap_glows_l3e_survival-probability-lo_20250101-repoint03684_{Version(old_major_version, 3)}.cdf', + f'imap_glows_l3e_survival-probability-lo_20250101-repoint03685_{Version(5, 3)}.cdf', + f'imap_glows_l3e_survival-probability-lo_20250101-repoint03735_{Version(5, 3)}.cdf' + ]), + create_mock_query_results([ + f'imap_glows_l3e_survival-probability-ul-sf_20250101-repoint03685_{Version(old_major_version, 4)}.cdf', + f'imap_glows_l3e_survival-probability-ul-sf_20250101-repoint03686_{Version(6, 4)}.cdf', + f'imap_glows_l3e_survival-probability-ul-sf_20250101-repoint03735_{Version(6, 4)}.cdf' + ]), + create_mock_query_results([ + f'imap_glows_l3e_survival-probability-ul-hf_20250101-repoint03686_{Version(old_major_version, 5)}.cdf', + f'imap_glows_l3e_survival-probability-ul-hf_20250101-repoint03687_{Version(7, 5)}.cdf', + f'imap_glows_l3e_survival-probability-ul-hf_20250101-repoint03735_{Version(7, 5)}.cdf' + ]) + ] + + result = identify_versions_for_l3e_output_files(start_cr_of_mission, end_cr_of_mission, + first_cr_updated_in_l3d, repointing_path, version_map) + + mock_imap_data_access.query.assert_has_calls([ + call(instrument='glows', data_level='l3e', version="latest", descriptor=GLOWS_L3E_HI_45_DESCRIPTOR), + call(instrument='glows', data_level='l3e', version="latest", descriptor=GLOWS_L3E_HI_90_DESCRIPTOR), + call(instrument='glows', data_level='l3e', version="latest", descriptor=GLOWS_L3E_LO_DESCRIPTOR), + call(instrument='glows', data_level='l3e', version="latest", descriptor=GLOWS_L3E_ULTRA_SF_DESCRIPTOR), + call(instrument='glows', data_level='l3e', version="latest", descriptor=GLOWS_L3E_ULTRA_HF_DESCRIPTOR) + ]) + + self.assertCountEqual(list(range(3682, 3735)), result.repointing_numbers) + + self.assertNotIn(3683, result.hi_45_repointings) + self.assertNotIn(3684, result.hi_90_repointings) + self.assertNotIn(3685, result.lo_repointings) + self.assertNotIn(3686, result.ultra_sf_repointings) + self.assertNotIn(3687, result.ultra_hf_repointings) + + self.assertEqual(Version(3, 2), result.hi_45_repointings[3682]) + self.assertEqual(Version(4, 3), result.hi_90_repointings[3683]) + self.assertEqual(Version(5, 4), result.lo_repointings[3684]) + self.assertEqual(Version(6, 5), result.ultra_sf_repointings[3685]) + self.assertEqual(Version(7, 6), result.ultra_hf_repointings[3686]) + + @patch('imap_l3_processing.glows.l3e.glows_l3e_utils.get_repoint_numbers_within_cr_window') + @patch('imap_l3_processing.glows.l3e.glows_l3e_utils.imap_data_access') + def test_identify_versions_for_l3e_output_files_does_not_process_if_input_major_version_is_none_and_existing_has_major(self, + mock_imap_data_access, + mock_get_repoint_numbers_within_cr_window): + start_cr_of_mission = 2093 + end_cr_of_mission = 2094 + first_cr_updated_in_l3d = None + repointing_path = get_test_data_path("fake_1_day_repointing_file.csv") + version_map = VersionMap({}, Version(None,1)) + + all_repointing_numbers = list(range(3682, 3736)) + updated_repointing_numbers = list() + + mock_get_repoint_numbers_within_cr_window.side_effect = [ + all_repointing_numbers, + updated_repointing_numbers + ] + + mock_imap_data_access.query.side_effect = [ + create_mock_query_results([ + f'imap_glows_l3e_survival-probability-hi-90_20250101-repoint{repoint:05d}_v001.0001.cdf' for repoint in all_repointing_numbers + ]), + create_mock_query_results([ + f'imap_glows_l3e_survival-probability-hi-45_20250101-repoint{repoint:05d}_v001.0002.cdf' for repoint in all_repointing_numbers + ]), + create_mock_query_results([ + f'imap_glows_l3e_survival-probability-lo_20250101-repoint{repoint:05d}_v001.0003.cdf' for repoint in all_repointing_numbers + ]), + create_mock_query_results([ + f'imap_glows_l3e_survival-probability-ul-sf_20250101-repoint{repoint:05d}_v001.0004.cdf' for repoint in all_repointing_numbers + ]), + create_mock_query_results([ + f'imap_glows_l3e_survival-probability-ul-hf_20250101-repoint{repoint:05d}_v001.0005.cdf' for repoint in all_repointing_numbers + ]) + ] + + result = identify_versions_for_l3e_output_files(start_cr_of_mission, end_cr_of_mission, + first_cr_updated_in_l3d, repointing_path, version_map) + + mock_imap_data_access.query.assert_has_calls([ + call(instrument='glows', data_level='l3e', version="latest", descriptor=GLOWS_L3E_HI_45_DESCRIPTOR), + call(instrument='glows', data_level='l3e', version="latest", descriptor=GLOWS_L3E_HI_90_DESCRIPTOR), + call(instrument='glows', data_level='l3e', version="latest", descriptor=GLOWS_L3E_LO_DESCRIPTOR), + call(instrument='glows', data_level='l3e', version="latest", descriptor=GLOWS_L3E_ULTRA_SF_DESCRIPTOR), + call(instrument='glows', data_level='l3e', version="latest", descriptor=GLOWS_L3E_ULTRA_HF_DESCRIPTOR) + ]) + + self.assertEqual(0, len(result.repointing_numbers)) + self.assertEqual({}, result.hi_45_repointings) + self.assertEqual({}, result.hi_90_repointings) + self.assertEqual({}, result.lo_repointings) + self.assertEqual({}, result.ultra_sf_repointings) + self.assertEqual({}, result.ultra_hf_repointings) + + @patch('imap_l3_processing.glows.l3e.glows_l3e_utils.get_repoint_numbers_within_cr_window') + @patch('imap_l3_processing.glows.l3e.glows_l3e_utils.imap_data_access') + def test_identify_versions_for_l3e_output_files_increments_minor_when_same_major_and_updated_l3d_covers_pointing( + self, mock_imap_data_access, mock_get_repoint_numbers_within_cr_window + ): + start_cr_of_mission = 2093 + end_cr_of_mission = 2095 + first_cr_updated_in_l3d = None + repointing_path = get_test_data_path("fake_1_day_repointing_file.csv") + version_map = VersionMap({desc: Version(3 + i, 5) for i, desc in enumerate(GLOWS_L3E_DESCRIPTORS)}) + + all_repointing_numbers = list(range(3682, 3763)) + updated_repointing_numbers = list(range(3709, 3763)) + + mock_get_repoint_numbers_within_cr_window.side_effect = [ + all_repointing_numbers, + updated_repointing_numbers + ] + + mock_imap_data_access.query.side_effect = [ + create_mock_query_results([ + f'imap_glows_l3e_survival-probability-hi-90_20250101-repoint{repoint:05d}_{Version(3, 1)}.cdf' for repoint in all_repointing_numbers + ]), + create_mock_query_results([ + f'imap_glows_l3e_survival-probability-hi-45_20250101-repoint{repoint:05d}_{Version(4, 2)}.cdf' for repoint in all_repointing_numbers + ]), + create_mock_query_results([ + f'imap_glows_l3e_survival-probability-lo_20250101-repoint{repoint:05d}_{Version(5, 3)}.cdf' for repoint in all_repointing_numbers + ]), + create_mock_query_results([ + f'imap_glows_l3e_survival-probability-ul-sf_20250101-repoint{repoint:05d}_{Version(6, 4)}.cdf' for repoint in all_repointing_numbers + ]), + create_mock_query_results([ + f'imap_glows_l3e_survival-probability-ul-hf_20250101-repoint{repoint:05d}_{Version(7, 5)}.cdf' for repoint in all_repointing_numbers + ]) + ] + + result = identify_versions_for_l3e_output_files(start_cr_of_mission, end_cr_of_mission, + first_cr_updated_in_l3d, repointing_path, version_map) + + mock_imap_data_access.query.assert_has_calls([ + call(instrument='glows', data_level='l3e', version="latest", descriptor=GLOWS_L3E_HI_45_DESCRIPTOR), + call(instrument='glows', data_level='l3e', version="latest", descriptor=GLOWS_L3E_HI_90_DESCRIPTOR), + call(instrument='glows', data_level='l3e', version="latest", descriptor=GLOWS_L3E_LO_DESCRIPTOR), + call(instrument='glows', data_level='l3e', version="latest", descriptor=GLOWS_L3E_ULTRA_SF_DESCRIPTOR), + call(instrument='glows', data_level='l3e', version="latest", descriptor=GLOWS_L3E_ULTRA_HF_DESCRIPTOR) + ]) + + expected_versions_for_hi45_repoint_number = {repoint_number: Version(3, 2) for repoint_number in + updated_repointing_numbers} + expected_versions_for_hi90_repoint_number = {repoint_number: Version(4, 3) for repoint_number in + updated_repointing_numbers} + expected_versions_for_lo_repoint_number = {repoint_number: Version(5, 4) for repoint_number in + updated_repointing_numbers} + expected_versions_for_ultra_sf_repoint_number = {repoint_number: Version(6, 5) for repoint_number in + updated_repointing_numbers} + expected_versions_for_ultra_hf_repoint_number = {repoint_number: Version(7, 6) for repoint_number in + updated_repointing_numbers} + + self.assertCountEqual(updated_repointing_numbers, result.repointing_numbers) + self.assertEqual(expected_versions_for_hi90_repoint_number, result.hi_90_repointings) + self.assertEqual(expected_versions_for_hi45_repoint_number, result.hi_45_repointings) + self.assertEqual(expected_versions_for_lo_repoint_number, result.lo_repointings) + self.assertEqual(expected_versions_for_ultra_sf_repoint_number, result.ultra_sf_repointings) + self.assertEqual(expected_versions_for_ultra_hf_repoint_number, result.ultra_hf_repointings) + + @patch('imap_l3_processing.glows.l3e.glows_l3e_utils.get_repoint_numbers_within_cr_window') + @patch('imap_l3_processing.glows.l3e.glows_l3e_utils.imap_data_access') + def test_identify_versions_for_l3e_output_files_increments_minor_for_legacy_versioning_and_updated_l3d_covers_pointing( + self, mock_imap_data_access, mock_get_repoint_numbers_within_cr_window + ): + start_cr_of_mission = 2093 + end_cr_of_mission = 2095 + first_cr_updated_in_l3d = None + repointing_path = get_test_data_path("fake_1_day_repointing_file.csv") + version_map = VersionMap({}, Version(None,1)) + + all_repointing_numbers = list(range(3682, 3763)) + updated_repointing_numbers = list(range(3709, 3763)) + + mock_get_repoint_numbers_within_cr_window.side_effect = [ + all_repointing_numbers, + updated_repointing_numbers + ] + + mock_imap_data_access.query.side_effect = [ + create_mock_query_results([ + f'imap_glows_l3e_survival-probability-hi-90_20250101-repoint{repoint:05d}_{Version(None, 1)}.cdf' for repoint in all_repointing_numbers + ]), + create_mock_query_results([ + f'imap_glows_l3e_survival-probability-hi-45_20250101-repoint{repoint:05d}_{Version(None, 2)}.cdf' for repoint in all_repointing_numbers + ]), + create_mock_query_results([ + f'imap_glows_l3e_survival-probability-lo_20250101-repoint{repoint:05d}_{Version(None, 3)}.cdf' for repoint in all_repointing_numbers + ]), + create_mock_query_results([ + f'imap_glows_l3e_survival-probability-ul-sf_20250101-repoint{repoint:05d}_{Version(None, 4)}.cdf' for repoint in all_repointing_numbers + ]), + create_mock_query_results([ + f'imap_glows_l3e_survival-probability-ul-hf_20250101-repoint{repoint:05d}_{Version(None, 5)}.cdf' for repoint in all_repointing_numbers + ]) + ] + + result = identify_versions_for_l3e_output_files(start_cr_of_mission, end_cr_of_mission, + first_cr_updated_in_l3d, repointing_path, version_map) + + mock_imap_data_access.query.assert_has_calls([ + call(instrument='glows', data_level='l3e', version="latest", descriptor=GLOWS_L3E_HI_45_DESCRIPTOR), + call(instrument='glows', data_level='l3e', version="latest", descriptor=GLOWS_L3E_HI_90_DESCRIPTOR), + call(instrument='glows', data_level='l3e', version="latest", descriptor=GLOWS_L3E_LO_DESCRIPTOR), + call(instrument='glows', data_level='l3e', version="latest", descriptor=GLOWS_L3E_ULTRA_SF_DESCRIPTOR), + call(instrument='glows', data_level='l3e', version="latest", descriptor=GLOWS_L3E_ULTRA_HF_DESCRIPTOR) + ]) + + expected_versions_for_hi45_repoint_number = {repoint_number: Version(None, 2) for repoint_number in + updated_repointing_numbers} + expected_versions_for_hi90_repoint_number = {repoint_number: Version(None, 3) for repoint_number in + updated_repointing_numbers} + expected_versions_for_lo_repoint_number = {repoint_number: Version(None, 4) for repoint_number in + updated_repointing_numbers} + expected_versions_for_ultra_sf_repoint_number = {repoint_number: Version(None, 5) for repoint_number in + updated_repointing_numbers} + expected_versions_for_ultra_hf_repoint_number = {repoint_number: Version(None, 6) for repoint_number in + updated_repointing_numbers} + + self.assertCountEqual(updated_repointing_numbers, result.repointing_numbers) + self.assertEqual(expected_versions_for_hi90_repoint_number, result.hi_90_repointings) + self.assertEqual(expected_versions_for_hi45_repoint_number, result.hi_45_repointings) + self.assertEqual(expected_versions_for_lo_repoint_number, result.lo_repointings) + self.assertEqual(expected_versions_for_ultra_sf_repoint_number, result.ultra_sf_repointings) + self.assertEqual(expected_versions_for_ultra_hf_repoint_number, result.ultra_hf_repointings) \ No newline at end of file diff --git a/tests/glows/test_glows_processor.py b/tests/glows/test_glows_processor.py index 9ce9faccc..a69e86b06 100644 --- a/tests/glows/test_glows_processor.py +++ b/tests/glows/test_glows_processor.py @@ -32,6 +32,7 @@ from imap_l3_processing.glows.l3bc.glows_l3bc_initializer import GlowsL3BCInitializerData from imap_l3_processing.glows.l3bc.models import ExternalDependencies from imap_l3_processing.glows.l3d.glows_l3d_dependencies import GlowsL3DDependencies +from imap_l3_processing.glows.l3d.models import GlowsL3DProcessorOutput from imap_l3_processing.glows.l3d.utils import PATH_TO_L3D_TOOLKIT from imap_l3_processing.glows.l3e.glows_l3e_call_arguments import GlowsL3eCallArguments from imap_l3_processing.glows.l3e.glows_l3e_dependencies import GlowsL3EDependencies @@ -39,7 +40,7 @@ from imap_l3_processing.glows.l3e.glows_l3e_initializer import GlowsL3EInitializerOutput from imap_l3_processing.glows.l3e.glows_l3e_lo_model import GlowsL3ELoData from imap_l3_processing.glows.l3e.glows_l3e_ultra_model import GlowsL3EUltraData -from imap_l3_processing.glows.l3e.glows_l3e_utils import GlowsL3eRepointings, LoPivotAngle +from imap_l3_processing.glows.l3e.glows_l3e_utils import GlowsL3eVersionsForRepointings, LoPivotAngle from imap_l3_processing.models import InputMetadata, VersionMap from imap_l3_processing.utils import save_data from tests.test_helpers import get_test_instrument_team_data_path, get_test_data_path, get_test_data_folder, \ @@ -63,16 +64,16 @@ def setUp(self): ) self.l3d_initializer_patcher = patch( - "imap_l3_processing.glows.glows_processor.GlowsL3DInitializer.should_process_l3d") + "imap_l3_processing.glows.glows_processor.GlowsL3DInitializer") self.mock_l3d_initializer = self.l3d_initializer_patcher.start() - self.mock_l3d_initializer.return_value = None + self.mock_l3d_initializer.should_process_l3d.return_value = None self.mock_l3e_initializer_patcher = patch( - "imap_l3_processing.glows.glows_processor.GlowsL3EInitializer.get_repointings_to_process") + "imap_l3_processing.glows.glows_processor.GlowsL3EInitializer") self.mock_l3e_initializer = self.mock_l3e_initializer_patcher.start() - self.mock_l3e_initializer.return_value = GlowsL3EInitializerOutput( + self.mock_l3e_initializer.get_repointings_to_process.return_value = GlowsL3EInitializerOutput( dependencies=Mock(), - repointings=GlowsL3eRepointings( + repointings=GlowsL3eVersionsForRepointings( repointing_numbers=[], hi_90_repointings={}, hi_45_repointings={}, @@ -1105,7 +1106,7 @@ def test_process_l3e(self, mock_process_ultra, mock_process_ultra_hf, mock_proce l3d_cdf_path = Path("path/to/l3d.cdf") initializer_data = GlowsL3EInitializerOutput( dependencies=mock_dependencies, - repointings=GlowsL3eRepointings( + repointings=GlowsL3eVersionsForRepointings( repointing_numbers=[25], hi_90_repointings={25: Version(None, 1)}, hi_45_repointings={25: Version(None, 2)}, @@ -1134,6 +1135,97 @@ def test_process_l3e(self, mock_process_ultra, mock_process_ultra_hf, mock_proce self.assertEqual(expected_l3e_products, actual_l3e_products) mock_get_lo_pivot_angles.assert_called_once_with([25]) + @patch('imap_l3_processing.glows.glows_processor.process_l3e_ul_hf') + @patch('imap_l3_processing.glows.glows_processor.process_l3e_ul_sf') + @patch('imap_l3_processing.glows.glows_processor.process_l3e_hi') + @patch('imap_l3_processing.glows.glows_processor.process_l3e_lo') + @patch('imap_l3_processing.glows.glows_processor.compute_glows_flags_for_window') + @patch('imap_l3_processing.glows.glows_processor.get_pointing_date_range') + @patch('imap_l3_processing.glows.glows_processor.process_l3d') + def test_process_l3e_through_process(self, mock_process_l3d, mock_get_pointing_date_range, mock_compute_flags, + mock_process_lo, mock_process_hi, mock_process_ul_sf, mock_process_ul_hf): + mock_compute_flags.return_value = 0 + self.mock_l3d_initializer.should_process_l3d.return_value = (sentinel.l3d_version, sentinel.glows_l3d_dependency, sentinel.old_l3d) + process_l3d_result = GlowsL3DProcessorOutput(sentinel.l3d_cdf_file_path, [sentinel.l3d_text_file_paths], + sentinel.last_processed_cr) + mock_process_l3d.return_value = process_l3d_result + + input_major_version = 5 + + input_version_map = VersionMap({ + GLOWS_L3B_DESCRIPTOR: Version(2, 1), + GLOWS_L3C_DESCRIPTOR: Version(2, 1), + GLOWS_L3D_DESCRIPTOR: Version(2, 1), + GLOWS_L3E_HI_45_DESCRIPTOR: Version(input_major_version, 1), + GLOWS_L3E_HI_90_DESCRIPTOR: Version(input_major_version, 1), + GLOWS_L3E_LO_DESCRIPTOR: Version(input_major_version, 1), + GLOWS_L3E_ULTRA_SF_DESCRIPTOR: Version(input_major_version, 1), + GLOWS_L3E_ULTRA_HF_DESCRIPTOR: Version(input_major_version, 1), + }) + input_metadata = InputMetadata('glows', "l3b", datetime(2024, 10, 7), None, version=input_version_map) + + mock_get_pointing_date_range.return_value = datetime(2025, 1, 1), datetime(2025, 1, 2) + + l3e_initialzer_repointings = GlowsL3eVersionsForRepointings( + [2902, 2903, 2904, 2905, 2906, 2907], + {2902: Version(None, 1), 2903: Version(1,11)}, + {2902: Version(None, 1), 2904: Version(2,12)}, + {2902: Version(None, 1), 2905: Version(3,13)}, + {2902: Version(None, 1), 2906: Version(4,14)}, + {2902: Version(None, 1), 2907: Version(5,15)}, + ) + + self.mock_l3e_initializer.get_repointings_to_process.return_value = GlowsL3EInitializerOutput(Mock(), l3e_initialzer_repointings, sentinel.l3d_cdf_file_path) + expected_lo_files = [ + [f'imap_glows_l3e_survival-probability-lo_20250101-repoint02902_v001.cdf', sentinel.lo_dat_1], + [f'imap_glows_l3e_survival-probability-lo_20250101-repoint02905_v003.0013.cdf', sentinel.lo_dat_2], + ] + mock_process_lo.side_effect = expected_lo_files + + expected_hi90_files = [ + [f'imap_glows_l3e_survival-probability-hi-90_20250101-repoint02902_v001.cdf',sentinel.hi90_dat_1], + [f'imap_glows_l3e_survival-probability-hi-90_20250101-repoint02903_v001.0011.cdf',sentinel.hi90_dat_2], + ] + expected_hi45_files = [ + [f'imap_glows_l3e_survival-probability-hi-45_20250101-repoint02902_v001.cdf', sentinel.hi45_dat_1], + [f'imap_glows_l3e_survival-probability-hi-45_20250101-repoint02904_v002.0012.cdf', sentinel.hi45_dat_2], + ] + mock_process_hi.side_effect = [expected_hi90_files[0], expected_hi45_files[0], expected_hi90_files[1], expected_hi45_files[1]] + + expected_ul_sf_files = [ + [f'imap_glows_l3e_survival-probability-ul-sf_20250101-repoint02902_v001.cdf', sentinel.ul_sf_dat_1], + [f'imap_glows_l3e_survival-probability-ul-sf_20250101-repoint02906_v004.0014.cdf', sentinel.ul_sf_dat_2], + ] + mock_process_ul_sf.side_effect = expected_ul_sf_files + + expected_ul_hf_files = [ + [f'imap_glows_l3e_survival-probability-ul-hf_20250101-repoint02902_v001.cdf', sentinel.ul_hf_dat_1], + [f'imap_glows_l3e_survival-probability-ul-hf_20250101-repoint02907_v005.0015.cdf', sentinel.ul_hf_dat_2], + ] + mock_process_ul_hf.side_effect = expected_ul_hf_files + + expected_products = [ + sentinel.l3d_text_file_paths, + sentinel.l3d_cdf_file_path, + f'imap_glows_l3e_survival-probability-lo_20250101-repoint02902_v001.cdf', sentinel.lo_dat_1, + f'imap_glows_l3e_survival-probability-hi-90_20250101-repoint02902_v001.cdf', sentinel.hi90_dat_1, + f'imap_glows_l3e_survival-probability-hi-45_20250101-repoint02902_v001.cdf', sentinel.hi45_dat_1, + f'imap_glows_l3e_survival-probability-ul-sf_20250101-repoint02902_v001.cdf', sentinel.ul_sf_dat_1, + f'imap_glows_l3e_survival-probability-ul-hf_20250101-repoint02902_v001.cdf', sentinel.ul_hf_dat_1, + f'imap_glows_l3e_survival-probability-hi-90_20250101-repoint02903_v001.0011.cdf', sentinel.hi90_dat_2, + f'imap_glows_l3e_survival-probability-hi-45_20250101-repoint02904_v002.0012.cdf', sentinel.hi45_dat_2, + f'imap_glows_l3e_survival-probability-lo_20250101-repoint02905_v003.0013.cdf', sentinel.lo_dat_2, + f'imap_glows_l3e_survival-probability-ul-sf_20250101-repoint02906_v004.0014.cdf', sentinel.ul_sf_dat_2, + f'imap_glows_l3e_survival-probability-ul-hf_20250101-repoint02907_v005.0015.cdf', sentinel.ul_hf_dat_2, + ] + + processor = GlowsProcessor(Mock(), input_metadata) + products = processor.process() + + + self.mock_l3e_initializer.get_repointings_to_process.assert_called_once_with(process_l3d_result, sentinel.old_l3d, self.mock_l3bc_initializer.return_value.repoint_file_path, input_version_map) + self.assertEqual(expected_products, products) + @patch('imap_l3_processing.glows.glows_processor.Processor.get_parent_file_names') @patch('imap_l3_processing.glows.glows_processor.save_data') @patch("imap_l3_processing.glows.glows_processor.GlowsL3EUltraData.convert_dat_to_glows_l3e_ul_product") @@ -1152,7 +1244,7 @@ def test_process_l3e_ultra_sf(self, mock_shutil, mock_determine_call_args, mock_ version = Version(1, 12) expected_input_metadata = InputMetadata('glows', "l3e", start_date=epoch_start_date, end_date=epoch_end_date, - version='v001.0012', descriptor=GLOWS_L3E_ULTRA_SF_DESCRIPTOR, + version=VersionMap({GLOWS_L3E_ULTRA_SF_DESCRIPTOR:version}), descriptor=GLOWS_L3E_ULTRA_SF_DESCRIPTOR, repointing=repointing) ultra_args = ["20241007_000000", "date.001", "vx", "vy", "vz", "30.000"] @@ -1211,7 +1303,7 @@ def test_process_l3e_ultra_hf(self, mock_shutil, mock_determine_call_args, mock_ version = Version(1, 12) input_metadata = InputMetadata('glows', "l3e", start_date=epoch_start_date, end_date=epoch_end_date, - version='v001.0012', descriptor=GLOWS_L3E_ULTRA_HF_DESCRIPTOR, + version=VersionMap({GLOWS_L3E_ULTRA_HF_DESCRIPTOR: version}), descriptor=GLOWS_L3E_ULTRA_HF_DESCRIPTOR, repointing=repointing) mock_determine_call_args.return_value = GlowsL3eCallArguments( @@ -1283,11 +1375,11 @@ def test_process_l3e_hi(self, mock_shutil, mock_determine_call_args, mock_run, mock_convert_dat_to_glows_l3e_hi_product, mock_save_data): test_cases = [ - ("90", 90), - ("45", 135) + ("90", 90, GLOWS_L3E_HI_90_DESCRIPTOR), + ("45", 135, GLOWS_L3E_HI_45_DESCRIPTOR) ] - for descriptor_elongation, elongation in test_cases: + for descriptor_elongation, elongation, descriptor in test_cases: with (self.subTest(elongation=elongation)): mock_determine_call_args.reset_mock() mock_run.reset_mock() @@ -1303,7 +1395,7 @@ def test_process_l3e_hi(self, mock_shutil, mock_determine_call_args, expected_input_metadata = InputMetadata( instrument='glows', data_level="l3e", start_date=epoch_start_date, end_date=epoch_end_date, - version=str(version), descriptor=f'survival-probability-hi-{descriptor_elongation}', + version=VersionMap({descriptor: version}), descriptor=f'survival-probability-hi-{descriptor_elongation}', repointing=repointing) l3e_dependencies = MagicMock(spec=GlowsL3EDependencies) @@ -1382,7 +1474,7 @@ def test_process_l3e_lo(self, mock_shutil, expected_input_metadata = InputMetadata( instrument='glows', data_level="l3e", start_date=epoch_start_date, end_date=epoch_end_date, - version=str(version), descriptor=f'survival-probability-lo', repointing=repointing) + version=VersionMap({GLOWS_L3E_LO_DESCRIPTOR: version}), descriptor=GLOWS_L3E_LO_DESCRIPTOR, repointing=repointing) lo_call_args = ["20241007_000000", "date.100", "vx", "vy", "vz", f"{elongation:.3f}"] @@ -1550,13 +1642,13 @@ def test_process_l3e_skips_repointing_on_exception(self, mock_process_ultra_sf, initializer_data = GlowsL3EInitializerOutput( dependencies=mock_dependencies, - repointings=GlowsL3eRepointings( + repointings=GlowsL3eVersionsForRepointings( repointing_numbers=[24, 25, 26, 27], - hi_90_repointings={24: 1, 25: 1, 26: 1, 27: 1}, - hi_45_repointings={24: 2, 25: 2, 26: 2, 27: 2}, - lo_repointings={24: 3, 25: 3, 26: 3, 27: 3}, - ultra_hf_repointings={24: 4, 25: 4, 26: 4, 27: 4}, - ultra_sf_repointings={24: 4, 25: 4, 26: 4, 27: 4}, + hi_90_repointings={24: Version(None, 1), 25: Version(None, 1), 26: Version(None, 1), 27: Version(None, 1)}, + hi_45_repointings={24: Version(None, 2), 25: Version(None, 2), 26: Version(None, 2), 27: Version(None, 2)}, + lo_repointings={24: Version(None, 3), 25: Version(None, 3), 26: Version(None, 3), 27: Version(None, 3)}, + ultra_hf_repointings={24: Version(None, 4), 25: Version(None, 4), 26: Version(None, 4), 27: Version(None, 4)}, + ultra_sf_repointings={24: Version(None, 4), 25: Version(None, 4), 26: Version(None, 4), 27: Version(None, 4)}, ), l3d_cdf_path=Path("path/to/l3d.cdf"), ) @@ -1566,34 +1658,34 @@ def test_process_l3e_skips_repointing_on_exception(self, mock_process_ultra_sf, mock_get_pointing_date_range.assert_has_calls([call(24), call(25), call(26), call(27)]) mock_process_hi.assert_has_calls([ - call(hi_parents, 24, start_epoch_1, epoch_delta_1, 90, 1, 0), - call(hi_parents, 24, start_epoch_1, epoch_delta_1, 135, 2, 0), - call(hi_parents, 25, start_epoch_2, epoch_delta_2, 90, 1, 0), - call(hi_parents, 25, start_epoch_2, epoch_delta_2, 135, 2, 0), - call(hi_parents, 26, start_epoch_3, epoch_delta_3, 90, 1, 0), - call(hi_parents, 26, start_epoch_3, epoch_delta_3, 135, 2, 0), - call(hi_parents, 27, start_epoch_4, epoch_delta_4, 90, 1, 0), - call(hi_parents, 27, start_epoch_4, epoch_delta_4, 135, 2, 0) + call(hi_parents, 24, start_epoch_1, epoch_delta_1, 90, Version(None, 1), 0), + call(hi_parents, 24, start_epoch_1, epoch_delta_1, 135, Version(None, 2), 0), + call(hi_parents, 25, start_epoch_2, epoch_delta_2, 90, Version(None, 1), 0), + call(hi_parents, 25, start_epoch_2, epoch_delta_2, 135, Version(None, 2), 0), + call(hi_parents, 26, start_epoch_3, epoch_delta_3, 90, Version(None, 1), 0), + call(hi_parents, 26, start_epoch_3, epoch_delta_3, 135, Version(None, 2), 0), + call(hi_parents, 27, start_epoch_4, epoch_delta_4, 90, Version(None, 1), 0), + call(hi_parents, 27, start_epoch_4, epoch_delta_4, 135, Version(None, 2), 0) ]) mock_process_lo.assert_has_calls([ - call(lo_parents + ["l1b_nhk_24"], 24, start_epoch_1, epoch_delta_1, 124, 3, 0), - call(lo_parents + ["l1b_nhk_25"], 25, start_epoch_2, epoch_delta_2, 125, 3, 0), - call(lo_parents + ["l1b_nhk_26"], 26, start_epoch_3, epoch_delta_3, 126, 3, 0), - call(lo_parents, 27, start_epoch_4, epoch_delta_4, 90, 3, 0), + call(lo_parents + ["l1b_nhk_24"], 24, start_epoch_1, epoch_delta_1, 124, Version(None, 3), 0), + call(lo_parents + ["l1b_nhk_25"], 25, start_epoch_2, epoch_delta_2, 125, Version(None, 3), 0), + call(lo_parents + ["l1b_nhk_26"], 26, start_epoch_3, epoch_delta_3, 126, Version(None, 3), 0), + call(lo_parents, 27, start_epoch_4, epoch_delta_4, 90, Version(None, 3), 0), ]) mock_process_ultra_sf.assert_has_calls([ - call(ultra_parents, 24, start_epoch_1, epoch_delta_1, 4, 0), - call(ultra_parents, 25, start_epoch_2, epoch_delta_2, 4, 0), - call(ultra_parents, 26, start_epoch_3, epoch_delta_3, 4, 0), - call(ultra_parents, 27, start_epoch_4, epoch_delta_4, 4, 0), + call(ultra_parents, 24, start_epoch_1, epoch_delta_1, Version(None, 4), 0), + call(ultra_parents, 25, start_epoch_2, epoch_delta_2, Version(None, 4), 0), + call(ultra_parents, 26, start_epoch_3, epoch_delta_3, Version(None, 4), 0), + call(ultra_parents, 27, start_epoch_4, epoch_delta_4, Version(None, 4), 0), ]) mock_process_ultra_hf.assert_has_calls([ - call(ultra_parents, 24, start_epoch_1, epoch_delta_1, 4, 0), - call(ultra_parents, 25, start_epoch_2, epoch_delta_2, 4, 0), - call(ultra_parents, 26, start_epoch_3, epoch_delta_3, 4, 0), - call(ultra_parents, 27, start_epoch_4, epoch_delta_4, 4, 0), + call(ultra_parents, 24, start_epoch_1, epoch_delta_1, Version(None, 4), 0), + call(ultra_parents, 25, start_epoch_2, epoch_delta_2, Version(None, 4), 0), + call(ultra_parents, 26, start_epoch_3, epoch_delta_3, Version(None, 4), 0), + call(ultra_parents, 27, start_epoch_4, epoch_delta_4, Version(None, 4), 0), ]) self.assertEqual(expected_l3e_products, actual_l3e_products) diff --git a/tests/integration/test_glows_processor_integration.py b/tests/integration/test_glows_processor_integration.py index 539e30e64..5d0058e69 100644 --- a/tests/integration/test_glows_processor_integration.py +++ b/tests/integration/test_glows_processor_integration.py @@ -16,16 +16,17 @@ import numpy as np from imap_data_access import ProcessingInputCollection, RepointInput from imap_data_access import ScienceInput, AncillaryInput -from imap_data_access.file_validation import ScienceFilePath, AncillaryFilePath +from imap_data_access.file_validation import ScienceFilePath, AncillaryFilePath, Version from spacepy.pycdf import CDF import imap_l3_data_processor import tests +from imap_l3_processing.glows.descriptors import GLOWS_L3BCDE_DESCRIPTORS from imap_l3_processing.glows.glows_processor import GlowsProcessor from imap_l3_processing.glows.l3a.glows_l3a_dependencies import GlowsL3ADependencies from imap_l3_processing.glows.l3a.utils import read_l2_glows_data, create_glows_l3a_from_dictionary from imap_l3_processing.glows.l3d.utils import PATH_TO_L3D_TOOLKIT -from imap_l3_processing.models import InputMetadata +from imap_l3_processing.models import InputMetadata, VersionMap from imap_l3_processing.utils import save_data from tests.integration.integration_test_helpers import mock_imap_data_access, run_istp_compliance_check from tests.test_helpers import get_test_data_path, get_test_instrument_team_data_path, \ @@ -141,8 +142,6 @@ def test_glows_l3a(self, tmp_dir): np.testing.assert_allclose(actual_dict['spin_angle_delta'], expected_dict['spin_angle_delta']) np.testing.assert_allclose(actual_dict['latitude'], expected_dict['latitude'], atol=1e-3) np.testing.assert_allclose(actual_dict['longitude'], expected_dict['longitude'], atol=1e-3) - # np.testing.assert_allclose(actual_dict['extra_heliospheric_background'], expected_dict['extra_heliospheric_background']) - # np.testing.assert_allclose(actual_dict['time_dependent_background'], expected_dict['time_dependent_background']) np.testing.assert_allclose(actual_dict['filter_temperature_average'], expected_dict['filter_temperature_average']) np.testing.assert_allclose(actual_dict['filter_temperature_std_dev'], @@ -190,7 +189,7 @@ def test_l3a_handles_l2_input_that_is_all_flagged(self): descriptor="hist", start_date=start_date, end_date=end_date, - version="v001", + version=VersionMap({}, Version(None, 1)), repointing=l2_science_file_path.repointing, ) @@ -266,8 +265,10 @@ def test_l3bcde_first_time_processing(self): shutil.rmtree(path) processing_input = ProcessingInputCollection(RepointInput("imap_2026_090_01.repoint")) + + version_map = VersionMap({descriptor: Version(i, 2) for i, descriptor in enumerate(GLOWS_L3BCDE_DESCRIPTORS)}) input_metadata = InputMetadata(instrument="glows", data_level="l3b", descriptor="ion-rate-profile", - version="v001", start_date=datetime(2000, 1, 1), + version=version_map, start_date=datetime(2000, 1, 1), end_date=datetime(2000, 1, 1)) processor = GlowsProcessor(processing_input, input_metadata) @@ -373,7 +374,7 @@ def test_l3bcde_reprocessing(self): with mock_imap_data_access(get_run_local_data_path("glows_reprocessing"), l3bcde_input_files): processing_input = ProcessingInputCollection(RepointInput("imap_2026_090_01.repoint")) input_metadata = InputMetadata(instrument="glows", data_level="l3b", descriptor="ion-rate-profile", - version="v001", start_date=datetime(2000, 1, 1), + version=VersionMap({}, Version(None, 1)), start_date=datetime(2000, 1, 1), end_date=datetime(2000, 1, 1)) processor = GlowsProcessor(processing_input, input_metadata) @@ -431,7 +432,7 @@ def test_glows_l3abcde_from_l2(self): data_level="l3a", start_date=start_date, end_date=end_date, - version="v001", + version=VersionMap({}, Version(None, 1)), descriptor="hist", repointing=l2_science_file_path.repointing, ) @@ -485,7 +486,7 @@ def test_glows_l3abcde_from_l2(self): with mock_imap_data_access(get_run_local_data_path("glows_l3bcde_with_prod_l2"), input_paths): processing_input = ProcessingInputCollection(RepointInput("imap_2026_139_01.repoint")) input_metadata = InputMetadata(instrument="glows", data_level="l3b", descriptor="ion-rate-profile", - version="v001", start_date=datetime(2000, 1, 1), + version=VersionMap({}, Version(None, 1)), start_date=datetime(2000, 1, 1), end_date=datetime(2000, 1, 1)) processor = GlowsProcessor(processing_input, input_metadata) From 8346b50d98c393967ac7bf339ea0d421b8ac2d35 Mon Sep 17 00:00:00 2001 From: Harrison Date: Fri, 10 Jul 2026 11:05:24 -0400 Subject: [PATCH 09/11] Harrison #118 - KJON/AFAH: GLOWS L3e can now be reprocessed if there are no changes between the output L3d and the existing L3d --- .../glows/l3e/glows_l3e_initializer.py | 21 +++-- pyproject.toml | 2 +- tests/glows/l3e/test_glows_l3e_initializer.py | 76 ++++++++++++------- 3 files changed, 58 insertions(+), 41 deletions(-) diff --git a/imap_l3_processing/glows/l3e/glows_l3e_initializer.py b/imap_l3_processing/glows/l3e/glows_l3e_initializer.py index b4a42b1ae..04cb06374 100644 --- a/imap_l3_processing/glows/l3e/glows_l3e_initializer.py +++ b/imap_l3_processing/glows/l3e/glows_l3e_initializer.py @@ -33,12 +33,6 @@ def get_repointings_to_process( repointing_file_path: Path, version_map: VersionMap, ) -> Optional[GlowsL3EInitializerOutput]: - first_updated_cr = None - if previous_l3d is not None: - first_updated_cr = find_first_updated_cr(l3d_output.l3d_cdf_file_path, previous_l3d) - if first_updated_cr is None: - return None - pipeline_settings_l3bcde = get_most_recently_uploaded_ancillary(imap_data_access.query(table='ancillary', instrument='glows', descriptor='pipeline-settings-l3bcde')) energy_grid_lo = get_most_recently_uploaded_ancillary(imap_data_access.query(table='ancillary', instrument='glows', descriptor='energy-grid-lo')) tess_xyz_8 = get_most_recently_uploaded_ancillary(imap_data_access.query(table='ancillary', instrument='glows', descriptor='tess-xyz-8')) @@ -63,12 +57,17 @@ def get_repointings_to_process( first_cr = l3e_deps.pipeline_settings["start_cr"] - if first_updated_cr is None: - first_updated_cr = first_cr - else: - first_updated_cr -= 1 + first_updated_cr = first_cr + if previous_l3d is not None: + first_updated_cr = find_first_updated_cr(l3d_output.l3d_cdf_file_path, previous_l3d) + if first_updated_cr is None: + first_updated_cr = first_cr + else: + first_updated_cr -= 1 + + - last_cr = ScienceFilePath(l3d_output.l3d_cdf_file_path).cr + last_cr = l3d_output.last_processed_cr glows_repointings = identify_versions_for_l3e_output_files(first_cr, last_cr, first_updated_cr, repointing_file_path, version_map) if len(glows_repointings.repointing_numbers) > 0: diff --git a/pyproject.toml b/pyproject.toml index b4cf591ad..1fb7ce0d6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -11,7 +11,7 @@ dependencies = [ "astropy==6.1.7", "astropy_healpix==1.1.2", "bitstring==4.3.0", - "imap-data-access>=0.42.0.dev0", + "imap-data-access==0.42.0.dev0", "lmfit==1.3.3", "netCDF4>=1.6", "numdifftools==0.9.41", diff --git a/tests/glows/l3e/test_glows_l3e_initializer.py b/tests/glows/l3e/test_glows_l3e_initializer.py index 1e4963799..e8245a386 100644 --- a/tests/glows/l3e/test_glows_l3e_initializer.py +++ b/tests/glows/l3e/test_glows_l3e_initializer.py @@ -13,14 +13,15 @@ from imap_l3_processing.models import VersionMap from tests.test_helpers import create_mock_query_results +MODULE = 'imap_l3_processing.glows.l3e.glows_l3e_initializer' class TestGlowsL3EInitializer(unittest.TestCase): - @patch('imap_l3_processing.glows.l3e.glows_l3e_initializer.get_pointing_date_range') - @patch('imap_l3_processing.glows.l3e.glows_l3e_initializer.GlowsL3EDependencies.fetch_dependencies') - @patch('imap_l3_processing.glows.l3e.glows_l3e_initializer.identify_versions_for_l3e_output_files') - @patch('imap_l3_processing.glows.l3e.glows_l3e_initializer.find_first_updated_cr') - @patch('imap_l3_processing.glows.l3e.glows_l3e_initializer.get_most_recently_uploaded_ancillary') - @patch('imap_l3_processing.glows.l3e.glows_l3e_initializer.imap_data_access.query') + @patch(f'{MODULE}.get_pointing_date_range') + @patch(f'{MODULE}.GlowsL3EDependencies.fetch_dependencies') + @patch(f'{MODULE}.identify_versions_for_l3e_output_files') + @patch(f'{MODULE}.find_first_updated_cr') + @patch(f'{MODULE}.get_most_recently_uploaded_ancillary') + @patch(f'{MODULE}.imap_data_access.query') def test_get_repointings_to_process(self, mock_query, mock_get_most_recently_uploaded_ancillary, mock_find_first_updated_cr, mock_identify_versions_for_l3e_output_files, mock_fetch_dependencies, mock_get_pointing_date_range): @@ -44,13 +45,13 @@ def test_get_repointings_to_process(self, mock_query, mock_get_most_recently_upl updated_l3d = Path('path/to/imap_glows_l3d_solar-hist_19470303-cr02091_v000.cdf') updated_l3d_text_file_path = Path("imap_glows_e-dens_19470303_20100101_v000.dat") - glows_l3d_processor_output = GlowsL3DProcessorOutput(updated_l3d, [updated_l3d_text_file_path], 2091) + glows_l3d_processor_output = GlowsL3DProcessorOutput(updated_l3d, [updated_l3d_text_file_path], sentinel.last_processed_cr) previous_l3d = 'imap_glows_l3d_solar-hist_19470303-cr02090_v000.cdf' mock_find_first_updated_cr.return_value = 2091 mock_l3e_dependencies = mock_fetch_dependencies.return_value - mock_l3e_dependencies.pipeline_settings = {"start_cr": 2089} + mock_l3e_dependencies.pipeline_settings = {"start_cr": sentinel.start_of_mission_cr} mock_l3e_dependencies.repointing_file = Path('path/to/repointing_file') expected_hi_45 = {1234: Version(None, 1), 2468: Version(None, 1)} @@ -90,7 +91,8 @@ def test_get_repointings_to_process(self, mock_query, mock_get_most_recently_upl mock_find_first_updated_cr.assert_called_once_with(updated_l3d, previous_l3d) - mock_identify_versions_for_l3e_output_files.assert_called_once_with(2089, 2091, 2090, repointing_file_path, input_major_version) + mock_identify_versions_for_l3e_output_files.assert_called_once_with(sentinel.start_of_mission_cr, sentinel.last_processed_cr, 2090, repointing_file_path, + input_major_version) mock_query.assert_has_calls([ call(table="ancillary", instrument='glows', descriptor='pipeline-settings-l3bcde'), @@ -101,7 +103,6 @@ def test_get_repointings_to_process(self, mock_query, mock_get_most_recently_upl call(table="ancillary", instrument='glows', descriptor='tess-ang-16'), ]) - mock_l3e_dependencies.copy_dependencies.assert_called_once() [fetch_dependencies_call] = mock_fetch_dependencies.call_args_list @@ -137,32 +138,48 @@ def test_get_repointings_to_process(self, mock_query, mock_get_most_recently_upl end_date=datetime(2011, 2, 2), ) - @patch('imap_l3_processing.glows.l3e.glows_l3e_initializer.find_first_updated_cr') - def test_get_repointings_to_process_with_identical_l3d_files(self, mock_find_first_updated_cr): + @patch(f'{MODULE}.GlowsL3EDependencies.fetch_dependencies') + @patch(f'{MODULE}.identify_versions_for_l3e_output_files') + @patch(f'{MODULE}.find_first_updated_cr') + def test_get_repointings_to_process_uses_mission_start_cr_as_first_updated_when_identical_l3d_files(self, + mock_find_first_updated_cr, + mock_identify_versions_for_l3e_output_files, + mock_fetch_dependencies): updated_l3d = Path('path/to/imap_glows_l3d_solar-hist_19470303-cr02091_v000.cdf') updated_l3d_text_file_path = Path("imap_glows_e-dens_19470303_20100101_v000.dat") - glows_l3d_processor_output = GlowsL3DProcessorOutput(updated_l3d, [updated_l3d_text_file_path], 2091) + expected_last_cr = 2091 + glows_l3d_processor_output = GlowsL3DProcessorOutput(updated_l3d, [updated_l3d_text_file_path], expected_last_cr) previous_l3d = 'imap_glows_l3d_solar-hist_19470303-cr02091_v000.cdf' + mock_l3e_dependencies = mock_fetch_dependencies.return_value + mock_l3e_dependencies.pipeline_settings = {"start_cr": sentinel.start_of_mission_cr} mock_find_first_updated_cr.return_value = None repointing_file_path = Path("imap_2026_105_01.repoint.csv") actual_initializer_output = GlowsL3EInitializer.get_repointings_to_process(glows_l3d_processor_output, previous_l3d, - repointing_file_path, sentinel.version_map) + repointing_file_path, + sentinel.version_map) + + expected_initializer_output = GlowsL3EInitializerOutput( + mock_l3e_dependencies, + mock_identify_versions_for_l3e_output_files.return_value, + glows_l3d_processor_output.l3d_cdf_file_path + ) mock_find_first_updated_cr.assert_called_once_with(glows_l3d_processor_output.l3d_cdf_file_path, previous_l3d) - self.assertIsNone(actual_initializer_output) - - @patch('imap_l3_processing.glows.l3e.glows_l3e_initializer.imap_data_access.query') - @patch('imap_l3_processing.glows.l3e.glows_l3e_initializer.get_most_recently_uploaded_ancillary') - @patch('imap_l3_processing.glows.l3e.glows_l3e_initializer.GlowsL3EDependencies.fetch_dependencies') - @patch('imap_l3_processing.glows.l3e.glows_l3e_initializer.identify_versions_for_l3e_output_files') - @patch('imap_l3_processing.glows.l3e.glows_l3e_initializer.find_first_updated_cr') - def test_get_repointings_to_process_handles_no_previous_l3d(self, mock_find_first_updated_cr, - mock_identify_versions_for_l3e_output_files, - mock_fetch_dependencies, - mock_get_most_recently_uploaded_ancillary, - _, ): + mock_identify_versions_for_l3e_output_files.assert_called_once_with(sentinel.start_of_mission_cr, expected_last_cr, sentinel.start_of_mission_cr, repointing_file_path, sentinel.version_map) + self.assertEqual(expected_initializer_output, actual_initializer_output) + + @patch(f'{MODULE}.imap_data_access.query') + @patch(f'{MODULE}.get_most_recently_uploaded_ancillary') + @patch(f'{MODULE}.GlowsL3EDependencies.fetch_dependencies') + @patch(f'{MODULE}.identify_versions_for_l3e_output_files') + @patch(f'{MODULE}.find_first_updated_cr') + def test_get_repointings_to_process_uses_mission_start_cr_when_no_previous_l3d(self, mock_find_first_updated_cr, + mock_identify_versions_for_l3e_output_files, + mock_fetch_dependencies, + mock_get_most_recently_uploaded_ancillary, + _, ): mock_get_most_recently_uploaded_ancillary.side_effect = [ create_mock_query_results(['imap_glows_pipeline-settings-l3bcde_20200101_v000.cdf'])[0], create_mock_query_results(['imap_glows_energy-grid-lo_20200101_v000.cdf'])[0], @@ -181,14 +198,15 @@ def test_get_repointings_to_process_handles_no_previous_l3d(self, mock_find_firs ) updated_l3d = Path('path/to/imap_glows_l3d_solar-hist_19470303-cr02091_v012.0001.cdf') - glows_l3d_processor_output = GlowsL3DProcessorOutput(updated_l3d, [], 2091) + glows_l3d_processor_output = GlowsL3DProcessorOutput(updated_l3d, [], sentinel.last_processed_cr) previous_l3d = None - mock_fetch_dependencies.return_value.pipeline_settings = {"start_cr": 2089} + mock_fetch_dependencies.return_value.pipeline_settings = {"start_cr": sentinel.start_of_mission_cr} repointing_file_path = Path("imap_2026_105_01.repoint.csv") _ = GlowsL3EInitializer.get_repointings_to_process(glows_l3d_processor_output, previous_l3d, repointing_file_path, sentinel.version_map) mock_find_first_updated_cr.assert_not_called() - mock_identify_versions_for_l3e_output_files.assert_called_once_with(2089, 2091, 2089, repointing_file_path, sentinel.version_map) + mock_identify_versions_for_l3e_output_files.assert_called_once_with(sentinel.start_of_mission_cr, sentinel.last_processed_cr, sentinel.start_of_mission_cr, repointing_file_path, + sentinel.version_map) From 8b9c90e071a23dff4f79d00f6ec27ed674ae330b Mon Sep 17 00:00:00 2001 From: Harrison Date: Mon, 13 Jul 2026 09:53:00 -0400 Subject: [PATCH 10/11] Harrison #118 - KJON/AFAH/LFAR: Update determination of version from query result to accommodate major_version and minor_version keys. --- .../glows/l3d/glows_l3d_initializer.py | 5 ++- imap_l3_processing/glows/l3d/utils.py | 9 ++++- .../glows/l3e/glows_l3e_utils.py | 10 ++++- imap_l3_processing/maps/map_initializer.py | 2 +- tests/periodically_run_tests.json | 8 ++-- tests/test_helpers.py | 37 +++++++++++++------ 6 files changed, 51 insertions(+), 20 deletions(-) diff --git a/imap_l3_processing/glows/l3d/glows_l3d_initializer.py b/imap_l3_processing/glows/l3d/glows_l3d_initializer.py index 0388d5550..01fda3f49 100644 --- a/imap_l3_processing/glows/l3d/glows_l3d_initializer.py +++ b/imap_l3_processing/glows/l3d/glows_l3d_initializer.py @@ -107,7 +107,10 @@ def should_process_l3d(external_deps: ExternalDependencies, l3bs: list[str], l3c old_l3d = Path(most_recent_l3d["file_path"]).name logger.info(f"Old L3d parents: {l3d_parents}, new L3d deps: {updated_input_files}") - most_recent_l3d_version = Version.from_version(most_recent_l3d['version']) + if 'major_version' in most_recent_l3d and 'minor_version' in most_recent_l3d: + most_recent_l3d_version = Version(most_recent_l3d['major_version'], most_recent_l3d['major_version']) + else: + most_recent_l3d_version = Version.from_version(most_recent_l3d['version']) same_major_version = most_recent_l3d_version.major == major_version if same_major_version and updated_input_files.issubset(l3d_parents): return None diff --git a/imap_l3_processing/glows/l3d/utils.py b/imap_l3_processing/glows/l3d/utils.py index 2fc4abd9a..f76ae4846 100644 --- a/imap_l3_processing/glows/l3d/utils.py +++ b/imap_l3_processing/glows/l3d/utils.py @@ -6,6 +6,7 @@ import imap_data_access import numpy as np +from imap_data_access.file_validation import Version from spacepy.pycdf import CDF import imap_l3_processing @@ -142,5 +143,11 @@ def rename_l3d_text_outputs(paths: list[Path], version: str) -> list[Path]: def query_for_most_recent_l3d(descriptor: str) -> Optional[dict]: query_result = imap_data_access.query(instrument="glows", data_level="l3d", descriptor=descriptor) - sorted_query_result = sorted(query_result, key=lambda qr: (qr["cr"], qr["version"]), reverse=True) + sorted_query_result = sorted( + query_result, + key=lambda qr: ( + qr["cr"], + (qr["minor_version"] if "minor_version" in qr else Version.from_version(qr["version"]).minor) + ), + reverse=True) return next(iter(sorted_query_result), None) diff --git a/imap_l3_processing/glows/l3e/glows_l3e_utils.py b/imap_l3_processing/glows/l3e/glows_l3e_utils.py index cebf7e538..da60b9b40 100644 --- a/imap_l3_processing/glows/l3e/glows_l3e_utils.py +++ b/imap_l3_processing/glows/l3e/glows_l3e_utils.py @@ -79,10 +79,16 @@ def identify_versions_for_l3e_output_files(start_cr_of_mission: int, end_cr_of_m updated_pointings_per_instruments = {} updated_pointing_numbers = {} - for descriptor in GLOWS_L3E_DESCRIPTORS: l3e_files = imap_data_access.query(instrument='glows', data_level='l3e', version="latest", descriptor=descriptor) - existing_file_versions = {int(l3e['repointing']): Version.from_version(l3e['version']) for l3e in l3e_files} + existing_file_versions = {} + for l3e in l3e_files: + if 'major_version' in l3e.keys() and 'minor_version' in l3e.keys(): + existing_file_versions[int(l3e['repointing'])] = Version(l3e["major_version"], l3e["minor_version"]) + elif 'version' in l3e.keys(): + existing_file_versions[int(l3e['repointing'])] = Version.from_version(l3e['version']) + else: + continue new_file_versions = {} for pointing_number in all_pointing_numbers: new_major_version = version_map.lookup(descriptor).major diff --git a/imap_l3_processing/maps/map_initializer.py b/imap_l3_processing/maps/map_initializer.py index 0c8c990c2..25116c32e 100644 --- a/imap_l3_processing/maps/map_initializer.py +++ b/imap_l3_processing/maps/map_initializer.py @@ -79,7 +79,7 @@ def get_latest_version_by_descriptor_and_start_date(query_results: list[dict]) - file_paths_by_descriptor = defaultdict(dict) for (descriptor, start_date), query_results in qr_by_descriptor_and_start_date.items(): - highest_version_qr = max(query_results, key=lambda x: x['version']) + highest_version_qr = max(query_results, key=lambda x: x['minor_version'] if 'minor_version' in x else Version.from_version(x['version']).minor) file_paths_by_descriptor[descriptor][start_date] = Path(highest_version_qr['file_path']).name return file_paths_by_descriptor diff --git a/tests/periodically_run_tests.json b/tests/periodically_run_tests.json index 79b15a720..6706b863f 100644 --- a/tests/periodically_run_tests.json +++ b/tests/periodically_run_tests.json @@ -1,14 +1,14 @@ { "test_l3bcde_first_time_processing": "2026-07-07T14:18:08.887025", - "test_spectral_fit_against_validation_data": "2026-07-07T13:55:28.665669", + "test_spectral_fit_against_validation_data": "2026-07-13T09:33:37.838756", "test_calculate_pickup_ions_with_minimize": "2026-05-20T10:52:46.704900", - "test_l3bcde_reprocessing": "2026-06-26T10:06:52.595047", - "test_glows_l3abcde_from_l2": "2026-06-26T10:04:11.001202", + "test_l3bcde_reprocessing": "2026-07-13T09:34:43.565666", + "test_glows_l3abcde_from_l2": "2026-07-13T09:34:43.073702", "test_codice_lo_3d_distributions": "2026-05-28T09:43:22.990231", "test_alpha_sigma_matches_empirical_std": null, "test_proton_sigma_matches_empirical_std": null, "test_swe_processor_with_local_data": "2026-07-07T14:22:38.114506", - "test_ultra_all_sp_maps": "2026-07-07T14:19:22.456659", + "test_ultra_all_sp_maps": "2026-07-13T09:35:49.849096", "test_ultra_combined_nsp_maps": "2026-07-07T14:20:08.230976", "test_ultra_combined_sp_maps": "2026-07-07T14:21:14.004464" } \ No newline at end of file diff --git a/tests/test_helpers.py b/tests/test_helpers.py index 97bffbad8..3afd3e9b0 100644 --- a/tests/test_helpers.py +++ b/tests/test_helpers.py @@ -229,17 +229,32 @@ def create_mock_query_results(file_names: list[Path | str], ingestion_dates: Opt match imap_file_path: case ScienceFilePath(): - file_paths.append({ - "instrument": imap_file_path.instrument, - "data_level": imap_file_path.data_level, - "descriptor": imap_file_path.descriptor, - "start_date": imap_file_path.start_date, - "ingestion_date": ingestion_date.strftime("%Y%m%d %H:%M:%S"), - "version": imap_file_path.version, - "cr": imap_file_path.cr, - "file_path": file_path, - "repointing": imap_file_path.repointing - }) + version_object = Version.from_version(imap_file_path.version) + if version_object.major is not None: + file_paths.append({ + "instrument": imap_file_path.instrument, + "data_level": imap_file_path.data_level, + "descriptor": imap_file_path.descriptor, + "start_date": imap_file_path.start_date, + "ingestion_date": ingestion_date.strftime("%Y%m%d %H:%M:%S"), + "major_version": version_object.major, + "minor_version": version_object.minor, + "cr": imap_file_path.cr, + "file_path": file_path, + "repointing": imap_file_path.repointing + }) + else: + file_paths.append({ + "instrument": imap_file_path.instrument, + "data_level": imap_file_path.data_level, + "descriptor": imap_file_path.descriptor, + "start_date": imap_file_path.start_date, + "ingestion_date": ingestion_date.strftime("%Y%m%d %H:%M:%S"), + "version": imap_file_path.version, + "cr": imap_file_path.cr, + "file_path": file_path, + "repointing": imap_file_path.repointing + }) case AncillaryFilePath(): file_paths.append({ "instrument": imap_file_path.instrument, From 106ccf64629a7aff2654525cd6b35f0bc0a484cc Mon Sep 17 00:00:00 2001 From: Harrison Date: Mon, 13 Jul 2026 10:45:45 -0400 Subject: [PATCH 11/11] Harrison #118 - KJON: Identical L3d files have a None first_updated_cr --- .../glows/l3e/glows_l3e_initializer.py | 6 +----- imap_l3_processing/glows/l3e/glows_l3e_utils.py | 5 +++-- tests/glows/l3e/test_glows_l3e_initializer.py | 4 ++-- tests/glows/l3e/test_glows_l3e_utils.py | 12 ++++++++++++ 4 files changed, 18 insertions(+), 9 deletions(-) diff --git a/imap_l3_processing/glows/l3e/glows_l3e_initializer.py b/imap_l3_processing/glows/l3e/glows_l3e_initializer.py index 04cb06374..7e4ae1317 100644 --- a/imap_l3_processing/glows/l3e/glows_l3e_initializer.py +++ b/imap_l3_processing/glows/l3e/glows_l3e_initializer.py @@ -60,13 +60,9 @@ def get_repointings_to_process( first_updated_cr = first_cr if previous_l3d is not None: first_updated_cr = find_first_updated_cr(l3d_output.l3d_cdf_file_path, previous_l3d) - if first_updated_cr is None: - first_updated_cr = first_cr - else: + if first_updated_cr is not None: first_updated_cr -= 1 - - last_cr = l3d_output.last_processed_cr glows_repointings = identify_versions_for_l3e_output_files(first_cr, last_cr, first_updated_cr, repointing_file_path, version_map) diff --git a/imap_l3_processing/glows/l3e/glows_l3e_utils.py b/imap_l3_processing/glows/l3e/glows_l3e_utils.py index da60b9b40..db45bdd0d 100644 --- a/imap_l3_processing/glows/l3e/glows_l3e_utils.py +++ b/imap_l3_processing/glows/l3e/glows_l3e_utils.py @@ -198,8 +198,9 @@ def get_lo_pivot_angles(repointings: list[int]) -> dict[int, LoPivotAngle]: result[repointing] = LoPivotAngle(parent_filename=None, pivot_angle=90.0) return result -def get_repoint_numbers_within_cr_window(start_cr_number: int, end_cr_number: int, repointing_data) -> list[int]: - +def get_repoint_numbers_within_cr_window(start_cr_number: int | None, end_cr_number: int, repointing_data) -> list[int]: + if start_cr_number is None: + return [] first_carrington_start_date = Time(jd_fm_Carrington(float(start_cr_number)), format='jd') last_cr_end_date = Time(jd_fm_Carrington(float(end_cr_number + 1)), format='jd') diff --git a/tests/glows/l3e/test_glows_l3e_initializer.py b/tests/glows/l3e/test_glows_l3e_initializer.py index e8245a386..708128a48 100644 --- a/tests/glows/l3e/test_glows_l3e_initializer.py +++ b/tests/glows/l3e/test_glows_l3e_initializer.py @@ -141,7 +141,7 @@ def test_get_repointings_to_process(self, mock_query, mock_get_most_recently_upl @patch(f'{MODULE}.GlowsL3EDependencies.fetch_dependencies') @patch(f'{MODULE}.identify_versions_for_l3e_output_files') @patch(f'{MODULE}.find_first_updated_cr') - def test_get_repointings_to_process_uses_mission_start_cr_as_first_updated_when_identical_l3d_files(self, + def test_get_repointings_to_process_identical_l3d_files(self, mock_find_first_updated_cr, mock_identify_versions_for_l3e_output_files, mock_fetch_dependencies): @@ -167,7 +167,7 @@ def test_get_repointings_to_process_uses_mission_start_cr_as_first_updated_when_ glows_l3d_processor_output.l3d_cdf_file_path ) mock_find_first_updated_cr.assert_called_once_with(glows_l3d_processor_output.l3d_cdf_file_path, previous_l3d) - mock_identify_versions_for_l3e_output_files.assert_called_once_with(sentinel.start_of_mission_cr, expected_last_cr, sentinel.start_of_mission_cr, repointing_file_path, sentinel.version_map) + mock_identify_versions_for_l3e_output_files.assert_called_once_with(sentinel.start_of_mission_cr, expected_last_cr, None, repointing_file_path, sentinel.version_map) self.assertEqual(expected_initializer_output, actual_initializer_output) @patch(f'{MODULE}.imap_data_access.query') diff --git a/tests/glows/l3e/test_glows_l3e_utils.py b/tests/glows/l3e/test_glows_l3e_utils.py index eac5f4ab2..d451387d4 100644 --- a/tests/glows/l3e/test_glows_l3e_utils.py +++ b/tests/glows/l3e/test_glows_l3e_utils.py @@ -291,6 +291,18 @@ def test_get_repoint_numbers_within_cr_window(self): actual_repoint_numbers = get_repoint_numbers_within_cr_window(start_cr, end_cr, repointing_data) np.testing.assert_array_equal(actual_repoint_numbers, expected_repoint_numbers) + def test_get_repoint_numbers_within_cr_window_returns_empty_for_none_start(self): + start_cr = None + end_cr = 2094 + + repointing_path = get_test_data_path("fake_1_day_repointing_file.csv") + + set_global_repoint_table_paths([repointing_path]) + repointing_data = get_repoint_data() + + actual_repoint_numbers = get_repoint_numbers_within_cr_window(start_cr, end_cr, repointing_data) + + np.testing.assert_array_equal(actual_repoint_numbers, []) @patch('imap_l3_processing.glows.l3e.glows_l3e_utils.get_repoint_numbers_within_cr_window') @patch('imap_l3_processing.glows.l3e.glows_l3e_utils.imap_data_access')