Coverage for pds_crawler/models/ode_ws_models.py: 81%
Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# -*- coding: utf-8 -*-
2# pds-crawler - ETL to index PDS data to pdssp
3# Copyright (C) 2023 - CNES (Jean-Christophe Malapert for Pôle Surfaces Planétaires)
4# This file is part of pds-crawler <https://github.com/pdssp/pds_crawler>
5# SPDX-License-Identifier: LGPL-3.0-or-later
6"""
7Module Name:
8 ode_ws_models
10Description:
11 ODE web service models
13Classes
15.. uml::
17 class PdsRegistryModel {
18 +ODEMetaDB: str
19 +IHID: str
20 +IHName: str
21 +IID: str
22 +IName: str
23 +PT: str
24 +PTName: str
25 +DataSetId: str
26 +NumberProducts: int
27 +ValidTargets: Dict[str, List[str]]
28 +MinOrbit: Optional[int]
29 +MaxOrbit: Optional[int]
30 +MinObservationTime: Optional[str]
31 +MaxObservationTime: Optional[str]
32 +NumberObservations: Optional[int]
33 +SpecialValue1: Optional[str]
34 +MinSpecialValue1: Optional[float]
35 +MaxSpecialValue1: Optional[float]
36 +SpecialValue2: Optional[str]
37 +MinSpecialValue2: Optional[float]
38 +MaxSpecialValue2: Optional[float]
39 }
41 class PdsRecordModel {
42 +ode_id: str
43 +pdsid: str
44 +ihid: str
45 +iid: str
46 +pt: str
47 +LabelFileName: str
48 +Product_creation_time: str
49 +Target_name: str
50 +Data_Set_Id: str
51 +Easternmost_longitude: float
52 +Maximum_latitude: float
53 +Minimum_latitude: float
54 +Westernmost_longitude: float
55 +Product_version_id: Optional[str]
56 +RelativePathtoVol: Optional[str]
57 +label: Optional[str]
58 +PDS4LabelURL: Optional[str]
59 +PDSVolume_Id: Optional[str]
60 +Label_product_type: Optional[str]
61 +Observation_id: Optional[str]
62 +Observation_number: Optional[int]
63 +Observation_type: Optional[str]
64 +Producer_id: Optional[str]
65 +Product_name: Optional[str]
66 +Product_release_date: Optional[str]
67 +Activity_id: Optional[str]
68 +Predicted_dust_opacity: Optional[float]
69 +Predicted_dust_opacity_text: Optional[str]
70 +Observation_time: Optional[str]
71 +SpaceCraft_clock_start_count: Optional[str]
72 +SpaceCraft_clock_stop_count: Optional[str]
73 +Start_orbit_number: Optional[int]
74 +Stop_orbit_number: Optional[int]
75 +UTC_start_time: Optional[str]
76 +UTC_stop_time: Optional[str]
77 +Emission_angle: Optional[float]
78 }
80 class ProductFile {
81 +FileName: str
82 +Type: Optional[str]
83 +KBytes: Optional[float]
84 +URL: Optional[str]
85 +Description: Optional[str]
86 +Creation_date: Optional[str]
87 }
89 PdsRecordModel --> ProductFile
90"""
91import inspect
92import logging
93import os
94from dataclasses import dataclass
95from dataclasses import field
96from datetime import datetime
97from typing import Any
98from typing import cast
99from typing import Dict
100from typing import Iterator
101from typing import List
102from typing import Optional
103from urllib.parse import urlparse
105import numpy as np
106import pystac
107from shapely import geometry
108from shapely import wkt
110from ..exception import CrawlerError
111from ..exception import DateConversionError
112from ..exception import PdsCollectionAttributeError
113from ..exception import PdsRecordAttributeError
114from ..exception import PlanetNotFound
115from ..utils import ProgressLogger
116from ..utils import utc_to_iso
117from ..utils import UtilsMath
118from .common import AbstractModel
119from .pds_models import Labo
120from .pdssp_models import PdsspModel
122logger = logging.getLogger(__name__)
125@dataclass(frozen=True, eq=True)
126class ProductFile(AbstractModel):
127 FileName: str
128 Type: Optional[str] = field(default=None, repr=True, compare=True)
129 KBytes: Optional[float] = field(default=None, repr=False, compare=False)
130 URL: Optional[str] = field(default=None, repr=False, compare=False)
131 Description: Optional[str] = field(default=None, repr=False, compare=False)
132 Creation_date: Optional[str] = field(
133 default=None, repr=False, compare=False
134 )
136 @classmethod
137 def from_dict(cls, env):
138 parameters = inspect.signature(cls).parameters
139 return cls(
140 **{
141 k: UtilsMath.convert_dt(v)
142 for k, v in env.items()
143 if k in parameters
144 }
145 )
148@dataclass(frozen=True, eq=True)
149class PdsRegistryModel(AbstractModel):
150 """ODE present products on an instrument host id, instrument id,
151 and product type structure.
153 see : https://oderest.rsl.wustl.edu/ODE_REST_V2.1.pdf
154 """
156 ODEMetaDB: str
157 """ODE Meta DB – can be used as a Target input"""
158 IHID: str
159 """Instrument Host Id"""
160 IHName: str = field(repr=False, compare=False)
161 """Instrument Host Name"""
162 IID: str
163 """Instrument Id"""
164 IName: str = field(repr=False, compare=False)
165 """Instrument Name"""
166 PT: str
167 """Product Type"""
168 PTName: str = field(repr=False, compare=False)
169 """Product Type Name"""
170 DataSetId: str
171 """Data Set Id"""
172 NumberProducts: int
173 """Number of products with this instrument host/instrument/product type"""
174 ValidTargets: Dict[str, List[str]] = field(repr=False, compare=False)
175 """Set of valid target values for the Target query parameter for a given
176 instrument host, instrument, product type. IIPTs are usually
177 targeted to the primary body for that ODE meta database. Example,
178 the products in the ODE Mars meta database primarily target Mars.
179 But some IIPTs have additional targets such as DIEMOS,
180 PHOBOS, or special calibration targets. These targets can then be
181 used in the query parameter TARGET=."""
182 MinOrbit: Optional[int] = field(default=None, repr=False, compare=False)
183 """Minimum orbit value for all products with this instrument
184 host/instrument/product type"""
185 MaxOrbit: Optional[int] = field(default=None, repr=False, compare=False)
186 """Maximum orbit value for all products with this instrument
187 host/instrument/product type"""
188 MinObservationTime: Optional[str] = field(
189 default=None, repr=False, compare=False
190 )
191 """Minimum observation time value for all products with this
192 instrument host/instrument/product type"""
193 MaxObservationTime: Optional[str] = field(
194 default=None, repr=False, compare=False
195 )
196 """Maximum observation time value for all products with this
197 instrument host/instrument/product type"""
198 NumberObservations: Optional[int] = field(
199 default=None, repr=False, compare=False
200 )
201 """Number of observation values found in the products (valid only for
202 selected instrument host/instrument/product types such as LOLA)"""
203 SpecialValue1: Optional[str] = field(
204 default=None, repr=False, compare=False
205 )
206 """Some sets have special values unique to that set. This is the name
207 or description of that value. Special values capture product type
208 specific information. For example, LRO LOLA RDRs include a
209 special value that holds the range of altimetry data. """
210 MinSpecialValue1: Optional[float] = field(
211 default=None, repr=False, compare=False
212 )
213 """Minimum special value 1"""
214 MaxSpecialValue1: Optional[float] = field(
215 default=None, repr=False, compare=False
216 )
217 """Maximum special value 1"""
218 SpecialValue2: Optional[str] = field(
219 default=None, repr=False, compare=False
220 )
221 """Some sets have a second special values unique to that set. This is
222 the name or description of that value."""
223 MinSpecialValue2: Optional[float] = field(
224 default=None, repr=False, compare=False
225 )
226 """Minimum special value 2"""
227 MaxSpecialValue2: Optional[float] = field(
228 default=None, repr=False, compare=False
229 )
230 """Maximum special value 2"""
232 def get_collection_id(self) -> str:
233 return PdsspModel.create_collection_id(Labo.ID, self.get_collection())
235 def get_collection(self) -> str:
236 return self.DataSetId
238 def get_instrument_id(self) -> str:
239 return PdsspModel.create_instru_id(Labo.ID, self.IID)
241 def get_instrument(self) -> str:
242 return self.IName
244 def get_plateform_id(self) -> str:
245 return PdsspModel.create_platform_id(Labo.ID, self.IHID)
247 def get_plateform(self) -> str:
248 return self.IHName
250 def get_mission_id(self) -> str:
251 return PdsspModel.create_mission_id(Labo.ID, self.IHID)
253 def get_mission(self) -> str:
254 return self.IHName
256 def get_body(self) -> str:
257 body: str = self.ODEMetaDB[0].upper() + self.ODEMetaDB[1:].lower()
258 return body
260 def get_body_id(self) -> str:
261 return PdsspModel.create_body_id(Labo.ID, self.get_body())
263 def get_range_orbit(self) -> Optional[pystac.RangeSummary]:
264 range: Optional[pystac.RangeSummary] = None
265 if self.MinOrbit is not None and self.MaxOrbit is not None:
266 range = pystac.RangeSummary(
267 minimum=int(self.MinOrbit), maximum=int(self.MaxOrbit) # type: ignore
268 )
269 return range
271 def get_range_special_value1(self) -> Optional[pystac.RangeSummary]:
272 range: Optional[pystac.RangeSummary] = None
273 if (
274 self.MinSpecialValue1 is not None
275 and self.MaxSpecialValue1 is not None
276 ):
277 range = pystac.RangeSummary(
278 minimum=float(self.MinSpecialValue1), # type: ignore
279 maximum=float(self.MaxSpecialValue1), # type: ignore
280 )
281 return range
283 def get_range_special_value2(self) -> Optional[pystac.RangeSummary]:
284 range: Optional[pystac.RangeSummary] = None
285 if (
286 self.MinSpecialValue2 is not None
287 and self.MaxSpecialValue2 is not None
288 ):
289 range = pystac.RangeSummary(
290 minimum=self.MinSpecialValue2, maximum=self.MaxSpecialValue2 # type: ignore
291 )
292 return range
294 def get_range_time(self) -> Optional[pystac.RangeSummary]:
295 range: Optional[pystac.RangeSummary] = None
296 if (
297 self.MinObservationTime is not None
298 and self.MaxObservationTime is not None
299 ):
300 range = pystac.RangeSummary(
301 minimum=self.MinObservationTime, # type: ignore
302 maximum=self.MaxObservationTime, # type: ignore
303 )
304 return range
306 def get_summaries(self) -> Optional[pystac.Summaries]:
307 summaries: Dict[str, Any] = dict()
308 range_orbits = self.get_range_orbit()
309 range_special_value1 = self.get_range_special_value1()
310 range_special_value2 = self.get_range_special_value2()
311 range_time = self.get_range_time()
312 if range_orbits is not None:
313 summaries["orbit"] = range_orbits
314 if range_special_value1 is not None:
315 summaries[self.SpecialValue1] = range_special_value1 # type: ignore
316 if range_special_value2 is not None:
317 summaries[self.SpecialValue2] = range_special_value2 # type: ignore
318 if range_time is not None:
319 summaries["observation_time"] = range_time
320 result: Optional[pystac.Summaries]
321 if len(summaries) > 0:
322 result = pystac.Summaries(summaries=summaries)
323 else:
324 result = None
325 return result
327 @classmethod
328 def from_dict(cls, env):
329 collection_id = (
330 f'{env["ODEMetaDB"]}_{env["IHID"]}_{env["IID"]}_{env["DataSetId"]}'
331 )
332 try:
333 if "ValidFootprints" in env and env["ValidFootprints"] == "F":
334 logger.warning(
335 f"Missing `Footprints` for {collection_id} IIPTSet: not added, return None."
336 )
337 return None
339 if "NumberProducts" in env and int(env["NumberProducts"]) == 0:
340 logger.warning(
341 f"Missing `NumberProducts` for {collection_id} IIPTSet: not added, return None."
342 )
343 return None
345 parameters = inspect.signature(cls).parameters
346 return cls(
347 **{
348 k: UtilsMath.convert_dt(v)
349 for k, v in env.items()
350 if k in parameters
351 }
352 )
353 except KeyError as err:
354 raise PdsCollectionAttributeError(
355 f"[KeyError] - {err} is missing for {collection_id}"
356 )
357 except TypeError as err:
358 raise PdsCollectionAttributeError(
359 f"[TypeError] - {err} is missing for {collection_id}"
360 )
362 def create_stac_collection(self) -> pystac.Collection:
363 collection = pystac.Collection(
364 id=self.get_collection_id(),
365 description=f"{self.PTName} products",
366 extent=pystac.Extent(
367 pystac.SpatialExtent(bboxes=[[]]),
368 pystac.TemporalExtent(intervals=[[None, None]]),
369 ),
370 title=self.get_collection(),
371 extra_fields={
372 "instruments": [self.get_instrument()],
373 "plateform": self.get_plateform(),
374 "mission": self.get_mission(),
375 },
376 license="CC0-1.0",
377 )
378 summaries = self.get_summaries()
379 if summaries is not None:
380 collection.summaries = summaries
381 return collection
383 def create_stac_instru_catalog(self) -> pystac.Catalog:
384 return pystac.Catalog(
385 id=self.get_instrument_id(),
386 title=self.get_instrument(),
387 description="",
388 )
390 def create_stac_platform_catalog(self) -> pystac.Catalog:
391 return pystac.Catalog(
392 id=self.get_plateform_id(),
393 title=self.get_plateform(),
394 description="",
395 )
397 def create_stac_mission_catalog(self) -> pystac.Catalog:
398 return pystac.Catalog(
399 id=self.get_mission_id(), title=self.get_mission(), description=""
400 )
402 def create_stac_body_catalog(self) -> pystac.Catalog:
403 url: Optional[str] = None
404 match self.ODEMetaDB.upper():
405 case "VENUS":
406 url = "https://solarsystem.nasa.gov/rails/active_storage/blobs/eyJfcmFpbHMiOnsibWVzc2FnZSI6IkJBaHBBcTBFIiwiZXhwIjpudWxsLCJwdXIiOiJibG9iX2lkIn19--1d5cefd65606b80f88a16ac6c3e4afde8d2e1ee6/PIA00271_detail.jpg?disposition=attachment"
407 case "MERCURY":
408 url = "https://www.nasa.gov/sites/default/files/mercury_1.jpg"
409 case "MARS":
410 url = "https://mars.nasa.gov/system/site_config_values/meta_share_images/1_mars-nasa-gov.jpg"
411 case "MOON":
412 url = "https://www.nasa.gov/sites/default/files/styles/full_width_feature/public/thumbnails/image/opo9914d.jpg"
413 case _:
414 raise PlanetNotFound(
415 f"Unexpected body to parse : {self.ODEMetaDB.upper()}"
416 )
417 extension: Dict = PdsspModel.create_ssys_extension(self.get_body())
418 catalog = pystac.Catalog(
419 id=self.get_body_id(),
420 title=self.get_body(),
421 description="",
422 stac_extensions=list(),
423 extra_fields=dict(),
424 )
425 catalog.stac_extensions.extend(extension["stac_extensions"])
426 catalog.extra_fields.update(extension["extra_fields"])
427 catalog.add_link(
428 pystac.Link(
429 rel=pystac.RelType.PREVIEW,
430 target=url,
431 media_type=pystac.MediaType.JPEG,
432 title=self.ODEMetaDB,
433 extra_fields={"credits": "NASA"},
434 )
435 )
436 return catalog
438 def to_hdf5(self, store_db: Any):
439 """Saves the information in the attributes of a HDF5 node
441 Args:
442 store_db (Any): HDF5 node
443 """
444 pds_collection_dict = self.__dict__
445 for key in pds_collection_dict.keys():
446 value = pds_collection_dict[key]
447 # when type is a dictionnary or list, a specific datatype
448 # is needed to encode an attribute in HDF5
449 if isinstance(value, dict) or isinstance(value, list):
450 store_db.attrs[key] = np.string_(str(value)) # type: ignore
451 elif value is not None:
452 store_db.attrs[key] = value
455@dataclass(frozen=True, eq=True)
456class PdsRecordModel(AbstractModel):
457 """ODE meta-data."""
459 ode_id: str = field(repr=False, compare=False)
460 """An internal ODE product identifier.
461 NOTE: This id is assigned by ODE when the product is
462 added to the ODE metadata database. It is
463 generally stable but can be changed when the
464 ODE metadatabase is rebuilt. In general, this id
465 should only be used shortly after acquisition."""
466 pdsid: str = field(repr=False, compare=False)
467 """PDS Product Id"""
468 ihid: str
469 """Instrument host id. See ODE for valid instrument host ids"""
470 iid: str
471 """Instrument id. See ODE for valid instrument ids"""
472 pt: str
473 """ODE Product type. This is ODE's product type.
474 In general, it is obtained from the label but can
475 be changed or depending on whether a label has
476 a product type, whether there are other products
477 in the same instrument with the same product
478 type in the label, etc. If this is not the same
479 product type as in the label, the return will
480 include a Label_Product_Type value as well"""
481 LabelFileName: str = field(repr=False, compare=False)
482 """The file name of the product label"""
483 Product_creation_time: str = field(repr=False, compare=False)
484 """Product creation time (UTC)"""
485 Target_name: str
486 """Product target (example: Mars)"""
487 Data_Set_Id: str = field(repr=True, compare=True)
488 """PDS Data Set Id"""
489 Easternmost_longitude: float = field(repr=False, compare=False)
490 """Longitude 0-360 Easternmost longitude of the footprint"""
491 Maximum_latitude: float = field(repr=False, compare=False)
492 """Planetocentric maximum latitude of the footprint"""
493 Minimum_latitude: float = field(repr=False, compare=False)
494 """Planetocentric minimum latitude of the footprint"""
495 Westernmost_longitude: float = field(repr=False, compare=False)
496 """Longitude 0-360 Westernmost longitude of the footprint"""
497 Product_version_id: Optional[str] = field(
498 default=None, repr=False, compare=False
499 )
500 """Product version"""
501 RelativePathtoVol: Optional[str] = field(
502 default=None, repr=False, compare=False
503 )
504 """The relative path from the volume root to the
505 product label file"""
506 label: Optional[str] = field(default=None, repr=False, compare=False)
507 """Complete product label for PDS3 product labels"""
508 PDS4LabelURL: Optional[str] = field(
509 default=None, repr=False, compare=False
510 )
511 """Pointer to PDS4 XML label for PDS4 products"""
512 PDSVolume_Id: Optional[str] = field(default=None)
513 """Volume Id"""
514 Label_product_type: Optional[str] = field(
515 default=None, repr=False, compare=False
516 )
517 """Label product type (if it exists in the label and is
518 different from the ODE_Product_Type)"""
519 Observation_id: Optional[str] = field(
520 default=None, repr=False, compare=False
521 )
522 """Identifies a scientific observation within a dataset."""
523 Observation_number: Optional[int] = field(
524 default=None, repr=False, compare=False
525 )
526 """Monotonically increasing ordinal counter of the
527 EDRs generated for a particular OBSERVATION_ID. """
528 Observation_type: Optional[str] = field(
529 default=None, repr=False, compare=False
530 )
531 """Identifies the general type of an observation"""
532 Producer_id: Optional[str] = field(default=None, repr=False, compare=False)
533 """Producer id"""
534 Product_name: Optional[str] = field(
535 default=None, repr=False, compare=False
536 )
537 """Product name"""
538 Product_release_date: Optional[str] = field(
539 default=None, repr=False, compare=False
540 )
541 """Product release date"""
542 Activity_id: Optional[str] = field(default=None, repr=False, compare=False)
543 """Label Activity id"""
544 Predicted_dust_opacity: Optional[float] = field(
545 default=None, repr=False, compare=False
546 )
547 """Predicted dust opacity"""
548 Predicted_dust_opacity_text: Optional[str] = field(
549 default=None, repr=False, compare=False
550 )
551 """Predicted dust opacity text."""
552 Observation_time: Optional[str] = field(
553 default=None, repr=False, compare=False
554 )
555 """Observation time (mid-point between the start
556 and end of the observation)"""
557 SpaceCraft_clock_start_count: Optional[str] = field(
558 default=None, repr=False, compare=False
559 )
560 """Spacecraft clock start"""
561 SpaceCraft_clock_stop_count: Optional[str] = field(
562 default=None, repr=False, compare=False
563 )
564 """Spacecraft clock end"""
565 Start_orbit_number: Optional[int] = field(
566 default=None, repr=False, compare=False
567 )
568 """Start orbit number"""
569 Stop_orbit_number: Optional[int] = field(
570 default=None, repr=False, compare=False
571 )
572 """End orbit number"""
573 UTC_start_time: Optional[str] = field(
574 default=None, repr=False, compare=False
575 )
576 """Observation start time in UTC"""
577 UTC_stop_time: Optional[str] = field(
578 default=None, repr=False, compare=False
579 )
580 """Observation end time in UTC"""
581 Emission_angle: Optional[float] = field(
582 default=None, repr=False, compare=False
583 )
584 """Emission angle"""
585 Emission_angle_text: Optional[str] = field(
586 default=None, repr=False, compare=False
587 )
588 """Emission angle text from the product label"""
589 Phase_angle: Optional[float] = field(
590 default=None, repr=False, compare=False
591 )
592 """Phase angle"""
593 Phase_angle_text: Optional[str] = field(
594 default=None, repr=False, compare=False
595 )
596 """Phase angle text from the product label"""
597 Incidence_angle: Optional[float] = field(
598 default=None, repr=False, compare=False
599 )
600 """Incidence angle"""
601 Incidence_angle_text: Optional[str] = field(
602 default=None, repr=False, compare=False
603 )
604 """Incidence angle text from the product label"""
605 Map_resolution: Optional[float] = field(
606 default=None, repr=False, compare=False
607 )
608 """Map resolution"""
609 Map_resolution_text: Optional[str] = field(
610 default=None, repr=False, compare=False
611 )
612 """Map resolution text from the product label"""
613 Map_scale: Optional[float] = field(default=None, repr=False, compare=False)
614 """Map scale"""
615 Map_scale_text: Optional[str] = field(
616 default=None, repr=False, compare=False
617 )
618 """Map scale text from the product label"""
619 Solar_distance: Optional[float] = field(
620 default=None, repr=False, compare=False
621 )
622 """Solar distance"""
623 Solar_distance_text: Optional[str] = field(
624 default=None, repr=False, compare=False
625 )
626 """Solar distance text from the product label"""
627 Solar_longitude: Optional[float] = field(
628 default=None, repr=False, compare=False
629 )
630 """Solar longitude"""
631 Center_georeferenced: Optional[bool] = field(
632 default=None, repr=False, compare=False
633 )
634 """T if the product has a footprint center"""
635 Center_latitude: Optional[float] = field(
636 default=None, repr=False, compare=False
637 )
638 """Planetocentric footprint center latitude"""
639 Center_longitude: Optional[float] = field(
640 default=None, repr=False, compare=False
641 )
642 """Longitude 0-360 center longitude"""
643 Center_latitude_text: Optional[str] = field(
644 default=None, repr=False, compare=False
645 )
646 """Text found in the center latitude label keyword
647 if the center latitude is not a valid number"""
648 Center_longitude_text: Optional[str] = field(
649 default=None, repr=False, compare=False
650 )
651 """Text found in the center longitude label keyword
652 if the center longitude is not a valid number"""
653 BB_georeferenced: Optional[bool] = field(
654 default=None, repr=False, compare=False
655 )
656 """T if the product has a footprint bounding box"""
657 Easternmost_longitude_text: Optional[str] = field(
658 default=None, repr=False, compare=False
659 )
660 """Longitude 0-360 Easternmost longitude of the footprint"""
661 Maximum_latitude_text: Optional[str] = field(
662 default=None, repr=False, compare=False
663 )
664 """Planetocentric maximum latitude of the footprint"""
665 Minimum_latitude_text: Optional[str] = field(
666 default=None, repr=False, compare=False
667 )
668 """Planetocentric minimum latitude of the footprint"""
669 Westernmost_longitude_text: Optional[str] = field(
670 default=None, repr=False, compare=False
671 )
672 """Longitude 0-360 Westernmost longitude of the footprint"""
673 Easternmost_longitude_text: Optional[str] = field(
674 default=None, repr=False, compare=False
675 )
676 """Text found in the easternmost longitude label
677 keyword if the easternmost longitude is not a
678 valid number"""
679 Maximum_latitude_text: Optional[str] = field(
680 default=None, repr=False, compare=False
681 )
682 """Text found in the maximum latitude label
683 keyword if the maximum latitude is not a valid
684 number"""
685 Minimum_latitude_text: Optional[str] = field(
686 default=None, repr=False, compare=False
687 )
688 """Text found in the minimum latitude label
689 keyword if the minimum latitude is not a valid
690 number"""
691 Westernmost_longitude_text: Optional[str] = field(
692 default=None, repr=False, compare=False
693 )
694 """Text found in the westernmost longitude label
695 keyword if the westernmost longitude is not a
696 valid number"""
697 Footprint_geometry: Optional[str] = field(
698 default=None, repr=False, compare=False
699 )
700 """Cylindrical projected planetocentric, longitude
701 0-360 product footprint in WKT format. Only if
702 there is a valid footprint. Note - this is a
703 cylindrical projected footprint. The footprint has
704 been split into multiple polygons when crossing
705 the 0/360 longitude line and any footprints that
706 cross the poles have been adjusted to add points
707 to and around the pole. It is meant for use in
708 cylindrical projects and is not appropriate for
709 spherical displays."""
710 Footprint_C0_geometry: Optional[str] = field(
711 default=None, repr=False, compare=False
712 )
713 """Planetocentric, longitude -180-180 product
714 footprint in WKT format. Only if there is a valid
715 footprint. Note - this is a cylindrical projected
716 footprint. The footprint has been split into
717 multiple polygons when crossing the -180/180
718 longitude line and any footprints that cross the
719 poles have been adjusted to add points to and
720 around the pole. It is meant for use in cylindrical
721 projects and is not appropriate for spherical
722 displays."""
723 Footprint_GL_geometry: Optional[str] = field(
724 default=None, repr=False, compare=False
725 )
726 """Planetocentric, longitude 0-360 product
727 footprint in WKT format. Only if there is a valid
728 footprint. This is not a projected footprint."""
729 Footprint_NP_geometry: Optional[str] = field(
730 default=None, repr=False, compare=False
731 )
732 """Stereographic south polar projected footprint in
733 WKT format. Only if there is a valid footprint.
734 This footprint has been projected into meters in
735 stereographic north polar projection"""
736 Footprint_SP_geometry: Optional[str] = field(
737 default=None, repr=False, compare=False
738 )
739 """Stereographic south polar projected footprint in
740 WKT format. Only if there is a valid footprint.
741 This footprint has been projected into meters in
742 stereographic south polar projection."""
743 Footprints_cross_meridian: Optional[str] = field(
744 default=None, repr=False, compare=False
745 )
746 """T if the footprint crosses the 0/360 longitude line"""
747 Pole_state: Optional[str] = field(default=None, repr=False, compare=False)
748 """String of "none", "north", or "south"""
749 Footprint_souce: Optional[str] = field(
750 default=None, repr=False, compare=False
751 )
752 """A brief description of where the footprint came from"""
753 USGS_Sites: Optional[str] = field(default=None, repr=False, compare=False)
754 """A USGS site that this product's footprint partially or completely covers"""
755 Comment: Optional[str] = field(default=None, repr=False, compare=False)
756 """Any associated comment"""
757 Description: Optional[str] = field(default=None, repr=False, compare=False)
758 """Label description"""
759 ODE_notes: Optional[str] = field(default=None, repr=False, compare=False)
760 """A note about how data has been entered into ODE"""
761 External_url: Optional[str] = field(
762 default=None, repr=False, compare=False
763 )
764 """URL to an external reference to the product.
765 Product type specific but usually something like
766 the HiRISE site."""
767 External_url2: Optional[str] = field(
768 default=None, repr=False, compare=False
769 )
770 """URL to an external reference to the product.
771 Product type specific but usually something like
772 the HiRISE site."""
773 External_url3: Optional[str] = field(
774 default=None, repr=False, compare=False
775 )
776 """URL to an external reference to the product.
777 Product type specific but usually something like
778 the HiRISE site"""
779 FilesURL: Optional[str] = field(default=None, repr=False, compare=False)
780 ProductURL: Optional[str] = field(default=None, repr=False, compare=False)
781 LabelURL: Optional[str] = field(default=None, repr=False, compare=False)
782 Product_files: Optional[List[ProductFile]] = field(
783 default=None, repr=False, compare=False
784 )
785 browse: Optional[str] = field(default=None, repr=False, compare=False)
786 """If there is an ODE browse image - returns a base64 string of the PNG image"""
787 thumbnail: Optional[str] = field(default=None, repr=False, compare=False)
788 """If there is an ODE thumbnail image - returns a base64 string of the PNG image"""
790 def get_id(self):
791 return str(self.ode_id)
793 def get_title(self):
794 return self.pdsid
796 def get_description(self):
797 return self.Description
799 def get_collection_id(self) -> str:
800 return PdsspModel.create_collection_id(Labo.ID, self.get_collection())
802 def get_collection(self) -> str:
803 return self.Data_Set_Id
805 def get_instrument_id(self) -> str:
806 return PdsspModel.create_instru_id(Labo.ID, self.iid)
808 def get_instrument(self, pds_registry_model: PdsRegistryModel) -> str:
809 return pds_registry_model.IName
811 def get_plateform_id(self) -> str:
812 return PdsspModel.create_platform_id(Labo.ID, self.ihid)
814 def get_plateform(self, pds_registry_model: PdsRegistryModel) -> str:
815 return pds_registry_model.IHName
817 def get_mission_id(self) -> str:
818 return PdsspModel.create_mission_id(Labo.ID, self.get_mission())
820 def get_mission(self) -> str:
821 return self.ihid
823 def get_body(self) -> str:
824 body: str = self.Target_name[0].upper() + self.Target_name[1:].lower()
825 return body
827 def get_body_id(self) -> str:
828 return PdsspModel.create_body_id(Labo.ID, self.get_body())
830 def get_start_date(self):
831 start_date: Optional[datetime] = None
832 if self.UTC_start_time is not None:
833 try:
834 start_date = datetime.fromisoformat(
835 utc_to_iso(self.UTC_start_time)
836 )
837 except: # noqa: E722
838 start_date = None
839 return start_date
841 def get_stop_date(self):
842 stop_date: Optional[datetime] = None
843 if self.UTC_stop_time is not None:
844 try:
845 stop_date = datetime.fromisoformat(
846 utc_to_iso(self.UTC_stop_time)
847 )
848 except: # noqa: E722
849 stop_date = None
850 return stop_date
852 def get_geometry(self) -> Dict[str, Any]:
853 return geometry.mapping(wkt.loads(self.Footprint_C0_geometry))
855 def get_bbox(self) -> list[float]:
856 return [
857 self.Westernmost_longitude,
858 self.Minimum_latitude,
859 self.Easternmost_longitude,
860 self.Maximum_latitude,
861 ]
863 def get_gsd(self) -> Optional[float]:
864 result: Optional[float] = None
865 if self.Map_resolution is not None:
866 result = self.Map_resolution
867 return result
869 def get_datetime(self) -> datetime:
870 date_obs: str
871 if self.Observation_time and not self.Observation_time.startswith(
872 "0000"
873 ):
874 date_obs = self.Observation_time
875 elif (
876 self.Product_creation_time
877 and not self.Product_creation_time.startswith("0000")
878 ):
879 date_obs = self.Product_creation_time
880 logger.warning(
881 f"Cannot find {self.Observation_time}, use Product_creation_time as datetime for {self}"
882 )
883 elif self.Product_release_date:
884 date_obs = self.Product_release_date
885 logger.warning(
886 f"Cannot find {self.Observation_time}, use Product_creation_time as datetime for {self}"
887 )
888 else:
889 raise ValueError("No datetime")
890 return datetime.fromisoformat(utc_to_iso(date_obs))
892 def get_properties(self) -> Dict[str, Any]:
893 properties = {
894 key: self.__dict__[key]
895 for key in self.__dict__.keys()
896 if key
897 in [
898 "pt",
899 "LabelFileName",
900 "Product_creation_time",
901 "Product_version_id",
902 "label",
903 "PDS4LabelURL",
904 "Data_Set_Id",
905 "PDSVolume_Id",
906 "Label_product_type",
907 "Observation_id",
908 "Observation_number",
909 "Observation_type",
910 "Producer_id",
911 "Product_name",
912 "Product_release_date",
913 "Activity_id",
914 "Predicted_dust_opacity",
915 "Predicted_dust_opacity_text",
916 "Observation_time",
917 "SpaceCraft_clock_start_count",
918 "SpaceCraft_clock_stop_count",
919 "Start_orbit_number",
920 "Stop_orbit_number",
921 "UTC_start_time",
922 "UTC_stop_time",
923 "Emission_angle",
924 "Emission_angle_text",
925 "Phase_angle",
926 "Phase_angle_text",
927 "Incidence_angle",
928 "Incidence_angle_text",
929 "Map_resolution_text",
930 "Map_scale",
931 "Map_scale_text",
932 "Solar_distance",
933 "Solar_distance_text",
934 "Solar_longitude",
935 "Center_latitude",
936 "Center_longitude",
937 "Center_latitude_text",
938 "USGS_Sites",
939 "Comment",
940 ]
941 and self.__dict__[key] is not None
942 }
943 season: Dict = PdsspModel.add_mars_keywords_if_mars(
944 body_id=self.get_body(),
945 ode_id=self.ode_id,
946 lat=self.Center_latitude,
947 slong=self.Solar_longitude,
948 date=self.get_datetime(),
949 )
950 properties.update(season)
951 return properties
953 def set_common_metadata(
954 self, item: pystac.Item, pds_registry: PdsRegistryModel
955 ):
956 item.common_metadata.license = "CC0-1.0"
957 item.common_metadata.instruments = [self.get_instrument(pds_registry)]
958 item.common_metadata.platform = self.get_plateform(pds_registry)
959 item.common_metadata.mission = self.get_mission()
960 item.common_metadata.description = self.get_description()
961 start_date = self.get_start_date()
962 stop_date = self.get_stop_date()
963 if start_date is not None:
964 item.common_metadata.start_datetime = start_date
965 if stop_date is not None:
966 item.common_metadata.end_datetime = stop_date
967 gsd = self.get_gsd()
968 if gsd is not None:
969 item.common_metadata.gsd = gsd
971 def add_assets_product_types(self, item: pystac.Item):
972 if not self.Product_files:
973 return
975 for product_file in self.Product_files:
976 if not product_file.URL:
977 continue
978 item.add_asset(
979 product_file.FileName,
980 pystac.Asset(
981 href=product_file.URL,
982 title=product_file.FileName,
983 description=product_file.Description,
984 roles=["metadata"],
985 ),
986 )
988 def add_assets_browse(self, item: pystac.Item):
989 if self.browse is not None:
990 parsed_url = urlparse(self.browse)
991 path: str = parsed_url.path
992 filename: str = os.path.basename(path)
993 item.add_asset(
994 filename,
995 pystac.Asset(
996 href=self.browse,
997 title=filename,
998 description="Browse image",
999 roles=["overview"],
1000 ),
1001 )
1003 def add_assets_thumbnail(self, item: pystac.Item):
1004 if self.thumbnail is not None:
1005 parsed_url = urlparse(self.thumbnail)
1006 path: str = parsed_url.path
1007 filename: str = os.path.basename(path)
1008 item.add_asset(
1009 filename,
1010 pystac.Asset(
1011 href=self.thumbnail,
1012 title=filename,
1013 description="Thumbnail image",
1014 roles=["thumbnail"],
1015 ),
1016 )
1018 def add_assets_data(self, item: pystac.Item):
1019 if self.LabelURL is not None:
1020 parsed_url = urlparse(self.LabelURL)
1021 path: str = parsed_url.path
1022 filename: str = os.path.basename(path)
1023 item.add_asset(
1024 filename,
1025 pystac.Asset(
1026 href=self.LabelURL,
1027 title=filename,
1028 description="Browse Label",
1029 roles=["metadata"],
1030 ),
1031 )
1033 if self.ProductURL is not None:
1034 parsed_url = urlparse(self.ProductURL)
1035 path: str = parsed_url.path
1036 filename: str = os.path.basename(path)
1037 item.add_asset(
1038 filename,
1039 pystac.Asset(
1040 href=self.ProductURL,
1041 title=filename,
1042 description="Product URL",
1043 roles=["data"],
1044 ),
1045 )
1047 if self.FilesURL is not None:
1048 parsed_url = urlparse(self.FilesURL)
1049 path: str = parsed_url.path
1050 filename: str = os.path.basename(path)
1051 item.add_asset(
1052 filename,
1053 pystac.Asset(
1054 href=self.FilesURL,
1055 title=filename,
1056 description="Files URL",
1057 roles=["metadata"],
1058 ),
1059 )
1061 def add_assets_external_files(self, item: pystac.Item):
1062 if self.External_url is not None:
1063 parsed_url = urlparse(self.External_url)
1064 path: str = parsed_url.path
1065 filename: str = os.path.basename(path)
1066 item.add_asset(
1067 filename,
1068 pystac.Asset(
1069 href=self.External_url,
1070 title=filename,
1071 description="External URL",
1072 roles=["metadata"],
1073 ),
1074 )
1076 if self.External_url2 is not None:
1077 parsed_url = urlparse(self.External_url2)
1078 path: str = parsed_url.path
1079 filename: str = os.path.basename(path)
1080 item.add_asset(
1081 filename,
1082 pystac.Asset(
1083 href=self.External_url2,
1084 title=filename,
1085 description="External URL2",
1086 roles=["metadata"],
1087 ),
1088 )
1090 if self.External_url3 is not None:
1091 parsed_url = urlparse(self.External_url3)
1092 path: str = parsed_url.path
1093 filename: str = os.path.basename(path)
1094 item.add_asset(
1095 filename,
1096 pystac.Asset(
1097 href=self.External_url3,
1098 title=filename,
1099 description="External URL3",
1100 roles=["metadata"],
1101 ),
1102 )
1104 def add_ssys_extention(self, item: pystac.Item):
1105 item.stac_extensions = [
1106 "https://raw.githubusercontent.com/thareUSGS/ssys/main/json-schema/schema.json"
1107 ]
1108 if item.properties is None:
1109 item.properties = {"ssys:targets": [self.Target_name]}
1110 else:
1111 item.properties["ssys:targets"] = [self.Target_name]
1113 @classmethod
1114 def from_dict(cls, env):
1115 try:
1116 if "Footprint_geometry" not in env:
1117 # logger.warning(f'Missing data = records.get_sample_records_pds(col.ODEMetaDB,col.IHID,col.IID,col.PT, col.NumberProducts, limit=1)`Footprint_geometry` for IIPTSet: not added, return None.')
1118 return None
1119 parameters = inspect.signature(cls).parameters
1120 data = env.copy()
1121 if "Product_files" in data:
1122 data["Product_files"] = [
1123 ProductFile.from_dict(item)
1124 for item in data["Product_files"]["Product_file"]
1125 ]
1126 return cls(
1127 **{
1128 k: UtilsMath.convert_dt(v)
1129 for k, v in data.items()
1130 if k in parameters
1131 }
1132 )
1133 except KeyError as err:
1134 logger.error(env)
1135 collection_id = f'{env["Target_name"]}_{env["ihid"]}_{env["iid"]}'
1136 if "Data_Set_Id" in env:
1137 collection_id = f"{collection_id}_{env['Data_Set_Id']}"
1138 raise PdsRecordAttributeError(
1139 f"[KeyError] - {err} is missing for {collection_id}"
1140 )
1141 except TypeError as err:
1142 logger.error(env)
1143 collection_id = f'{env["Target_name"]}_{env["ihid"]}_{env["iid"]}'
1144 if "Data_Set_Id" in env:
1145 collection_id = f"{collection_id}_{env['Data_Set_Id']}"
1146 raise PdsRecordAttributeError(
1147 f"[TypeError] - {err} is missing for {collection_id}"
1148 )
1150 def to_stac_item(self, pds_registry: PdsRegistryModel) -> pystac.Item:
1151 try:
1152 item = pystac.Item(
1153 id=self.get_id(),
1154 geometry=self.get_geometry(),
1155 bbox=self.get_bbox(),
1156 datetime=self.get_datetime(),
1157 properties=self.get_properties(),
1158 collection=self.get_collection_id(),
1159 )
1160 self.set_common_metadata(item, pds_registry)
1161 self.add_assets_product_types(item)
1162 self.add_assets_browse(item)
1163 self.add_assets_thumbnail(item)
1164 self.add_assets_data(item)
1165 self.add_assets_external_files(item)
1166 self.add_ssys_extention(item)
1168 return item
1169 except PdsCollectionAttributeError as err:
1170 raise CrawlerError(
1171 f"""
1172 error : {err.__class__.__name__}
1173 Message : {err.__cause__}
1174 class: {self}
1175 data: {self.__dict__}
1176 """
1177 )
1178 except PdsRecordAttributeError as err:
1179 raise CrawlerError(
1180 f"""
1181 error : {err.__class__.__name__}
1182 Message : {err.__cause__}
1183 class: {self}
1184 data: {self.__dict__}
1185 """
1186 )
1187 except PlanetNotFound as err:
1188 raise CrawlerError(
1189 f"""
1190 error : {err.__class__.__name__}
1191 Message : {err.__cause__}
1192 class: {self}
1193 data: {self.__dict__}
1194 """
1195 )
1196 except DateConversionError as err:
1197 raise CrawlerError(
1198 f"""
1199 error : {err.__class__.__name__}
1200 Message : {err.__cause__}
1201 class: {self}
1202 data: {self.__dict__}
1203 """
1204 )
1205 except Exception as err:
1206 logger.exception(f"Unexpected error : {err}")
1207 raise CrawlerError(
1208 f"""
1209 Unexpected error : {err.__cause__}
1210 class: {self}
1211 data: {self.__dict__}
1212 """
1213 )
1216@dataclass(frozen=True, eq=True)
1217class PdsRecordsModel(AbstractModel):
1218 pds_records_model: List[PdsRecordModel]
1219 target_name: str
1220 plateform_id: str
1221 instrument_id: str
1222 dataset_id: str
1224 @classmethod
1225 def from_dict(cls, env) -> Optional["PdsRecordsModel"]:
1226 result: Optional[PdsRecordsModel]
1227 pds_records_model = list()
1228 for item in env:
1229 pds_record_model: PdsRecordModel = PdsRecordModel.from_dict(item)
1230 if pds_record_model is not None:
1231 pds_records_model.append(pds_record_model)
1232 if len(pds_records_model) == 0:
1233 result = None
1234 else:
1235 result = PdsRecordsModel(
1236 pds_records_model=pds_records_model,
1237 target_name=pds_records_model[0].Target_name,
1238 plateform_id=pds_records_model[0].ihid,
1239 instrument_id=pds_records_model[0].iid,
1240 dataset_id=pds_records_model[0].Data_Set_Id,
1241 )
1242 return result
1244 def to_stac_item_iter(
1245 self, pds_registry: PdsRegistryModel, progress_bar: bool = True
1246 ) -> Iterator[pystac.Item]:
1247 pds_records = self.pds_records_model
1248 with ProgressLogger(
1249 total=len(pds_records),
1250 iterable=pds_records,
1251 logger=logger,
1252 description="Downloaded responses from the collection",
1253 position=1,
1254 leave=False,
1255 disable_tqdm=not progress_bar,
1256 ) as progress_logger:
1257 for pds_record_model in progress_logger:
1258 yield cast(PdsRecordModel, pds_record_model).to_stac_item(
1259 pds_registry
1260 )
1262 def __repr__(self) -> str:
1263 return f"PdsRecordsModel({self.target_name}/{self.plateform_id}/{self.instrument_id}/{self.dataset_id}, nb_records={len(self.pds_records_model)})"