Index

_

__init__() (opdi.ingestion.AircraftDatabaseIngestion method) (opdi.ingestion.osn_aircraft_db.AircraftDatabaseIngestion method) (opdi.ingestion.osn_statevectors.StateVectorIngestion method) (opdi.ingestion.ourairports.OurAirportsIngestion method) (opdi.ingestion.StateVectorIngestion method) (opdi.monitoring.advanced_stats.AdvancedStatsCollector method) (opdi.monitoring.AdvancedStatsCollector method) (opdi.monitoring.basic_stats.BasicStatsCollector method) (opdi.monitoring.BasicStatsCollector method) (opdi.output.csv_exporter.CSVExporter method) (opdi.output.CSVExporter method) (opdi.output.parquet_exporter.ParquetExporter method) (opdi.output.ParquetExporter method) (opdi.pipeline.events.FlightEventProcessor method) (opdi.pipeline.flights.FlightListProcessor method) (opdi.pipeline.TrackProcessor method) (opdi.pipeline.tracks.TrackProcessor method) (opdi.reference.h3_airport_layouts.AirportLayoutGenerator method) (opdi.reference.h3_airport_zones.AirportDetectionZoneGenerator method) (opdi.reference.h3_airspaces.AirspaceH3Generator method)

A

add_cumulative_distance() (in module opdi.utils) (in module opdi.utils.geospatial) add_distance_measure() (in module opdi.pipeline.events) add_time_measure() (in module opdi.pipeline.events) AdvancedStatsCollector (class in opdi.monitoring) (class in opdi.monitoring.advanced_stats) AircraftDatabaseIngestion (class in opdi.ingestion) (class in opdi.ingestion.osn_aircraft_db) airport_detection_resolution (opdi.config.H3Config attribute) (opdi.H3Config attribute) airport_layout_resolution (opdi.config.H3Config attribute) (opdi.H3Config attribute) AIRPORT_SCHEMA (opdi.reference.AirportDetectionZoneGenerator attribute) (opdi.reference.h3_airport_zones.AirportDetectionZoneGenerator attribute)

AirportDetectionZoneGenerator (class in opdi.reference) (class in opdi.reference.h3_airport_zones) AirportLayoutGenerator (class in opdi.reference) (class in opdi.reference.h3_airport_layouts) airspace_resolution (opdi.config.H3Config attribute) (opdi.H3Config attribute) AirspaceH3Generator (class in opdi.reference) (class in opdi.reference.h3_airspaces) altitude_smoothing_window_minutes (opdi.config.IngestionConfig attribute) (opdi.IngestionConfig attribute) analyze_missing_data() (opdi.monitoring.advanced_stats.AdvancedStatsCollector method) (opdi.monitoring.AdvancedStatsCollector method) app_name (opdi.config.SparkConfig attribute) (opdi.SparkConfig attribute)

B

BasicStatsCollector (class in opdi.monitoring) (class in opdi.monitoring.basic_stats) batch_size (opdi.config.IngestionConfig attribute) (opdi.IngestionConfig attribute)

BBOX_OFFSET (opdi.reference.AirportDetectionZoneGenerator attribute) (opdi.reference.AirportLayoutGenerator attribute) (opdi.reference.h3_airport_layouts.AirportLayoutGenerator attribute) (opdi.reference.h3_airport_zones.AirportDetectionZoneGenerator attribute) buffer_geometry() (in module opdi.reference.h3_airport_layouts)

C

calculate_airport_events() (in module opdi.pipeline.events) calculate_bearing() (in module opdi.utils) (in module opdi.utils.geospatial) calculate_firstseen_lastseen_events() (in module opdi.pipeline.events) calculate_horizontal_segment_events() (in module opdi.pipeline.events) calculate_vertical_crossing_events() (in module opdi.pipeline.events) check_known_outage() (opdi.monitoring.advanced_stats.AdvancedStatsCollector method) (opdi.monitoring.AdvancedStatsCollector method) clean_and_export_all() (opdi.output.csv_exporter.CSVExporter method) (opdi.output.CSVExporter method) clean_and_export_file() (opdi.output.csv_exporter.CSVExporter method) (opdi.output.CSVExporter method) clean_and_save_data() (in module opdi.output) (in module opdi.output.csv_exporter) clean_str() (in module opdi.reference.h3_airport_layouts) cleanup_local_files() (opdi.ingestion.osn_statevectors.StateVectorIngestion method) (opdi.ingestion.StateVectorIngestion method) COLUMN_MAPPING (opdi.ingestion.osn_statevectors.StateVectorIngestion attribute) (opdi.ingestion.StateVectorIngestion attribute) compact_h3_set() (in module opdi.utils) (in module opdi.utils.h3_helpers) convert_to_polygon() (in module opdi.reference.h3_airport_layouts)

create_local_session() (opdi.utils.spark_helpers.SparkSessionManager static method) (opdi.utils.SparkSessionManager static method) create_session() (opdi.utils.spark_helpers.SparkSessionManager static method) (opdi.utils.SparkSessionManager static method) create_table_if_not_exists() (opdi.ingestion.AircraftDatabaseIngestion method) (opdi.ingestion.osn_aircraft_db.AircraftDatabaseIngestion method) (opdi.ingestion.osn_statevectors.StateVectorIngestion method) (opdi.ingestion.StateVectorIngestion method) (opdi.pipeline.FlightListProcessor method) (opdi.pipeline.flights.FlightListProcessor method) (opdi.pipeline.TrackProcessor method) (opdi.pipeline.tracks.TrackProcessor method) (opdi.reference.AirportLayoutGenerator method) (opdi.reference.AirspaceH3Generator method) (opdi.reference.h3_airport_layouts.AirportLayoutGenerator method) (opdi.reference.h3_airspaces.AirspaceH3Generator method) create_tables() (opdi.ingestion.ourairports.OurAirportsIngestion method) (opdi.ingestion.OurAirportsIngestion method) create_tables_if_not_exist() (opdi.pipeline.events.FlightEventProcessor method) (opdi.pipeline.FlightEventProcessor method) CSVExporter (class in opdi.output) (class in opdi.output.csv_exporter)

D

default() (opdi.config.OPDIConfig class method) (opdi.OPDIConfig class method) DEFAULT_KNOWN_OUTAGES (opdi.monitoring.advanced_stats.AdvancedStatsCollector attribute) (opdi.monitoring.AdvancedStatsCollector attribute) DEFAULT_TABLES (opdi.monitoring.basic_stats.BasicStatsCollector attribute) (opdi.monitoring.BasicStatsCollector attribute) destination_point() (in module opdi.utils) (in module opdi.utils.geospatial) download_and_convert() (opdi.ingestion.AircraftDatabaseIngestion method) (opdi.ingestion.osn_aircraft_db.AircraftDatabaseIngestion method)

download_files() (opdi.ingestion.osn_statevectors.StateVectorIngestion method) (opdi.ingestion.StateVectorIngestion method) driver_cores (opdi.config.SparkConfig attribute) (opdi.SparkConfig attribute) driver_max_result_size (opdi.config.SparkConfig attribute) (opdi.SparkConfig attribute) driver_memory (opdi.config.SparkConfig attribute) (opdi.SparkConfig attribute) dynamic_allocation_max_executors (opdi.config.SparkConfig attribute) (opdi.SparkConfig attribute)

E

executor_cores (opdi.config.SparkConfig attribute) (opdi.SparkConfig attribute) executor_heartbeat_interval (opdi.config.SparkConfig attribute) (opdi.SparkConfig attribute) executor_instances (opdi.config.SparkConfig attribute) (opdi.SparkConfig attribute) executor_memory (opdi.config.SparkConfig attribute) (opdi.SparkConfig attribute) executor_memory_overhead (opdi.config.SparkConfig attribute) (opdi.SparkConfig attribute)

export_all() (opdi.output.parquet_exporter.ParquetExporter method) (opdi.output.ParquetExporter method) export_flight_events() (opdi.output.parquet_exporter.ParquetExporter method) (opdi.output.ParquetExporter method) export_flight_list() (opdi.output.parquet_exporter.ParquetExporter method) (opdi.output.ParquetExporter method) export_measurements() (opdi.output.parquet_exporter.ParquetExporter method) (opdi.output.ParquetExporter method)

F

fetch_airport_list() (opdi.reference.AirportLayoutGenerator method) (opdi.reference.h3_airport_layouts.AirportLayoutGenerator method) fetch_daily_row_counts() (opdi.monitoring.advanced_stats.AdvancedStatsCollector method) (opdi.monitoring.AdvancedStatsCollector method) fill_geometry() (in module opdi.reference) (in module opdi.reference.h3_airspaces) fill_geometry_compact() (in module opdi.reference) (in module opdi.reference.h3_airspaces) fill_missing_dates() (opdi.monitoring.advanced_stats.AdvancedStatsCollector method) (opdi.monitoring.AdvancedStatsCollector method)

fill_missing_width() (in module opdi.reference.h3_airport_layouts) flight_level_to_meters() (in module opdi.utils) (in module opdi.utils.geospatial) FlightEventProcessor (class in opdi.pipeline) (class in opdi.pipeline.events) FlightListProcessor (class in opdi.pipeline) (class in opdi.pipeline.flights) for_environment() (opdi.config.OPDIConfig class method) (opdi.OPDIConfig class method)

G

gaussmf() (in module opdi.pipeline.events) generate() (opdi.reference.AirportDetectionZoneGenerator method) (opdi.reference.h3_airport_zones.AirportDetectionZoneGenerator method) generate_circle_polygon() (in module opdi.reference) (in module opdi.reference.h3_airport_zones) generate_intervals() (in module opdi.utils) (in module opdi.utils.datetime_helpers) (opdi.output.parquet_exporter.ParquetExporter method) (opdi.output.ParquetExporter method) generate_month_intervals() (opdi.output.parquet_exporter.ParquetExporter method) (opdi.output.ParquetExporter method) generate_months() (in module opdi.utils) (in module opdi.utils.datetime_helpers) generate_quality_report() (opdi.monitoring.advanced_stats.AdvancedStatsCollector method) (opdi.monitoring.AdvancedStatsCollector method) get_aircraft_info() (opdi.ingestion.AircraftDatabaseIngestion method) (opdi.ingestion.osn_aircraft_db.AircraftDatabaseIngestion method) get_all_table_counts() (opdi.monitoring.basic_stats.BasicStatsCollector method) (opdi.monitoring.BasicStatsCollector method)

get_coords() (in module opdi.reference.h3_airspaces) get_data_within_interval() (in module opdi.utils) (in module opdi.utils.datetime_helpers) get_data_within_timeframe() (in module opdi.utils) (in module opdi.utils.datetime_helpers) get_h3_coords() (in module opdi.utils) (in module opdi.utils.h3_helpers) get_h3_resolution() (in module opdi.utils.h3_helpers) get_minio_files_for_date() (opdi.monitoring.advanced_stats.AdvancedStatsCollector method) (opdi.monitoring.AdvancedStatsCollector method) get_or_create() (opdi.utils.spark_helpers.SparkSessionManager static method) (opdi.utils.SparkSessionManager static method) get_spark() (in module opdi.utils) (in module opdi.utils.spark_helpers) get_start_end_of_month() (in module opdi.utils) (in module opdi.utils.datetime_helpers) get_table_count() (opdi.monitoring.basic_stats.BasicStatsCollector method) (opdi.monitoring.BasicStatsCollector method) get_table_schema() (opdi.monitoring.basic_stats.BasicStatsCollector method) (opdi.monitoring.BasicStatsCollector method)

H

h3 (opdi.config.OPDIConfig attribute) (opdi.OPDIConfig attribute) h3_distance() (in module opdi.utils) (in module opdi.utils.h3_helpers) h3_list_prep() (in module opdi.utils) (in module opdi.utils.h3_helpers) h3_to_children() (in module opdi.utils.h3_helpers) h3_to_parent() (in module opdi.utils.h3_helpers) H3Config (class in opdi) (class in opdi.config) hadoop_filesystem (opdi.config.ProjectConfig attribute) (opdi.ProjectConfig attribute)

hadoop_group (opdi.config.SparkConfig attribute) (opdi.SparkConfig attribute) handle_timestamp_without_timezone (opdi.config.SparkConfig attribute) (opdi.SparkConfig attribute) haversine_distance() (in module opdi.utils) (in module opdi.utils.geospatial) hex_ring() (in module opdi.utils) (in module opdi.utils.h3_helpers) hexagonify_airport() (in module opdi.reference) (in module opdi.reference.h3_airport_layouts)

I

iceberg_jar_path (opdi.config.SparkConfig attribute) (opdi.SparkConfig attribute) ingest() (opdi.ingestion.AircraftDatabaseIngestion method) (opdi.ingestion.osn_aircraft_db.AircraftDatabaseIngestion method) (opdi.ingestion.osn_statevectors.StateVectorIngestion method) (opdi.ingestion.StateVectorIngestion method) ingest_all() (opdi.ingestion.ourairports.OurAirportsIngestion method) (opdi.ingestion.OurAirportsIngestion method)

ingest_dataset() (opdi.ingestion.ourairports.OurAirportsIngestion method) (opdi.ingestion.OurAirportsIngestion method) ingestion (opdi.config.OPDIConfig attribute) (opdi.OPDIConfig attribute) IngestionConfig (class in opdi) (class in opdi.config) is_number() (in module opdi.reference.h3_airport_layouts) is_valid_h3_index() (in module opdi.utils) (in module opdi.utils.h3_helpers)

K

k_ring() (in module opdi.utils) (in module opdi.utils.h3_helpers)

L

LAT_MAX (opdi.reference.AirportDetectionZoneGenerator attribute) (opdi.reference.AirportLayoutGenerator attribute) (opdi.reference.h3_airport_layouts.AirportLayoutGenerator attribute) (opdi.reference.h3_airport_zones.AirportDetectionZoneGenerator attribute) LAT_MIN (opdi.reference.AirportDetectionZoneGenerator attribute) (opdi.reference.AirportLayoutGenerator attribute) (opdi.reference.h3_airport_layouts.AirportLayoutGenerator attribute) (opdi.reference.h3_airport_zones.AirportDetectionZoneGenerator attribute) list_available_files() (opdi.ingestion.osn_statevectors.StateVectorIngestion method) (opdi.ingestion.StateVectorIngestion method)

load_processed_files() (opdi.ingestion.osn_statevectors.StateVectorIngestion method) (opdi.ingestion.StateVectorIngestion method) LON_MAX (opdi.reference.AirportDetectionZoneGenerator attribute) (opdi.reference.AirportLayoutGenerator attribute) (opdi.reference.h3_airport_layouts.AirportLayoutGenerator attribute) (opdi.reference.h3_airport_zones.AirportDetectionZoneGenerator attribute) LON_MIN (opdi.reference.AirportDetectionZoneGenerator attribute) (opdi.reference.AirportLayoutGenerator attribute) (opdi.reference.h3_airport_layouts.AirportLayoutGenerator attribute) (opdi.reference.h3_airport_zones.AirportDetectionZoneGenerator attribute)

M

mark_files_processed() (opdi.ingestion.osn_statevectors.StateVectorIngestion method) (opdi.ingestion.StateVectorIngestion method) MAX_FL (opdi.pipeline.FlightListProcessor attribute) (opdi.pipeline.flights.FlightListProcessor attribute) max_vertical_rate_mps (opdi.config.IngestionConfig attribute) (opdi.IngestionConfig attribute) meters_to_flight_level() (in module opdi.utils) (in module opdi.utils.geospatial) minio_endpoint (opdi.config.IngestionConfig attribute) (opdi.IngestionConfig attribute) module opdi opdi.config opdi.ingestion opdi.ingestion.osn_aircraft_db opdi.ingestion.osn_statevectors opdi.ingestion.ourairports opdi.monitoring opdi.monitoring.advanced_stats opdi.monitoring.basic_stats opdi.output opdi.output.csv_exporter opdi.output.parquet_exporter opdi.pipeline opdi.pipeline.events opdi.pipeline.flights opdi.pipeline.tracks opdi.reference opdi.reference.h3_airport_layouts opdi.reference.h3_airport_zones opdi.reference.h3_airspaces opdi.utils opdi.utils.datetime_helpers opdi.utils.geospatial opdi.utils.h3_helpers opdi.utils.spark_helpers

N

network_timeout (opdi.config.SparkConfig attribute) (opdi.SparkConfig attribute)

O

opdi module opdi.config module opdi.ingestion module opdi.ingestion.osn_aircraft_db module opdi.ingestion.osn_statevectors module opdi.ingestion.ourairports module opdi.monitoring module opdi.monitoring.advanced_stats module opdi.monitoring.basic_stats module opdi.output module opdi.output.csv_exporter module opdi.output.parquet_exporter module opdi.pipeline module opdi.pipeline.events module opdi.pipeline.flights module

opdi.pipeline.tracks module opdi.reference module opdi.reference.h3_airport_layouts module opdi.reference.h3_airport_zones module opdi.reference.h3_airspaces module opdi.utils module opdi.utils.datetime_helpers module opdi.utils.geospatial module opdi.utils.h3_helpers module opdi.utils.spark_helpers module OPDIConfig (class in opdi) (class in opdi.config) osn_aircraft_db_url (opdi.config.IngestionConfig attribute) (opdi.IngestionConfig attribute) ourairports_base_url (opdi.config.IngestionConfig attribute) (opdi.IngestionConfig attribute) ourairports_datasets (opdi.config.IngestionConfig attribute) (opdi.IngestionConfig attribute) OurAirportsIngestion (class in opdi.ingestion) (class in opdi.ingestion.ourairports)

P

ParquetExporter (class in opdi.output) (class in opdi.output.parquet_exporter) polyfill_geojson() (in module opdi.utils) (in module opdi.utils.h3_helpers) polygon_to_h3() (in module opdi.reference.h3_airport_layouts) prepare_for_flight_list() (opdi.reference.AirportDetectionZoneGenerator method) (opdi.reference.h3_airport_zones.AirportDetectionZoneGenerator method) print_summary() (opdi.monitoring.basic_stats.BasicStatsCollector method) (opdi.monitoring.BasicStatsCollector method) print_table_counts() (in module opdi.monitoring) (in module opdi.monitoring.basic_stats) process_airport() (opdi.reference.AirportLayoutGenerator method) (opdi.reference.h3_airport_layouts.AirportLayoutGenerator method) process_all() (opdi.reference.AirportLayoutGenerator method) (opdi.reference.AirspaceH3Generator method) (opdi.reference.h3_airport_layouts.AirportLayoutGenerator method) (opdi.reference.h3_airspaces.AirspaceH3Generator method) process_and_write_batch() (opdi.ingestion.osn_statevectors.StateVectorIngestion method) (opdi.ingestion.StateVectorIngestion method) process_ansp() (opdi.reference.AirspaceH3Generator method) (opdi.reference.h3_airspaces.AirspaceH3Generator method) process_countries() (opdi.reference.AirspaceH3Generator method) (opdi.reference.h3_airspaces.AirspaceH3Generator method)

process_dai() (opdi.pipeline.FlightListProcessor method) (opdi.pipeline.flights.FlightListProcessor method) process_date_range() (opdi.pipeline.events.FlightEventProcessor method) (opdi.pipeline.FlightEventProcessor method) (opdi.pipeline.FlightListProcessor method) (opdi.pipeline.flights.FlightListProcessor method) (opdi.pipeline.TrackProcessor method) (opdi.pipeline.tracks.TrackProcessor method) process_fir() (opdi.reference.AirspaceH3Generator method) (opdi.reference.h3_airspaces.AirspaceH3Generator method) process_month() (opdi.pipeline.events.FlightEventProcessor method) (opdi.pipeline.FlightEventProcessor method) (opdi.pipeline.TrackProcessor method) (opdi.pipeline.tracks.TrackProcessor method) process_overflights() (opdi.pipeline.FlightListProcessor method) (opdi.pipeline.flights.FlightListProcessor method) project (opdi.config.OPDIConfig attribute) (opdi.OPDIConfig attribute) project_name (opdi.config.ProjectConfig attribute) (opdi.ProjectConfig attribute) ProjectConfig (class in opdi) (class in opdi.config)

R

remove_partial_files() (opdi.ingestion.osn_statevectors.StateVectorIngestion method) (opdi.ingestion.StateVectorIngestion method)

retrieve_osm_data() (in module opdi.reference) (in module opdi.reference.h3_airport_layouts)

S

safe_convert_to_float() (in module opdi.reference.h3_airport_layouts) safe_to_pandas() (opdi.output.parquet_exporter.ParquetExporter static method) (opdi.output.ParquetExporter static method) save_to_parquet() (opdi.reference.AirportDetectionZoneGenerator method) (opdi.reference.h3_airport_zones.AirportDetectionZoneGenerator method) setup_minio_client() (opdi.ingestion.osn_statevectors.StateVectorIngestion method) (opdi.ingestion.StateVectorIngestion method) (opdi.monitoring.advanced_stats.AdvancedStatsCollector method) (opdi.monitoring.AdvancedStatsCollector method) shuffle_compress (opdi.config.SparkConfig attribute) (opdi.SparkConfig attribute) shuffle_spill_compress (opdi.config.SparkConfig attribute) (opdi.SparkConfig attribute)

smf() (in module opdi.pipeline.events) spark (opdi.config.OPDIConfig attribute) (opdi.OPDIConfig attribute) SparkConfig (class in opdi) (class in opdi.config) SparkSessionManager (class in opdi.utils) (class in opdi.utils.spark_helpers) StateVectorIngestion (class in opdi.ingestion) (class in opdi.ingestion.osn_statevectors) stop_session() (opdi.utils.spark_helpers.SparkSessionManager static method) (opdi.utils.SparkSessionManager static method)

T

table_exists() (opdi.monitoring.basic_stats.BasicStatsCollector method) (opdi.monitoring.BasicStatsCollector method) to_spark_config() (opdi.config.SparkConfig method) (opdi.SparkConfig method) track_gap_low_altitude_meters (opdi.config.IngestionConfig attribute) (opdi.IngestionConfig attribute) track_gap_low_altitude_minutes (opdi.config.IngestionConfig attribute) (opdi.IngestionConfig attribute)

track_gap_threshold_minutes (opdi.config.IngestionConfig attribute) (opdi.IngestionConfig attribute) track_resolutions (opdi.config.H3Config attribute) (opdi.H3Config attribute) TrackProcessor (class in opdi.pipeline) (class in opdi.pipeline.tracks)

U

ui_show_console_progress (opdi.config.SparkConfig attribute) (opdi.SparkConfig attribute)

uncompact_h3_set() (in module opdi.utils) (in module opdi.utils.h3_helpers)

V

visualize_daily_counts() (opdi.monitoring.advanced_stats.AdvancedStatsCollector method) (opdi.monitoring.AdvancedStatsCollector method)

W

warehouse_path (opdi.config.ProjectConfig attribute) (opdi.ProjectConfig attribute)

write_to_table() (opdi.ingestion.AircraftDatabaseIngestion method) (opdi.ingestion.osn_aircraft_db.AircraftDatabaseIngestion method)

Z

zmf() (in module opdi.pipeline.events)