From 367b6630338b50e32f71a429cc4b29bfa49162de Mon Sep 17 00:00:00 2001 From: Jake Rosenberg Date: Fri, 2 Feb 2024 15:07:17 -0600 Subject: [PATCH] Task/DES-2623: Add Pydantic models and integration tests for entity metadata (#1137) * Add Pydantic models and integration tests for entity metadata * linting * Default to null values for fields we plan to deprecate * Add utils for constructing graphs from projects/pubs * refactor to use constants for entity names * add paths to publication graph constructor * add utils for converting legacy data * improve conciseness of update_file_tag_paths method * Update BaseProject to use the new facility model * validate dropdown options as id/name pairs for type Other * consolidate dropdown values in Experiments * clean up code for handling dropdowns * improve types and add post-validator for fields in base project metadata * Represent all dropdown value fields as id/name pairs * update alias for experimental facility * store string-type award numbers under 'number' instead of 'name * construct graphs for all pubs. TODO: handle file tags when no path is provided. * Fix bugs in file tag transforms * factor out models/validators and populate publication graph with user list * consolidate fields and parse keywords as array * Publish Other metadata as a child node for consistent versioning. * Factor out method for constructing entity paths * move projects_v2 into the designsafe api module * add deduplication for published paths * add version information to pub graphs * add leading slash to published base paths * Add database model and ingest workflow for project metadata * add database-enforced uniqueness constraint for project IDs. * update project model and migration scripts * add utilities for updating metadata and graphs * consolidate migrations and add tests for project metadata init * fix type ambiguities in schema models * Refactor and add more unit tests for operations * improve re-ordering algo * formatting * bugfixes and tests for file/tag association --------- Co-authored-by: Jake Rosenberg Co-authored-by: Jake Rosenberg Co-authored-by: Jake Rosenberg Co-authored-by: Jake Rosenberg --- .pylintrc | 5 +- designsafe/apps/api/projects_v2/__init__.py | 0 designsafe/apps/api/projects_v2/apps.py | 10 + designsafe/apps/api/projects_v2/constants.py | 601 ++++++++++++++++++ .../projects_v2/migration_utils/__init__.py | 0 .../migration_utils/graph_constructor.py | 346 ++++++++++ .../migration_utils/project_db_ingest.py | 108 ++++ .../migration_utils/publication_transforms.py | 260 ++++++++ .../projects_v2/migrations/0001_initial.py | 108 ++++ .../api/projects_v2/migrations/__init__.py | 0 .../apps/api/projects_v2/models/__init__.py | 2 + .../projects_v2/models/project_metadata.py | 147 +++++ .../api/projects_v2/operations/__init__.py | 0 .../_tests/project_meta_unit_test.py | 340 ++++++++++ .../operations/graph_operations.py | 177 ++++++ .../operations/project_meta_operations.py | 176 +++++ .../operations/project_system_operations.py | 1 + .../api/projects_v2/schema_models/__init__.py | 83 +++ .../schema_models/_field_models.py | 178 ++++++ .../schema_models/_field_transforms.py | 91 +++ .../api/projects_v2/schema_models/base.py | 171 +++++ .../projects_v2/schema_models/experimental.py | 199 ++++++ .../projects_v2/schema_models/field_recon.py | 194 ++++++ .../projects_v2/schema_models/hybrid_sim.py | 204 ++++++ .../projects_v2/schema_models/simulation.py | 136 ++++ .../projects_v2/tests/schema_integration.py | 81 +++ designsafe/apps/api/projects_v2/urls.py | 1 + designsafe/apps/api/projects_v2/views.py | 1 + designsafe/conftest.py | 11 + designsafe/settings/common_settings.py | 2 + designsafe/settings/test_settings.py | 2 + 31 files changed, 3633 insertions(+), 2 deletions(-) create mode 100644 designsafe/apps/api/projects_v2/__init__.py create mode 100644 designsafe/apps/api/projects_v2/apps.py create mode 100644 designsafe/apps/api/projects_v2/constants.py create mode 100644 designsafe/apps/api/projects_v2/migration_utils/__init__.py create mode 100644 designsafe/apps/api/projects_v2/migration_utils/graph_constructor.py create mode 100644 designsafe/apps/api/projects_v2/migration_utils/project_db_ingest.py create mode 100644 designsafe/apps/api/projects_v2/migration_utils/publication_transforms.py create mode 100644 designsafe/apps/api/projects_v2/migrations/0001_initial.py create mode 100644 designsafe/apps/api/projects_v2/migrations/__init__.py create mode 100644 designsafe/apps/api/projects_v2/models/__init__.py create mode 100644 designsafe/apps/api/projects_v2/models/project_metadata.py create mode 100644 designsafe/apps/api/projects_v2/operations/__init__.py create mode 100644 designsafe/apps/api/projects_v2/operations/_tests/project_meta_unit_test.py create mode 100644 designsafe/apps/api/projects_v2/operations/graph_operations.py create mode 100644 designsafe/apps/api/projects_v2/operations/project_meta_operations.py create mode 100644 designsafe/apps/api/projects_v2/operations/project_system_operations.py create mode 100644 designsafe/apps/api/projects_v2/schema_models/__init__.py create mode 100644 designsafe/apps/api/projects_v2/schema_models/_field_models.py create mode 100644 designsafe/apps/api/projects_v2/schema_models/_field_transforms.py create mode 100644 designsafe/apps/api/projects_v2/schema_models/base.py create mode 100644 designsafe/apps/api/projects_v2/schema_models/experimental.py create mode 100644 designsafe/apps/api/projects_v2/schema_models/field_recon.py create mode 100644 designsafe/apps/api/projects_v2/schema_models/hybrid_sim.py create mode 100644 designsafe/apps/api/projects_v2/schema_models/simulation.py create mode 100644 designsafe/apps/api/projects_v2/tests/schema_integration.py create mode 100644 designsafe/apps/api/projects_v2/urls.py create mode 100644 designsafe/apps/api/projects_v2/views.py diff --git a/.pylintrc b/.pylintrc index 01cf9bb20c..8f56da028e 100644 --- a/.pylintrc +++ b/.pylintrc @@ -52,7 +52,7 @@ ignore=CVS,tests.py # ignore-list. The regex matches against paths and can be in Posix or Windows # format. Because '\\' represents the directory delimiter on Windows systems, # it can't be used as an escape character. -ignore-paths=^.*migrations/.*$ +ignore-paths=^.*migrations/.*$,^.*_tests/.*$ # Files or directories matching the regular expression patterns are skipped. # The regex matches against base names, not paths. The default value ignores @@ -429,7 +429,8 @@ disable=raw-checker-failed, suppressed-message, useless-suppression, deprecated-pragma, - use-symbolic-message-instead + use-symbolic-message-instead, + duplicate-code # Enable the message, report, category or checker with the given id(s). You can # either give multiple identifier separated by comma (,) or put this option diff --git a/designsafe/apps/api/projects_v2/__init__.py b/designsafe/apps/api/projects_v2/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/designsafe/apps/api/projects_v2/apps.py b/designsafe/apps/api/projects_v2/apps.py new file mode 100644 index 0000000000..91c812e709 --- /dev/null +++ b/designsafe/apps/api/projects_v2/apps.py @@ -0,0 +1,10 @@ +"""Projects V2 API""" +from django.apps import AppConfig + + +class ProjectsV2AppConfig(AppConfig): + """App config for Projects V2 API""" + + name = "designsafe.apps.api.projects_v2" + label = "projects_v2_api" + verbose_name = "Designsafe Projects V2" diff --git a/designsafe/apps/api/projects_v2/constants.py b/designsafe/apps/api/projects_v2/constants.py new file mode 100644 index 0000000000..3a56def7cb --- /dev/null +++ b/designsafe/apps/api/projects_v2/constants.py @@ -0,0 +1,601 @@ +# pylint: disable=line-too-long +"""Mapping of possible metadata names.""" + +PROJECT = "designsafe.project" +PROJECT_GRAPH = "designsafe.project.graph" +# Experimental +EXPERIMENT = "designsafe.project.experiment" +EXPERIMENT_REPORT = "designsafe.project.report" +EXPERIMENT_ANALYSIS = "designsafe.project.analysis" +EXPERIMENT_MODEL_CONFIG = "designsafe.project.model_config" +EXPERIMENT_SENSOR = "designsafe.project.sensor_list" +EXPERIMENT_EVENT = "designsafe.project.event" +# Simulation +SIMULATION = "designsafe.project.simulation" +SIMULATION_REPORT = "designsafe.project.simulation.report" +SIMULATION_ANALYSIS = "designsafe.project.simulation.analysis" +SIMULATION_MODEL = "designsafe.project.simulation.model" +SIMULATION_INPUT = "designsafe.project.simulation.input" +SIMULATION_OUTPUT = "designsafe.project.simulation.output" +# Field Research +FIELD_RECON_MISSION = "designsafe.project.field_recon.mission" +FIELD_RECON_REPORT = "designsafe.project.field_recon.report" +FIELD_RECON_COLLECTION = "designsafe.project.field_recon.collection" +FIELD_RECON_SOCIAL_SCIENCE = "designsafe.project.field_recon.social_science" +FIELD_RECON_PLANNING = "designsafe.project.field_recon.planning" +FIELD_RECON_GEOSCIENCE = "designsafe.project.field_recon.geoscience" +# Hybrid Sim +HYBRID_SIM = "designsafe.project.hybrid_simulation" +HYBRID_SIM_GLOBAL_MODEL = "designsafe.project.hybrid_simulation.global_model" +HYBRID_SIM_COORDINATOR = "designsafe.project.hybrid_simulation.coordinator" +HYBRID_SIM_SIM_SUBSTRUCTURE = "designsafe.project.hybrid_simulation.sim_substructure" +HYBRID_SIM_EXP_SUBSTRUCTURE = "designsafe.project.hybrid_simulation.exp_substructure" +HYBRID_SIM_COORDINATOR_OUTPUT = ( + "designsafe.project.hybrid_simulation.coordinator_output" +) +HYBRID_SIM_SIM_OUTPUT = "designsafe.project.hybrid_simulation.sim_output" +HYBRID_SIM_EXP_OUTPUT = "designsafe.project.hybrid_simulation.exp_output" +HYBRID_SIM_ANALYSIS = "designsafe.project.hybrid_simulation.analysis" +HYBRID_SIM_REPORT = "designsafe.project.hybrid_simulation.report" + +FACILITY_OPTIONS = [ + { + "id": "rapid-uw", + "name": ( + "RAPID - Natural Hazard and Disasters Reconnaissance Facility " + "- University of Washington" + ), + }, + { + "id": "converge-boulder", + "name": ( + "CONVERGE - Social Science/Interdisciplinary Resources and Extreme " + "Events Coordination - University of Colorado Boulder" + ), + }, + {"id": "geer", "name": "GEER - Geotechnical Extreme Event Reconnaissance"}, + { + "id": "iseeer", + "name": "ISEEER - Interdisciplinary Science and Engineering Extreme Events Research", + }, + {"id": "neer", "name": "NEER - Nearshore Extreme Event Reconnaissance"}, + { + "id": "oseer", + "name": "OSEER - Operations and Systems Engineering Extreme Events Research", + }, + {"id": "pheer", "name": "PHEER - Public Health Extreme Events Research"}, + { + "id": "summeer", + "name": "SUMMEER - Sustainable Material Management Extreme Events Reconnaissance", + }, + {"id": "sseer", "name": "SSEER - Social Science Extreme Events Research"}, + { + "id": "steer", + "name": "StEER - Structural Engineering Extreme Event Reconnaissance", + }, + { + "id": "ohhwrl-oregon", + "name": "Large Wave Flume and Directional Wave Basin - Oregon State University", + }, + { + "id": "eqss-utaustin", + "name": "Mobile Field Shakers - University of Texas at Austin", + }, + { + "id": "cgm-ucdavis", + "name": "Center for Geotechnical Modeling - University of California, Davis", + }, + {"id": "cgm-ucdavis", "name": "Center for Geotechnical Modeling, UC Davis"}, + { + "id": "lhpost-sandiego", + "name": ( + "Six Degree of Freedom Large High-Performance Outdoor Shake Table " + "(LHPOST6) - University of California, San Diego" + ), + }, + {"id": "wwhr-florida", "name": "Wall of Wind - Florida International University"}, + { + "id": "niche", + "name": ( + ( + "National Full-Scale Testing Infrastructure for Community Hardening " + "in Extreme Wind, Surge, and Wave Events (NICHE)" + ) + ), + }, + { + "id": "pfsml-florida", + "name": "Boundary Layer Wind Tunnel - University of Florida", + }, + { + "id": "rtmd-lehigh", + "name": ( + "Real-Time Multi-Directional (RTMD) Experimental Facility with " + "Large-Scale Hybrid Simulation Testing Capabilities - LeHigh University" + ), + }, + {"id": "simcntr", "name": "SimCenter"}, + {"id": "nco-purdue", "name": "Network Coordination Office - Purdue University"}, + {"id": "crbcrp", "name": "Center for Risk-Based Community Resilience Planning"}, + {"id": "uc-berkeley", "name": "UC Berkeley"}, + {"id": "ut-austin", "name": "University of Texas at Austin"}, + { + "id": "oh-hinsdale-osu", + "name": "O.H. Hinsdale Wave Research Laboratory, Oregon State University", + }, + { + "id": "seel-ucla", + "name": ( + "University of California, Los Angeles, " + "Structural/Earthquake Engineering Laboratory" + ), + }, +] + +EQUIPMENT_TYPES = equipment_types = { + "rtmd-lehigh": [ + {"id": "hybrid_simulation", "name": "Hybrid Simulation"}, + { + "id": "lsrthcshas", + "name": "Large-Scale, Real-Time/Hybrid Capable Servo-Hydraulic Actuator System", + }, + { + "id": "ssrthcshas", + "name": "Small-Scale, Real-Time/Hybrid Capable Servo-Hydraulic Actuator System ", + }, + {"id": "rssb", "name": "Reduced-Scale Soil Box"}, + {"id": "ssihb", "name": "Soil-Structure Interaction Hybrid Simulation"}, + {"id": "other", "name": "Other"}, + ], + "cgm-ucdavis": [ + { + "id": "9-m_radius_dynamic_geotechnical_centrifuge", + "name": "9m Radius Dynamic Geotechnical Centrifuge", + }, + { + "id": "1-m_radius_dynamic_geotechnical_centrifuge", + "name": "1m Radius Dynamic Geotechnical Centrifuge", + }, + {"id": "other", "name": "Other"}, + ], + "eqss-utaustin": [ + { + "id": "liquidator", + "name": "Low Frequency, Two Axis Shaker (Liquidator)", + }, + {"id": "t-rex", "name": "High Force Three Axis Shaker (T Rex)"}, + { + "id": "tractor-t-rex", + "name": "Tractor-Trailer Rig, Big Rig, with T-Rex", + }, + {"id": "raptor", "name": "Single Axis Vertical Shaker (Raptor)"}, + {"id": "rattler", "name": "Single Axis Horizontal Shaker (Rattler)"}, + {"id": "thumper", "name": "Urban, Three axis Shaker (Thumper)"}, + {"id": "other", "name": "Other"}, + ], + "pfsml-florida": [ + {"id": "blwt", "name": "Boundary Layer Wind Tunnel (BLWT)"}, + { + "id": "abl", + "name": "Atmospheric Boundary Layer Wind Tunnel Test (ABL)", + }, + {"id": "wdrt", "name": "Wind Driven Rain Test"}, + {"id": "wtdt", "name": "Wind Tunnel Destructive Test"}, + {"id": "dfs", "name": "Dynamic Flow Simulator (DFS)"}, + { + "id": "hapla", + "name": "High Airflow Pressure Loading Actuator (HAPLA)", + }, + { + "id": "spla", + "name": "Spatiotemporal Pressure Loading Actuator (SPLA)", + }, + {"id": "other", "name": "Other"}, + ], + "wwhr-florida": [ + {"id": "pmtp", "name": "Physical Measurement Test Protocol"}, + {"id": "fmtp", "name": "Failure Mode Test Protocol"}, + {"id": "wdrtp", "name": "Wind Driven Rain Test Protocol"}, + {"id": "other", "name": "Other"}, + ], + "lhpost-sandiego": [ + { + "id": "lhpost", + "name": "Large High Performance Outdoor Shake Table (LHPOST)", + }, + {"id": "other", "name": "Other"}, + ], + "ohhwrl-oregon": [ + {"id": "lwf", "name": "Large Wave Flume (LWF)"}, + {"id": "dwb", "name": "Directional Wave Basin (DWB)"}, + {"id": "mobs", "name": "Mobile Shaker"}, + {"id": "pla", "name": "Pressure Loading Actuator"}, + {"id": "other", "name": "Other"}, + ], + "other": [{"id": "other", "name": "Other"}], +} + + +EXPERIMENT_TYPES = { + "rtmd-lehigh": [ + {"id": "hybrid_simulation", "name": "Hybrid Simulation"}, + {"id": "large", "name": "Large-Scale"}, + {"id": "small", "name": "Small-Scale"}, + {"id": "char", "name": "Characterization"}, + {"id": "qstatic", "name": "Quasi-Static"}, + {"id": "dynamic", "name": "Dynamic"}, + {"id": "ssi", "name": "Soil-Structure Interaction"}, + {"id": "other", "name": "Other"}, + ], + "cgm-ucdavis": [ + {"id": "centrifuge", "name": "Centrifuge"}, + {"id": "other", "name": "Other"}, + ], + "eqss-utaustin": [ + {"id": "mobile_shaker", "name": "Mobile Shaker"}, + {"id": "other", "name": "Other"}, + ], + "pfsml-florida": [ + {"id": "wind", "name": "Wind"}, + {"id": "other", "name": "Other"}, + ], + "wwhr-florida": [ + {"id": "wind", "name": "Wind"}, + {"id": "other", "name": "Other"}, + ], + "lhpost-sandiego": [ + {"id": "shake", "name": "Shake"}, + {"id": "other", "name": "Other"}, + ], + "ohhwrl-oregon": [ + {"id": "wave", "name": "Wave"}, + {"id": "other", "name": "Other"}, + ], + "other": [{"id": "other", "name": "Other"}], +} + +NATURAL_HAZARD_TYPES = [ + {"id": "drought", "name": "Drought"}, + {"id": "earthquake", "name": "Earthquake"}, + {"id": "extreme temperatures", "name": "Extreme Temperatures"}, + {"id": "fire", "name": "Wildfire"}, + {"id": "flood", "name": "Flood"}, + {"id": "hurricane/tropical storm", "name": "Hurricane/Tropical Storm"}, + {"id": "landslide", "name": "Landslide"}, + {"id": "tornado", "name": "Tornado"}, + {"id": "tsunami", "name": "Tsunami"}, + {"id": "thunderstorm", "name": "Thunderstorm"}, + {"id": "storm surge", "name": "Storm Surge"}, + {"id": "pandemic", "name": "Pandemic"}, + {"id": "wind", "name": "Wind"}, + {"id": "fire", "name": "Fire"}, + {"id": "hurricane/tropical storm", "name": "Hurricane"}, + {"id": "hurricane/tropical storm", "name": "Tropical Storm"}, +] + +FIELD_RESEARCH_TYPES = [ + {"id": "engineering", "name": "Engineering"}, + {"id": "geosciences", "name": "Geosciences"}, + {"id": "public health", "name": "Public Health"}, + {"id": "social sciences", "name": "Social Sciences"}, + {"id": "interdisciplinary", "name": "Interdisciplinary"}, + {"id": "field experiment", "name": "Field Experiment"}, + {"id": "cross-sectional study", "name": "Cross-Sectional Study"}, + {"id": "longitudinal study", "name": "Longitudinal Study"}, + {"id": "reconnaissance", "name": "Reconnaissance"}, + {"id": "other", "name": "Other"}, +] + +OTHER_DATA_TYPES = [ + {"id": "archival materials", "name": "Archival Materials"}, + {"id": "audio", "name": "Audio"}, + {"id": "benchmark dataset", "name": "Benchmark Dataset"}, + {"id": "check sheet", "name": "Check Sheet"}, + {"id": "code", "name": "Code"}, + {"id": "database", "name": "Database"}, + {"id": "dataset", "name": "Dataset"}, + {"id": "engineering", "name": "Engineering"}, + {"id": "image", "name": "Image"}, + {"id": "interdisciplinary", "name": "Interdisciplinary"}, + {"id": "jupyter notebook", "name": "Jupyter Notebook"}, + {"id": "learning object", "name": "Learning Object"}, + {"id": "model", "name": "Model"}, + {"id": "paper", "name": "Paper"}, + {"id": "proceeding", "name": "Proceeding"}, + {"id": "poster", "name": "Poster"}, + {"id": "presentation", "name": "Presentation"}, + {"id": "report", "name": "Report"}, + { + "id": "research experience for undergraduates", + "name": "Research Experience for Undergraduates", + }, + {"id": "simcenter testbed", "name": "SimCenter Testbed"}, + {"id": "social sciences", "name": "Social Sciences"}, + {"id": "survey instrument", "name": "Survey Instrument"}, + {"id": "testbed", "name": "Testbed"}, + {"id": "video", "name": "Video"}, +] + +SIMULATION_TYPES = [ + {"id": "Geotechnical", "name": "Geotechnical"}, + {"id": "Structural", "name": "Structural"}, + {"id": "Soil Structure System", "name": "Soil Structure System"}, + {"id": "Storm Surge", "name": "Storm Surge"}, + {"id": "Wind", "name": "Wind"}, + {"id": "Other", "name": "Other"}, +] + +HYBRID_SIM_TYPES = [ + {"id": "Earthquake", "name": "Earthquake"}, + {"id": "Wind", "name": "Wind"}, + {"id": "Other", "name": "Other"}, +] + + +FR_EQUIPMENT_TYPES = { + "General": [{"id": "none", "name": "None"}, {"id": "other", "name": "Other"}], + "IT/Electronics": [ + { + "id": "ipad pro 10.5 | 256 gb, wifi+cellular ", + "name": "iPad Pro 10.5 | 256 GB, WiFi+Cellular ", + }, + { + "id": "ipad rugged case w/keyboard | zagg rugged messenger", + "name": "iPad Rugged Case w/Keyboard | ZAGG Rugged Messenger", + }, + { + "id": "ipad accessories | tripod, hd microphone, external battery, apple pencil", + "name": "iPAD accessories | Tripod, HD Microphone, External battery, Apple Pencil", + }, + { + "id": "flash drive | for backing up ipad (sandisk wireless stick 256gb)", + "name": "Flash Drive | For backing up iPad (SanDisk Wireless Stick 256GB)", + }, + { + "id": "external hard drive | 2 tb drive for drone footage, protective bumpers, integrated microsd, usb-c", + "name": "External Hard Drive | 2 TB Drive for drone footage, protective bumpers, integrated microSD, USB-C", + }, + { + "id": "custom processing desktop pc | 3.3 ghz i9-7900x, 128gb ddr4 ram, nvidia geforce rtx 2080 ti", + "name": "Custom Processing Desktop PC | 3.3 GHz i9-7900X, 128GB DDR4 RAM, Nvidia GeForce RTX 2080 Ti", + }, + { + "id": "processing laptop | 2.9ghz intel i9-8950hk, 64gb ddr4 ram, nvidia geforce rtx 2070", + "name": "Processing Laptop | 2.9GHz Intel i9-8950HK, 64GB DDR4 RAM, NVIDIA GeForce RTX 2070", + }, + ], + "Terrestrial Laser Scanning": [ + { + "id": "short range scanner | leica blk360", + "name": "Short Range Scanner | Leica BLK360", + }, + { + "id": "medium range scanner | leica rtc360 ", + "name": "Medium Range Scanner | Leica RTC360 ", + }, + { + "id": "medium range scanner | leica scanstation p50", + "name": "Medium Range Scanner | Leica Scanstation P50", + }, + { + "id": "long range scanner | maptek i-site xr3", + "name": "Long Range Scanner | Maptek I-Site XR3", + }, + { + "id": "long range scanner | maptek i-site lr3", + "name": "Long Range Scanner | Maptek I-Site LR3", + }, + ], + "Survey": [ + { + "id": "robotic total station | leica nova ts16i instrument package", + "name": "Robotic Total Station | Leica Nova TS16I instrument package", + }, + { + "id": "survey (digital level) | leica ls15 digital level package", + "name": "Survey (Digital level) | Leica LS15 digital level package", + }, + { + "id": "survey (gnss receiver) | leica gs18t smartantenna package", + "name": "Survey (GNSS receiver) | Leica GS18T SmartAntenna package", + }, + { + "id": "thermometer/digital compass/barometer/humidity sensor | kestrel meter 5500 weather meter", + "name": "Thermometer/Digital Compass/Barometer/Humidity Sensor | Kestrel Meter 5500 Weather Meter", + }, + ], + "Unmanned Aerial Systems": [ + { + "id": "small portable (backpack-size) drone | dji mavic pro", + "name": "Small Portable (backpack-size) Drone | DJI Mavic Pro", + }, + { + "id": "small portable (backpack-size) drone | dji mavic pro 2", + "name": "Small Portable (backpack-size) Drone | DJI Mavic Pro 2", + }, + { + "id": "small to medium-sized drone | dji phantom 4 pro+ ", + "name": "Small to Medium-Sized Drone | DJI Phantom 4 Pro+ ", + }, + { + "id": "small to medium-sized drone | dji phantom 4 pro+ rtk ", + "name": "Small to Medium-Sized Drone | DJI Phantom 4 Pro+ RTK ", + }, + { + "id": "medium size drone | dji inspire2", + "name": "Medium Size Drone | DJI Inspire2", + }, + { + "id": "mid-grade industral weather resistant drone | dji matrice 210 ", + "name": "Mid-grade Industral Weather Resistant Drone | DJI Matrice 210 ", + }, + { + "id": "high-end industrial weather resistant drone | dji matrice 210 rtk", + "name": "High-end Industrial Weather Resistant Drone | DJI Matrice 210 RTK", + }, + { + "id": "large fixed wing drone | sense fly ebee with rtk and base station", + "name": "Large Fixed Wing Drone | Sense fly Ebee with RTK and base station", + }, + { + "id": "kite balloon with tether | skyshot hybrid helikite (already purchased)", + "name": "Kite Balloon with Tether | SkyShot Hybrid HeliKite (already purchased)", + }, + { + "id": "high-end drone with uas lidar system | dji m600 drone with phoenix aerial mini ranger lidar system", + "name": "High-end Drone with UAS LiDAR System | DJI M600 drone with Phoenix Aerial Mini Ranger LiDAR system", + }, + { + "id": "drone imaging systems (compatible with all matrice platforms) | zenmuse x4s, x5s, and x30 (zoom) camera. micasense altum multispectral sensor.", + "name": "Drone Imaging systems (compatible with all Matrice platforms) | Zenmuse X4S, X5S, and X30 (zoom) camera. MicaSense Altum Multispectral Sensor.", + }, + ], + "Imaging": [ + { + "id": "digital slr camera | canon 7d mark ii, with narrow and wide angle lenses", + "name": "Digital SLR Camera | Canon 7D Mark II, with narrow and wide angle lenses", + }, + { + "id": "robotic camera mount | gigapan epic pro v (for dslr) and tripod", + "name": "Robotic Camera Mount | GigaPan Epic Pro V (for DSLR) and Tripod", + }, + { + "id": "360 degree camera | insta360 one for apple products", + "name": "360 Degree Camera | Insta360 One for Apple Products", + }, + { + "id": "car camera system | applied streetview 360 degree panorama gps camera system", + "name": "Car Camera System | Applied Streetview 360 degree panorama GPS camera system", + }, + { + "id": "car camera system | nctech istar pulsar+ (streetview) 360 degree panorama gps camera system", + "name": "Car Camera System | NCTech iSTAR Pulsar+ (streetview) 360 degree Panorama GPS Camera system", + }, + { + "id": "thermal camera | flir one pro", + "name": "Thermal Camera | Flir One Pro", + }, + ], + "3D Visualization": [ + { + "id": "computer automated virtual environment (cave) system | at rapid facility headquarters only. b363d tv, tv stand, computer, tracking camera and accessories. ", + "name": "Computer Automated Virtual Environment (CAVE) System | at RAPID Facility Headquarters ONLY. B363D TV, TV stand, computer, tracking camera and accessories. ", + } + ], + "Site Characterization": [ + { + "id": "seismometers | nanometrics triaxial, 20s, trillium seismometer with centaur digital recorder", + "name": "Seismometers | Nanometrics triaxial, 20s, Trillium seismometer with Centaur digital recorder", + }, + { + "id": "multichannel analysis of surface waves (masw) system | atom wireless seismic system, 24-channel system with gps and wifi", + "name": "Multichannel Analysis of Surface Waves (MASW) System | ATOM wireless seismic system, 24-channel system with GPS and WiFi", + }, + ], + "Structural": [ + { + "id": "accelerograph system | nanometrics standalone accelerographs with batteries, cases, and data loggers", + "name": "Accelerograph system | Nanometrics standalone accelerographs with batteries, cases, and data loggers", + } + ], + "Ground Investigation": [ + { + "id": "hand operated dynamic cone penetrometer (dcp) system | smartdcp digital recording system and hand operated dcp", + "name": "Hand Operated Dynamic Cone Penetrometer (DCP) System | SmartDCP digital recording system and hand operated DCP", + }, + { + "id": "lightweight, portable dcp | sol'solution panda dcp", + "name": "Lightweight, Portable DCP | Sol'Solution Panda DCP", + }, + { + "id": "pocket penetrometer | geotester pocket penetrometer", + "name": "Pocket Penetrometer | Geotester pocket penetrometer", + }, + { + "id": "schmidt hammer | digi-schmidt 2000, digital measuring system", + "name": "Schmidt Hammer | Digi-Schmidt 2000, digital measuring system", + }, + { + "id": "basic soil sampling kit | ams 3-1/4-inch basic soil sampling kit, hand augers and samplers", + "name": "Basic Soil Sampling Kit | AMS 3-1/4-inch basic soil sampling kit, hand augers and samplers", + }, + ], + "Coastal": [ + { + "id": "remote-operated hydrographic survey boat | z-boat 1800 with single beam echo sounder", + "name": "Remote-Operated Hydrographic Survey Boat | Z-boat 1800 with single beam echo sounder", + }, + { + "id": "acousitc beacons | ulb 350/37 underwater acoustic beacon", + "name": "Acousitc Beacons | ULB 350/37 underwater acoustic beacon", + }, + { + "id": "beacon locator - diver operated pinger receiver system | dpr-275 diver pinger receiver", + "name": "Beacon Locator - Diver Operated Pinger Receiver System | DPR-275 Diver Pinger Receiver", + }, + { + "id": "water level data logger | trublue 255", + "name": "Water Level Data Logger | TruBlue 255", + }, + { + "id": "acoustic doppler velocimeter | aquadopp profiler with 90deg angle head", + "name": "Acoustic Doppler Velocimeter | AquaDopp profiler with 90deg angle head", + }, + { + "id": 'grab sampler | petite ponar 6"x6" w/ 50 ft of rope', + "name": 'Grab Sampler | Petite Ponar 6"x6" w/ 50 ft of rope', + }, + ], + "Social": [ + { + "id": "eeg headset | emotiv 14-channel headset", + "name": "EEG Headset | Emotiv 14-channel headset", + }, + { + "id": "eeg headset | emotiv 5-channel headset", + "name": "EEG Headset | Emotiv 5-channel headset", + }, + {"id": "pen and paper", "name": "Pen and Paper"}, + ], + "Support": [ + { + "id": "smaller battery | goal zero yeti 150", + "name": "Smaller Battery | Goal Zero Yeti 150", + }, + { + "id": "high capacity battery | goal zero yeti 400", + "name": "High Capacity Battery | Goal Zero Yeti 400", + }, + { + "id": "safety vest | custom nheri rapid safety vests with ipad pocket", + "name": "Safety Vest | Custom NHERI Rapid safety vests with iPad pocket", + }, + { + "id": "safety helmet | rock climbing helmet", + "name": "Safety Helmet | Rock climbing helmet", + }, + { + "id": "power inverters | 300w dc 12v to 110v car outlet", + "name": "Power Inverters | 300W DC 12V to 110V car outlet", + }, + { + "id": "walkie talkies (pair) | waterproof, weather proof, long range", + "name": "Walkie Talkies (pair) | Waterproof, weather proof, long range", + }, + { + "id": "rapp pack | swiss gear scansmart backpack w/plumb bob, measuring tape, digital caliper, safety glasses, hand level, pocket rod, first aid kit, weld gauge, crack gauge, range finder,", + "name": "RAPP Pack | Swiss Gear ScanSmart backpack w/Plumb Bob, Measuring tape, Digital Caliper, Safety Glasses, Hand level, Pocket Rod, First Aid Kit, Weld Gauge, Crack Gauge, Range Finder,", + }, + ], +} + +FR_OBSERVATION_TYPES = [ + {"id": "wind", "name": "Wind"}, + {"id": "structural", "name": "Structural"}, + {"id": "storm surge / coastal", "name": "Storm Surge / Coastal"}, + {"id": "social science", "name": "Social Science"}, + {"id": "geotechnical", "name": "Geotechnical"}, + {"id": "field sensors", "name": "Field Sensors"}, + {"id": "coastal", "name": "Coastal"}, + {"id": "other", "name": "Other"}, +] diff --git a/designsafe/apps/api/projects_v2/migration_utils/__init__.py b/designsafe/apps/api/projects_v2/migration_utils/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/designsafe/apps/api/projects_v2/migration_utils/graph_constructor.py b/designsafe/apps/api/projects_v2/migration_utils/graph_constructor.py new file mode 100644 index 0000000000..9764db1c1d --- /dev/null +++ b/designsafe/apps/api/projects_v2/migration_utils/graph_constructor.py @@ -0,0 +1,346 @@ +"""Utils for constructing project trees from legacy association IDs.""" +import json +from typing import TypedDict, Optional +from uuid import uuid4 +from pathlib import Path +import networkx as nx +from django.utils.text import slugify +from designsafe.apps.api.agave import service_account +from designsafe.apps.data.models.elasticsearch import IndexedPublication +from designsafe.apps.projects.managers.publication import FIELD_MAP +from designsafe.apps.api.projects_v2.schema_models import PATH_SLUGS +from designsafe.apps.projects.models.categories import Category +from designsafe.apps.api.projects_v2 import constants as names +from designsafe.apps.api.projects_v2.models.project_metadata import ProjectMetadata +from designsafe.apps.api.projects_v2.migration_utils.publication_transforms import ( + transform_entity, + construct_users, +) + +# map metadata 'name' field to allowed (direct) children +ALLOWED_RELATIONS = { + names.PROJECT: [ + names.EXPERIMENT, + names.SIMULATION, + names.HYBRID_SIM, + names.FIELD_RECON_MISSION, + names.FIELD_RECON_REPORT, + ], + # Experimental + names.EXPERIMENT: [ + names.EXPERIMENT_ANALYSIS, + names.EXPERIMENT_REPORT, + names.EXPERIMENT_MODEL_CONFIG, + ], + names.EXPERIMENT_MODEL_CONFIG: [names.EXPERIMENT_SENSOR], + names.EXPERIMENT_SENSOR: [names.EXPERIMENT_EVENT], + # Simulation + names.SIMULATION: [ + names.SIMULATION_ANALYSIS, + names.SIMULATION_REPORT, + names.SIMULATION_MODEL, + ], + names.SIMULATION_MODEL: [names.SIMULATION_INPUT], + names.SIMULATION_INPUT: [names.SIMULATION_OUTPUT], + # Hybrid sim + names.HYBRID_SIM: [ + names.HYBRID_SIM_REPORT, + names.HYBRID_SIM_GLOBAL_MODEL, + names.HYBRID_SIM_ANALYSIS, + ], + names.HYBRID_SIM_GLOBAL_MODEL: [names.HYBRID_SIM_COORDINATOR], + names.HYBRID_SIM_COORDINATOR: [ + names.HYBRID_SIM_COORDINATOR_OUTPUT, + names.HYBRID_SIM_SIM_SUBSTRUCTURE, + names.HYBRID_SIM_EXP_SUBSTRUCTURE, + ], + names.HYBRID_SIM_SIM_SUBSTRUCTURE: [names.HYBRID_SIM_SIM_OUTPUT], + names.HYBRID_SIM_EXP_SUBSTRUCTURE: [names.HYBRID_SIM_EXP_OUTPUT], + # Field Recon + names.FIELD_RECON_MISSION: [ + names.FIELD_RECON_COLLECTION, + names.FIELD_RECON_PLANNING, + names.FIELD_RECON_SOCIAL_SCIENCE, + names.FIELD_RECON_GEOSCIENCE, + ], +} + + +def get_entities_by_project_id(project_id: str) -> list[dict]: + """Return all entities matching a project ID, with the root as element 0.""" + client = service_account() + base_query = {"value.projectId": project_id, "name": "designsafe.project"} + root_project_listing = client.meta.listMetadata(q=json.dumps(base_query)) + root_project_meta = root_project_listing[0] + + project_uuid = root_project_meta["uuid"] + + associations_query = {"associationIds": project_uuid} + associated_entities = client.meta.listMetadata(q=json.dumps(associations_query)) + + return list(map(dict, root_project_listing + associated_entities)) + + +def get_path(graph: nx.DiGraph, node_id: str): + """Iterate through all predecessors in the graph (inclusive)""" + shortest_path = nx.shortest_path(graph, "NODE_ROOT", node_id) + path_uuids = map(lambda node: graph.nodes[node]["uuid"], shortest_path) + + return list(path_uuids) + + +def construct_graph(project_id) -> nx.DiGraph: + """Construct a directed graph from a project's association IDs.""" + entity_listing = get_entities_by_project_id(project_id) + root_entity = entity_listing[0] + + project_graph = nx.DiGraph() + root_node_id = "NODE_ROOT" + root_node_data = { + "uuid": root_entity["uuid"], + "name": root_entity["name"], + } + project_graph.add_node(root_node_id, **root_node_data) + + construct_graph_recurse(project_graph, entity_listing, root_entity, root_node_id) + + deprecated_mission_nodes = [ + node + for (node, data) in project_graph.nodes.data() + if data["name"] == names.FIELD_RECON_COLLECTION + ] + project_graph.remove_nodes_from(deprecated_mission_nodes) + + return project_graph + + +def construct_graph_from_db(project_id) -> nx.DiGraph: + """Construct a directed graph from a project's association IDs using the db.""" + prj_entities = ProjectMetadata.get_entities_by_project_id(project_id) + root_entity = prj_entities.get(name="designsafe.project").to_dict() + entity_listing = [e.to_dict() for e in prj_entities] + + project_graph = nx.DiGraph() + root_node_id = "NODE_ROOT" + root_node_data = { + "uuid": root_entity["uuid"], + "name": root_entity["name"], + } + project_graph.add_node(root_node_id, **root_node_data) + + construct_graph_recurse(project_graph, entity_listing, root_entity, root_node_id) + + deprecated_mission_nodes = [ + node + for (node, data) in project_graph.nodes.data() + if data["name"] == names.FIELD_RECON_COLLECTION + ] + project_graph.remove_nodes_from(deprecated_mission_nodes) + + return project_graph + + +def construct_graph_recurse( + graph: nx.DiGraph, + entity_list: list[dict], + parent: dict, + parent_node_id: str, +): + """Recurse through an entity's children and add nodes/edges. B is a child of A if + all of A's descendants are referenced in B's association IDs.""" + + # Handle legacy hybrid sim projects with incomplete associationIds arrays. + association_path = get_path(graph, parent_node_id) + if parent["name"] in [ + "designsafe.project.hybrid_simulation.sim_substructure", + "designsafe.project.hybrid_simulation.exp_substructure", + ]: + association_path.pop(-2) + + children = filter( + lambda child: ( + child["name"] in ALLOWED_RELATIONS.get(parent["name"], []) + and set(child["associationIds"]) >= set(association_path) + ), + entity_list, + ) + + for idx, child in enumerate(children): + child_node_id = f"NODE_{uuid4()}" + child_order = next( + ( + order["value"] + for order in get_entity_orders(child) + if order["parent"] == parent["uuid"] + ), + idx, + ) + + child_data = { + "uuid": child["uuid"], + "name": child["name"], + "order": child_order, + } + graph.add_node(child_node_id, **child_data) + graph.add_edge(parent_node_id, child_node_id) + construct_graph_recurse(graph, entity_list, child, child_node_id) + + +def get_entities_from_publication(project_id: str, version=None): + """Loop through a publication's fields and construct a flat entity list.""" + entity_fields: list[str] = list(FIELD_MAP.values()) + pub = IndexedPublication.from_id(project_id, revision=version) + + entity_list = [pub.project.to_dict()] + for field in entity_fields: + field_entities = [e.to_dict() for e in getattr(pub, field, [])] + entity_list += field_entities + + return entity_list + + +def construct_publication_graph(project_id, version=None) -> nx.DiGraph: + """Construct a directed graph from a publications's association IDs.""" + entity_listing = get_entities_from_publication(project_id, version=version) + root_entity = entity_listing[0] + + pub_graph = nx.DiGraph() + + project_type = root_entity["value"]["projectType"] + + root_node_id = "NODE_ROOT" + if project_type == "other": + root_node_data = {"uuid": None, "name": None, "projectType": "other"} + else: + root_node_data = { + "uuid": root_entity["uuid"], + "name": root_entity["name"], + "projectType": root_entity["value"]["projectType"], + } + pub_graph.add_node(root_node_id, **root_node_data) + if project_type == "other": + base_node_data = { + "uuid": root_entity["uuid"], + "name": root_entity["name"], + "projectType": root_entity["value"]["projectType"], + } + base_node_id = f"NODE_{uuid4()}" + pub_graph.add_node(base_node_id, **base_node_data) + pub_graph.add_edge(root_node_id, base_node_id) + construct_graph_recurse(pub_graph, entity_listing, root_entity, root_node_id) + + pub_graph.nodes["NODE_ROOT"]["basePath"] = f"/{project_id}" + pub_graph = construct_entity_filepaths(entity_listing, pub_graph, version) + + return pub_graph + + +def construct_entity_filepaths( + entity_listing: list[dict], pub_graph: nx.DiGraph, version: Optional[int] = None +): + """ + Walk the publication graph and construct base file paths for each node. + The file path for a node contains the titles of each entity above it in the + hierarchy. Returns the publication graph with basePath data added for each node. + """ + for parent_node, child_node in nx.bfs_edges(pub_graph, "NODE_ROOT"): + # Construct paths based on the entity hierarchy + parent_base_path = pub_graph.nodes[parent_node]["basePath"] + entity_name_slug = PATH_SLUGS.get(pub_graph.nodes[child_node]["name"]) + entity_title = next( + e["value"]["title"] + for e in entity_listing + if e["uuid"] == pub_graph.nodes[child_node]["uuid"] + ) + entity_dirname = f"{entity_name_slug}--{slugify(entity_title)}" + + if version and child_node in pub_graph.successors("NODE_ROOT"): + # Version datasets if the containing publication is versioned. + child_path = Path(parent_base_path) / f"{entity_dirname}--v{version}" + elif parent_node in pub_graph.successors("NODE_ROOT"): + # Publishable entities have a "data" folder in Bagit ontology. + child_path = Path(parent_base_path) / "data" / entity_dirname + else: + child_path = Path(parent_base_path) / entity_dirname + + pub_graph.nodes[child_node]["basePath"] = str(child_path) + return pub_graph + + +class EntityOrder(TypedDict): + """Representation for UI orders stored in legacy metadata.""" + + value: int + parent: str + + +def get_entity_orders( + entity: dict, +) -> list[EntityOrder]: + """extract ordering metadata for a project or pub.""" + pub_orders = entity.get("_ui", None) + if pub_orders: + return entity["_ui"].get("orders", []) + + prj_orders = Category.objects.filter(uuid=entity["uuid"]).first() + if not prj_orders: + return [] + return prj_orders.to_dict().get("orders", []) + + +def transform_pub_entities(project_id: str, version: Optional[int] = None): + """Validate publication entities against their corresponding model.""" + entity_listing = get_entities_from_publication(project_id, version=version) + base_pub_meta = IndexedPublication.from_id(project_id, revision=version).to_dict() + pub_graph = construct_publication_graph(project_id, version) + + for _, node_data in pub_graph.nodes.items(): + node_entity = next( + (e for e in entity_listing if e["uuid"] == node_data["uuid"]), None + ) + if not node_entity: + continue + data_path = str(Path(node_data["basePath"]) / "data") + new_entity_value = transform_entity(node_entity, base_pub_meta, data_path) + node_data["value"] = new_entity_value + + project_users = construct_users(entity_listing[0]) + base_node = next( + node + for (node, node_data) in pub_graph.nodes.items() + if node_data["uuid"] == entity_listing[0]["uuid"] + ) + pub_graph.nodes[base_node]["value"]["users"] = project_users + + for pub in pub_graph.successors("NODE_ROOT"): + if version and version > 1: + pub_graph.nodes[pub]["version"] = version + pub_graph.nodes[pub]["versionInfo"] = base_pub_meta.get( + "revisionText", None + ) + pub_graph.nodes[pub]["versionDate"] = base_pub_meta.get( + "revisionDate", None + ) + else: + pub_graph.nodes[pub]["version"] = 1 + + return pub_graph + + +def combine_pub_versions(project_id: str) -> nx.DiGraph: + """Construct a tree of all versions of published datasets in a project.""" + latest_version: int = IndexedPublication.max_revision(project_id) + + pub_graph = transform_pub_entities(project_id) + if not latest_version: + return pub_graph + + versions = range(2, latest_version + 1) + for version in versions: + version_graph = transform_pub_entities(project_id, version) + version_pubs = version_graph.successors("NODE_ROOT") + pub_graph: nx.DiGraph = nx.compose(pub_graph, version_graph) + for node_id in version_pubs: + pub_graph.add_edge("NODE_ROOT", node_id) + + return pub_graph diff --git a/designsafe/apps/api/projects_v2/migration_utils/project_db_ingest.py b/designsafe/apps/api/projects_v2/migration_utils/project_db_ingest.py new file mode 100644 index 0000000000..b2f69d508d --- /dev/null +++ b/designsafe/apps/api/projects_v2/migration_utils/project_db_ingest.py @@ -0,0 +1,108 @@ +"""Utilities for ingesting projects using the ProjectMetadata db model.""" +from pydantic import ValidationError +import networkx as nx +from designsafe.apps.api.projects_v2.models.project_metadata import ProjectMetadata +from designsafe.apps.api.projects_v2.migration_utils.graph_constructor import ( + get_entities_by_project_id, + construct_graph_from_db, +) +from designsafe.apps.api.projects_v2.schema_models import SCHEMA_MAPPING +from designsafe.apps.api.projects_v2.tests.schema_integration import iterate_entities + + +def ingest_project_to_db(project_id): + """Ingest all entities within a project into the database.""" + (base_project, *entities) = get_entities_by_project_id(project_id) + + base_model = SCHEMA_MAPPING[base_project["name"]].model_validate( + base_project["value"] + ) + db_base_model, _ = ProjectMetadata.objects.update_or_create( + uuid=base_project["uuid"], + name=base_project["name"], + defaults={"value": base_model.model_dump(mode="json")}, + ) + db_base_model.save() + for entity in entities: + schema_model = SCHEMA_MAPPING.get(entity["name"], None) + if not schema_model: + continue + value_model = schema_model.model_validate(entity["value"]) + + ProjectMetadata.objects.update_or_create( + uuid=entity["uuid"], + name=entity["name"], + defaults={ + "value": value_model.model_dump(mode="json"), + "base_project": db_base_model, + }, + ) + + +def ingest_base_projects(): + """Ingest all top-level project metadata.""" + name = "designsafe.project" + project_schema = SCHEMA_MAPPING["designsafe.project"] + for project_meta in iterate_entities(name): + value_model = project_schema.model_validate(project_meta["value"]) + ProjectMetadata.objects.update_or_create( + uuid=project_meta["uuid"], + name=project_meta["name"], + defaults={ + "value": value_model.model_dump(mode="json"), + "created": project_meta["created"], + }, + ) + + +def ingest_entities_by_name(name): + """Ingest metadata entities under a given `name` field.""" + entities = iterate_entities(name) + for entity in entities: + schema_model = SCHEMA_MAPPING[entity["name"]] + try: + value_model = schema_model.model_validate(entity["value"]) + except ValidationError as err: + print(entity) + raise err + try: + prj = ProjectMetadata.objects.get( + name="designsafe.project", uuid__in=entity["associationIds"] + ) + ProjectMetadata.objects.update_or_create( + uuid=entity["uuid"], + name=entity["name"], + defaults={ + "value": value_model.model_dump(mode="json"), + "base_project": prj, + "created": entity["created"], + "association_ids": entity["associationIds"], + }, + ) + except ProjectMetadata.DoesNotExist: + print(entity["uuid"]) + + +def ingest_sub_entities(): + """Ingest all entities other than base project metadata. + Run AFTER ingesting projects.""" + for name in SCHEMA_MAPPING: + if name != "designsafe.project": + print(f"ingesting for type: {name}") + ingest_entities_by_name(name) + + +def ingest_graphs(): + """Construct project graphs and ingest into the db. + Run AFTER ingesting projects/entities""" + + base_projects = ProjectMetadata.objects.filter(name="designsafe.project") + for project in base_projects: + prj_graph = construct_graph_from_db(project.value["projectId"]) + graph_json = nx.node_link_data(prj_graph) + + ProjectMetadata.objects.update_or_create( + name="designsafe.project.graph", + base_project__uuid=project.uuid, + defaults={"value": graph_json, "base_project": project}, + ) diff --git a/designsafe/apps/api/projects_v2/migration_utils/publication_transforms.py b/designsafe/apps/api/projects_v2/migration_utils/publication_transforms.py new file mode 100644 index 0000000000..2c6a33f882 --- /dev/null +++ b/designsafe/apps/api/projects_v2/migration_utils/publication_transforms.py @@ -0,0 +1,260 @@ +"""Utilities to convert published entitities to a consistent schema.""" +from pathlib import Path +from typing import TypedDict +from django.conf import settings +from django.contrib.auth import get_user_model +from designsafe.apps.api.projects_v2.schema_models import SCHEMA_MAPPING + +user_model = get_user_model() + + +def convert_v1_user(user): + """Convert legacy user representation to the V2 schema.""" + if not isinstance(user, dict): + user = user.to_dict() + return { + "fname": user["first_name"], + "lname": user["last_name"], + "name": user["username"], + "username": user["username"], + "email": user["email"], + "inst": user["profile"]["institution"], + "order": user["_ui"]["order"], + "guest": False, + "authorship": True, + "role": "team_member", + } + + +def get_user_info(username: str, role: str = None) -> dict: + """Construct a user object from info in the db.""" + try: + user_obj = user_model.objects.get(username=username) + except user_model.DoesNotExist: + return { + "username": username, + "fname": "N/A", + "lname": "N/A", + "email": "N/A", + "inst": "N/A", + "role": role, + } + user_info = { + "username": username, + "fname": user_obj.first_name, + "lname": user_obj.last_name, + "email": user_obj.email, + "inst": user_obj.profile.institution, + } + if role: + user_info["role"] = role + return user_info + + +def construct_users(entity: dict) -> list[dict]: + """Get users associated with an entity.""" + _users = [] + _users.append(get_user_info(entity["value"].get("pi"), role="pi")) + for co_pi in entity["value"].get("coPis", []): + _users.append(get_user_info(co_pi, role="co_pi")) + for team_member in entity["value"].get("teamMembers", []): + _users.append(get_user_info(team_member, role="team_member")) + for guest_member in entity["value"].get("guestMembers", []): + _users.append({**guest_member, "username": None, "role": "guest"}) + + return _users + + +def convert_v2_user(user): + """Convert v2 publication user to fill in all fields""" + is_guest = user.get("guest", False) + if is_guest: + role = "guest" + username = None + else: + role = "team_member" + username = user["name"] + + return { + "fname": user["fname"], + "lname": user["lname"], + "name": user["name"], + "username": username, + "role": role, + "guest": is_guest, + "order": user["order"], + "email": user.get("email", None), + "inst": user.get("inst", None), + "authorship": user["authorship"], + } + + +def get_v1_authors(entity: dict, pub_users: list[dict]): + """Convert V1 publication authors to a standard format.""" + authors = entity["value"].get("authors", []) + updated_authors = [] + for username in authors: + author_data = next( + (user for user in pub_users if user["username"] == username), None + ) + if not author_data: + continue + updated_authors.append(convert_v1_user(author_data)) + return updated_authors + + +class FileTagDict(TypedDict): + """{"tagName": str, "fileUuid": str, "path": str}""" + + tagName: str + fileUuid: str + path: str + + +def convert_legacy_tags(entity: dict) -> list[FileTagDict]: + """For V1 pubs, file tags are stored in a convoluted way.""" + reconstructed_tags = [] + + tags = entity["value"]["tags"] + tag_options = entity.get("tagsAsOptions", []) + file_objs = entity.get("fileObjs", []) + association_hrefs = entity["_links"]["associationIds"] + + def get_file_obj_by_uuid(uuid): + association = next( + (assoc for assoc in association_hrefs if assoc["rel"] == uuid), None + ) + if not association: + return None + file_obj = next( + ( + file_obj + for file_obj in file_objs + if association["href"].endswith(file_obj["path"]) + ), + None, + ) + return file_obj + + for inner_dict in tags.values(): + for key in inner_dict.keys(): + tag_name = next( + (tag["label"] for tag in tag_options if tag["name"] == key), None + ) + if not tag_name: + continue + for tag in inner_dict[key]: + for file_uuid in tag["file"]: + file_obj = get_file_obj_by_uuid(file_uuid) + if not file_obj: + continue + reconstructed_tags.append( + { + "tagName": tag_name, + "fileUuid": file_uuid, + "path": file_obj["path"], + } + ) + return reconstructed_tags + + +def update_file_tag_paths(entity: dict, base_path: str) -> list[FileTagDict]: + """ + Updates tag paths to reflect what the paths should be in the published dataset. + e.g. if file "/data/xyz" is going to move to "/PRJ-1234/entity1/xyz" + then all tags with "/data/xyz" as a path prefix should have it updated to the final + path. + + Before: [...{"pathName": "/data/xyz/taggedFile.json}, "tagName": "Record"}] + After: [...{"pathName": "/PRJ-1234/entity1/xyz/taggedFile.json}, "tagName": "Record"}] + """ + tags = entity["value"].get("fileTags", None) + if not tags: + tags = convert_legacy_tags(entity) + + updated_tags = [] + path_mapping = get_path_mapping(entity, base_path) + + for tag in tags: + if not tag.get("path", None): + # If there is no path, we can't recover the tag. + continue + tag_path_prefixes = [p for p in path_mapping if tag["path"].startswith(p)] + + for prefix in tag_path_prefixes: + updated_tags.append( + {**tag, "path": tag["path"].replace(prefix, path_mapping[prefix], 1)} + ) + + return updated_tags + + +def get_path_mapping(entity: dict, base_path: str): + """map fileObj paths to published paths and handle duplicate names""" + file_objs = entity["value"]["fileObjs"] + is_type_other = entity["value"].get("projectType", None) == "other" + + if is_type_other: + # type Other is a special case since all files are associated at the root. + return {"": base_path} + path_mapping = {} + duplicate_counts = {} + for file_obj in file_objs: + pub_path = str(Path(base_path) / Path(file_obj["path"]).name) + if pub_path in path_mapping.values(): + duplicate_counts[pub_path] = duplicate_counts.get(pub_path, 0) + 1 + # splice dupe count into name, e.g. "myfile(1).txt" + [base_name, *ext] = Path(pub_path).name.split(".", 1) + deduped_name = f"{base_name}({duplicate_counts[pub_path]})" + + pub_path = str(Path(base_path) / ".".join([deduped_name, *ext])) + path_mapping[file_obj["path"]] = pub_path + + return path_mapping + + +def update_file_objs( + entity: dict, base_path: str, system_id="designsafe.storage.published" +): + """Return an updated file_objs array relative to a new base path.""" + file_objs = entity["value"]["fileObjs"] + path_mapping = get_path_mapping(entity, base_path) + updated_file_objs = [] + for file_obj in file_objs: + updated_file_objs.append( + {**file_obj, "path": path_mapping[file_obj["path"]], "system": system_id} + ) + return updated_file_objs + + +def transform_entity(entity: dict, base_pub_meta: dict, base_path: str): + """Convert published entity to use our Pydantic schema. Returns a serialized + reprsentation of the `value` attribute.""" + model = SCHEMA_MAPPING[entity["name"]] + authors = entity["value"].get("authors", None) + schema_version = base_pub_meta.get("version", 1) + if authors and schema_version == 1: + updated_authors = get_v1_authors(entity, base_pub_meta["users"]) + entity["value"]["authors"] = updated_authors + if authors and schema_version > 1: + fixed_authors = list(map(convert_v2_user, entity["authors"])) + entity["value"]["authors"] = sorted(fixed_authors, key=lambda a: a["order"]) + + old_tags = entity["value"].get("tags", None) + if old_tags: + new_style_tags = convert_legacy_tags(entity) + entity["value"]["fileTags"] = new_style_tags + + file_objs = entity.get("fileObjs", None) + # Some legacy experiment/hybrid sim entities have file_objs incorrectly + # populated from their children. In these cases, _filepaths is empty. + if file_objs and entity.get("_filePaths", None) != []: + entity["value"]["fileObjs"] = file_objs + if entity["value"].get("fileTags", False): + entity["value"]["fileTags"] = update_file_tag_paths(entity, base_path) + entity["value"]["fileObjs"] = update_file_objs( + entity, base_path, system_id=settings.PUBLISHED_SYSTEM + ) + + validated_model = model.model_validate(entity["value"]) + return validated_model.model_dump() diff --git a/designsafe/apps/api/projects_v2/migrations/0001_initial.py b/designsafe/apps/api/projects_v2/migrations/0001_initial.py new file mode 100644 index 0000000000..9c85d0e9bc --- /dev/null +++ b/designsafe/apps/api/projects_v2/migrations/0001_initial.py @@ -0,0 +1,108 @@ +# Generated by Django 4.2.6 on 2024-01-19 01:24 + +import designsafe.apps.api.projects_v2.models.project_metadata +from django.conf import settings +import django.core.serializers.json +import django.core.validators +from django.db import migrations, models +import django.db.models.deletion +import django.utils.timezone + + +class Migration(migrations.Migration): + initial = True + + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ] + + operations = [ + migrations.CreateModel( + name="ProjectMetadata", + fields=[ + ( + "uuid", + models.CharField( + default=designsafe.apps.api.projects_v2.models.project_metadata.uuid_pk, + editable=False, + max_length=100, + primary_key=True, + serialize=False, + ), + ), + ( + "name", + models.CharField( + help_text="Metadata namespace, e.g. 'designsafe.project'", + max_length=100, + validators=[django.core.validators.MinLengthValidator(1)], + ), + ), + ( + "value", + models.JSONField( + encoder=django.core.serializers.json.DjangoJSONEncoder, + help_text="JSON document containing file metadata, including title/description", + ), + ), + ("created", models.DateTimeField(default=django.utils.timezone.now)), + ("last_updated", models.DateTimeField(auto_now=True)), + ( + "association_ids", + models.JSONField( + default=list, + help_text="(DEPRECATE IN V3) Tapis V2 association IDs.", + ), + ), + ( + "base_project", + models.ForeignKey( + help_text="Base project containing this entity.For top-level project metadata, this is `self`.", + on_delete=django.db.models.deletion.CASCADE, + to="projects_v2_api.projectmetadata", + ), + ), + ( + "users", + models.ManyToManyField( + help_text="Users who have access to a project.", + related_name="projects", + to=settings.AUTH_USER_MODEL, + ), + ), + ], + options={ + "indexes": [ + models.Index(models.F("value__projectId"), name="value_project_id"), + models.Index(fields=["name"], name="projects_v2_name_2b10f5_idx"), + ], + }, + ), + migrations.AddConstraint( + model_name="projectmetadata", + constraint=models.UniqueConstraint( + models.F("value__projectId"), + condition=models.Q(("name", "designsafe.project")), + name="unique_id_per_project", + ), + ), + migrations.AddConstraint( + model_name="projectmetadata", + constraint=models.UniqueConstraint( + condition=models.Q(("name", "designsafe.project.graph")), + fields=("base_project_id",), + name="unique_graph_per_project", + ), + ), + migrations.AddConstraint( + model_name="projectmetadata", + constraint=models.CheckConstraint( + check=models.Q( + ("name", "designsafe.project"), + ("value__projectId__isnull", True), + _negated=True, + ), + name="base_projectId_not_null", + ), + ), + ] diff --git a/designsafe/apps/api/projects_v2/migrations/__init__.py b/designsafe/apps/api/projects_v2/migrations/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/designsafe/apps/api/projects_v2/models/__init__.py b/designsafe/apps/api/projects_v2/models/__init__.py new file mode 100644 index 0000000000..58b001e946 --- /dev/null +++ b/designsafe/apps/api/projects_v2/models/__init__.py @@ -0,0 +1,2 @@ +"""Projects V2 API models.""" +from designsafe.apps.api.projects_v2.models.project_metadata import ProjectMetadata diff --git a/designsafe/apps/api/projects_v2/models/project_metadata.py b/designsafe/apps/api/projects_v2/models/project_metadata.py new file mode 100644 index 0000000000..b6c95bbd62 --- /dev/null +++ b/designsafe/apps/api/projects_v2/models/project_metadata.py @@ -0,0 +1,147 @@ +"""Models for representing project metadata""" +import uuid +from django.utils import timezone +from django.db import models +from django.core.validators import MinLengthValidator +from django.core.serializers.json import DjangoJSONEncoder +from django.contrib.auth import get_user_model +from django.conf import settings +from django.dispatch import receiver +from designsafe.apps.api.projects_v2 import constants + +user_model = get_user_model() + + +def uuid_pk(): + """Generate a string UUID for use as a primary key.""" + return str(uuid.uuid4()) + + +class ProjectMetadata(models.Model): + """This model represents project metadata. Each entity has a foreign-key relation + to the base project metadata (the base project relates to itself). + + Some useful operations: + - list all projects for a user: + `.projects.filter(name="designsafe.project")` + - Get all entities with a given project ID: + `ProjectMetadata.objects.filter(base_project__value__projectId="PRJ-1234")` + + """ + + uuid = models.CharField( + max_length=100, primary_key=True, default=uuid_pk, editable=False + ) + name = models.CharField( + max_length=100, + validators=[MinLengthValidator(1)], + help_text="Metadata namespace, e.g. 'designsafe.project'", + ) + value = models.JSONField( + encoder=DjangoJSONEncoder, + help_text=( + "JSON document containing file metadata, including title/description" + ), + ) + + users = models.ManyToManyField( + to=user_model, + related_name="projects", + help_text="Users who have access to a project.", + ) + base_project = models.ForeignKey( + "self", + on_delete=models.CASCADE, + help_text=( + "Base project containing this entity." + "For top-level project metadata, this is `self`." + ), + ) + created = models.DateTimeField(default=timezone.now) + last_updated = models.DateTimeField(auto_now=True) + + association_ids = models.JSONField( + default=list, + help_text="(DEPRECATE IN V3) Tapis V2 association IDs.", + ) + + @property + def project_id(self) -> str: + """Convenience method for retrieving project IDs.""" + return self.base_project.value.get("projectId") + + @property + def project_graph(self): + """Convenience method for returning the project graph metadata""" + return self.__class__.objects.get( + name=constants.PROJECT_GRAPH, base_project=self.base_project + ) + + @classmethod + def get_project_by_id(cls, project_id: str): + """Return base project metadata matching a project ID value.""" + return cls.objects.get(name=constants.PROJECT, value__projectId=project_id) + + @classmethod + def get_entities_by_project_id(cls, project_id: str): + """Return an iterable of all metadata objects for a given project ID.""" + return cls.objects.filter(base_project__value__projectId=project_id) + + def to_dict(self): + """dict representation.""" + return { + "uuid": self.uuid, + "name": self.name, + "value": self.value, + "created": self.created, + "lastUpdated": self.last_updated, + "associationIds": self.association_ids, + } + + def sync_users(self): + """Sync associated users with the user list in the metadata.""" + if self.name != constants.PROJECT: + return + prj_users = [user.get("username") for user in self.value.get("users", [])] + project_user_query = models.Q(username__in=prj_users) + admin_user_query = models.Q(username__in=settings.PROJECT_ADMIN_USERS) + self.users.set(user_model.objects.filter(project_user_query | admin_user_query)) + + def save(self, *args, **kwargs): + if self.name == constants.PROJECT: + self.base_project = self + super().save(*args, **kwargs) + + class Meta: + # Create indexes on name and project ID, since these will be used for lookups. + indexes = [ + models.Index(models.F("value__projectId"), name="value_project_id"), + models.Index(fields=["name"]), + ] + + constraints = [ + # Each project has a unique ID + models.UniqueConstraint( + models.F("value__projectId"), + condition=models.Q(name=constants.PROJECT), + name="unique_id_per_project", + ), + # A project can have at most 1 graph associated to it. + models.UniqueConstraint( + fields=["base_project_id"], + condition=models.Q(name=constants.PROJECT_GRAPH), + name="unique_graph_per_project", + ), + # Top-level project metadata cannot be saved without a project ID. + models.CheckConstraint( + check=~models.Q(name=constants.PROJECT, value__projectId__isnull=True), + name="base_projectId_not_null", + ), + ] + + +@receiver(models.signals.post_save, sender=ProjectMetadata) +def handle_save(instance: ProjectMetadata, **_): + """After saving a project, update the associated users so that listings can + be performed.""" + instance.sync_users() diff --git a/designsafe/apps/api/projects_v2/operations/__init__.py b/designsafe/apps/api/projects_v2/operations/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/designsafe/apps/api/projects_v2/operations/_tests/project_meta_unit_test.py b/designsafe/apps/api/projects_v2/operations/_tests/project_meta_unit_test.py new file mode 100644 index 0000000000..c49ddd17ec --- /dev/null +++ b/designsafe/apps/api/projects_v2/operations/_tests/project_meta_unit_test.py @@ -0,0 +1,340 @@ +"""Tests for operations that create/update project metadata.""" +import pytest +import networkx as nx +from django import db +from designsafe.apps.api.projects_v2 import constants +from designsafe.apps.api.projects_v2.operations.project_meta_operations import ( + create_project_metdata, + create_entity_metadata, + add_file_associations, + remove_file_associations, + add_file_tags, + remove_file_tags, + ProjectMetadata, + FileObj, + FileTag, +) +from designsafe.apps.api.projects_v2.operations.graph_operations import ( + initialize_project_graph, + add_node_to_project, + remove_nodes_from_project, + reorder_project_nodes, +) + + +@pytest.mark.django_db +def test_project_meta_creation(project_admin_user): + project_value = {"title": "Test Project", "projectId": "PRJ-1234"} + prj_meta = create_project_metdata(project_value) + + assert prj_meta.value["projectId"] == "PRJ-1234" + assert project_admin_user in prj_meta.users.all() + + with pytest.raises(db.IntegrityError): + create_project_metdata(project_value) + + +@pytest.mark.django_db +def test_entity_creation(project_admin_user): + project_value = {"title": "Test Project", "projectId": "PRJ-1234", "users": []} + create_project_metdata(project_value) + + entity_meta = create_entity_metadata( + "PRJ-1234", + name="designsafe.project.experiment", + value={"title": "My Experiment", "description": "Test Experiment"}, + ) + + entity_obj = ProjectMetadata.objects.get(uuid=entity_meta.uuid) + assert entity_obj.base_project.value["projectId"] == "PRJ-1234" + assert ProjectMetadata.get_entities_by_project_id("PRJ-1234").count() == 2 + + +@pytest.mark.django_db +def test_graph_init(): + project_value = {"title": "Test Project", "projectId": "PRJ-1234", "users": []} + create_project_metdata(project_value) + initialize_project_graph("PRJ-1234") + + graph_value = ProjectMetadata.get_project_by_id("PRJ-1234").project_graph.value + nx_graph = nx.node_link_graph(graph_value) + assert len(nx_graph) == 1 + assert nx_graph.nodes.get("NODE_ROOT")["name"] == "designsafe.project" + + +@pytest.mark.django_db +def test_graph_init_other(): + project_value = { + "title": "Test Project", + "projectId": "PRJ-1234", + "users": [], + "projectType": "other", + } + create_project_metdata(project_value) + initialize_project_graph("PRJ-1234") + + graph_value = ProjectMetadata.get_project_by_id("PRJ-1234").project_graph.value + nx_graph: nx.DiGraph = nx.node_link_graph(graph_value) + assert len(nx_graph) == 2 + assert nx_graph.nodes.get("NODE_ROOT")["name"] == None + + +@pytest.mark.django_db +def test_graph_add_nodes(): + project_value = { + "title": "Test Project", + "projectId": "PRJ-1234", + "users": [], + "projectType": "experimental", + } + create_project_metdata(project_value) + initialize_project_graph("PRJ-1234") + + new_node_1 = add_node_to_project( + "PRJ-1234", "NODE_ROOT", "UUID1", constants.EXPERIMENT + ) + new_node_2 = add_node_to_project( + "PRJ-1234", new_node_1, "UUID2", constants.EXPERIMENT_MODEL_CONFIG + ) + + graph = ProjectMetadata.get_project_by_id("PRJ-1234").project_graph + nx_graph: nx.DiGraph = nx.node_link_graph(graph.value) + + assert new_node_1 in nx_graph + assert new_node_2 in nx_graph.successors(new_node_1) + + +@pytest.mark.django_db +def test_graph_remove_nodes(): + project_value = { + "title": "Test Project", + "projectId": "PRJ-1234", + "users": [], + "projectType": "experimental", + } + create_project_metdata(project_value) + initialize_project_graph("PRJ-1234") + + new_node_1 = add_node_to_project( + "PRJ-1234", "NODE_ROOT", "UUID1", constants.EXPERIMENT + ) + new_node_2 = add_node_to_project( + "PRJ-1234", new_node_1, "UUID2", constants.EXPERIMENT_MODEL_CONFIG + ) + + remove_nodes_from_project("PRJ-1234", [new_node_1]) + + graph = ProjectMetadata.get_project_by_id("PRJ-1234").project_graph + nx_graph: nx.DiGraph = nx.node_link_graph(graph.value) + + assert len(nx_graph.nodes) == 1 + assert new_node_1 not in nx_graph + assert new_node_2 not in nx_graph + + +@pytest.mark.django_db +def test_graph_remove_nodes_renormalizes_order(): + project_value = { + "title": "Test Project", + "projectId": "PRJ-1234", + "users": [], + "projectType": "experimental", + } + create_project_metdata(project_value) + initialize_project_graph("PRJ-1234") + + new_node_1 = add_node_to_project( + "PRJ-1234", "NODE_ROOT", "UUID1", constants.EXPERIMENT + ) + new_node_2 = add_node_to_project( + "PRJ-1234", new_node_1, "UUID2", constants.EXPERIMENT_MODEL_CONFIG + ) + new_node_3 = add_node_to_project( + "PRJ-1234", new_node_1, "UUID3", constants.EXPERIMENT_MODEL_CONFIG + ) + new_node_4 = add_node_to_project( + "PRJ-1234", new_node_1, "UUID4", constants.EXPERIMENT_MODEL_CONFIG + ) + + graph = ProjectMetadata.get_project_by_id("PRJ-1234").project_graph + nx_graph: nx.DiGraph = nx.node_link_graph(graph.value) + assert nx_graph.nodes[new_node_2]["order"] == 0 + assert nx_graph.nodes[new_node_3]["order"] == 1 + assert nx_graph.nodes[new_node_4]["order"] == 2 + + remove_nodes_from_project("PRJ-1234", [new_node_2]) + graph_post_delete = ProjectMetadata.get_project_by_id("PRJ-1234").project_graph + nx_graph_post_delete: nx.DiGraph = nx.node_link_graph(graph_post_delete.value) + + assert nx_graph_post_delete.nodes[new_node_3]["order"] == 0 + assert nx_graph_post_delete.nodes[new_node_4]["order"] == 1 + + +@pytest.mark.django_db +def test_graph_node_reorder(): + project_value = { + "title": "Test Project", + "projectId": "PRJ-1234", + "users": [], + "projectType": "experimental", + } + create_project_metdata(project_value) + initialize_project_graph("PRJ-1234") + + new_node_1 = add_node_to_project( + "PRJ-1234", "NODE_ROOT", "UUID1", constants.EXPERIMENT + ) + new_node_2 = add_node_to_project( + "PRJ-1234", new_node_1, "UUID2", constants.EXPERIMENT_MODEL_CONFIG + ) + new_node_3 = add_node_to_project( + "PRJ-1234", new_node_1, "UUID3", constants.EXPERIMENT_MODEL_CONFIG + ) + new_node_4 = add_node_to_project( + "PRJ-1234", new_node_1, "UUID4", constants.EXPERIMENT_MODEL_CONFIG + ) + new_node_5 = add_node_to_project( + "PRJ-1234", new_node_1, "UUID4", constants.EXPERIMENT_MODEL_CONFIG + ) + + graph = ProjectMetadata.get_project_by_id("PRJ-1234").project_graph + nx_graph: nx.DiGraph = nx.node_link_graph(graph.value) + + assert nx_graph.nodes[new_node_2]["order"] == 0 + assert nx_graph.nodes[new_node_3]["order"] == 1 + assert nx_graph.nodes[new_node_4]["order"] == 2 + assert nx_graph.nodes[new_node_5]["order"] == 3 + + reorder_project_nodes("PRJ-1234", new_node_2, 2) + + graph = ProjectMetadata.get_project_by_id("PRJ-1234").project_graph + nx_graph: nx.DiGraph = nx.node_link_graph(graph.value) + + assert nx_graph.nodes[new_node_3]["order"] == 0 + assert nx_graph.nodes[new_node_4]["order"] == 1 + assert nx_graph.nodes[new_node_2]["order"] == 2 + assert nx_graph.nodes[new_node_5]["order"] == 3 + + reorder_project_nodes("PRJ-1234", new_node_5, 0) + + graph = ProjectMetadata.get_project_by_id("PRJ-1234").project_graph + nx_graph: nx.DiGraph = nx.node_link_graph(graph.value) + + assert nx_graph.nodes[new_node_5]["order"] == 0 + assert nx_graph.nodes[new_node_3]["order"] == 1 + assert nx_graph.nodes[new_node_4]["order"] == 2 + assert nx_graph.nodes[new_node_2]["order"] == 3 + + # Test idempotency. + reorder_project_nodes("PRJ-1234", new_node_5, 0) + + graph = ProjectMetadata.get_project_by_id("PRJ-1234").project_graph + nx_graph: nx.DiGraph = nx.node_link_graph(graph.value) + + assert nx_graph.nodes[new_node_5]["order"] == 0 + assert nx_graph.nodes[new_node_3]["order"] == 1 + assert nx_graph.nodes[new_node_4]["order"] == 2 + assert nx_graph.nodes[new_node_2]["order"] == 3 + + reorder_project_nodes("PRJ-1234", new_node_5, 3) + + graph = ProjectMetadata.get_project_by_id("PRJ-1234").project_graph + nx_graph: nx.DiGraph = nx.node_link_graph(graph.value) + + assert nx_graph.nodes[new_node_3]["order"] == 0 + assert nx_graph.nodes[new_node_4]["order"] == 1 + assert nx_graph.nodes[new_node_2]["order"] == 2 + assert nx_graph.nodes[new_node_5]["order"] == 3 + + +@pytest.mark.django_db +def test_add_file_associations(): + project_value = { + "title": "Test Project", + "projectId": "PRJ-1234", + "users": [], + "projectType": "experimental", + } + entity_value = { + "title": "Test Entity", + "description": "Entity with file associations", + } + create_project_metdata(project_value) + + entity_meta = create_entity_metadata( + "PRJ-1234", name=constants.EXPERIMENT_EVENT, value=entity_value + ) + + file_objs = [ + FileObj( + system="project.system", name="file1", path="/path/to/file1", type="file" + ), + FileObj( + system="project.system", name="file2", path="/path/to/file2", type="file" + ), + ] + + assert len(entity_meta.value["fileObjs"]) == 0 + updated_entity = add_file_associations(entity_meta.uuid, file_objs) + + assert len(updated_entity.value["fileObjs"]) == 2 + + more_file_objs = [ + FileObj( + system="project.system", name="file2", path="/path/to/file2", type="file" + ), + FileObj( + system="project.system", name="file3", path="/path/to/file3", type="file" + ), + ] + updated_entity = add_file_associations(entity_meta.uuid, more_file_objs) + + assert len(updated_entity.value["fileObjs"]) == 3 + + updated_entity = remove_file_associations( + entity_meta.uuid, ["/path/to/file1", "/path/to/file2", "/path/to/file3"] + ) + assert len(updated_entity.value["fileObjs"]) == 0 + + +@pytest.mark.django_db +def test_add_file_tags(): + project_value = { + "title": "Test Project", + "projectId": "PRJ-1234", + "users": [], + "projectType": "experimental", + } + entity_value = { + "title": "Test Entity", + "description": "Entity with file associations", + } + create_project_metdata(project_value) + + entity_meta = create_entity_metadata( + "PRJ-1234", name=constants.EXPERIMENT_EVENT, value=entity_value + ) + + file_tags = [ + FileTag(tag_name="tag1", path="/path/to/file1"), + FileTag(tag_name="tag2", path="/path/to/file1"), + FileTag(tag_name="tag1", path="/path/to/file2"), + ] + + assert len(entity_meta.value["fileTags"]) == 0 + updated_entity = add_file_tags(entity_meta.uuid, file_tags) + + assert len(updated_entity.value["fileTags"]) == 3 + + more_file_tags = [ + FileTag(tag_name="tag1", path="/path/to/file1"), + FileTag(tag_name="tag2", path="/path/to/file1"), + FileTag(tag_name="tag2", path="/path/to/file2"), + ] + updated_entity = add_file_tags(entity_meta.uuid, more_file_tags) + + assert len(updated_entity.value["fileTags"]) == 4 + + updated_entity = remove_file_tags(entity_meta.uuid, (file_tags + more_file_tags)) + + assert len(updated_entity.value["fileTags"]) == 0 diff --git a/designsafe/apps/api/projects_v2/operations/graph_operations.py b/designsafe/apps/api/projects_v2/operations/graph_operations.py new file mode 100644 index 0000000000..5de30eb78f --- /dev/null +++ b/designsafe/apps/api/projects_v2/operations/graph_operations.py @@ -0,0 +1,177 @@ +"""Utilities for adding, removing, and reordering nodes in the project graph.""" +import uuid +import copy +import networkx as nx +from django.db import transaction +from designsafe.apps.api.projects_v2 import constants +from designsafe.apps.api.projects_v2.models import ProjectMetadata + + +def _get_next_child_order(graph: nx.DiGraph, parent_node: str) -> int: + child_nodes = graph.successors(parent_node) + max_order = max((graph.nodes[child]["order"] for child in child_nodes), default=-1) + return int(max_order) + 1 + + +def _renormalize_ordering(graph: nx.DiGraph) -> nx.DiGraph: + """ + Update order attributes on graph nodes so that the orders for each node's children + form a sequence from 0..(n-1) where n is the number of children. + """ + for node in graph.nodes: + sorted_children = sorted( + graph.successors(node), key=lambda n: graph.nodes[n]["order"] + ) + for i, node_id in enumerate(sorted_children): + graph.nodes[node_id]["order"] = i + return graph + + +def _add_node_to_graph( + graph: nx.DiGraph, parent_node_id: str, meta_uuid: str, name: str +) -> tuple[nx.DiGraph, str]: + """Add a node with data to a graph, and return the graph.""" + if not graph.has_node(parent_node_id): + raise nx.exception.NodeNotFound + + _graph: nx.DiGraph = copy.deepcopy(graph) + order = _get_next_child_order(_graph, parent_node_id) + child_node_id = f"NODE_{name}_{uuid.uuid4()}" + _graph.add_node(child_node_id, uuid=meta_uuid, name=name, order=order) + _graph.add_edge(parent_node_id, child_node_id) + return (_graph, child_node_id) + + +def _remove_nodes_from_graph(graph: nx.DiGraph, node_ids: list[str]) -> nx.DiGraph: + """Remove a node from a graph, including its successors.""" + + _graph: nx.DiGraph = copy.deepcopy(graph) + nodes_to_remove = set() + for node_id in node_ids: + # dfs_preorder_nodes iterates over all nodes reachable from node_id. + reachable_nodes = nx.dfs_preorder_nodes(_graph, node_id) + nodes_to_remove.update(reachable_nodes) + + _graph.remove_nodes_from(nodes_to_remove) + _graph = _renormalize_ordering(_graph) + return _graph + + +def _reorder_nodes(graph: nx.DiGraph, node_id: str, new_index: int): + """ + Update `order` values so that the given node has the `order` specified by the + `new_index` arg. Any other children of the node's parent are also updated to + preserve the sequential ordering. + """ + _graph: nx.DiGraph = copy.deepcopy(graph) + + parent = next(_graph.predecessors(node_id)) + old_index = _graph.nodes[node_id]["order"] + + sorted_siblings: list[str] = sorted( + list(_graph.successors(parent)), key=lambda n: _graph.nodes[n]["order"] + ) + + if new_index > old_index: + for node_to_dec in sorted_siblings[old_index + 1 : new_index + 1]: + _graph.nodes[node_to_dec]["order"] -= 1 + if new_index < old_index: + for node_to_inc in sorted_siblings[new_index:old_index]: + _graph.nodes[node_to_inc]["order"] += 1 + + _graph.nodes[node_id]["order"] = new_index + + return _graph + + +def initialize_project_graph(project_id: str): + """ + Initialize the entity graph in a default state for a project. For type Other, the + default graph has an "empty" root node that contains the base entity as its child. + This is to allow multiple versions to be published as siblings in the graph. + Otherwise, the graph is initialized as a single node pointing to the project root. + This method should be called when creating a new project AND when changing a + project's type. + """ + project_model = ProjectMetadata.get_project_by_id(project_id) + project_graph = nx.DiGraph() + + root_node_id = "NODE_ROOT" + project_type = project_model.value.get("projectType", None) + base_node_data = { + "uuid": project_model.uuid, + "name": project_model.name, + "projectType": project_type, + } + + if project_type == "other": + # type Other projects have a "null" parent node above the project root, to + # support multiple versions. + project_graph.add_node( + root_node_id, **{"uuid": None, "name": None, "projectType": "other"} + ) + base_node_id = f"NODE_project_{uuid.uuid4()}" + project_graph.add_node(base_node_id, **base_node_data) + project_graph.add_edge(root_node_id, base_node_id) + else: + project_graph.add_node(root_node_id, **base_node_data) + + graph_model_value = nx.node_link_data(project_graph) + res, _ = ProjectMetadata.objects.update_or_create( + name=constants.PROJECT_GRAPH, + base_project=project_model, + defaults={"value": graph_model_value}, + ) + return res + + +def add_node_to_project(project_id: str, parent_node: str, meta_uuid: str, name: str): + """Update the database entry for a project graph to add a node.""" + # Lock the project graph's tale row to prevent conflicting updates. + with transaction.atomic(): + graph_model = ProjectMetadata.objects.select_for_update().get( + name=constants.PROJECT_GRAPH, base_project__value__projectId=project_id + ) + project_graph = nx.node_link_graph(graph_model.value) + + (updated_graph, new_node_id) = _add_node_to_graph( + project_graph, parent_node, meta_uuid, name + ) + + graph_model.value = nx.node_link_data(updated_graph) + graph_model.save() + return new_node_id + + +def remove_nodes_from_project(project_id: str, node_ids: list[str]): + """ + Update the database entry for the project graph to remove nodes. + This method takes an array of node IDs because if we delete an entity, it may need + to be removed simultaneously from multiple positions in the graph. + """ + # Lock the project graph's tale row to prevent conflicting updates. + with transaction.atomic(): + graph_model = ProjectMetadata.objects.select_for_update().get( + name=constants.PROJECT_GRAPH, base_project__value__projectId=project_id + ) + project_graph = nx.node_link_graph(graph_model.value) + + updated_graph = _remove_nodes_from_graph(project_graph, node_ids) + + graph_model.value = nx.node_link_data(updated_graph) + graph_model.save() + + +def reorder_project_nodes(project_id: str, node_id: str, new_index: int): + """Update the database entry for the project graph to reorder nodes.""" + # Lock the project graph's tale row to prevent conflicting updates. + with transaction.atomic(): + graph_model = ProjectMetadata.objects.select_for_update().get( + name=constants.PROJECT_GRAPH, base_project__value__projectId=project_id + ) + project_graph = nx.node_link_graph(graph_model.value) + + updated_graph = _reorder_nodes(project_graph, node_id, new_index) + + graph_model.value = nx.node_link_data(updated_graph) + graph_model.save() diff --git a/designsafe/apps/api/projects_v2/operations/project_meta_operations.py b/designsafe/apps/api/projects_v2/operations/project_meta_operations.py new file mode 100644 index 0000000000..77836ff8ed --- /dev/null +++ b/designsafe/apps/api/projects_v2/operations/project_meta_operations.py @@ -0,0 +1,176 @@ +"""Utilities for creating and managing project metadata objects/associations.""" +import operator +from django.db import models, transaction +from designsafe.apps.api.projects_v2.schema_models import SCHEMA_MAPPING +from designsafe.apps.api.projects_v2.schema_models.base import ( + FileObj, + FileTag, + PartialEntityWithFiles, +) +from designsafe.apps.api.projects_v2 import constants +from designsafe.apps.api.projects_v2.models import ProjectMetadata + + +def create_project_metdata(value): + """Create a project metadata object in the database.""" + schema_model = SCHEMA_MAPPING[constants.PROJECT] + validated_model = schema_model.model_validate(value) + + project_db_model = ProjectMetadata( + name=constants.PROJECT, value=validated_model.model_dump() + ) + project_db_model.save() + return project_db_model + + +def create_entity_metadata(project_id, name, value): + """Create entity metadata associated with an existing project.""" + base_project = ProjectMetadata.get_project_by_id(project_id) + schema_model = SCHEMA_MAPPING[name] + validated_model = schema_model.model_validate(value) + + entity_db_model = ProjectMetadata( + name=name, value=validated_model.model_dump(), base_project=base_project + ) + entity_db_model.save() + return entity_db_model + + +def patch_metadata(uuid, value): + """Update an entity's `value` attribute. This method patches the metadata + so that only fields in the payload are overwritten.""" + entity = ProjectMetadata.objects.get(uuid=uuid) + schema_model = SCHEMA_MAPPING[entity.name] + validated_model = schema_model.model_validate(value) + + patched_metadata = {**entity.value, **validated_model.model_dump()} + entity.value = patched_metadata + entity.save() + return entity + + +def delete_entity(uuid: str): + """Delete a non-root entity.""" + entity = ProjectMetadata.objects.get(uuid=uuid) + if entity.name in (constants.PROJECT, constants.PROJECT_GRAPH): + raise ValueError("Cannot delete a top-level project or graph object.") + entity.delete() + return "OK" + + +def clear_entities(project_id): + """Delete all entities except the project root and graph. Used when changing project + type, so that file associations don't get stuck on unreachable entities. + """ + + ProjectMetadata.get_entities_by_project_id(project_id).filter( + ~models.Q(name__in=[constants.PROJECT, constants.PROJECT_GRAPH]) + ).delete() + + return "OK" + + +def _merge_file_objs( + prev_file_objs: list[FileObj], new_file_objs: list[FileObj] +) -> list[FileObj]: + """Combine two arrays of FileObj models, overwriting the first if there are conflicts.""" + new_file_paths = [f.path for f in new_file_objs] + deduped_file_objs = [fo for fo in prev_file_objs if fo.path not in new_file_paths] + + return sorted( + [*deduped_file_objs, *new_file_objs], key=operator.attrgetter("name", "path") + ) + + +def _filter_file_objs( + prev_file_objs: list[FileObj], paths_to_remove: list[str] +) -> list[FileObj]: + return sorted( + [fo for fo in prev_file_objs if fo.path not in paths_to_remove], + key=operator.attrgetter("name", "path"), + ) + + +def add_file_associations(uuid: str, new_file_objs: list[FileObj]): + """Associate one or more file objects to an entity.""" + # Use atomic transaction here to prevent multiple calls from clobbering each other + with transaction.atomic(): + entity = ProjectMetadata.objects.select_for_update().get(uuid=uuid) + entity_file_model = PartialEntityWithFiles.model_validate(entity.value) + + merged_file_objs = _merge_file_objs(entity_file_model.file_objs, new_file_objs) + entity.value["fileObjs"] = [f.model_dump() for f in merged_file_objs] + + entity.save() + return entity + + +def remove_file_associations(uuid: str, file_paths: list[str]): + """Remove file associations from an entity by their paths.""" + with transaction.atomic(): + entity = ProjectMetadata.objects.select_for_update().get(uuid=uuid) + entity_file_model = PartialEntityWithFiles.model_validate(entity.value) + + filtered_file_objs = _filter_file_objs(entity_file_model.file_objs, file_paths) + entity.value["fileObjs"] = [f.model_dump() for f in filtered_file_objs] + entity.save() + return entity + + +def _check_file_tag_included(tag: FileTag, tag_list: list[FileTag]): + return next( + (t for t in tag_list if t.path == tag.path and t.tag_name == tag.tag_name), + False, + ) + + +def _merge_file_tags( + prev_file_tags: list[FileTag], new_file_tags: list[FileTag] +) -> list[FileTag]: + deduped_file_tags = [ + tag + for tag in prev_file_tags + if not _check_file_tag_included(tag, new_file_tags) + ] + return [*deduped_file_tags, *new_file_tags] + + +def _filter_file_tags( + prev_file_tags: list[FileTag], tags_to_remove: list[FileTag] +) -> list[FileTag]: + return [ + tag + for tag in prev_file_tags + if not _check_file_tag_included(tag, tags_to_remove) + ] + + +def add_file_tags(uuid: str, file_tags: list[FileTag]): + """Add a file tag to an entity""" + with transaction.atomic(): + entity = ProjectMetadata.objects.select_for_update().get(uuid=uuid) + entity_file_model = PartialEntityWithFiles.model_validate(entity.value) + # If a file with this path already exists, remove it from the list. + + entity.value["fileTags"] = [ + t.model_dump() + for t in _merge_file_tags(entity_file_model.file_tags, file_tags) + ] + entity.save() + return entity + + +def remove_file_tags(uuid: str, file_tags: list[FileTag]): + """Remove file tags from an entity""" + + with transaction.atomic(): + entity = ProjectMetadata.objects.select_for_update().get(uuid=uuid) + entity_file_model = PartialEntityWithFiles.model_validate(entity.value) + # If a file with this path already exists, remove it from the list. + + entity.value["fileTags"] = [ + t.model_dump() + for t in _filter_file_tags(entity_file_model.file_tags, file_tags) + ] + entity.save() + return entity diff --git a/designsafe/apps/api/projects_v2/operations/project_system_operations.py b/designsafe/apps/api/projects_v2/operations/project_system_operations.py new file mode 100644 index 0000000000..af5e2966dc --- /dev/null +++ b/designsafe/apps/api/projects_v2/operations/project_system_operations.py @@ -0,0 +1 @@ +"""Utilities for creating project systems and managing access permissions.""" diff --git a/designsafe/apps/api/projects_v2/schema_models/__init__.py b/designsafe/apps/api/projects_v2/schema_models/__init__.py new file mode 100644 index 0000000000..556f7a4512 --- /dev/null +++ b/designsafe/apps/api/projects_v2/schema_models/__init__.py @@ -0,0 +1,83 @@ +"""Pydantic models for all project entities.""" +from designsafe.apps.api.projects_v2 import constants +from designsafe.apps.api.projects_v2.schema_models import ( + base, + experimental, + hybrid_sim, + simulation, + field_recon, +) +from designsafe.apps.api.projects_v2.schema_models._field_models import MetadataModel + + +SCHEMA_MAPPING: dict[str, MetadataModel] = { + constants.PROJECT: base.BaseProject, + # Experimental + constants.EXPERIMENT: experimental.Experiment, + constants.EXPERIMENT_ANALYSIS: experimental.ExperimentAnalysis, + constants.EXPERIMENT_REPORT: experimental.ExperimentReport, + constants.EXPERIMENT_MODEL_CONFIG: experimental.ExperimentModelConfig, + constants.EXPERIMENT_SENSOR: experimental.ExperimentSensor, + constants.EXPERIMENT_EVENT: experimental.ExperimentSensor, + # Simulation + constants.SIMULATION: simulation.Simulation, + constants.SIMULATION_REPORT: simulation.SimulationReport, + constants.SIMULATION_ANALYSIS: simulation.SimulationAnalysis, + constants.SIMULATION_MODEL: simulation.SimulationModel, + constants.SIMULATION_INPUT: simulation.SimulationInput, + constants.SIMULATION_OUTPUT: simulation.SimulationOutput, + # Field Research + constants.FIELD_RECON_MISSION: field_recon.Mission, + constants.FIELD_RECON_COLLECTION: field_recon.FieldReconCollection, + constants.FIELD_RECON_PLANNING: field_recon.PlanningCollection, + constants.FIELD_RECON_GEOSCIENCE: field_recon.GeoscienceCollection, + constants.FIELD_RECON_SOCIAL_SCIENCE: field_recon.SocialScienceCollection, + constants.FIELD_RECON_REPORT: field_recon.FieldReconReport, + # Hybrid Sim + constants.HYBRID_SIM: hybrid_sim.HybridSimulation, + constants.HYBRID_SIM_ANALYSIS: hybrid_sim.HybridSimAnalysis, + constants.HYBRID_SIM_REPORT: hybrid_sim.HybridSimReport, + constants.HYBRID_SIM_GLOBAL_MODEL: hybrid_sim.HybridSimGlobalModel, + constants.HYBRID_SIM_COORDINATOR: hybrid_sim.HybridSimCoordinator, + constants.HYBRID_SIM_COORDINATOR_OUTPUT: hybrid_sim.HybridSimCoordinatorOutput, + constants.HYBRID_SIM_SIM_SUBSTRUCTURE: hybrid_sim.HybridSimSimSubstructure, + constants.HYBRID_SIM_SIM_OUTPUT: hybrid_sim.HybridSimSimOutput, + constants.HYBRID_SIM_EXP_SUBSTRUCTURE: hybrid_sim.HybridSimExpSubstructure, + constants.HYBRID_SIM_EXP_OUTPUT: hybrid_sim.HybridSimExpOutput, +} + +PATH_SLUGS = { + constants.PROJECT: "Project", + # Experimental + constants.EXPERIMENT: "Experiment", + constants.EXPERIMENT_ANALYSIS: "Analysis", + constants.EXPERIMENT_REPORT: "Report", + constants.EXPERIMENT_MODEL_CONFIG: "Model-config", + constants.EXPERIMENT_SENSOR: "Sensor", + constants.EXPERIMENT_EVENT: "Event", + # Simulation + constants.SIMULATION: "Simulation", + constants.SIMULATION_REPORT: "Report", + constants.SIMULATION_ANALYSIS: "Analysis", + constants.SIMULATION_MODEL: "Model", + constants.SIMULATION_INPUT: "Input", + constants.SIMULATION_OUTPUT: "Output", + # Field Research + constants.FIELD_RECON_MISSION: "Mission", + constants.FIELD_RECON_COLLECTION: "Collection", + constants.FIELD_RECON_PLANNING: "Planning-collection", + constants.FIELD_RECON_GEOSCIENCE: "Geoscience-collection", + constants.FIELD_RECON_SOCIAL_SCIENCE: "Social-science-collection", + constants.FIELD_RECON_REPORT: "Report", + # Hybrid Sim + constants.HYBRID_SIM: "Hybrid-simulation", + constants.HYBRID_SIM_ANALYSIS: "Analysis", + constants.HYBRID_SIM_REPORT: "Report", + constants.HYBRID_SIM_GLOBAL_MODEL: "Global-model", + constants.HYBRID_SIM_COORDINATOR: "Coordinator", + constants.HYBRID_SIM_COORDINATOR_OUTPUT: "Coordinator-output", + constants.HYBRID_SIM_SIM_SUBSTRUCTURE: "Simulation-substructure", + constants.HYBRID_SIM_SIM_OUTPUT: "Simulation-output", + constants.HYBRID_SIM_EXP_SUBSTRUCTURE: "Experimental-substructure", + constants.HYBRID_SIM_EXP_OUTPUT: "Experimental-output", +} diff --git a/designsafe/apps/api/projects_v2/schema_models/_field_models.py b/designsafe/apps/api/projects_v2/schema_models/_field_models.py new file mode 100644 index 0000000000..867a98c3fb --- /dev/null +++ b/designsafe/apps/api/projects_v2/schema_models/_field_models.py @@ -0,0 +1,178 @@ +"""Utiity models used in multiple field types""" +from datetime import datetime +from functools import partial +from typing import Annotated, Literal, Optional +from pydantic import AliasChoices, BaseModel, BeforeValidator, ConfigDict, Field +from pydantic.alias_generators import to_camel +from django.contrib.auth import get_user_model +from pytas.http import TASClient + + +class MetadataModel(BaseModel): + """Subclass BaseModel with custom config.""" + + model_config = ConfigDict( + alias_generator=to_camel, + populate_by_name=True, + from_attributes=True, + extra="forbid", + coerce_numbers_to_str=True, + ) + + def model_dump(self, *args, **kwargs): + # default by_alias to true for camelCase serialization + return partial(super().model_dump, by_alias=True)(*args, **kwargs) + + +class ProjectUser(MetadataModel): + """Model for project users.""" + + order: Optional[int] = None + guest: Optional[bool] = None + fname: Optional[str] = None + lname: Optional[str] = None + name: Optional[str] = None + email: Optional[str] = None + inst: Optional[str] = None + user: Optional[str] = None + username: Optional[str] = None + role: Optional[Literal["pi", "co_pi", "team_member", "guest"]] = None + + authorship: Optional[bool] = None + + @classmethod + def from_username(cls, username: str, role: str = "team_member", **kwargs): + """Fill in a user object using values from the db.""" + user_model = get_user_model() + try: + user_obj = user_model.objects.get(username=username) + return cls( + username=username, + fname=user_obj.first_name, + lname=user_obj.last_name, + email=user_obj.email, + inst=user_obj.profile.institution, + role=role, + **kwargs + ) + except user_model.DoesNotExist: + try: + tas_client = TASClient() + tas_user: Optional[dict] = tas_client.get_user(username=username) + if not tas_user: + return cls(username=username, role=role, guest=False) + return cls( + username=username, + fname=tas_user["firstName"], + lname=tas_user["lastName"], + email=tas_user["email"], + inst=tas_user["institution"], + role=role, + **kwargs + ) + # pylint:disable=broad-exception-caught + except Exception as _: + print(username) + print("unrecoverable username") + return cls(username=username, role=role, guest=False) + + +class GuestMember(MetadataModel): + """Model for guest members.""" + + order: Optional[int] = None + guest: bool = True + fname: str + lname: str + inst: Optional[str] = None + email: Optional[str] = None + user: str + + +class ProjectAward(MetadataModel): + """Model for awards.""" + + order: int = 0 + name: Annotated[ + str, BeforeValidator(lambda n: n if isinstance(n, str) else "") + ] = "" + number: str = "" + funding_source: Optional[str] = None + + +class AssociatedProject(MetadataModel): + """model for associated projects.""" + + # only title guaranteed + type: str = "Linked Dataset" + title: str + href: Optional[str] = "" + href_type: str = "URL" + order: Optional[int] = None + # Some test projects have this weird attribute. + delete: Optional[bool] = None + # Some legacy projects have a doi attribute. + doi: str = "" + + +class ReferencedWork(MetadataModel): + """Model for referenced works.""" + + title: str + doi: str = Field(validation_alias=AliasChoices("doi", "url")) + href_type: str = "URL" + + +class FileTag(MetadataModel): + """Model for file tags.""" + + file_uuid: Optional[str] = Field(default=None, exclude=True) + tag_name: str + path: Optional[str] = None + + +class FileObj(MetadataModel): + """Model for associated files""" + + system: str + name: str + path: str + type: Literal["file", "dir"] + length: Optional[int] = None + last_modified: Optional[str] = None + uuid: Optional[str] = None + + +class HazmapperMap(MetadataModel): + """Model for Hazmapper maps.""" + + name: str + uuid: str + path: str + deployment: str + href: Optional[str] = None + + +class Ref(MetadataModel): + """Model for refs attached to legacy analysis entities""" + + referencedoi: str = "" + reference: str = "" + + +class DropdownValue(MetadataModel): + """Model for a dropdown option with an ID and name""" + + id: str + name: str + + +class NaturalHazardEvent(MetadataModel): + """Model for natural hazard events""" + + event_name: str + event_start: datetime + event_end: Optional[datetime] = None + location: str + latitude: str + longitude: str diff --git a/designsafe/apps/api/projects_v2/schema_models/_field_transforms.py b/designsafe/apps/api/projects_v2/schema_models/_field_transforms.py new file mode 100644 index 0000000000..ac234b4366 --- /dev/null +++ b/designsafe/apps/api/projects_v2/schema_models/_field_transforms.py @@ -0,0 +1,91 @@ +"""Transforms for converting legacy metadata to fit our Pydantic schemas.""" +from typing import Optional, TypedDict + + +def handle_award_number(award: list[dict] | list[str] | str) -> list[dict]: + """Handle the case where awards are saved as strings.""" + if isinstance(award, list): + if award and isinstance(award[0], str): + return [{"number": "".join(award)}] + return award + if isinstance(award, str): + return [{"number": award}] + return award + + +def handle_array_of_none(field): + """Convert `[None]` and `[{}]` to empty lists.""" + if field in ([None], [{}]): + return [] + return field + + +def handle_legacy_authors(author_list: list): + """Handle the case where the author field is an array of usernames.""" + if not bool(author_list): + return [] + if isinstance(author_list[0], str): + author_map = map( + lambda author: {"name": author, "guest": False, "authorship": True}, + author_list, + ) + return list(author_map) + return author_list + + +def handle_keywords(keywords: str | list[str]) -> list[str]: + """Split keywords into an array.""" + if isinstance(keywords, str): + return [ + keyword.strip() + for keyword in keywords.split(",") + if keyword not in ("", "None") + ] + return keywords + + +class DropdownValueDict(TypedDict): + """{"id": "X", "name": "Y"}""" + + id: str + name: str + + +def handle_dropdown_value( + dropdown_value: Optional[str | DropdownValueDict], + options: list[DropdownValueDict], + fallback: Optional[DropdownValueDict] = None, +) -> Optional[DropdownValueDict]: + """Look up value if a string id/value is passed.""" + if not dropdown_value: + return None + + if isinstance(dropdown_value, str): + if dropdown_value.lower() == "other": + return fallback or {"id": "other", "name": ""} + return next( + ( + option + for option in options + if dropdown_value in (option["id"], option["name"]) + ), + fallback, + ) + return dropdown_value + + +def handle_dropdown_values( + dropdown_values: list[str | DropdownValueDict | None], + options: list[DropdownValueDict], +) -> list[DropdownValueDict]: + """Handle an array of dropdown values.""" + dropdown_values_validated = [] + for value in dropdown_values: + if isinstance(value, str): + dropdown_value = handle_dropdown_value( + value, options, fallback={"id": "other", "name": value} + ) + dropdown_values_validated.append(dropdown_value) + else: + dropdown_values_validated.append(value) + return list(filter(bool, dropdown_values_validated)) diff --git a/designsafe/apps/api/projects_v2/schema_models/base.py b/designsafe/apps/api/projects_v2/schema_models/base.py new file mode 100644 index 0000000000..4d4f4c91a5 --- /dev/null +++ b/designsafe/apps/api/projects_v2/schema_models/base.py @@ -0,0 +1,171 @@ +"""Pydantic schema models for base-level project entities.""" +from datetime import datetime +from typing import Literal, Optional, Annotated +from pydantic import ( + BeforeValidator, + AliasChoices, + ConfigDict, + model_validator, + Field, +) +from designsafe.apps.api.projects_v2.constants import ( + NATURAL_HAZARD_TYPES, + FIELD_RESEARCH_TYPES, + OTHER_DATA_TYPES, +) +from designsafe.apps.api.projects_v2.schema_models._field_models import MetadataModel +from designsafe.apps.api.projects_v2.schema_models._field_models import ( + AssociatedProject, + DropdownValue, + FileTag, + FileObj, + GuestMember, + HazmapperMap, + NaturalHazardEvent, + ProjectAward, + ProjectUser, + ReferencedWork, +) +from designsafe.apps.api.projects_v2.schema_models._field_transforms import ( + handle_array_of_none, + handle_award_number, + handle_dropdown_value, + handle_dropdown_values, + handle_keywords, +) + + +class PartialEntityWithFiles(MetadataModel): + """Model for representing an entity with associated files.""" + + model_config = ConfigDict(extra="ignore") + + file_tags: list[FileTag] = [] + file_objs: list[FileObj] = [] + + +class BaseProject(MetadataModel): + """Model for project root metadata.""" + + project_id: str + project_type: Literal[ + "other", + "experimental", + "simulation", + "hybrid_simulation", + "field_recon", + "field_reconnaissance", + "None", + ] = "None" + title: str + description: str = "" + team_members: list[str] = Field( + default=[], validation_alias=AliasChoices("teamMembers", "teamMember") + ) + users: list[ProjectUser] = [] + + guest_members: Annotated[ + list[GuestMember], BeforeValidator(handle_array_of_none) + ] = [] + pi: Optional[str] = None + co_pis: list[str] = [] + data_type: Annotated[ + Optional[DropdownValue], + BeforeValidator( + lambda v: handle_dropdown_value( + v, OTHER_DATA_TYPES, fallback={"id": "other", "name": v} + ) + ), + ] = None + data_types: list[DropdownValue] = [] + + authors: list[ProjectUser] = Field( + default=[], validation_alias=AliasChoices("authors", "teamOrder") + ) + + # This field is stored as awardNumber in projects and awardNumbers in pubs + award_number: Annotated[ + list[ProjectAward], BeforeValidator(handle_award_number) + ] = Field(default=[], exclude=True) + award_numbers: Annotated[ + list[ProjectAward], BeforeValidator(handle_award_number) + ] = [] + associated_projects: list[AssociatedProject] = [] + referenced_data: list[ReferencedWork] = [] + ef: Optional[str] = None + keywords: Annotated[list[str], BeforeValidator(handle_keywords)] = [] + + nh_event: str = "" + nh_event_start: Annotated[ + Optional[datetime], BeforeValidator(lambda v: v or None) + ] = None + nh_event_end: Annotated[ + Optional[datetime], BeforeValidator(lambda v: v or None) + ] = None + nh_location: str = "" + nh_latitude: str = "" + nh_longitude: str = "" + nh_events: list[NaturalHazardEvent] = [] + + nh_types: Annotated[ + list[DropdownValue], + BeforeValidator(lambda v: handle_dropdown_values(v, NATURAL_HAZARD_TYPES)), + ] = [] + + fr_types: Annotated[ + list[DropdownValue], + BeforeValidator(lambda v: handle_dropdown_values(v, FIELD_RESEARCH_TYPES)), + ] = [] + dois: list[str] = [] + + file_tags: list[FileTag] = [] + file_objs: list[FileObj] = [] + hazmapper_maps: list[HazmapperMap] = [] + + facilities: list[DropdownValue] = [] + + # These fields are ONLY present on publication PRJ-1665 + natural_hazard_type: Optional[str] = None + natural_hazard_event: Optional[str] = None + coverage_temporal: Optional[str] = None + lat_long_name: Optional[str] = None + + def construct_users(self) -> list[ProjectUser]: + """Fill in missing user information from the database.""" + users = [] + if self.pi and self.pi != "None": + users.append(ProjectUser.from_username(self.pi, role="pi")) + for co_pi in self.co_pis: + if len(co_pi) == 1 or co_pi == "None": + continue + users.append(ProjectUser.from_username(co_pi, role="co_pi")) + for team_member in self.team_members: + if len(team_member) == 1 or team_member == "None": + continue + users.append(ProjectUser.from_username(team_member, role="team_member")) + return users + + @model_validator(mode="after") + def post_validate(self): + """Populate derived fields if they don't exist yet.""" + _authors = sorted(self.authors or [], key=lambda a: getattr(a, "order", 0)) + if self.authors != _authors: + self.authors = _authors + if self.data_type and not self.data_types: + self.data_types = [self.data_type] + if self.nh_event and self.nh_event_start and not self.nh_events: + self.nh_events = [ + NaturalHazardEvent( + event_name=self.nh_event, + event_start=self.nh_event_start, + event_end=self.nh_event_end, + location=self.nh_location, + latitude=self.nh_latitude, + longitude=self.nh_longitude, + ) + ] + if self.award_number and not self.award_numbers: + self.award_numbers = self.award_number + if (not self.users) and (users := self.construct_users()): + self.users = users + return self diff --git a/designsafe/apps/api/projects_v2/schema_models/experimental.py b/designsafe/apps/api/projects_v2/schema_models/experimental.py new file mode 100644 index 0000000000..661fcd217c --- /dev/null +++ b/designsafe/apps/api/projects_v2/schema_models/experimental.py @@ -0,0 +1,199 @@ +"""Pydantic models for Experimental entities""" +import itertools +from typing import Optional, Annotated +from pydantic import BeforeValidator, Field, ConfigDict, model_validator, AliasChoices +from designsafe.apps.api.projects_v2.schema_models._field_models import MetadataModel +from designsafe.apps.api.projects_v2.schema_models._field_models import ( + AssociatedProject, + DropdownValue, + FileObj, + FileTag, + ProjectUser, + Ref, + ReferencedWork, +) +from designsafe.apps.api.projects_v2.schema_models._field_transforms import ( + handle_array_of_none, + handle_legacy_authors, + handle_dropdown_value, +) +from designsafe.apps.api.projects_v2.constants import ( + FACILITY_OPTIONS, + EQUIPMENT_TYPES, + EXPERIMENT_TYPES, +) + +equipment_type_options = list(itertools.chain(*EQUIPMENT_TYPES.values())) +experiment_type_options = list(itertools.chain(*EXPERIMENT_TYPES.values())) + + +class Experiment(MetadataModel): + """Model for Experiments.""" + + title: str + description: str = "" + + referenced_data: list[ReferencedWork] = [] + related_work: list[AssociatedProject] = [] + + facility: Annotated[ + Optional[DropdownValue], + BeforeValidator(lambda v: handle_dropdown_value(v, FACILITY_OPTIONS)), + Field( + exclude=True, + validation_alias=AliasChoices("facility", "experimentalFacility"), + ), + ] = None + experimental_facility_other: str = Field(default="", exclude=True) + + experiment_type: Annotated[ + Optional[DropdownValue], + BeforeValidator(lambda v: handle_dropdown_value(v, experiment_type_options)), + ] = None + experiment_type_other: str = Field(default="", exclude=True) + + equipment_type: Annotated[ + Optional[DropdownValue], + BeforeValidator(lambda v: handle_dropdown_value(v, equipment_type_options)), + ] = None + equipment_type_other: str = Field(default="", exclude=True) + + procedure_start: str = "" + procedure_end: str = "" + + authors: Annotated[list[ProjectUser], BeforeValidator(handle_legacy_authors)] = [] + project: list[str] = [] + dois: list[str] = [] + + @model_validator(mode="after") + def handle_other(self): + """Use values of XXX_other fields to fill in dropdown values.""" + if ( + self.equipment_type_other + and self.equipment_type + and not self.equipment_type.name + ): + self.equipment_type.name = self.equipment_type_other + if ( + self.experiment_type_other + and self.experiment_type + and not self.experiment_type.name + ): + self.experiment_type.name = self.experiment_type_other + if ( + self.experimental_facility_other + and self.facility + and not self.facility.name + ): + self.facility.name = self.experimental_facility_other + return self + + +class ExperimentModelConfig(MetadataModel): + """Model for model configurations.""" + + title: str + description: str = "" + project: list[str] = [] + experiments: list[str] = [] + files: list[str] = [] + file_tags: list[FileTag] = [] + file_objs: list[FileObj] = [] + + # Deprecated/legacy fields + drawing: Optional[list[str]] = Field( + default=None, exclude=True, alias="modelDrawing" + ) + tags: Optional[dict] = Field(default=None, exclude=True) + image: Optional[dict] = Field(default=None, exclude=True) + lat: Optional[str] = Field(default=None, exclude=True) + lon: Optional[str] = Field(default=None, exclude=True) + video: Optional[dict] = Field(default=None, exclude=True) + spatial: Optional[str] = Field(default=None, exclude=True) + coverage: Optional[str] = Field(default=None, exclude=True) + + +class ExperimentSensor(MetadataModel): + """Model for sensors.""" + + model_config = ConfigDict(protected_namespaces=()) + + title: str + description: str = "" + project: list[str] = [] + experiments: list[str] = [] + model_configs: list[str] = [] + files: list[str] = [] + file_tags: list[FileTag] = [] + file_objs: list[FileObj] = [] + + # Deprecated legacy fields + sensor_list_type: Optional[str] = None + sensor_drawing: Optional[list[str]] = None + tags: Optional[dict] = Field(default=None, exclude=True) + + sensor_lists: Optional[list[str]] = Field(default=None, exclude=True) + event_type: Optional[str] = None + + # This field is ONLY present on pub PRJ-1649 + analysis: Optional[list[str]] = None + + # This field ONLY Present on sensor 8078182091498319385-242ac11c-0001-012 + load: Optional[list[str]] = Field(default=None, exclude=True) + + +class ExperimentEvent(MetadataModel): + """Model for experimental events.""" + + model_config = ConfigDict(protected_namespaces=()) + + title: str + description: str = "" + event_type: str = "" + project: list[str] = [] + experiments: list[str] = [] + sensor_lists: list[str] = [] + analysis: list[str] = [] + model_configs: list[str] = [] + + files: list[str] = [] + file_tags: list[FileTag] = [] + file_objs: list[FileObj] = [] + + tags: Optional[dict] = Field(default=None, exclude=True) + load: Optional[list[str]] = Field(default=None, exclude=True) + + +class ExperimentAnalysis(MetadataModel): + """Model for experimental analysis.""" + + title: str + description: str = "" + analysis_type: str = "" + refs: Annotated[list[Ref], BeforeValidator(handle_array_of_none)] = [] + + analysis_data: str = "" + application: str = "" + script: list[str] = [] + project: list[str] = [] + experiments: list[str] = [] + files: list[str] = [] + file_tags: list[FileTag] = [] + file_objs: list[FileObj] = [] + + tags: Optional[dict] = Field(default=None, exclude=True) + reference: Optional[str] = Field(default=None, exclude=True) + referencedoi: Optional[str] = Field(default=None, exclude=True) + + +class ExperimentReport(MetadataModel): + """Model for experimental reports.""" + + title: str + description: str = "" + + project: list[str] = [] + experiments: list[str] = [] + files: list[str] = [] + file_tags: list[FileTag] = [] + file_objs: list[FileObj] = [] diff --git a/designsafe/apps/api/projects_v2/schema_models/field_recon.py b/designsafe/apps/api/projects_v2/schema_models/field_recon.py new file mode 100644 index 0000000000..b8b62b6af3 --- /dev/null +++ b/designsafe/apps/api/projects_v2/schema_models/field_recon.py @@ -0,0 +1,194 @@ +"""Pydantic schema models for Field Recon entities""" +from typing import Annotated, Optional +import itertools +from pydantic import BeforeValidator, Field, AliasChoices +from designsafe.apps.api.projects_v2.schema_models._field_models import MetadataModel +from designsafe.apps.api.projects_v2.schema_models._field_models import ( + AssociatedProject, + DropdownValue, + FileObj, + FileTag, + ProjectUser, + ReferencedWork, +) +from designsafe.apps.api.projects_v2.schema_models._field_transforms import ( + handle_array_of_none, + handle_legacy_authors, + handle_dropdown_values, +) +from designsafe.apps.api.projects_v2.constants import ( + FR_EQUIPMENT_TYPES, + FR_OBSERVATION_TYPES, +) + +equipment_type_options = list(itertools.chain(*FR_EQUIPMENT_TYPES.values())) + + +class Mission(MetadataModel): + """Model for field recon missions.""" + + title: str + description: str = "" + + referenced_data: list[ReferencedWork] = [] + related_work: list[AssociatedProject] = [] + + event: str = "" + date_start: str = "" + date_end: str = "" + location: str = "" + latitude: str = "" + longitude: str = "" + elevation: str = "" + + authors: Annotated[list[ProjectUser], BeforeValidator(handle_legacy_authors)] = [] + project: list[str] = [] + dois: list[str] = [] + + # Deprecate these later + facility: Optional[DropdownValue] = None + + +class FieldReconReport(MetadataModel): + """Model for field recon reports.""" + + title: str + description: str = "" + + referenced_data: Annotated[ + list[ReferencedWork], BeforeValidator(handle_array_of_none) + ] = [] + related_work: list[AssociatedProject] = [] + + file_tags: list[FileTag] = [] + authors: Annotated[ + list[ProjectUser], BeforeValidator(handle_legacy_authors) + ] = Field(default=[], validation_alias=AliasChoices("authors", "dataCollectors")) + + guest_data_collectors: list[str] = [] + project: list[str] = [] + files: list[str] = [] + file_objs: list[FileObj] = [] + dois: list[str] = [] + + # deprecated, only appears in test projects + facility: Optional[DropdownValue] = None + missions: list[str] = Field(default=[], exclude=True) + referenced_datas: list[ReferencedWork] = Field(default=[], exclude=True) + + +class Instrument(MetadataModel): + """model for instruments used in field recon projects.""" + + model: str = "other" + name: str = "" + + +class FieldReconCollection(MetadataModel): + """Model for field recon collections without a specific type (deprecated)""" + + title: str + description: str = "" + + observation_types: list[str | None] = [] + date_start: str = "" + date_end: str = "" + + data_collectors: list[ProjectUser] = [] + guest_data_collectors: list[str] = [] + + location: str = "" + latitude: str = "" + longitude: str = "" + elevation: str = "" + instruments: list[Instrument] = [] + referenced_datas: Annotated[ + list[ReferencedWork], BeforeValidator(handle_array_of_none) + ] = [] + + project: list[str] = [] + missions: list[str] = [] + files: list[str] = [] + file_objs: list[FileObj] = [] + file_tags: list[FileTag] = [] + + +class SocialScienceCollection(MetadataModel): + """Model for social science collections""" + + title: str + description: str + unit: str = "" + modes: Annotated[list[str], BeforeValidator(handle_array_of_none)] = [] + sample_approach: Annotated[list[str], BeforeValidator(handle_array_of_none)] = [] + sample_size: str + date_start: str + date_end: str + data_collectors: list[ProjectUser] = [] + location: str = "" + latitude: str = "" + longitude: str = "" + equipment: Annotated[ + list[DropdownValue], + BeforeValidator(lambda v: handle_dropdown_values(v, equipment_type_options)), + ] = [] + restriction: str = "" + referenced_data: Annotated[ + list[ReferencedWork], BeforeValidator(handle_array_of_none) + ] = [] + project: list[str] = [] + missions: list[str] = [] + files: list[str] = [] + file_objs: list[FileObj] = [] + file_tags: list[FileTag] = [] + + # Deprecated test fields + methods: list[str | None] = Field(default=[], exclude=True) + + +class PlanningCollection(MetadataModel): + """Model for planning collections.""" + + title: str + description: str = "" + data_collectors: list[ProjectUser] = [] + referenced_data: Annotated[ + list[ReferencedWork], BeforeValidator(handle_array_of_none) + ] = [] + + project: list[str] = [] + missions: list[str] = [] + files: list[str] = [] + file_objs: list[FileObj] = [] + file_tags: list[FileTag] = [] + + +class GeoscienceCollection(MetadataModel): + """Model for geoscience collections.""" + + title: str + description: str = "" + data_collectors: list[ProjectUser] = [] + referenced_data: Annotated[ + list[ReferencedWork], BeforeValidator(handle_array_of_none) + ] = [] + + observation_types: Annotated[ + list[DropdownValue], + BeforeValidator(lambda v: handle_dropdown_values(v, FR_OBSERVATION_TYPES)), + ] = [] + date_start: str + date_end: str + location: str = "" + latitude: str = "" + longitude: str = "" + equipment: Annotated[ + list[DropdownValue], + BeforeValidator(lambda v: handle_dropdown_values(v, equipment_type_options)), + ] = [] + + project: list[str] = [] + missions: list[str] = [] + files: list[str] = [] + file_objs: list[FileObj] = [] + file_tags: list[FileTag] = [] diff --git a/designsafe/apps/api/projects_v2/schema_models/hybrid_sim.py b/designsafe/apps/api/projects_v2/schema_models/hybrid_sim.py new file mode 100644 index 0000000000..bda56feeeb --- /dev/null +++ b/designsafe/apps/api/projects_v2/schema_models/hybrid_sim.py @@ -0,0 +1,204 @@ +"""Pydantic schema models for Hybrid Simulation entities""" +from typing import Annotated, Optional +from pydantic import BeforeValidator, Field, model_validator +from designsafe.apps.api.projects_v2.schema_models._field_models import MetadataModel +from designsafe.apps.api.projects_v2.schema_models._field_models import ( + AssociatedProject, + DropdownValue, + FileObj, + FileTag, + ProjectUser, + Ref, + ReferencedWork, +) +from designsafe.apps.api.projects_v2.schema_models._field_transforms import ( + handle_array_of_none, + handle_legacy_authors, + handle_dropdown_value, +) +from designsafe.apps.api.projects_v2.constants import HYBRID_SIM_TYPES + + +class HybridSimulation(MetadataModel): + """Model for a base simulation.""" + + title: str + description: str = "" + simulation_type: Annotated[ + DropdownValue, + BeforeValidator(lambda v: handle_dropdown_value(v, HYBRID_SIM_TYPES)), + ] + simulation_type_other: str = Field(exclude=True) + procedure_start: str = "" + procedure_end: str = "" + referenced_data: list[ReferencedWork] = [] + related_work: list[AssociatedProject] = [] + authors: Annotated[list[ProjectUser], BeforeValidator(handle_legacy_authors)] = [] + project: list[str] = [] + dois: list[str] = [] + + facility: Optional[DropdownValue] = None + + @model_validator(mode="after") + def handle_other(self): + """Use values of XXX_other fields to fill in dropdown values.""" + if ( + self.simulation_type_other + and self.simulation_type + and not self.simulation_type.name + ): + self.simulation_type.name = self.simulation_type_other + return self + + +class HybridSimGlobalModel(MetadataModel): + """Model for hybrid sim global models.""" + + title: str + description: str = "" + + project: list[str] = [] + hybrid_simulations: list[str] = [] + files: list[str] = [] + file_tags: list[FileTag] = [] + file_objs: list[FileObj] = [] + + tags: Optional[dict] = Field(default=None, exclude=True) + + +class HybridSimCoordinator(MetadataModel): + """Model for coordinators.""" + + title: str + description: str = "" + application_version: str = "" + + project: list[str] = [] + hybrid_simulations: list[str] = [] + global_models: list[str] = [] + files: list[str] = [] + file_tags: list[FileTag] = [] + file_objs: list[FileObj] = [] + + tags: Optional[dict] = Field(default=None, exclude=True) + + +class HybridSimSimSubstructure(MetadataModel): + """Model for simulation substructures.""" + + title: str + description: str = "" + application_version: str = "" + + project: list[str] = [] + hybrid_simulations: list[str] = [] + global_models: list[str] = [] + coordinators: list[str] = [] + files: list[str] = [] + file_tags: list[FileTag] = [] + file_objs: list[FileObj] = [] + + tags: Optional[dict] = Field(default=None, exclude=True) + + +class HybridSimExpSubstructure(MetadataModel): + """Model for experimental substructures.""" + + title: str + description: str = "" + + project: list[str] = [] + hybrid_simulations: list[str] = [] + global_models: list[str] = [] + coordinators: list[str] = [] + files: list[str] = [] + file_tags: list[FileTag] = [] + file_objs: list[FileObj] = [] + + tags: Optional[dict] = Field(default=None, exclude=True) + + +class HybridSimCoordinatorOutput(MetadataModel): + """Model for coordinator output.""" + + title: str + description: str = "" + + project: list[str] = [] + hybrid_simulations: list[str] = [] + global_models: list[str] = [] + coordinators: list[str] = [] + files: list[str] = [] + file_tags: list[FileTag] = [] + file_objs: list[FileObj] = [] + + tags: Optional[dict] = Field(default=None, exclude=True) + + +class HybridSimSimOutput(MetadataModel): + """Model for coordinator output.""" + + title: str + description: str = "" + + project: list[str] = [] + hybrid_simulations: list[str] = [] + global_models: list[str] = [] + sim_substructures: list[str] = [] + coordinators: list[str] = [] + files: list[str] = [] + file_tags: list[FileTag] = [] + file_objs: list[FileObj] = [] + + tags: Optional[dict] = Field(default=None, exclude=True) + + +class HybridSimExpOutput(MetadataModel): + """Model for experimental substructure output.""" + + title: str + description: str = "" + + project: list[str] = [] + hybrid_simulations: list[str] = [] + global_models: list[str] = [] + exp_substructures: list[str] = [] + coordinators: list[str] = [] + files: list[str] = [] + file_tags: list[FileTag] = [] + file_objs: list[FileObj] = [] + + tags: Optional[dict] = Field(default=None, exclude=True) + + +class HybridSimAnalysis(MetadataModel): + """Model for hybrid sim analysis entities.""" + + title: str + description: str = "" + refs: Annotated[list[Ref], BeforeValidator(handle_array_of_none)] = [] + + project: list[str] = [] + hybrid_simulations: list[str] = [] + files: list[str] = [] + file_tags: list[FileTag] = [] + file_objs: list[FileObj] = [] + + tags: Optional[dict] = Field(default=None, exclude=True) + reference: Optional[str] = None + referencedoi: Optional[str] = None + + +class HybridSimReport(MetadataModel): + """Model for hybrid sim reports.""" + + title: str + description: str = "" + + project: list[str] = [] + hybrid_simulations: list[str] = [] + files: list[str] = [] + file_tags: list[FileTag] = [] + file_objs: list[FileObj] = [] + + tags: Optional[dict] = Field(default=None, exclude=True) diff --git a/designsafe/apps/api/projects_v2/schema_models/simulation.py b/designsafe/apps/api/projects_v2/schema_models/simulation.py new file mode 100644 index 0000000000..c614464bf6 --- /dev/null +++ b/designsafe/apps/api/projects_v2/schema_models/simulation.py @@ -0,0 +1,136 @@ +"""Pydantic models for Simulation entities.""" + +from typing import Optional, Annotated +from pydantic import BeforeValidator, ConfigDict, Field, model_validator +from designsafe.apps.api.projects_v2.schema_models._field_models import MetadataModel +from designsafe.apps.api.projects_v2.schema_models._field_models import ( + AssociatedProject, + DropdownValue, + FileObj, + FileTag, + ProjectUser, + Ref, + ReferencedWork, +) +from designsafe.apps.api.projects_v2.schema_models._field_transforms import ( + handle_array_of_none, + handle_legacy_authors, + handle_dropdown_value, +) +from designsafe.apps.api.projects_v2.constants import SIMULATION_TYPES + + +class Simulation(MetadataModel): + """Model for a base simulation.""" + + title: str + description: str = "" + simulation_type: Annotated[ + DropdownValue, + BeforeValidator(lambda v: handle_dropdown_value(v, SIMULATION_TYPES)), + ] + simulation_type_other: str = Field(exclude=True) + referenced_data: list[ReferencedWork] = [] + related_work: list[AssociatedProject] = [] + authors: Annotated[list[ProjectUser], BeforeValidator(handle_legacy_authors)] = [] + project: list[str] = [] + dois: list[str] = [] + + facility: Optional[DropdownValue] = None + + @model_validator(mode="after") + def handle_other(self): + """Use values of XXX_other fields to fill in dropdown values.""" + if ( + self.simulation_type_other + and self.simulation_type + and not self.simulation_type.name + ): + self.simulation_type.name = self.simulation_type_other + return self + + +class SimulationModel(MetadataModel): + """Model for a simulation model.""" + + title: str + application_version: str = "" + application_version_other: str = "" + application_version_desc: str = "" + nh_type: str = "" + nh_type_other: str = "" + simulated_system: str = "" + description: str = "" + + project: list[str] = [] + simulations: list[str] = [] + files: list[str] = [] + file_tags: list[FileTag] = [] + file_objs: list[FileObj] = [] + + +class SimulationInput(MetadataModel): + """Model for simulation input.""" + + model_config = ConfigDict(protected_namespaces=()) + + title: str + description: str = "" + + project: list[str] = [] + model_configs: list[str] = [] + simulations: list[str] = [] + files: list[str] = [] + file_tags: list[FileTag] = [] + file_objs: list[FileObj] = [] + + +class SimulationOutput(MetadataModel): + """Model for simulation output.""" + + model_config = ConfigDict(protected_namespaces=()) + + title: str + description: str = "" + + project: list[str] = [] + model_configs: list[str] = [] + sim_inputs: list[str] = [] + simulations: list[str] = [] + files: list[str] = [] + file_tags: list[FileTag] = [] + file_objs: list[FileObj] = [] + + +class SimulationAnalysis(MetadataModel): + """Model for simulation analysis.""" + + title: str + description: str = "" + + refs: Annotated[list[Ref], BeforeValidator(handle_array_of_none)] = [] + project: list[str] = [] + sim_inputs: list[str] = [] + sim_outputs: list[str] = [] + simulations: list[str] = [] + files: list[str] = [] + file_tags: list[FileTag] = [] + file_objs: list[FileObj] = [] + + reference: Optional[str] = None + referencedoi: Optional[str] = None + + +class SimulationReport(MetadataModel): + """Model for simulation reports.""" + + title: str + description: str = "" + + project: list[str] = [] + sim_inputs: list[str] = [] + sim_outputs: list[str] = [] + simulations: list[str] = [] + files: list[str] = [] + file_tags: list[FileTag] = [] + file_objs: list[FileObj] = [] diff --git a/designsafe/apps/api/projects_v2/tests/schema_integration.py b/designsafe/apps/api/projects_v2/tests/schema_integration.py new file mode 100644 index 0000000000..2473d4c1b5 --- /dev/null +++ b/designsafe/apps/api/projects_v2/tests/schema_integration.py @@ -0,0 +1,81 @@ +"""Integration-type tests to confirm that Pydantic schemas are exhaustive.""" +import json +from typing import Iterator +from pydantic import BaseModel, ValidationError +from designsafe.apps.api.projects_v2.schema_models.base import BaseProject +from designsafe.apps.api.projects_v2.migration_utils.graph_constructor import ( + transform_pub_entities, +) +from designsafe.apps.api.agave import service_account +from designsafe.apps.api.publications.operations import listing as list_pubs + + +def update_project(uuid, new_value): + """Utility for patching a dict into entity metadata.""" + client = service_account() + meta = client.meta.getMetadata(uuid=uuid) + meta["value"] = {**meta["value"], **new_value} + + meta["lastUpdated"] = meta["lastUpdated"].isoformat() + meta["created"] = meta["created"].isoformat() + return client.meta.updateMetadata(uuid=uuid, body=dict(meta)) + + +def iterate_entities(name="designsafe.project") -> Iterator[dict]: + """Yield all entities for a given namespace""" + client = service_account() + query = {"name": name} + offset = 0 + limit = 100 + while True: + listing = client.meta.listMetadata( + q=json.dumps(query), offset=offset, limit=limit + ) + yield from listing + + if len(listing) < limit: + break + + offset += limit + + +def validate_entities(name: str = "designsafe.project", model: BaseModel = BaseProject): + """Validate a Pydantic model against all instances of an entity namespace.""" + gen = iterate_entities(name) + for entity in gen: + try: + validated_model = model.model_validate(entity["value"]) + model_json = validated_model.model_dump() + # Assert that subsequent validation does not affect the data. + assert model.model_validate(model_json).model_dump() == model_json + except ValidationError as exc: + print(entity["uuid"]) + print(entity) + print(exc) + + +def iterate_pubs(): + """Returns an iterator over all publications.""" + offset = 0 + limit = 100 + while True: + listing = list_pubs(offset=offset, limit=limit) + res = listing["listing"] + yield from res + + if len(res) < limit: + break + + offset += limit + + +def validate_publications(): + """Attempt graph construction for all publications.""" + all_pubs = iterate_pubs() + for pub in all_pubs: + # print(pub["projectId"]) + try: + transform_pub_entities(pub["projectId"]) + except ValidationError as exc: + print(pub["projectId"]) + print(exc) diff --git a/designsafe/apps/api/projects_v2/urls.py b/designsafe/apps/api/projects_v2/urls.py new file mode 100644 index 0000000000..61b6ada40e --- /dev/null +++ b/designsafe/apps/api/projects_v2/urls.py @@ -0,0 +1 @@ +"""Placeholder API routes""" diff --git a/designsafe/apps/api/projects_v2/views.py b/designsafe/apps/api/projects_v2/views.py new file mode 100644 index 0000000000..f850a859fd --- /dev/null +++ b/designsafe/apps/api/projects_v2/views.py @@ -0,0 +1 @@ +"""Placeholder views module""" diff --git a/designsafe/conftest.py b/designsafe/conftest.py index 1fd8729e16..3b327245b1 100644 --- a/designsafe/conftest.py +++ b/designsafe/conftest.py @@ -33,6 +33,17 @@ def regular_user(django_user_model, mock_agave_client): yield user +@pytest.fixture +def project_admin_user(django_user_model): + django_user_model.objects.create_user(username="test_prjadmin", + password="password", + first_name="Project", + last_name="Admin", + ) + user = django_user_model.objects.get(username="test_prjadmin") + yield user + + @pytest.fixture def authenticated_user(client, regular_user): client.force_login(regular_user) diff --git a/designsafe/settings/common_settings.py b/designsafe/settings/common_settings.py index 3a7934de78..af2b46e187 100644 --- a/designsafe/settings/common_settings.py +++ b/designsafe/settings/common_settings.py @@ -85,6 +85,7 @@ 'designsafe.apps.api', 'designsafe.apps.api.notifications', 'designsafe.apps.api.datafiles', + 'designsafe.apps.api.projects_v2', 'designsafe.apps.accounts', 'designsafe.apps.cms_plugins', 'designsafe.apps.box_integration', @@ -464,6 +465,7 @@ MEETING_REQUEST_EMAIL = os.environ.get('MEETING_REQUEST_EMAIL', 'info@designsafe-ci.org') NEW_ACCOUNT_ALERT_EMAILS = os.environ.get('NEW_ACCOUNT_ALERT_EMAILS', 'no-reply@designsafe-ci.org,') +PROJECT_ADMIN_USERS = ['ds_admin', 'prjadmin'] PROJECT_ADMINS_EMAIL = ['maria@tacc.utexas.edu', 'gendlerk@tacc.utexas.edu'] DEV_PROJECT_ADMINS_EMAIL = ['tbrown@tacc.utexas.edu', 'sgray@tacc.utexas.edu', 'vgonzalez@tacc.utexas.edu'] diff --git a/designsafe/settings/test_settings.py b/designsafe/settings/test_settings.py index 6b61aceda7..e88bd5e0be 100644 --- a/designsafe/settings/test_settings.py +++ b/designsafe/settings/test_settings.py @@ -84,6 +84,7 @@ 'designsafe.apps.auth', 'designsafe.apps.api', 'designsafe.apps.api.notifications', + 'designsafe.apps.api.projects_v2', 'designsafe.apps.accounts', 'designsafe.apps.cms_plugins', 'designsafe.apps.box_integration', @@ -473,6 +474,7 @@ 'auth': json.loads(os.environ.get('PROJECT_SYSTEM_STORAGE_CREDENTIALS', '{}')) } } +PROJECT_ADMIN_USERS = ["test_prjadmin"] PUBLISHED_SYSTEM = 'designsafe.storage.published'