deploy: 8fb16d6

_downloads/819e10305ddd6839cd7da05935b17060/mass-inference.txt

@@ -1,17 +1,17 @@
-2024-08-03 02:13:05 (INFO): Running in non-distributed local mode
-2024-08-03 02:13:05 (INFO): Setting env PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True
-2024-08-03 02:13:05 (INFO): Project root: /home/runner/work/fairchem/fairchem/src/fairchem
-2024-08-03 02:13:06 (INFO): amp: true
+2024-08-14 00:54:09 (INFO): Running in non-distributed local mode
+2024-08-14 00:54:09 (INFO): Setting env PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True
+2024-08-14 00:54:09 (INFO): Project root: /home/runner/work/fairchem/fairchem/src/fairchem
+2024-08-14 00:54:10 (INFO): amp: true
 cmd:
-  checkpoint_dir: ./checkpoints/2024-08-03-02-12-16
-  commit: 08b8c1e
+  checkpoint_dir: ./checkpoints/2024-08-14-00-53-20
+  commit: 8fb16d6
   identifier: ''
-  logs_dir: ./logs/tensorboard/2024-08-03-02-12-16
+  logs_dir: ./logs/tensorboard/2024-08-14-00-53-20
   print_every: 10
-  results_dir: ./results/2024-08-03-02-12-16
+  results_dir: ./results/2024-08-14-00-53-20
   seed: 0
-  timestamp_id: 2024-08-03-02-12-16
-  version: 0.1.dev1+g08b8c1e
+  timestamp_id: 2024-08-14-00-53-20
+  version: 1.1.0
 dataset: {}
 evaluation_metrics:
   metrics:
@@ -37,8 +37,7 @@ loss_functions:
 - forces:
     coefficient: 1
     fn: l2mae
-model: gemnet_t
-model_attributes:
+model:
   activation: silu
   cbf:
     name: spherical_harmonics
@@ -55,6 +54,7 @@ model_attributes:
     name: polynomial
   extensive: true
   max_neighbors: 50
+  name: gemnet_t
   num_after_skip: 2
   num_atom: 3
   num_before_skip: 1
@@ -114,23 +114,24 @@ test_dataset:
 trainer: ocp
 val_dataset: {}
 
-2024-08-03 02:13:06 (WARNING): Could not find dataset metadata.npz files in '[PosixPath('data.db')]'
-2024-08-03 02:13:06 (WARNING): Disabled BalancedBatchSampler because num_replicas=1.
-2024-08-03 02:13:06 (WARNING): Failed to get data sizes, falling back to uniform partitioning. BalancedBatchSampler requires a dataset that has a metadata attributed with number of atoms.
-2024-08-03 02:13:06 (INFO): rank: 0: Sampler created...
-2024-08-03 02:13:06 (INFO): Created BalancedBatchSampler with sampler=<fairchem.core.common.data_parallel.StatefulDistributedSampler object at 0x7f2bf30d1310>, batch_size=16, drop_last=False
-2024-08-03 02:13:06 (INFO): Loading model: gemnet_t
-2024-08-03 02:13:08 (INFO): Loaded GemNetT with 31671825 parameters.
-2024-08-03 02:13:08 (WARNING): log_summary for Tensorboard not supported
-2024-08-03 02:13:08 (INFO): Loading checkpoint from: /tmp/fairchem_checkpoints/gndt_oc22_all_s2ef.pt
-2024-08-03 02:13:08 (INFO): Overwriting scaling factors with those loaded from checkpoint. If you're generating predictions with a pretrained checkpoint, this is the correct behavior. To disable this, delete `scale_dict` from the checkpoint. 
-2024-08-03 02:13:08 (WARNING): Scale factor comment not found in model
-2024-08-03 02:13:08 (INFO): Predicting on test.
+2024-08-14 00:54:10 (WARNING): Could not find dataset metadata.npz files in '[PosixPath('data.db')]'
+2024-08-14 00:54:10 (WARNING): Disabled BalancedBatchSampler because num_replicas=1.
+2024-08-14 00:54:10 (WARNING): Failed to get data sizes, falling back to uniform partitioning. BalancedBatchSampler requires a dataset that has a metadata attributed with number of atoms.
+2024-08-14 00:54:10 (INFO): rank: 0: Sampler created...
+2024-08-14 00:54:10 (INFO): Created BalancedBatchSampler with sampler=<fairchem.core.common.data_parallel.StatefulDistributedSampler object at 0x7f18d61afd90>, batch_size=16, drop_last=False
+2024-08-14 00:54:10 (INFO): Loading model: gemnet_t
+2024-08-14 00:54:12 (INFO): Loaded GemNetT with 31671825 parameters.
+2024-08-14 00:54:12 (WARNING): log_summary for Tensorboard not supported
+2024-08-14 00:54:12 (INFO): Attemping to load user specified checkpoint at /tmp/fairchem_checkpoints/gndt_oc22_all_s2ef.pt
+2024-08-14 00:54:12 (INFO): Loading checkpoint from: /tmp/fairchem_checkpoints/gndt_oc22_all_s2ef.pt
+2024-08-14 00:54:12 (INFO): Overwriting scaling factors with those loaded from checkpoint. If you're generating predictions with a pretrained checkpoint, this is the correct behavior. To disable this, delete `scale_dict` from the checkpoint. 
+2024-08-14 00:54:12 (WARNING): Scale factor comment not found in model
+2024-08-14 00:54:12 (INFO): Predicting on test.
 device 0:   0%|                                           | 0/3 [00:00<?, ?it/s]/opt/hostedtoolcache/Python/3.11.9/x64/lib/python3.11/site-packages/torch_geometric/data/collate.py:145: UserWarning: TypedStorage is deprecated. It will be removed in the future and UntypedStorage will be the only storage class. This should only matter to you if you are using storages directly.  To access UntypedStorage directly, use tensor.untyped_storage() instead of tensor.storage()
   storage = elem.storage()._new_shared(numel)
 /opt/hostedtoolcache/Python/3.11.9/x64/lib/python3.11/site-packages/torch_geometric/data/collate.py:145: UserWarning: TypedStorage is deprecated. It will be removed in the future and UntypedStorage will be the only storage class. This should only matter to you if you are using storages directly.  To access UntypedStorage directly, use tensor.untyped_storage() instead of tensor.storage()
   storage = elem.storage()._new_shared(numel)
-device 0:  33%|███████████▋                       | 1/3 [00:02<00:05,  2.56s/it]device 0:  67%|███████████████████████▎           | 2/3 [00:06<00:03,  3.21s/it]device 0: 100%|███████████████████████████████████| 3/3 [00:07<00:00,  2.10s/it]device 0: 100%|███████████████████████████████████| 3/3 [00:07<00:00,  2.35s/it]
-2024-08-03 02:13:15 (INFO): Writing results to ./results/2024-08-03-02-12-16/ocp_predictions.npz
-2024-08-03 02:13:15 (INFO): Total time taken: 7.171952247619629
-Elapsed time = 13.2 seconds
+device 0:  33%|███████████▋                       | 1/3 [00:02<00:04,  2.02s/it]device 0:  67%|███████████████████████▎           | 2/3 [00:04<00:02,  2.18s/it]device 0: 100%|███████████████████████████████████| 3/3 [00:06<00:00,  2.33s/it]device 0: 100%|███████████████████████████████████| 3/3 [00:06<00:00,  2.28s/it]
+2024-08-14 00:54:19 (INFO): Writing results to ./results/2024-08-14-00-53-20/ocp_predictions.npz
+2024-08-14 00:54:19 (INFO): Total time taken: 6.979607820510864
+Elapsed time = 13.0 seconds

_sources/autoapi/core/common/distutils/index.rst

@@ -34,6 +34,7 @@ Functions
    core.common.distutils.is_master
    core.common.distutils.synchronize
    core.common.distutils.broadcast
+   core.common.distutils.broadcast_object_list
    core.common.distutils.all_reduce
    core.common.distutils.all_gather
    core.common.distutils.gather_objects
@@ -62,6 +63,8 @@ Module Contents
 
 .. py:function:: broadcast(tensor: torch.Tensor, src, group=dist.group.WORLD, async_op: bool = False) -> None
 
+.. py:function:: broadcast_object_list(object_list: list[Any], src: int, group=dist.group.WORLD, device: str | None = None) -> None
+
 .. py:function:: all_reduce(data, group=dist.group.WORLD, average: bool = False, device=None) -> torch.Tensor
 
 .. py:function:: all_gather(data, group=dist.group.WORLD, device=None) -> list[torch.Tensor]

_sources/autoapi/core/common/index.rst

@@ -35,6 +35,7 @@ Submodules
    /autoapi/core/common/logger/index
    /autoapi/core/common/profiler_utils/index
    /autoapi/core/common/registry/index
+   /autoapi/core/common/slurm/index
    /autoapi/core/common/test_utils/index
    /autoapi/core/common/transforms/index
    /autoapi/core/common/tutorial_utils/index

_sources/autoapi/core/common/slurm/index.rst

@@ -0,0 +1,19 @@
+core.common.slurm
+=================
+
+.. py:module:: core.common.slurm
+
+
+Functions
+---------
+
+.. autoapisummary::
+
+   core.common.slurm.add_timestamp_id_to_submission_pickle
+
+
+Module Contents
+---------------
+
+.. py:function:: add_timestamp_id_to_submission_pickle(slurm_folder: str, slurm_job_id: str, timestamp_id: str)
+

_sources/autoapi/core/common/utils/index.rst

@@ -66,6 +66,7 @@ Functions
    core.common.utils.new_trainer_context
    core.common.utils._resolve_scale_factor_submodule
    core.common.utils._report_incompat_keys
+   core.common.utils.match_state_dict
    core.common.utils.load_state_dict
    core.common.utils.scatter_det
    core.common.utils.get_commit_hash
@@ -255,6 +256,8 @@ Module Contents
 
 .. py:function:: _report_incompat_keys(model: torch.nn.Module, keys: torch.nn.modules.module._IncompatibleKeys, strict: bool = False) -> tuple[list[str], list[str]]
 
+.. py:function:: match_state_dict(model_state_dict: collections.abc.Mapping[str, torch.Tensor], checkpoint_state_dict: collections.abc.Mapping[str, torch.Tensor]) -> dict
+
 .. py:function:: load_state_dict(module: torch.nn.Module, state_dict: collections.abc.Mapping[str, torch.Tensor], strict: bool = True) -> tuple[list[str], list[str]]
 
 .. py:function:: scatter_det(*args, **kwargs)

_sources/autoapi/core/models/base/index.rst

@@ -21,6 +21,7 @@ Classes
    core.models.base.GraphModelMixin
    core.models.base.HeadInterface
    core.models.base.BackboneInterface
+   core.models.base.HydraInterface
    core.models.base.HydraModel
 
 
@@ -75,7 +76,7 @@ Module Contents
    Mixin Model class implementing some general convenience properties and methods.
 
 
-   .. py:method:: generate_graph(data, cutoff=None, max_neighbors=None, use_pbc=None, otf_graph=None, enforce_max_neighbors_strictly=None)
+   .. py:method:: generate_graph(data, cutoff=None, max_neighbors=None, use_pbc=None, otf_graph=None, enforce_max_neighbors_strictly=None, use_pbc_single=False)
 
 
    .. py:property:: num_params
@@ -123,9 +124,28 @@ Module Contents
 
 
 
+.. py:class:: HydraInterface
+
+   Bases: :py:obj:`abc.ABC`
+
+
+   Helper class that provides a standard way to create an ABC using
+   inheritance.
+
+
+   .. py:method:: get_backbone() -> BackboneInterface
+      :abstractmethod:
+
+
+
+   .. py:method:: get_heads() -> dict[str, HeadInterface]
+      :abstractmethod:
+
+
+
 .. py:class:: HydraModel(backbone: dict, heads: dict, otf_graph: bool = True)
 
-   Bases: :py:obj:`torch.nn.Module`, :py:obj:`GraphModelMixin`
+   Bases: :py:obj:`torch.nn.Module`, :py:obj:`GraphModelMixin`, :py:obj:`HydraInterface`
 
 
    Base class for all neural network modules.
@@ -163,11 +183,13 @@ Module Contents
    .. py:attribute:: otf_graph
 
 
-   .. py:attribute:: backbone_model_name
+   .. py:attribute:: backbone
 
 
-   .. py:attribute:: backbone
-      :type:  BackboneInterface
+   .. py:attribute:: heads
+
+
+   .. py:attribute:: backbone_model_name
 
 
    .. py:attribute:: output_heads
@@ -180,3 +202,9 @@ Module Contents
    .. py:method:: forward(data: torch_geometric.data.Batch)
 
 
+   .. py:method:: get_backbone() -> BackboneInterface
+
+
+   .. py:method:: get_heads() -> dict[str, HeadInterface]
+
+

_sources/autoapi/core/models/dimenet_plus_plus/index.rst

@@ -336,7 +336,7 @@ Module Contents
 
 
 
-.. py:class:: DimeNetPlusPlusWrap(use_pbc: bool = True, regress_forces: bool = True, hidden_channels: int = 128, num_blocks: int = 4, int_emb_size: int = 64, basis_emb_size: int = 8, out_emb_channels: int = 256, num_spherical: int = 7, num_radial: int = 6, otf_graph: bool = False, cutoff: float = 10.0, envelope_exponent: int = 5, num_before_skip: int = 1, num_after_skip: int = 2, num_output_layers: int = 3)
+.. py:class:: DimeNetPlusPlusWrap(use_pbc: bool = True, use_pbc_single: bool = False, regress_forces: bool = True, hidden_channels: int = 128, num_blocks: int = 4, int_emb_size: int = 64, basis_emb_size: int = 8, out_emb_channels: int = 256, num_spherical: int = 7, num_radial: int = 6, otf_graph: bool = False, cutoff: float = 10.0, envelope_exponent: int = 5, num_before_skip: int = 1, num_after_skip: int = 2, num_output_layers: int = 3)
 
    Bases: :py:obj:`DimeNetPlusPlus`, :py:obj:`fairchem.core.models.base.GraphModelMixin`
 
@@ -380,6 +380,9 @@ Module Contents
    .. py:attribute:: use_pbc
 
 
+   .. py:attribute:: use_pbc_single
+
+
    .. py:attribute:: cutoff
 
 
@@ -402,7 +405,7 @@ Module Contents
 
 
 
-.. py:class:: DimeNetPlusPlusWrapBackbone(use_pbc: bool = True, regress_forces: bool = True, hidden_channels: int = 128, num_blocks: int = 4, int_emb_size: int = 64, basis_emb_size: int = 8, out_emb_channels: int = 256, num_spherical: int = 7, num_radial: int = 6, otf_graph: bool = False, cutoff: float = 10.0, envelope_exponent: int = 5, num_before_skip: int = 1, num_after_skip: int = 2, num_output_layers: int = 3)
+.. py:class:: DimeNetPlusPlusWrapBackbone(use_pbc: bool = True, use_pbc_single: bool = False, regress_forces: bool = True, hidden_channels: int = 128, num_blocks: int = 4, int_emb_size: int = 64, basis_emb_size: int = 8, out_emb_channels: int = 256, num_spherical: int = 7, num_radial: int = 6, otf_graph: bool = False, cutoff: float = 10.0, envelope_exponent: int = 5, num_before_skip: int = 1, num_after_skip: int = 2, num_output_layers: int = 3)
 
    Bases: :py:obj:`DimeNetPlusPlusWrap`, :py:obj:`fairchem.core.models.base.BackboneInterface`
 

_sources/autoapi/core/models/equiformer_v2/equiformer_v2/index.rst

@@ -35,7 +35,7 @@ Module Contents
    :value: 23.395238876342773
 
 
-.. py:class:: EquiformerV2(use_pbc: bool = True, regress_forces: bool = True, otf_graph: bool = True, max_neighbors: int = 500, max_radius: float = 5.0, max_num_elements: int = 90, num_layers: int = 12, sphere_channels: int = 128, attn_hidden_channels: int = 128, num_heads: int = 8, attn_alpha_channels: int = 32, attn_value_channels: int = 16, ffn_hidden_channels: int = 512, norm_type: str = 'rms_norm_sh', lmax_list: list[int] | None = None, mmax_list: list[int] | None = None, grid_resolution: int | None = None, num_sphere_samples: int = 128, edge_channels: int = 128, use_atom_edge_embedding: bool = True, share_atom_edge_embedding: bool = False, use_m_share_rad: bool = False, distance_function: str = 'gaussian', num_distance_basis: int = 512, attn_activation: str = 'scaled_silu', use_s2_act_attn: bool = False, use_attn_renorm: bool = True, ffn_activation: str = 'scaled_silu', use_gate_act: bool = False, use_grid_mlp: bool = False, use_sep_s2_act: bool = True, alpha_drop: float = 0.1, drop_path_rate: float = 0.05, proj_drop: float = 0.0, weight_init: str = 'normal', enforce_max_neighbors_strictly: bool = True, avg_num_nodes: float | None = None, avg_degree: float | None = None, use_energy_lin_ref: bool | None = False, load_energy_lin_ref: bool | None = False)
+.. py:class:: EquiformerV2(use_pbc: bool = True, use_pbc_single: bool = False, regress_forces: bool = True, otf_graph: bool = True, max_neighbors: int = 500, max_radius: float = 5.0, max_num_elements: int = 90, num_layers: int = 12, sphere_channels: int = 128, attn_hidden_channels: int = 128, num_heads: int = 8, attn_alpha_channels: int = 32, attn_value_channels: int = 16, ffn_hidden_channels: int = 512, norm_type: str = 'rms_norm_sh', lmax_list: list[int] | None = None, mmax_list: list[int] | None = None, grid_resolution: int | None = None, num_sphere_samples: int = 128, edge_channels: int = 128, use_atom_edge_embedding: bool = True, share_atom_edge_embedding: bool = False, use_m_share_rad: bool = False, distance_function: str = 'gaussian', num_distance_basis: int = 512, attn_activation: str = 'scaled_silu', use_s2_act_attn: bool = False, use_attn_renorm: bool = True, ffn_activation: str = 'scaled_silu', use_gate_act: bool = False, use_grid_mlp: bool = False, use_sep_s2_act: bool = True, alpha_drop: float = 0.1, drop_path_rate: float = 0.05, proj_drop: float = 0.0, weight_init: str = 'normal', enforce_max_neighbors_strictly: bool = True, avg_num_nodes: float | None = None, avg_degree: float | None = None, use_energy_lin_ref: bool | None = False, load_energy_lin_ref: bool | None = False)
 
    Bases: :py:obj:`torch.nn.Module`, :py:obj:`fairchem.core.models.base.GraphModelMixin`
 
@@ -44,6 +44,8 @@ Module Contents
 
    :param use_pbc: Use periodic boundary conditions
    :type use_pbc: bool
+   :param use_pbc_single: Process batch PBC graphs one at a time
+   :type use_pbc_single: bool
    :param regress_forces: Compute forces
    :type regress_forces: bool
    :param otf_graph: Compute graph On The Fly (OTF)
@@ -129,6 +131,9 @@ Module Contents
    .. py:attribute:: use_pbc
 
 
+   .. py:attribute:: use_pbc_single
+
+
    .. py:attribute:: regress_forces
 
 
@@ -326,7 +331,7 @@ Module Contents
 
 
 
-.. py:class:: EquiformerV2Backbone(use_pbc: bool = True, regress_forces: bool = True, otf_graph: bool = True, max_neighbors: int = 500, max_radius: float = 5.0, max_num_elements: int = 90, num_layers: int = 12, sphere_channels: int = 128, attn_hidden_channels: int = 128, num_heads: int = 8, attn_alpha_channels: int = 32, attn_value_channels: int = 16, ffn_hidden_channels: int = 512, norm_type: str = 'rms_norm_sh', lmax_list: list[int] | None = None, mmax_list: list[int] | None = None, grid_resolution: int | None = None, num_sphere_samples: int = 128, edge_channels: int = 128, use_atom_edge_embedding: bool = True, share_atom_edge_embedding: bool = False, use_m_share_rad: bool = False, distance_function: str = 'gaussian', num_distance_basis: int = 512, attn_activation: str = 'scaled_silu', use_s2_act_attn: bool = False, use_attn_renorm: bool = True, ffn_activation: str = 'scaled_silu', use_gate_act: bool = False, use_grid_mlp: bool = False, use_sep_s2_act: bool = True, alpha_drop: float = 0.1, drop_path_rate: float = 0.05, proj_drop: float = 0.0, weight_init: str = 'normal', enforce_max_neighbors_strictly: bool = True, avg_num_nodes: float | None = None, avg_degree: float | None = None, use_energy_lin_ref: bool | None = False, load_energy_lin_ref: bool | None = False)
+.. py:class:: EquiformerV2Backbone(*args, **kwargs)
 
    Bases: :py:obj:`EquiformerV2`, :py:obj:`fairchem.core.models.base.BackboneInterface`
 
@@ -335,6 +340,8 @@ Module Contents
 
    :param use_pbc: Use periodic boundary conditions
    :type use_pbc: bool
+   :param use_pbc_single: Process batch PBC graphs one at a time
+   :type use_pbc_single: bool
    :param regress_forces: Compute forces
    :type regress_forces: bool
    :param otf_graph: Compute graph On The Fly (OTF)
@@ -417,6 +424,16 @@ Module Contents
    :type load_energy_lin_ref: bool
 
 
+   .. py:attribute:: energy_block
+      :value: None
+
+
+
+   .. py:attribute:: force_block
+      :value: None
+
+
+
    .. py:method:: forward(data: torch_geometric.data.batch.Batch) -> dict[str, torch.Tensor]
 
       Backbone forward.