SDK 0.11.0 fixes

README.md

@@ -1,37 +1,47 @@
 # GroqFlow 🚀
 
-GroqFlow™ is the easiest way to get started with Groq's technology. GroqFlow provides an automated workflow for compiling Machine Learning, Artifical Intelligence, and High-Performance Computing workloads into Groq programs and executing those programs on the Groq Language Processing Unit™ (LPU).
+GroqFlow™ is the easiest way to get started with Groq's technology. GroqFlow provides an automated tool flow for compiling machine learning and linear algebra workloads into Groq programs and executing those programs on GroqChip™ processors.
+
+We recommend that your system meets the following hardware requirements:
+
+- To build models: 32GB or more of RAM.
+- To run models: 8 GroqChip processors is recommended, especially for larger models.
 
 ---
 
-## System Requirements
+## Installation Guide
+
+Sign-up on [support.groq.com](https://support.groq.com) to download and install GroqWare™ Suite version >=0.9.2.1.
+
+For installation instructions, please have a look at our [Install Guide](docs/install.md).
+
 
-To begin, we recommend that your system meets the following software and hardware requirements:
+## Getting Started
 
-- Ubuntu 22.04 or Rocky 8.4 Linux distribution.
-- 32GB RAM (or more) to build models.
-- 8 LPUs (especially for larger models) to run models.
-- GroqWare Suite™ version >=0.9.2.1 installation*:
-  - Groq Developer Tools Package (groq-devtools) for building and compiling models.
-  - Groq Runtime Package (groq-runtime) for running compiled models on Groq hardware.
+To Groq a PyTorch model, simply provide your model and inputs to the `groqit()` function. Once `groqit()` has built your model, you can execute your Groq model the same way you execute a PyTorch model.
 
-*For information on how to install GroqWare Suite on your system, create an account on our [portal](https://support.groq.com/) and view the [GroqWare Quick Start Guide](https://support.groq.com/#/downloads/view/groqware-qsg) for installation instructions.
+<img src="https://github.com/groq/groqflow/raw/main/docs/img/groqflow.gif"  width="800"/>
+
+
+`groqit()` also works with ONNX files and provides many customization options. You can find more information about how to use groqit() in our [User Guide](docs/user_guide.md).
 
 ---
 
 ## Navigating GroqFlow
 
-* [Documentation](docs/): All GroqFlow documentation, including the installation guide, user guide, known issues, and versioning.
+* [demo_helpers](demo_helpers/): Scripts used for GroqFlow demos and proof points.
 
-* [Examples](examples/): Includes various GroqFlow examples.
+* [docs](docs/): All information you'd need to be successful with GroqFlow.
 
-* [GroqFlow](groqflow/): The source code for the `groqflow` package.
+* [examples](examples/): Includes various GroqFlow examples.
 
-* [Proof Points](proof_points/): Machine learning proof points using GroqFlow.
+* [groqflow](groqflow/): The source code for the `groqflow` package.
 
-* [README.md](readme.md): This README.
+* [proof_points](proof_points/): Machine learning proof points using GroqFlow.
 
----
+* [readme.md](readme.md): This readme.
+
+* [setup.py](setup.py): GroqFlow setup script for installation.
 
 ## Contributors
 

demo_helpers/demo_helpers/compute_performance.py

@@ -182,24 +182,16 @@ def pytorch_model_inference(dataset, model):
             out = model(**inputs)
 
             if not isinstance(out, torch.Tensor):
-                if isinstance(out, tuple):
-                    if len(out) == 1:
-                        out = out[0]
-                    else:
-                        raise ValueError("Cannot handle tuple with len", len(out))
-                elif isinstance(out, dict):
-                    if "logits" in out:
-                        out = out.logits
-                    elif "start_logits" in out and "end_logits" in out:
-                        out = torch.vstack((out["start_logits"], out["end_logits"]))
-                    elif "last_hidden_state" in out:
-                        out = out.last_hidden_state
-                    else:
-                        raise ValueError(
-                            "Unknown output key. List of keys:", list(out.keys())
-                        )
+                if "logits" in out:
+                    out = out.logits
+                elif "start_logits" in out and "end_logits" in out:
+                    out = torch.vstack((out["start_logits"], out["end_logits"]))
+                elif "last_hidden_state" in out:
+                    out = out.last_hidden_state
                 else:
-                    raise ValueError("Unknown output type", type(out))
+                    raise ValueError(
+                        "Unknown output key. List of keys:", list(out.keys())
+                    )
             pred.append(out)
 
     return dataset.postprocess(pred)

demo_helpers/demo_helpers/dataset.py

@@ -856,7 +856,7 @@ def _download_dataset(self):
 
     def preprocess(self):
         return [
-            {"images": self.coco[i][0].unsqueeze(0).numpy()}
+            {"image_arrays": self.coco[i][0].unsqueeze(0).numpy()}
             for i in range(len(self.coco))
         ]
 

demo_helpers/demo_helpers/model_download.py

@@ -1,23 +1,19 @@
 import os
-import zipfile
 
 from datasets.utils.file_utils import cached_path
 from groqflow.common.build import DEFAULT_CACHE_DIR
 
 
-YOLOV6N_MODEL = "yolov6n_model"
-YOLOV6N_SOURCE = "yolov6n_source"
+YOLOV6N_ONNX = "yolov6n_onnx"
 
 
 DATA_URLS = {
-    YOLOV6N_MODEL: "https://github.com/meituan/YOLOv6/releases/download/0.4.0/yolov6n.pt",
-    YOLOV6N_SOURCE: "https://github.com/meituan/YOLOv6/archive/refs/tags/0.4.0.zip",
+    YOLOV6N_ONNX: "https://github.com/meituan/YOLOv6/releases/download/0.1.0/yolov6n.onnx",
 }
 
 
 DST_PATHS = {
-    YOLOV6N_MODEL: "pytorch_models/yolov6_nano/yolov6n.pt",
-    YOLOV6N_SOURCE: "pytorch_models/yolov6_nano/YOLOv6",
+    YOLOV6N_ONNX: "onnx_models/yolov6n.onnx",
 }
 
 
@@ -31,18 +27,3 @@ def download_model(model):
     download_path = cached_path(url)
     os.symlink(download_path, dst_path)
     return dst_path
-
-
-def download_source(source):
-    dst_path = os.path.join(DEFAULT_CACHE_DIR, DST_PATHS[source])
-    if os.path.exists(dst_path):
-        return dst_path
-
-    os.makedirs(os.path.dirname(dst_path), exist_ok=True)
-    url = DATA_URLS[source]
-    download_path = cached_path(url)
-    with zipfile.ZipFile(download_path, "r") as zip_ref:
-        extracted_dir = os.path.dirname(dst_path)
-        zip_ref.extractall(extracted_dir)
-        os.rename(os.path.join(extracted_dir, zip_ref.infolist()[0].filename), dst_path)
-    return dst_path

demo_helpers/demo_helpers/models.py

@@ -1,18 +1,8 @@
 import os
-import subprocess
-import sys
-
 import torch
 import torch.nn as nn
 import torch.nn.functional as F
 
-from demo_helpers.model_download import (
-    YOLOV6N_MODEL,
-    YOLOV6N_SOURCE,
-    download_model,
-    download_source,
-)
-
 
 class M5(nn.Module):
     def __init__(self, n_input=1, n_output=35, stride=16, n_channel=32):
@@ -140,33 +130,6 @@ def forward(self, input):
         return self.logsoftmax(output)
 
 
-def get_yolov6n_model():
-    weights = download_model(YOLOV6N_MODEL)
-    source = download_source(YOLOV6N_SOURCE)
-    export_script = os.path.join(source, "deploy/ONNX/export_onnx.py")
-
-    cmd = [
-        sys.executable,
-        export_script,
-        "--weights",
-        weights,
-        "--img",
-        "640",
-        "--batch",
-        "1",
-        "--simplify",
-    ]
-    p = subprocess.Popen(
-        cmd, cwd=source, stdout=subprocess.PIPE, stderr=subprocess.PIPE
-    )
-    p.communicate()
-    if p.returncode != 0:
-        raise RuntimeError("Unable to get ONNX model")
-
-    onnx_file = weights.replace(".pt", ".onnx")
-    return onnx_file
-
-
 def load_pretrained(model_name):
     """Loads a pre-trained model
 

demo_helpers/setup.py

@@ -12,13 +12,14 @@
     ),
     install_requires=[
         "charset-normalizer==2.1.0",
+        "torch>=1.12.0",
         "transformers>=4.20.0",
         "datasets>=2.3.2",
         "prettytable>=3.3.0",
         "wget>=3.2",
         "setuptools==57.2.0",
-        "torchvision==0.16.0",
-        "torchaudio==2.1.0",
+        "torchvision>=0.11.3",
+        "torchaudio>=0.12.1",
         "path>=16.4.0",
     ],
     classifiers=[],

docs/readme.md

@@ -1,13 +1,11 @@
 # Documentation
 
-The following are links to GroqFlow documentation:
+1. [Install Guide](install.md): Provides detailed instructions on how to install GroqFlow.
 
-- [Install Guide](install.md): Instructions on how to install GroqFlow.
+2. [Known Issues](known_issues.md): Lists the current known issues that may occur when using GroqFlow.
 
-- [User Guide](user_guide.md): Overview and examples for all of GroqFlow's methods, flags, and options.
+3. [Read Me](readme.md): This readme.
 
-- [Known Issues](known_issues.md): Currently known issues that may occur when using GroqFlow.
+4. [User Guide](user_guide.md): Provides an overview and examples for all of GroqFlow's flags and options.
 
-- [Versioning](versioning.md): Explanation of GroqFlow's versioning scheme.
-
-- [README.md](readme.md): This README.
+5. [Versioning](versioning.md): Explanation of GroqFlow's versioning scheme.

groqflow/common/build.py

@@ -192,8 +192,8 @@ class GroqInfo(of_build.Info):
     estimated_pcie_output_latency: Optional[float] = None
     estimated_throughput: Optional[float] = None
     estimated_latency: Optional[float] = None
-    compiled_model_input_bytes: Optional[int] = None
-    compiled_model_output_bytes: Optional[int] = None
+    compiled_onnx_input_bytes: Optional[int] = None
+    compiled_onnx_output_bytes: Optional[int] = None
     compiler_ram_bytes: Optional[float] = None
 
 

groqflow/common/sdk_helpers.py

@@ -33,11 +33,7 @@ def get_num_chips_available(pci_devices=None):
 
     # Capture the list of pci devices on the system using the linux lspci utility
     if pci_devices is None:
-        pci_devices = (
-            subprocess.check_output([lspci, "-n"], stderr=subprocess.DEVNULL)
-            .decode("utf-8")
-            .split("\n")
-        )
+        pci_devices = subprocess.check_output([lspci, "-n"]).decode("utf-8").split("\n")
 
     # Unique registered vendor id: 1de0, and device id: "0000"
     groq_card_id = "1de0:0000"
@@ -78,11 +74,7 @@ def _installed_package_version(package: str, os_version: OS) -> Union[bool, str]
         # Get package info
         try:
             cmd = ["apt-cache", "policy", package]
-            package_info = (
-                subprocess.check_output(cmd, stderr=subprocess.DEVNULL)
-                .decode("utf-8")
-                .split("\n")
-            )
+            package_info = subprocess.check_output(cmd).decode("utf-8").split("\n")
         except (FileNotFoundError, subprocess.CalledProcessError) as e:
             raise exp.Error("apt-cache policy command failed") from e
 
@@ -97,11 +89,7 @@ def _installed_package_version(package: str, os_version: OS) -> Union[bool, str]
         # Get package info
         cmd = ["dnf", "info", package]
         try:
-            package_info = (
-                subprocess.check_output(cmd, stderr=subprocess.DEVNULL)
-                .decode("utf-8")
-                .split("\n")
-            )
+            package_info = subprocess.check_output(cmd).decode("utf-8").split("\n")
         except FileNotFoundError as e:
             raise exp.Error("dnf info command failed") from e
         except subprocess.CalledProcessError as e:

groqflow/groqmodel/groqmodel.py

@@ -139,13 +139,13 @@ def estimate_performance(self) -> GroqEstimatedPerformance:
 
         # Calculate compute latency and estimate PCIe latency
         self.state.info.estimated_pcie_input_latency = (
-            self.state.info.compiled_model_input_bytes / pcie_bandwidth
+            self.state.info.compiled_onnx_input_bytes / pcie_bandwidth
         ) + pcie_latency
         self.state.info.deterministic_compute_latency = on_chip_compute_cycles / (
             frequency
         )
         self.state.info.estimated_pcie_output_latency = (
-            self.state.info.compiled_model_output_bytes / pcie_bandwidth
+            self.state.info.compiled_onnx_output_bytes / pcie_bandwidth
         ) + pcie_latency
 
         # When pipelined, the reported cycle is the duration of a single pipelining stage

groqflow/justgroqit/compile.py

@@ -40,8 +40,8 @@ def analyze_onnx(state: build.GroqState):
     input_onnx = state.intermediate_results[0]
 
     (
-        state.info.compiled_model_input_bytes,
-        state.info.compiled_model_output_bytes,
+        state.info.compiled_onnx_input_bytes,
+        state.info.compiled_onnx_output_bytes,
     ) = onnx_helpers.io_bytes(input_onnx)
 
     # Count the number of trained model parameters
@@ -51,12 +51,6 @@ def analyze_onnx(state: build.GroqState):
 
 def analyze_torch_script(state: build.GroqState):
     model = torch.jit.load(state.torch_script_file)
-    state.info.compiled_model_input_bytes = sum(
-        t.element_size() for t in state.inputs.values()
-    )
-    outputs = model(**state.inputs)
-    state.info.compiled_model_output_bytes = sum(t.element_size() for t in outputs)
-
     state.info.num_parameters = sum(
         p.numel() for p in model.parameters() if p.requires_grad
     )
@@ -67,7 +61,7 @@ def torch_types_to_str(type: torch.dtype):
         return "f16"
     elif type == torch.float32:
         return "f32"
-    elif type == torch.float64:
+    elif type == torch.flaot64:
         return "f64"
     elif type == torch.uint8:
         return "ui8"
@@ -298,7 +292,7 @@ def fire(self, state: build.GroqState):
             state.intermediate_results = [gten_file]
         else:
             msg = f"""
-            Attempted to use Groq Torch Importer to import TorchSript model into
+            Attempted use Groq Torch Importer to impor TorchSript model into
             Groq's Tensor(GTen) dialect format. However, this operation did not
             succeed. Please contact GroqFlow support to determine a path forwards.
             More information may be available in the log file at **{self.logfile_path}**

groqflow/version.py

@@ -1 +1 @@
-__version__ = "4.3.0"
+__version__ = "4.2.1"

proof_points/computer_vision/deit/deit_tiny.py

@@ -15,9 +15,7 @@
 
 def evaluate_deit_tiny(rebuild_policy=None, should_execute=True):
     # load torch model
-    model = ViTForImageClassification.from_pretrained(
-        "facebook/deit-tiny-patch16-224", torchscript=True
-    )
+    model = ViTForImageClassification.from_pretrained("facebook/deit-tiny-patch16-224")
     model.eval()
 
     # create dummy inputs to prime groq model
@@ -28,13 +26,12 @@ def evaluate_deit_tiny(rebuild_policy=None, should_execute=True):
 
     # compute performance on CPU and GroqChip
     if should_execute:
-        compute_performance(
+        return compute_performance(
             groq_model,
             model,
             dataset="sampled_imagenet",
             task="classification",
         )
-    print(f"Proof point {__file__} finished!")
 
 
 if __name__ == "__main__":

proof_points/computer_vision/googlenet/googlenet.py

@@ -40,8 +40,6 @@ def evaluate_googlenet(rebuild_policy=None, should_execute=None):
             groq_model, torch_model, "sampled_imagenet", task="classification"
         )
 
-    print(f"Proof point {__file__} finished!")
-
 
 if __name__ == "__main__":
     evaluate_googlenet(**parse_args())

proof_points/computer_vision/mobilenetv2/mobilenetv2.py

@@ -43,8 +43,6 @@ def evaluate_mobilenetv2(rebuild_policy=None, should_execute=None):
             groq_model, torch_model, "sampled_imagenet", task="classification"
         )
 
-    print(f"Proof point {__file__} finished!")
-
 
 if __name__ == "__main__":
     evaluate_mobilenetv2(**parse_args())