Merge branch 'main' into vv-update-witgen

.github/workflows/build-prover-template.yml

@@ -45,7 +45,7 @@ jobs:
       RUNNER_COMPOSE_FILE: "docker-compose-runner-nightly.yml"
       ERA_BELLMAN_CUDA_RELEASE: ${{ inputs.ERA_BELLMAN_CUDA_RELEASE }}
       CUDA_ARCH: ${{ inputs.CUDA_ARCH }}
-    runs-on: [ matterlabs-ci-runner-high-performance ]
+    runs-on: [matterlabs-ci-runner-high-performance]
     strategy:
       matrix:
         component:
@@ -56,6 +56,7 @@ jobs:
           - prover-fri-gateway
           - prover-job-monitor
           - proof-fri-gpu-compressor
+          - prover-autoscaler
     outputs:
       protocol_version: ${{ steps.protocolversion.outputs.protocol_version }}
     steps:
@@ -91,7 +92,6 @@ jobs:
         run: |
           ci_run run_retried curl -LO https://storage.googleapis.com/matterlabs-setup-keys-us/setup-keys/setup_2\^24.key
 
-
       - name: login to Docker registries
         if: github.event_name != 'pull_request' && (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/'))
         run: |

.github/workflows/ci.yml

@@ -42,6 +42,9 @@ jobs:
               - '!prover/extract-setup-data-keys.sh'
               - 'docker/prover*/**'
               - '.github/workflows/build-prover-template.yml'
+              - '.github/workflows/new-build-prover-template.yml'
+              - '.github/workflows/build-witness-generator-template.yml'
+              - '.github/workflows/new-build-witness-generator-template.yml'
               - '.github/workflows/ci-prover-reusable.yml'
               - 'docker-compose-runner-nightly.yml'
               - '!**/*.md'
@@ -53,7 +56,9 @@ jobs:
               - 'docker/external-node/**'
               - 'docker/server/**'
               - '.github/workflows/build-core-template.yml'
+              - '.github/workflows/new-build-core-template.yml'
               - '.github/workflows/build-contract-verifier-template.yml'
+              - '.github/workflows/new-build-contract-verifier-template.yml'
               - '.github/workflows/ci-core-reusable.yml'
               - '.github/workflows/ci-core-lint-reusable.yml'
               - 'Cargo.toml'

.github/workflows/new-build-contract-verifier-template.yml

@@ -74,36 +74,30 @@ jobs:
           tar -C ./contracts -zxf system-contracts.tar.gz
 
       - name: Install Apt dependencies
+        if: env.BUILD_CONTRACTS == 'true'
         shell: bash
         run: |
           sudo apt-get update && sudo apt-get install -y libssl-dev pkg-config
 
       - name: Install Node
+        if: env.BUILD_CONTRACTS == 'true'
         uses: actions/setup-node@1e60f620b9541d16bece96c5465dc8ee9832be0b # v4.0.3
         with:
           node-version: 20
           cache: 'npm'
 
       - name: Install Yarn
+        if: env.BUILD_CONTRACTS == 'true'
         run: npm install -g yarn
 
       - name: Setup rust
+        if: env.BUILD_CONTRACTS == 'true'
         uses: actions-rust-lang/setup-rust-toolchain@1fbea72663f6d4c03efaab13560c8a24cfd2a7cc # v1.9.0
         with:
           toolchain: nightly-2024-08-01
 
-      - name: Install cargo-nextest from crates.io
-        uses: baptiste0928/cargo-install@904927dbe77864e0f2281519fe9d5bd097a220b3 # v3.1.1
-        with:
-          crate: cargo-nextest
-
-      - name: Install sqlx-cli from crates.io
-        uses: baptiste0928/cargo-install@904927dbe77864e0f2281519fe9d5bd097a220b3 # v3.1.1
-        with:
-          crate: sqlx-cli
-          tag: 0.8.1
-
       - name: Install foundry-zksync
+        if: env.BUILD_CONTRACTS == 'true'
         run: |
           mkdir ./foundry-zksync
           curl -LO https://github.com/matter-labs/foundry-zksync/releases/download/nightly-15bec2f861b3b4c71e58f85e2b2c9dd722585aa8/foundry_nightly_linux_amd64.tar.gz
@@ -112,6 +106,7 @@ jobs:
           echo "$PWD/foundry-zksync" >> $GITHUB_PATH
 
       - name: Pre-download compilers
+        if: env.BUILD_CONTRACTS == 'true'
         shell: bash
         run: |
           # Download needed versions of vyper compiler
@@ -129,18 +124,14 @@ jobs:
             chmod +x "./hardhat-nodejs/compilers-v2/$compiler/${compiler}-v${version}"
           done
 
-      - name: init
-        shell: bash
-        run: |
-          mkdir -p ./volumes/postgres
-          docker compose up -d postgres
-
       - name: Install zkstack
+        if: env.BUILD_CONTRACTS == 'true'
         run: |
           ./zkstack_cli/zkstackup/install --path ./zkstack_cli/zkstackup/zkstackup
           zkstackup --local || true
 
       - name: build contracts
+        if: env.BUILD_CONTRACTS == 'true'
         shell: bash
         run: |
           cp etc/tokens/{test,localhost}.json

.github/workflows/new-build-core-template.yml

@@ -79,36 +79,30 @@ jobs:
           tar -C ./contracts -zxf system-contracts.tar.gz
 
       - name: Install Apt dependencies
+        if: env.BUILD_CONTRACTS == 'true'
         shell: bash
         run: |
           sudo apt-get update && sudo apt-get install -y libssl-dev pkg-config
 
       - name: Install Node
+        if: env.BUILD_CONTRACTS == 'true'
         uses: actions/setup-node@1e60f620b9541d16bece96c5465dc8ee9832be0b # v4.0.3
         with:
           node-version: 20
           cache: 'npm'
 
       - name: Install Yarn
+        if: env.BUILD_CONTRACTS == 'true'
         run: npm install -g yarn
 
       - name: Setup rust
+        if: env.BUILD_CONTRACTS == 'true'
         uses: actions-rust-lang/setup-rust-toolchain@1fbea72663f6d4c03efaab13560c8a24cfd2a7cc # v1.9.0
         with:
           toolchain: nightly-2024-08-01
 
-      - name: Install cargo-nextest from crates.io
-        uses: baptiste0928/cargo-install@904927dbe77864e0f2281519fe9d5bd097a220b3 # v3.1.1
-        with:
-          crate: cargo-nextest
-
-      - name: Install sqlx-cli from crates.io
-        uses: baptiste0928/cargo-install@904927dbe77864e0f2281519fe9d5bd097a220b3 # v3.1.1
-        with:
-          crate: sqlx-cli
-          tag: 0.8.1
-
       - name: Install foundry-zksync
+        if: env.BUILD_CONTRACTS == 'true'
         run: |
           mkdir ./foundry-zksync
           curl -LO https://github.com/matter-labs/foundry-zksync/releases/download/nightly-15bec2f861b3b4c71e58f85e2b2c9dd722585aa8/foundry_nightly_linux_amd64.tar.gz
@@ -117,6 +111,7 @@ jobs:
           echo "$PWD/foundry-zksync" >> $GITHUB_PATH
 
       - name: Pre-download compilers
+        if: env.BUILD_CONTRACTS == 'true'
         shell: bash
         run: |
           # Download needed versions of vyper compiler
@@ -134,18 +129,14 @@ jobs:
             chmod +x "./hardhat-nodejs/compilers-v2/$compiler/${compiler}-v${version}"
           done
 
-      - name: init
-        shell: bash
-        run: |
-          mkdir -p ./volumes/postgres
-          docker compose up -d postgres
-
       - name: Install zkstack
+        if: env.BUILD_CONTRACTS == 'true'
         run: |
           ./zkstack_cli/zkstackup/install --path ./zkstack_cli/zkstackup/zkstackup
           zkstackup --local || true
 
       - name: build contracts
+        if: env.BUILD_CONTRACTS == 'true'
         shell: bash
         run: |
           cp etc/tokens/{test,localhost}.json

Cargo.toml

@@ -35,7 +35,6 @@ members = [
   "core/node/consensus",
   "core/node/contract_verification_server",
   "core/node/api_server",
-  "core/node/tee_verifier_input_producer",
   "core/node/base_token_adjuster",
   "core/node/external_proof_integration_api",
   "core/node/logs_bloom_backfill",
@@ -309,6 +308,5 @@ zksync_node_storage_init = { version = "0.1.0", path = "core/node/node_storage_i
 zksync_node_consensus = { version = "0.1.0", path = "core/node/consensus" }
 zksync_contract_verification_server = { version = "0.1.0", path = "core/node/contract_verification_server" }
 zksync_node_api_server = { version = "0.1.0", path = "core/node/api_server" }
-zksync_tee_verifier_input_producer = { version = "0.1.0", path = "core/node/tee_verifier_input_producer" }
 zksync_base_token_adjuster = { version = "0.1.0", path = "core/node/base_token_adjuster" }
 zksync_logs_bloom_backfill = { version = "0.1.0", path = "core/node/logs_bloom_backfill" }

core/bin/zksync_server/src/main.rs

@@ -45,7 +45,7 @@ struct Cli {
     /// Comma-separated list of components to launch.
     #[arg(
         long,
-        default_value = "api,tree,eth,state_keeper,housekeeper,tee_verifier_input_producer,commitment_generator,da_dispatcher,vm_runner_protective_reads"
+        default_value = "api,tree,eth,state_keeper,housekeeper,commitment_generator,da_dispatcher,vm_runner_protective_reads"
     )]
     components: ComponentsToRun,
     /// Path to the yaml config. If set, it will be used instead of env vars.

core/bin/zksync_server/src/node_builder.rs

@@ -55,7 +55,6 @@ use zksync_node_framework::{
             main_batch_executor::MainBatchExecutorLayer, mempool_io::MempoolIOLayer,
             output_handler::OutputHandlerLayer, RocksdbStorageOptions, StateKeeperLayer,
         },
-        tee_verifier_input_producer::TeeVerifierInputProducerLayer,
         vm_runner::{
             bwip::BasicWitnessInputProducerLayer, playground::VmPlaygroundLayer,
             protective_reads::ProtectiveReadsWriterLayer,
@@ -288,6 +287,7 @@ impl MainNodeBuilder {
         self.node.add_layer(ProofDataHandlerLayer::new(
             try_load_config!(self.configs.proof_data_handler_config),
             self.genesis_config.l1_batch_commit_data_generator_mode,
+            self.genesis_config.l2_chain_id,
         ));
         Ok(self)
     }
@@ -493,14 +493,6 @@ impl MainNodeBuilder {
         Ok(self)
     }
 
-    fn add_tee_verifier_input_producer_layer(mut self) -> anyhow::Result<Self> {
-        self.node.add_layer(TeeVerifierInputProducerLayer::new(
-            self.genesis_config.l2_chain_id,
-        ));
-
-        Ok(self)
-    }
-
     fn add_da_client_layer(mut self) -> anyhow::Result<Self> {
         let Some(da_client_config) = self.configs.da_client_config.clone() else {
             tracing::warn!("No config for DA client, using the NoDA client");
@@ -727,9 +719,6 @@ impl MainNodeBuilder {
                 Component::EthTxManager => {
                     self = self.add_eth_tx_manager_layer()?;
                 }
-                Component::TeeVerifierInputProducer => {
-                    self = self.add_tee_verifier_input_producer_layer()?;
-                }
                 Component::Housekeeper => {
                     self = self
                         .add_house_keeper_layer()?

core/bin/zksync_tee_prover/src/api_client.rs

@@ -1,13 +1,10 @@
-use reqwest::Client;
+use reqwest::{Client, Response, StatusCode};
 use secp256k1::{ecdsa::Signature, PublicKey};
-use serde::{de::DeserializeOwned, Serialize};
+use serde::Serialize;
 use url::Url;
 use zksync_basic_types::H256;
 use zksync_prover_interface::{
-    api::{
-        RegisterTeeAttestationRequest, RegisterTeeAttestationResponse, SubmitTeeProofRequest,
-        SubmitTeeProofResponse, TeeProofGenerationDataRequest, TeeProofGenerationDataResponse,
-    },
+    api::{RegisterTeeAttestationRequest, SubmitTeeProofRequest, TeeProofGenerationDataRequest},
     inputs::TeeVerifierInput,
     outputs::L1BatchTeeProofForL1,
 };
@@ -31,10 +28,9 @@ impl TeeApiClient {
         }
     }
 
-    async fn post<Req, Resp, S>(&self, endpoint: S, request: Req) -> Result<Resp, reqwest::Error>
+    async fn post<Req, S>(&self, endpoint: S, request: Req) -> Result<Response, reqwest::Error>
     where
         Req: Serialize + std::fmt::Debug,
-        Resp: DeserializeOwned,
         S: AsRef<str>,
     {
         let url = self.api_base_url.join(endpoint.as_ref()).unwrap();
@@ -46,9 +42,7 @@ impl TeeApiClient {
             .json(&request)
             .send()
             .await?
-            .error_for_status()?
-            .json::<Resp>()
-            .await
+            .error_for_status()
     }
 
     /// Registers the attestation quote with the TEE prover interface API, effectively proving that
@@ -63,8 +57,7 @@ impl TeeApiClient {
             attestation: attestation_quote_bytes,
             pubkey: public_key.serialize().to_vec(),
         };
-        self.post::<_, RegisterTeeAttestationResponse, _>("/tee/register_attestation", request)
-            .await?;
+        self.post("/tee/register_attestation", request).await?;
         tracing::info!(
             "Attestation quote was successfully registered for the public key {}",
             public_key
@@ -77,12 +70,17 @@ impl TeeApiClient {
     pub async fn get_job(
         &self,
         tee_type: TeeType,
-    ) -> Result<Option<Box<TeeVerifierInput>>, TeeProverError> {
+    ) -> Result<Option<TeeVerifierInput>, TeeProverError> {
         let request = TeeProofGenerationDataRequest { tee_type };
-        let response = self
-            .post::<_, TeeProofGenerationDataResponse, _>("/tee/proof_inputs", request)
-            .await?;
-        Ok(response.0)
+        let response = self.post("/tee/proof_inputs", request).await?;
+        match response.status() {
+            StatusCode::OK => Ok(Some(response.json::<TeeVerifierInput>().await?)),
+            StatusCode::NO_CONTENT => Ok(None),
+            _ => response
+                .json::<Option<TeeVerifierInput>>()
+                .await
+                .map_err(TeeProverError::Request),
+        }
     }
 
     /// Submits the successfully verified proof to the TEE prover interface API.
@@ -101,7 +99,7 @@ impl TeeApiClient {
             tee_type,
         }));
         let observer = METRICS.proof_submitting_time.start();
-        self.post::<_, SubmitTeeProofResponse, _>(
+        self.post(
             format!("/tee/submit_proofs/{batch_number}").as_str(),
             request,
         )