From d5d9e650b1e1563aa6d940fe16be8f8e59b92dda Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Thu, 12 Dec 2024 00:20:13 +0000
Subject: [PATCH] Bump golang.org/x/crypto from 0.25.0 to 0.31.0

Bumps [golang.org/x/crypto](https://github.com/golang/crypto) from 0.25.0 to 0.31.0.
- [Commits](https://github.com/golang/crypto/compare/v0.25.0...v0.31.0)

---
updated-dependencies:
- dependency-name: golang.org/x/crypto
  dependency-type: indirect
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 go.mod                                        |    10 +-
 go.sum                                        |    20 +-
 vendor/golang.org/x/crypto/LICENSE            |     4 +-
 .../x/crypto/chacha20/chacha_noasm.go         |     2 +-
 .../{chacha_ppc64le.go => chacha_ppc64x.go}   |     2 +-
 .../{chacha_ppc64le.s => chacha_ppc64x.s}     |   114 +-
 .../chacha20poly1305/chacha20poly1305_amd64.s | 11503 +++++++++++++---
 .../x/crypto/internal/poly1305/mac_noasm.go   |     2 +-
 .../x/crypto/internal/poly1305/sum_amd64.s    |   133 +-
 .../{sum_ppc64le.go => sum_ppc64x.go}         |     2 +-
 .../poly1305/{sum_ppc64le.s => sum_ppc64x.s}  |    30 +-
 .../x/crypto/salsa20/salsa/salsa20_amd64.s    |  1742 +--
 vendor/golang.org/x/crypto/ssh/client_auth.go |     5 +
 vendor/golang.org/x/crypto/ssh/keys.go        |    44 +-
 vendor/golang.org/x/crypto/ssh/server.go      |    19 +-
 vendor/golang.org/x/sync/LICENSE              |     4 +-
 vendor/golang.org/x/sys/LICENSE               |     4 +-
 .../golang.org/x/sys/cpu/asm_darwin_x86_gc.s  |    17 +
 vendor/golang.org/x/sys/cpu/cpu.go            |    21 +
 vendor/golang.org/x/sys/cpu/cpu_arm64.go      |    12 +
 vendor/golang.org/x/sys/cpu/cpu_darwin_x86.go |    61 +
 vendor/golang.org/x/sys/cpu/cpu_gc_x86.go     |     4 +-
 .../x/sys/cpu/{cpu_x86.s => cpu_gc_x86.s}     |     2 +-
 vendor/golang.org/x/sys/cpu/cpu_gccgo_x86.go  |     6 -
 .../golang.org/x/sys/cpu/cpu_linux_arm64.go   |     4 +
 .../golang.org/x/sys/cpu/cpu_linux_noinit.go  |     2 +-
 .../golang.org/x/sys/cpu/cpu_linux_riscv64.go |   137 +
 vendor/golang.org/x/sys/cpu/cpu_other_x86.go  |    11 +
 vendor/golang.org/x/sys/cpu/cpu_riscv64.go    |    11 +-
 vendor/golang.org/x/sys/cpu/cpu_x86.go        |     6 +-
 .../x/sys/cpu/syscall_darwin_x86_gc.go        |    98 +
 vendor/golang.org/x/sys/unix/README.md        |     2 +-
 vendor/golang.org/x/sys/unix/ioctl_linux.go   |    96 +
 vendor/golang.org/x/sys/unix/mkerrors.sh      |    18 +-
 vendor/golang.org/x/sys/unix/syscall_aix.go   |     2 +-
 .../golang.org/x/sys/unix/syscall_darwin.go   |    49 +
 vendor/golang.org/x/sys/unix/syscall_hurd.go  |     1 +
 vendor/golang.org/x/sys/unix/syscall_linux.go |    65 +-
 .../x/sys/unix/syscall_linux_arm64.go         |     2 +
 .../x/sys/unix/syscall_linux_loong64.go       |     2 +
 .../x/sys/unix/syscall_linux_riscv64.go       |     2 +
 .../golang.org/x/sys/unix/syscall_openbsd.go  |     1 +
 .../x/sys/unix/syscall_zos_s390x.go           |   104 +-
 .../golang.org/x/sys/unix/vgetrandom_linux.go |    13 +
 .../x/sys/unix/vgetrandom_unsupported.go      |    11 +
 .../x/sys/unix/zerrors_darwin_amd64.go        |    12 +
 .../x/sys/unix/zerrors_darwin_arm64.go        |    12 +
 vendor/golang.org/x/sys/unix/zerrors_linux.go |    82 +-
 .../x/sys/unix/zerrors_linux_386.go           |    27 +
 .../x/sys/unix/zerrors_linux_amd64.go         |    27 +
 .../x/sys/unix/zerrors_linux_arm.go           |    27 +
 .../x/sys/unix/zerrors_linux_arm64.go         |    28 +
 .../x/sys/unix/zerrors_linux_loong64.go       |    27 +
 .../x/sys/unix/zerrors_linux_mips.go          |    27 +
 .../x/sys/unix/zerrors_linux_mips64.go        |    27 +
 .../x/sys/unix/zerrors_linux_mips64le.go      |    27 +
 .../x/sys/unix/zerrors_linux_mipsle.go        |    27 +
 .../x/sys/unix/zerrors_linux_ppc.go           |    27 +
 .../x/sys/unix/zerrors_linux_ppc64.go         |    27 +
 .../x/sys/unix/zerrors_linux_ppc64le.go       |    27 +
 .../x/sys/unix/zerrors_linux_riscv64.go       |    27 +
 .../x/sys/unix/zerrors_linux_s390x.go         |    27 +
 .../x/sys/unix/zerrors_linux_sparc64.go       |    27 +
 .../x/sys/unix/zerrors_zos_s390x.go           |     2 +
 .../x/sys/unix/zsyscall_darwin_amd64.go       |    68 +
 .../x/sys/unix/zsyscall_darwin_amd64.s        |    15 +
 .../x/sys/unix/zsyscall_darwin_arm64.go       |    68 +
 .../x/sys/unix/zsyscall_darwin_arm64.s        |    15 +
 .../golang.org/x/sys/unix/zsyscall_linux.go   |    43 +-
 .../x/sys/unix/zsyscall_openbsd_386.go        |    24 +
 .../x/sys/unix/zsyscall_openbsd_386.s         |     5 +
 .../x/sys/unix/zsyscall_openbsd_amd64.go      |    24 +
 .../x/sys/unix/zsyscall_openbsd_amd64.s       |     5 +
 .../x/sys/unix/zsyscall_openbsd_arm.go        |    24 +
 .../x/sys/unix/zsyscall_openbsd_arm.s         |     5 +
 .../x/sys/unix/zsyscall_openbsd_arm64.go      |    24 +
 .../x/sys/unix/zsyscall_openbsd_arm64.s       |     5 +
 .../x/sys/unix/zsyscall_openbsd_mips64.go     |    24 +
 .../x/sys/unix/zsyscall_openbsd_mips64.s      |     5 +
 .../x/sys/unix/zsyscall_openbsd_ppc64.go      |    24 +
 .../x/sys/unix/zsyscall_openbsd_ppc64.s       |     6 +
 .../x/sys/unix/zsyscall_openbsd_riscv64.go    |    24 +
 .../x/sys/unix/zsyscall_openbsd_riscv64.s     |     5 +
 .../x/sys/unix/zsysnum_linux_386.go           |     1 +
 .../x/sys/unix/zsysnum_linux_amd64.go         |     2 +
 .../x/sys/unix/zsysnum_linux_arm.go           |     1 +
 .../x/sys/unix/zsysnum_linux_arm64.go         |     3 +-
 .../x/sys/unix/zsysnum_linux_loong64.go       |     3 +
 .../x/sys/unix/zsysnum_linux_mips.go          |     1 +
 .../x/sys/unix/zsysnum_linux_mips64.go        |     1 +
 .../x/sys/unix/zsysnum_linux_mips64le.go      |     1 +
 .../x/sys/unix/zsysnum_linux_mipsle.go        |     1 +
 .../x/sys/unix/zsysnum_linux_ppc.go           |     1 +
 .../x/sys/unix/zsysnum_linux_ppc64.go         |     1 +
 .../x/sys/unix/zsysnum_linux_ppc64le.go       |     1 +
 .../x/sys/unix/zsysnum_linux_riscv64.go       |     3 +-
 .../x/sys/unix/zsysnum_linux_s390x.go         |     1 +
 .../x/sys/unix/zsysnum_linux_sparc64.go       |     1 +
 .../x/sys/unix/ztypes_darwin_amd64.go         |    73 +
 .../x/sys/unix/ztypes_darwin_arm64.go         |    73 +
 .../x/sys/unix/ztypes_freebsd_386.go          |     1 +
 .../x/sys/unix/ztypes_freebsd_amd64.go        |     1 +
 .../x/sys/unix/ztypes_freebsd_arm.go          |     1 +
 .../x/sys/unix/ztypes_freebsd_arm64.go        |     1 +
 .../x/sys/unix/ztypes_freebsd_riscv64.go      |     1 +
 vendor/golang.org/x/sys/unix/ztypes_linux.go  |   230 +-
 .../x/sys/unix/ztypes_linux_riscv64.go        |    33 +
 .../golang.org/x/sys/unix/ztypes_zos_s390x.go |     6 +
 .../golang.org/x/sys/windows/dll_windows.go   |     2 +-
 .../x/sys/windows/security_windows.go         |     2 +-
 .../x/sys/windows/syscall_windows.go          |    52 +-
 .../golang.org/x/sys/windows/types_windows.go |   199 +-
 .../x/sys/windows/zsyscall_windows.go         |   158 +-
 vendor/golang.org/x/term/LICENSE              |     4 +-
 vendor/golang.org/x/term/README.md            |    11 +-
 vendor/golang.org/x/term/term_windows.go      |     1 +
 vendor/golang.org/x/text/LICENSE              |     4 +-
 .../x/text/internal/catmsg/codec.go           |     2 +-
 vendor/modules.txt                            |    10 +-
 119 files changed, 12831 insertions(+), 3383 deletions(-)
 rename vendor/golang.org/x/crypto/chacha20/{chacha_ppc64le.go => chacha_ppc64x.go} (89%)
 rename vendor/golang.org/x/crypto/chacha20/{chacha_ppc64le.s => chacha_ppc64x.s} (76%)
 rename vendor/golang.org/x/crypto/internal/poly1305/{sum_ppc64le.go => sum_ppc64x.go} (95%)
 rename vendor/golang.org/x/crypto/internal/poly1305/{sum_ppc64le.s => sum_ppc64x.s} (89%)
 create mode 100644 vendor/golang.org/x/sys/cpu/asm_darwin_x86_gc.s
 create mode 100644 vendor/golang.org/x/sys/cpu/cpu_darwin_x86.go
 rename vendor/golang.org/x/sys/cpu/{cpu_x86.s => cpu_gc_x86.s} (94%)
 create mode 100644 vendor/golang.org/x/sys/cpu/cpu_linux_riscv64.go
 create mode 100644 vendor/golang.org/x/sys/cpu/cpu_other_x86.go
 create mode 100644 vendor/golang.org/x/sys/cpu/syscall_darwin_x86_gc.go
 create mode 100644 vendor/golang.org/x/sys/unix/vgetrandom_linux.go
 create mode 100644 vendor/golang.org/x/sys/unix/vgetrandom_unsupported.go

diff --git a/go.mod b/go.mod
index 22253cb96..44be8a3db 100644
--- a/go.mod
+++ b/go.mod
@@ -123,12 +123,12 @@ require (
 	go.uber.org/multierr v1.11.0 // indirect
 	go.uber.org/zap v1.27.0 // indirect
 	go4.org v0.0.0-20230225012048-214862532bf5 // indirect
-	golang.org/x/crypto v0.25.0 // indirect
+	golang.org/x/crypto v0.31.0 // indirect
 	golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 // indirect
-	golang.org/x/sync v0.7.0 // indirect
-	golang.org/x/sys v0.22.0 // indirect
-	golang.org/x/term v0.22.0 // indirect
-	golang.org/x/text v0.16.0 // indirect
+	golang.org/x/sync v0.10.0 // indirect
+	golang.org/x/sys v0.28.0 // indirect
+	golang.org/x/term v0.27.0 // indirect
+	golang.org/x/text v0.21.0 // indirect
 	golang.org/x/tools v0.23.0 // indirect
 	gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect
 	google.golang.org/genproto v0.0.0-20240730163845-b1a4ccb954bf // indirect
diff --git a/go.sum b/go.sum
index c4a25ce0b..2526dba33 100755
--- a/go.sum
+++ b/go.sum
@@ -367,8 +367,8 @@ golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8U
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
-golang.org/x/crypto v0.25.0 h1:ypSNr+bnYL2YhwoMt2zPxHFmbAN1KZs/njMG3hxUp30=
-golang.org/x/crypto v0.25.0/go.mod h1:T+wALwcMOSE0kXgUAnPAHqTLW+XHgcELELW8VaDgm/M=
+golang.org/x/crypto v0.31.0 h1:ihbySMvVjLAeSH1IbfcRTkD/iNscyz8rGzjF/E5hV6U=
+golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
@@ -436,8 +436,8 @@ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M=
-golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
+golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ=
+golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -457,13 +457,13 @@ golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI=
-golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA=
+golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
-golang.org/x/term v0.22.0 h1:BbsgPEJULsl2fV/AT3v15Mjva5yXKQDyKf+TbDz7QJk=
-golang.org/x/term v0.22.0/go.mod h1:F3qCibpT5AMpCRfhfT53vVJwhLtIVHhB9XDjfFvnMI4=
+golang.org/x/term v0.27.0 h1:WP60Sv1nlK1T6SupCHbXzSaN0b9wUmsPoRS9b61A23Q=
+golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM=
 golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@@ -471,8 +471,8 @@ golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
 golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
-golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4=
-golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI=
+golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo=
+golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ=
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.6.0 h1:eTDhh4ZXt5Qf0augr54TN6suAUudPcawVZeIAPU7D4U=
diff --git a/vendor/golang.org/x/crypto/LICENSE b/vendor/golang.org/x/crypto/LICENSE
index 6a66aea5e..2a7cf70da 100644
--- a/vendor/golang.org/x/crypto/LICENSE
+++ b/vendor/golang.org/x/crypto/LICENSE
@@ -1,4 +1,4 @@
-Copyright (c) 2009 The Go Authors. All rights reserved.
+Copyright 2009 The Go Authors.
 
 Redistribution and use in source and binary forms, with or without
 modification, are permitted provided that the following conditions are
@@ -10,7 +10,7 @@ notice, this list of conditions and the following disclaimer.
 copyright notice, this list of conditions and the following disclaimer
 in the documentation and/or other materials provided with the
 distribution.
-   * Neither the name of Google Inc. nor the names of its
+   * Neither the name of Google LLC nor the names of its
 contributors may be used to endorse or promote products derived from
 this software without specific prior written permission.
 
diff --git a/vendor/golang.org/x/crypto/chacha20/chacha_noasm.go b/vendor/golang.org/x/crypto/chacha20/chacha_noasm.go
index db42e6676..c709b7284 100644
--- a/vendor/golang.org/x/crypto/chacha20/chacha_noasm.go
+++ b/vendor/golang.org/x/crypto/chacha20/chacha_noasm.go
@@ -2,7 +2,7 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-//go:build (!arm64 && !s390x && !ppc64le) || !gc || purego
+//go:build (!arm64 && !s390x && !ppc64 && !ppc64le) || !gc || purego
 
 package chacha20
 
diff --git a/vendor/golang.org/x/crypto/chacha20/chacha_ppc64le.go b/vendor/golang.org/x/crypto/chacha20/chacha_ppc64x.go
similarity index 89%
rename from vendor/golang.org/x/crypto/chacha20/chacha_ppc64le.go
rename to vendor/golang.org/x/crypto/chacha20/chacha_ppc64x.go
index 3a4287f99..bd183d9ba 100644
--- a/vendor/golang.org/x/crypto/chacha20/chacha_ppc64le.go
+++ b/vendor/golang.org/x/crypto/chacha20/chacha_ppc64x.go
@@ -2,7 +2,7 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-//go:build gc && !purego
+//go:build gc && !purego && (ppc64 || ppc64le)
 
 package chacha20
 
diff --git a/vendor/golang.org/x/crypto/chacha20/chacha_ppc64le.s b/vendor/golang.org/x/crypto/chacha20/chacha_ppc64x.s
similarity index 76%
rename from vendor/golang.org/x/crypto/chacha20/chacha_ppc64le.s
rename to vendor/golang.org/x/crypto/chacha20/chacha_ppc64x.s
index c672ccf69..a660b4112 100644
--- a/vendor/golang.org/x/crypto/chacha20/chacha_ppc64le.s
+++ b/vendor/golang.org/x/crypto/chacha20/chacha_ppc64x.s
@@ -19,7 +19,7 @@
 // The differences in this and the original implementation are
 // due to the calling conventions and initialization of constants.
 
-//go:build gc && !purego
+//go:build gc && !purego && (ppc64 || ppc64le)
 
 #include "textflag.h"
 
@@ -36,32 +36,68 @@
 // for VPERMXOR
 #define MASK  R18
 
-DATA consts<>+0x00(SB)/8, $0x3320646e61707865
-DATA consts<>+0x08(SB)/8, $0x6b20657479622d32
-DATA consts<>+0x10(SB)/8, $0x0000000000000001
-DATA consts<>+0x18(SB)/8, $0x0000000000000000
-DATA consts<>+0x20(SB)/8, $0x0000000000000004
-DATA consts<>+0x28(SB)/8, $0x0000000000000000
-DATA consts<>+0x30(SB)/8, $0x0a0b08090e0f0c0d
-DATA consts<>+0x38(SB)/8, $0x0203000106070405
-DATA consts<>+0x40(SB)/8, $0x090a0b080d0e0f0c
-DATA consts<>+0x48(SB)/8, $0x0102030005060704
-DATA consts<>+0x50(SB)/8, $0x6170786561707865
-DATA consts<>+0x58(SB)/8, $0x6170786561707865
-DATA consts<>+0x60(SB)/8, $0x3320646e3320646e
-DATA consts<>+0x68(SB)/8, $0x3320646e3320646e
-DATA consts<>+0x70(SB)/8, $0x79622d3279622d32
-DATA consts<>+0x78(SB)/8, $0x79622d3279622d32
-DATA consts<>+0x80(SB)/8, $0x6b2065746b206574
-DATA consts<>+0x88(SB)/8, $0x6b2065746b206574
-DATA consts<>+0x90(SB)/8, $0x0000000100000000
-DATA consts<>+0x98(SB)/8, $0x0000000300000002
-DATA consts<>+0xa0(SB)/8, $0x5566774411223300
-DATA consts<>+0xa8(SB)/8, $0xddeeffcc99aabb88
-DATA consts<>+0xb0(SB)/8, $0x6677445522330011
-DATA consts<>+0xb8(SB)/8, $0xeeffccddaabb8899
+DATA consts<>+0x00(SB)/4, $0x61707865
+DATA consts<>+0x04(SB)/4, $0x3320646e
+DATA consts<>+0x08(SB)/4, $0x79622d32
+DATA consts<>+0x0c(SB)/4, $0x6b206574
+DATA consts<>+0x10(SB)/4, $0x00000001
+DATA consts<>+0x14(SB)/4, $0x00000000
+DATA consts<>+0x18(SB)/4, $0x00000000
+DATA consts<>+0x1c(SB)/4, $0x00000000
+DATA consts<>+0x20(SB)/4, $0x00000004
+DATA consts<>+0x24(SB)/4, $0x00000000
+DATA consts<>+0x28(SB)/4, $0x00000000
+DATA consts<>+0x2c(SB)/4, $0x00000000
+DATA consts<>+0x30(SB)/4, $0x0e0f0c0d
+DATA consts<>+0x34(SB)/4, $0x0a0b0809
+DATA consts<>+0x38(SB)/4, $0x06070405
+DATA consts<>+0x3c(SB)/4, $0x02030001
+DATA consts<>+0x40(SB)/4, $0x0d0e0f0c
+DATA consts<>+0x44(SB)/4, $0x090a0b08
+DATA consts<>+0x48(SB)/4, $0x05060704
+DATA consts<>+0x4c(SB)/4, $0x01020300
+DATA consts<>+0x50(SB)/4, $0x61707865
+DATA consts<>+0x54(SB)/4, $0x61707865
+DATA consts<>+0x58(SB)/4, $0x61707865
+DATA consts<>+0x5c(SB)/4, $0x61707865
+DATA consts<>+0x60(SB)/4, $0x3320646e
+DATA consts<>+0x64(SB)/4, $0x3320646e
+DATA consts<>+0x68(SB)/4, $0x3320646e
+DATA consts<>+0x6c(SB)/4, $0x3320646e
+DATA consts<>+0x70(SB)/4, $0x79622d32
+DATA consts<>+0x74(SB)/4, $0x79622d32
+DATA consts<>+0x78(SB)/4, $0x79622d32
+DATA consts<>+0x7c(SB)/4, $0x79622d32
+DATA consts<>+0x80(SB)/4, $0x6b206574
+DATA consts<>+0x84(SB)/4, $0x6b206574
+DATA consts<>+0x88(SB)/4, $0x6b206574
+DATA consts<>+0x8c(SB)/4, $0x6b206574
+DATA consts<>+0x90(SB)/4, $0x00000000
+DATA consts<>+0x94(SB)/4, $0x00000001
+DATA consts<>+0x98(SB)/4, $0x00000002
+DATA consts<>+0x9c(SB)/4, $0x00000003
+DATA consts<>+0xa0(SB)/4, $0x11223300
+DATA consts<>+0xa4(SB)/4, $0x55667744
+DATA consts<>+0xa8(SB)/4, $0x99aabb88
+DATA consts<>+0xac(SB)/4, $0xddeeffcc
+DATA consts<>+0xb0(SB)/4, $0x22330011
+DATA consts<>+0xb4(SB)/4, $0x66774455
+DATA consts<>+0xb8(SB)/4, $0xaabb8899
+DATA consts<>+0xbc(SB)/4, $0xeeffccdd
 GLOBL consts<>(SB), RODATA, $0xc0
 
+#ifdef GOARCH_ppc64
+#define BE_XXBRW_INIT() \
+		LVSL (R0)(R0), V24 \
+		VSPLTISB $3, V25   \
+		VXOR V24, V25, V24 \
+
+#define BE_XXBRW(vr) VPERM vr, vr, V24, vr
+#else
+#define BE_XXBRW_INIT()
+#define BE_XXBRW(vr)
+#endif
+
 //func chaCha20_ctr32_vsx(out, inp *byte, len int, key *[8]uint32, counter *uint32)
 TEXT ·chaCha20_ctr32_vsx(SB),NOSPLIT,$64-40
 	MOVD out+0(FP), OUT
@@ -94,6 +130,8 @@ TEXT ·chaCha20_ctr32_vsx(SB),NOSPLIT,$64-40
 	// Clear V27
 	VXOR V27, V27, V27
 
+	BE_XXBRW_INIT()
+
 	// V28
 	LXVW4X (CONSTBASE)(R11), VS60
 
@@ -299,6 +337,11 @@ loop_vsx:
 	VADDUWM V8, V18, V8
 	VADDUWM V12, V19, V12
 
+	BE_XXBRW(V0)
+	BE_XXBRW(V4)
+	BE_XXBRW(V8)
+	BE_XXBRW(V12)
+
 	CMPU LEN, $64
 	BLT tail_vsx
 
@@ -327,6 +370,11 @@ loop_vsx:
 	VADDUWM V9, V18, V8
 	VADDUWM V13, V19, V12
 
+	BE_XXBRW(V0)
+	BE_XXBRW(V4)
+	BE_XXBRW(V8)
+	BE_XXBRW(V12)
+
 	CMPU  LEN, $64
 	BLT   tail_vsx
 
@@ -334,8 +382,8 @@ loop_vsx:
 	LXVW4X (INP)(R8), VS60
 	LXVW4X (INP)(R9), VS61
 	LXVW4X (INP)(R10), VS62
-	VXOR   V27, V0, V27
 
+	VXOR V27, V0, V27
 	VXOR V28, V4, V28
 	VXOR V29, V8, V29
 	VXOR V30, V12, V30
@@ -354,6 +402,11 @@ loop_vsx:
 	VADDUWM V10, V18, V8
 	VADDUWM V14, V19, V12
 
+	BE_XXBRW(V0)
+	BE_XXBRW(V4)
+	BE_XXBRW(V8)
+	BE_XXBRW(V12)
+
 	CMPU LEN, $64
 	BLT  tail_vsx
 
@@ -381,6 +434,11 @@ loop_vsx:
 	VADDUWM V11, V18, V8
 	VADDUWM V15, V19, V12
 
+	BE_XXBRW(V0)
+	BE_XXBRW(V4)
+	BE_XXBRW(V8)
+	BE_XXBRW(V12)
+
 	CMPU  LEN, $64
 	BLT   tail_vsx
 
@@ -408,9 +466,9 @@ loop_vsx:
 
 done_vsx:
 	// Increment counter by number of 64 byte blocks
-	MOVD (CNT), R14
+	MOVWZ (CNT), R14
 	ADD  BLOCKS, R14
-	MOVD R14, (CNT)
+	MOVWZ R14, (CNT)
 	RET
 
 tail_vsx:
diff --git a/vendor/golang.org/x/crypto/chacha20poly1305/chacha20poly1305_amd64.s b/vendor/golang.org/x/crypto/chacha20poly1305/chacha20poly1305_amd64.s
index 731d2ac6d..fd5ee845f 100644
--- a/vendor/golang.org/x/crypto/chacha20poly1305/chacha20poly1305_amd64.s
+++ b/vendor/golang.org/x/crypto/chacha20poly1305/chacha20poly1305_amd64.s
@@ -1,2715 +1,9762 @@
-// Copyright 2016 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-// This file was originally from https://golang.org/cl/24717 by Vlad Krasnov of CloudFlare.
+// Code generated by command: go run chacha20poly1305_amd64_asm.go -out ../chacha20poly1305_amd64.s -pkg chacha20poly1305. DO NOT EDIT.
 
 //go:build gc && !purego
 
 #include "textflag.h"
-// General register allocation
-#define oup DI
-#define inp SI
-#define inl BX
-#define adp CX // free to reuse, after we hash the additional data
-#define keyp R8 // free to reuse, when we copy the key to stack
-#define itr2 R9 // general iterator
-#define itr1 CX // general iterator
-#define acc0 R10
-#define acc1 R11
-#define acc2 R12
-#define t0 R13
-#define t1 R14
-#define t2 R15
-#define t3 R8
-// Register and stack allocation for the SSE code
-#define rStore (0*16)(BP)
-#define sStore (1*16)(BP)
-#define state1Store (2*16)(BP)
-#define state2Store (3*16)(BP)
-#define tmpStore (4*16)(BP)
-#define ctr0Store (5*16)(BP)
-#define ctr1Store (6*16)(BP)
-#define ctr2Store (7*16)(BP)
-#define ctr3Store (8*16)(BP)
-#define A0 X0
-#define A1 X1
-#define A2 X2
-#define B0 X3
-#define B1 X4
-#define B2 X5
-#define C0 X6
-#define C1 X7
-#define C2 X8
-#define D0 X9
-#define D1 X10
-#define D2 X11
-#define T0 X12
-#define T1 X13
-#define T2 X14
-#define T3 X15
-#define A3 T0
-#define B3 T1
-#define C3 T2
-#define D3 T3
-// Register and stack allocation for the AVX2 code
-#define rsStoreAVX2 (0*32)(BP)
-#define state1StoreAVX2 (1*32)(BP)
-#define state2StoreAVX2 (2*32)(BP)
-#define ctr0StoreAVX2 (3*32)(BP)
-#define ctr1StoreAVX2 (4*32)(BP)
-#define ctr2StoreAVX2 (5*32)(BP)
-#define ctr3StoreAVX2 (6*32)(BP)
-#define tmpStoreAVX2 (7*32)(BP) // 256 bytes on stack
-#define AA0 Y0
-#define AA1 Y5
-#define AA2 Y6
-#define AA3 Y7
-#define BB0 Y14
-#define BB1 Y9
-#define BB2 Y10
-#define BB3 Y11
-#define CC0 Y12
-#define CC1 Y13
-#define CC2 Y8
-#define CC3 Y15
-#define DD0 Y4
-#define DD1 Y1
-#define DD2 Y2
-#define DD3 Y3
-#define TT0 DD3
-#define TT1 AA3
-#define TT2 BB3
-#define TT3 CC3
-// ChaCha20 constants
-DATA ·chacha20Constants<>+0x00(SB)/4, $0x61707865
-DATA ·chacha20Constants<>+0x04(SB)/4, $0x3320646e
-DATA ·chacha20Constants<>+0x08(SB)/4, $0x79622d32
-DATA ·chacha20Constants<>+0x0c(SB)/4, $0x6b206574
-DATA ·chacha20Constants<>+0x10(SB)/4, $0x61707865
-DATA ·chacha20Constants<>+0x14(SB)/4, $0x3320646e
-DATA ·chacha20Constants<>+0x18(SB)/4, $0x79622d32
-DATA ·chacha20Constants<>+0x1c(SB)/4, $0x6b206574
-// <<< 16 with PSHUFB
-DATA ·rol16<>+0x00(SB)/8, $0x0504070601000302
-DATA ·rol16<>+0x08(SB)/8, $0x0D0C0F0E09080B0A
-DATA ·rol16<>+0x10(SB)/8, $0x0504070601000302
-DATA ·rol16<>+0x18(SB)/8, $0x0D0C0F0E09080B0A
-// <<< 8 with PSHUFB
-DATA ·rol8<>+0x00(SB)/8, $0x0605040702010003
-DATA ·rol8<>+0x08(SB)/8, $0x0E0D0C0F0A09080B
-DATA ·rol8<>+0x10(SB)/8, $0x0605040702010003
-DATA ·rol8<>+0x18(SB)/8, $0x0E0D0C0F0A09080B
-
-DATA ·avx2InitMask<>+0x00(SB)/8, $0x0
-DATA ·avx2InitMask<>+0x08(SB)/8, $0x0
-DATA ·avx2InitMask<>+0x10(SB)/8, $0x1
-DATA ·avx2InitMask<>+0x18(SB)/8, $0x0
-
-DATA ·avx2IncMask<>+0x00(SB)/8, $0x2
-DATA ·avx2IncMask<>+0x08(SB)/8, $0x0
-DATA ·avx2IncMask<>+0x10(SB)/8, $0x2
-DATA ·avx2IncMask<>+0x18(SB)/8, $0x0
-// Poly1305 key clamp
-DATA ·polyClampMask<>+0x00(SB)/8, $0x0FFFFFFC0FFFFFFF
-DATA ·polyClampMask<>+0x08(SB)/8, $0x0FFFFFFC0FFFFFFC
-DATA ·polyClampMask<>+0x10(SB)/8, $0xFFFFFFFFFFFFFFFF
-DATA ·polyClampMask<>+0x18(SB)/8, $0xFFFFFFFFFFFFFFFF
-
-DATA ·sseIncMask<>+0x00(SB)/8, $0x1
-DATA ·sseIncMask<>+0x08(SB)/8, $0x0
-// To load/store the last < 16 bytes in a buffer
-DATA ·andMask<>+0x00(SB)/8, $0x00000000000000ff
-DATA ·andMask<>+0x08(SB)/8, $0x0000000000000000
-DATA ·andMask<>+0x10(SB)/8, $0x000000000000ffff
-DATA ·andMask<>+0x18(SB)/8, $0x0000000000000000
-DATA ·andMask<>+0x20(SB)/8, $0x0000000000ffffff
-DATA ·andMask<>+0x28(SB)/8, $0x0000000000000000
-DATA ·andMask<>+0x30(SB)/8, $0x00000000ffffffff
-DATA ·andMask<>+0x38(SB)/8, $0x0000000000000000
-DATA ·andMask<>+0x40(SB)/8, $0x000000ffffffffff
-DATA ·andMask<>+0x48(SB)/8, $0x0000000000000000
-DATA ·andMask<>+0x50(SB)/8, $0x0000ffffffffffff
-DATA ·andMask<>+0x58(SB)/8, $0x0000000000000000
-DATA ·andMask<>+0x60(SB)/8, $0x00ffffffffffffff
-DATA ·andMask<>+0x68(SB)/8, $0x0000000000000000
-DATA ·andMask<>+0x70(SB)/8, $0xffffffffffffffff
-DATA ·andMask<>+0x78(SB)/8, $0x0000000000000000
-DATA ·andMask<>+0x80(SB)/8, $0xffffffffffffffff
-DATA ·andMask<>+0x88(SB)/8, $0x00000000000000ff
-DATA ·andMask<>+0x90(SB)/8, $0xffffffffffffffff
-DATA ·andMask<>+0x98(SB)/8, $0x000000000000ffff
-DATA ·andMask<>+0xa0(SB)/8, $0xffffffffffffffff
-DATA ·andMask<>+0xa8(SB)/8, $0x0000000000ffffff
-DATA ·andMask<>+0xb0(SB)/8, $0xffffffffffffffff
-DATA ·andMask<>+0xb8(SB)/8, $0x00000000ffffffff
-DATA ·andMask<>+0xc0(SB)/8, $0xffffffffffffffff
-DATA ·andMask<>+0xc8(SB)/8, $0x000000ffffffffff
-DATA ·andMask<>+0xd0(SB)/8, $0xffffffffffffffff
-DATA ·andMask<>+0xd8(SB)/8, $0x0000ffffffffffff
-DATA ·andMask<>+0xe0(SB)/8, $0xffffffffffffffff
-DATA ·andMask<>+0xe8(SB)/8, $0x00ffffffffffffff
-
-GLOBL ·chacha20Constants<>(SB), (NOPTR+RODATA), $32
-GLOBL ·rol16<>(SB), (NOPTR+RODATA), $32
-GLOBL ·rol8<>(SB), (NOPTR+RODATA), $32
-GLOBL ·sseIncMask<>(SB), (NOPTR+RODATA), $16
-GLOBL ·avx2IncMask<>(SB), (NOPTR+RODATA), $32
-GLOBL ·avx2InitMask<>(SB), (NOPTR+RODATA), $32
-GLOBL ·polyClampMask<>(SB), (NOPTR+RODATA), $32
-GLOBL ·andMask<>(SB), (NOPTR+RODATA), $240
-// No PALIGNR in Go ASM yet (but VPALIGNR is present).
-#define shiftB0Left BYTE $0x66; BYTE $0x0f; BYTE $0x3a; BYTE $0x0f; BYTE $0xdb; BYTE $0x04 // PALIGNR $4, X3, X3
-#define shiftB1Left BYTE $0x66; BYTE $0x0f; BYTE $0x3a; BYTE $0x0f; BYTE $0xe4; BYTE $0x04 // PALIGNR $4, X4, X4
-#define shiftB2Left BYTE $0x66; BYTE $0x0f; BYTE $0x3a; BYTE $0x0f; BYTE $0xed; BYTE $0x04 // PALIGNR $4, X5, X5
-#define shiftB3Left BYTE $0x66; BYTE $0x45; BYTE $0x0f; BYTE $0x3a; BYTE $0x0f; BYTE $0xed; BYTE $0x04 // PALIGNR $4, X13, X13
-#define shiftC0Left BYTE $0x66; BYTE $0x0f; BYTE $0x3a; BYTE $0x0f; BYTE $0xf6; BYTE $0x08 // PALIGNR $8, X6, X6
-#define shiftC1Left BYTE $0x66; BYTE $0x0f; BYTE $0x3a; BYTE $0x0f; BYTE $0xff; BYTE $0x08 // PALIGNR $8, X7, X7
-#define shiftC2Left BYTE $0x66; BYTE $0x45; BYTE $0x0f; BYTE $0x3a; BYTE $0x0f; BYTE $0xc0; BYTE $0x08 // PALIGNR $8, X8, X8
-#define shiftC3Left BYTE $0x66; BYTE $0x45; BYTE $0x0f; BYTE $0x3a; BYTE $0x0f; BYTE $0xf6; BYTE $0x08 // PALIGNR $8, X14, X14
-#define shiftD0Left BYTE $0x66; BYTE $0x45; BYTE $0x0f; BYTE $0x3a; BYTE $0x0f; BYTE $0xc9; BYTE $0x0c // PALIGNR $12, X9, X9
-#define shiftD1Left BYTE $0x66; BYTE $0x45; BYTE $0x0f; BYTE $0x3a; BYTE $0x0f; BYTE $0xd2; BYTE $0x0c // PALIGNR $12, X10, X10
-#define shiftD2Left BYTE $0x66; BYTE $0x45; BYTE $0x0f; BYTE $0x3a; BYTE $0x0f; BYTE $0xdb; BYTE $0x0c // PALIGNR $12, X11, X11
-#define shiftD3Left BYTE $0x66; BYTE $0x45; BYTE $0x0f; BYTE $0x3a; BYTE $0x0f; BYTE $0xff; BYTE $0x0c // PALIGNR $12, X15, X15
-#define shiftB0Right BYTE $0x66; BYTE $0x0f; BYTE $0x3a; BYTE $0x0f; BYTE $0xdb; BYTE $0x0c // PALIGNR $12, X3, X3
-#define shiftB1Right BYTE $0x66; BYTE $0x0f; BYTE $0x3a; BYTE $0x0f; BYTE $0xe4; BYTE $0x0c // PALIGNR $12, X4, X4
-#define shiftB2Right BYTE $0x66; BYTE $0x0f; BYTE $0x3a; BYTE $0x0f; BYTE $0xed; BYTE $0x0c // PALIGNR $12, X5, X5
-#define shiftB3Right BYTE $0x66; BYTE $0x45; BYTE $0x0f; BYTE $0x3a; BYTE $0x0f; BYTE $0xed; BYTE $0x0c // PALIGNR $12, X13, X13
-#define shiftC0Right shiftC0Left
-#define shiftC1Right shiftC1Left
-#define shiftC2Right shiftC2Left
-#define shiftC3Right shiftC3Left
-#define shiftD0Right BYTE $0x66; BYTE $0x45; BYTE $0x0f; BYTE $0x3a; BYTE $0x0f; BYTE $0xc9; BYTE $0x04 // PALIGNR $4, X9, X9
-#define shiftD1Right BYTE $0x66; BYTE $0x45; BYTE $0x0f; BYTE $0x3a; BYTE $0x0f; BYTE $0xd2; BYTE $0x04 // PALIGNR $4, X10, X10
-#define shiftD2Right BYTE $0x66; BYTE $0x45; BYTE $0x0f; BYTE $0x3a; BYTE $0x0f; BYTE $0xdb; BYTE $0x04 // PALIGNR $4, X11, X11
-#define shiftD3Right BYTE $0x66; BYTE $0x45; BYTE $0x0f; BYTE $0x3a; BYTE $0x0f; BYTE $0xff; BYTE $0x04 // PALIGNR $4, X15, X15
-
-// Some macros
-
-// ROL rotates the uint32s in register R left by N bits, using temporary T.
-#define ROL(N, R, T) \
-	MOVO R, T; PSLLL $(N), T; PSRLL $(32-(N)), R; PXOR T, R
-
-// ROL16 rotates the uint32s in register R left by 16, using temporary T if needed.
-#ifdef GOAMD64_v2
-#define ROL16(R, T) PSHUFB ·rol16<>(SB), R
-#else
-#define ROL16(R, T) ROL(16, R, T)
-#endif
-
-// ROL8 rotates the uint32s in register R left by 8, using temporary T if needed.
-#ifdef GOAMD64_v2
-#define ROL8(R, T) PSHUFB ·rol8<>(SB), R
-#else
-#define ROL8(R, T) ROL(8, R, T)
-#endif
-
-#define chachaQR(A, B, C, D, T) \
-	PADDD B, A; PXOR A, D; ROL16(D, T) \
-	PADDD D, C; PXOR C, B; MOVO B, T; PSLLL $12, T; PSRLL $20, B; PXOR T, B \
-	PADDD B, A; PXOR A, D; ROL8(D, T) \
-	PADDD D, C; PXOR C, B; MOVO B, T; PSLLL $7, T; PSRLL $25, B; PXOR T, B
-
-#define chachaQR_AVX2(A, B, C, D, T) \
-	VPADDD B, A, A; VPXOR A, D, D; VPSHUFB ·rol16<>(SB), D, D                         \
-	VPADDD D, C, C; VPXOR C, B, B; VPSLLD $12, B, T; VPSRLD $20, B, B; VPXOR T, B, B \
-	VPADDD B, A, A; VPXOR A, D, D; VPSHUFB ·rol8<>(SB), D, D                          \
-	VPADDD D, C, C; VPXOR C, B, B; VPSLLD $7, B, T; VPSRLD $25, B, B; VPXOR T, B, B
-
-#define polyAdd(S) ADDQ S, acc0; ADCQ 8+S, acc1; ADCQ $1, acc2
-#define polyMulStage1 MOVQ (0*8)(BP), AX; MOVQ AX, t2; MULQ acc0; MOVQ AX, t0; MOVQ DX, t1; MOVQ (0*8)(BP), AX; MULQ acc1; IMULQ acc2, t2; ADDQ AX, t1; ADCQ DX, t2
-#define polyMulStage2 MOVQ (1*8)(BP), AX; MOVQ AX, t3; MULQ acc0; ADDQ AX, t1; ADCQ $0, DX; MOVQ DX, acc0; MOVQ (1*8)(BP), AX; MULQ acc1; ADDQ AX, t2; ADCQ $0, DX
-#define polyMulStage3 IMULQ acc2, t3; ADDQ acc0, t2; ADCQ DX, t3
-#define polyMulReduceStage MOVQ t0, acc0; MOVQ t1, acc1; MOVQ t2, acc2; ANDQ $3, acc2; MOVQ t2, t0; ANDQ $-4, t0; MOVQ t3, t1; SHRQ $2, t3, t2; SHRQ $2, t3; ADDQ t0, acc0; ADCQ t1, acc1; ADCQ $0, acc2; ADDQ t2, acc0; ADCQ t3, acc1; ADCQ $0, acc2
-
-#define polyMulStage1_AVX2 MOVQ (0*8)(BP), DX; MOVQ DX, t2; MULXQ acc0, t0, t1; IMULQ acc2, t2; MULXQ acc1, AX, DX; ADDQ AX, t1; ADCQ DX, t2
-#define polyMulStage2_AVX2 MOVQ (1*8)(BP), DX; MULXQ acc0, acc0, AX; ADDQ acc0, t1; MULXQ acc1, acc1, t3; ADCQ acc1, t2; ADCQ $0, t3
-#define polyMulStage3_AVX2 IMULQ acc2, DX; ADDQ AX, t2; ADCQ DX, t3
-
-#define polyMul polyMulStage1; polyMulStage2; polyMulStage3; polyMulReduceStage
-#define polyMulAVX2 polyMulStage1_AVX2; polyMulStage2_AVX2; polyMulStage3_AVX2; polyMulReduceStage
-// ----------------------------------------------------------------------------
+
+// func polyHashADInternal<>()
 TEXT polyHashADInternal<>(SB), NOSPLIT, $0
-	// adp points to beginning of additional data
-	// itr2 holds ad length
-	XORQ acc0, acc0
-	XORQ acc1, acc1
-	XORQ acc2, acc2
-	CMPQ itr2, $13
-	JNE  hashADLoop
-
-openFastTLSAD:
-	// Special treatment for the TLS case of 13 bytes
-	MOVQ (adp), acc0
-	MOVQ 5(adp), acc1
-	SHRQ $24, acc1
-	MOVQ $1, acc2
-	polyMul
+	// Hack: Must declare #define macros inside of a function due to Avo constraints
+	// ROL rotates the uint32s in register R left by N bits, using temporary T.
+	#define ROL(N, R, T) \
+		MOVO R, T; \
+		PSLLL $(N), T; \
+		PSRLL $(32-(N)), R; \
+		PXOR T, R
+
+	// ROL8 rotates the uint32s in register R left by 8, using temporary T if needed.
+	#ifdef GOAMD64_v2
+		#define ROL8(R, T) PSHUFB ·rol8<>(SB), R
+	#else
+		#define ROL8(R, T) ROL(8, R, T)
+	#endif
+
+	// ROL16 rotates the uint32s in register R left by 16, using temporary T if needed.
+	#ifdef GOAMD64_v2
+		#define ROL16(R, T) PSHUFB ·rol16<>(SB), R
+	#else
+		#define ROL16(R, T) ROL(16, R, T)
+	#endif
+	XORQ  R10, R10
+	XORQ  R11, R11
+	XORQ  R12, R12
+	CMPQ  R9, $0x0d
+	JNE   hashADLoop
+	MOVQ  (CX), R10
+	MOVQ  5(CX), R11
+	SHRQ  $0x18, R11
+	MOVQ  $0x00000001, R12
+	MOVQ  (BP), AX
+	MOVQ  AX, R15
+	MULQ  R10
+	MOVQ  AX, R13
+	MOVQ  DX, R14
+	MOVQ  (BP), AX
+	MULQ  R11
+	IMULQ R12, R15
+	ADDQ  AX, R14
+	ADCQ  DX, R15
+	MOVQ  8(BP), AX
+	MOVQ  AX, R8
+	MULQ  R10
+	ADDQ  AX, R14
+	ADCQ  $0x00, DX
+	MOVQ  DX, R10
+	MOVQ  8(BP), AX
+	MULQ  R11
+	ADDQ  AX, R15
+	ADCQ  $0x00, DX
+	IMULQ R12, R8
+	ADDQ  R10, R15
+	ADCQ  DX, R8
+	MOVQ  R13, R10
+	MOVQ  R14, R11
+	MOVQ  R15, R12
+	ANDQ  $0x03, R12
+	MOVQ  R15, R13
+	ANDQ  $-4, R13
+	MOVQ  R8, R14
+	SHRQ  $0x02, R8, R15
+	SHRQ  $0x02, R8
+	ADDQ  R13, R10
+	ADCQ  R14, R11
+	ADCQ  $0x00, R12
+	ADDQ  R15, R10
+	ADCQ  R8, R11
+	ADCQ  $0x00, R12
 	RET
 
 hashADLoop:
 	// Hash in 16 byte chunks
-	CMPQ itr2, $16
-	JB   hashADTail
-	polyAdd(0(adp))
-	LEAQ (1*16)(adp), adp
-	SUBQ $16, itr2
-	polyMul
-	JMP  hashADLoop
+	CMPQ  R9, $0x10
+	JB    hashADTail
+	ADDQ  (CX), R10
+	ADCQ  8(CX), R11
+	ADCQ  $0x01, R12
+	LEAQ  16(CX), CX
+	SUBQ  $0x10, R9
+	MOVQ  (BP), AX
+	MOVQ  AX, R15
+	MULQ  R10
+	MOVQ  AX, R13
+	MOVQ  DX, R14
+	MOVQ  (BP), AX
+	MULQ  R11
+	IMULQ R12, R15
+	ADDQ  AX, R14
+	ADCQ  DX, R15
+	MOVQ  8(BP), AX
+	MOVQ  AX, R8
+	MULQ  R10
+	ADDQ  AX, R14
+	ADCQ  $0x00, DX
+	MOVQ  DX, R10
+	MOVQ  8(BP), AX
+	MULQ  R11
+	ADDQ  AX, R15
+	ADCQ  $0x00, DX
+	IMULQ R12, R8
+	ADDQ  R10, R15
+	ADCQ  DX, R8
+	MOVQ  R13, R10
+	MOVQ  R14, R11
+	MOVQ  R15, R12
+	ANDQ  $0x03, R12
+	MOVQ  R15, R13
+	ANDQ  $-4, R13
+	MOVQ  R8, R14
+	SHRQ  $0x02, R8, R15
+	SHRQ  $0x02, R8
+	ADDQ  R13, R10
+	ADCQ  R14, R11
+	ADCQ  $0x00, R12
+	ADDQ  R15, R10
+	ADCQ  R8, R11
+	ADCQ  $0x00, R12
+	JMP   hashADLoop
 
 hashADTail:
-	CMPQ itr2, $0
+	CMPQ R9, $0x00
 	JE   hashADDone
 
 	// Hash last < 16 byte tail
-	XORQ t0, t0
-	XORQ t1, t1
-	XORQ t2, t2
-	ADDQ itr2, adp
+	XORQ R13, R13
+	XORQ R14, R14
+	XORQ R15, R15
+	ADDQ R9, CX
 
 hashADTailLoop:
-	SHLQ $8, t0, t1
-	SHLQ $8, t0
-	MOVB -1(adp), t2
-	XORQ t2, t0
-	DECQ adp
-	DECQ itr2
-	JNE  hashADTailLoop
-
-hashADTailFinish:
-	ADDQ t0, acc0; ADCQ t1, acc1; ADCQ $1, acc2
-	polyMul
-
-	// Finished AD
+	SHLQ  $0x08, R13, R14
+	SHLQ  $0x08, R13
+	MOVB  -1(CX), R15
+	XORQ  R15, R13
+	DECQ  CX
+	DECQ  R9
+	JNE   hashADTailLoop
+	ADDQ  R13, R10
+	ADCQ  R14, R11
+	ADCQ  $0x01, R12
+	MOVQ  (BP), AX
+	MOVQ  AX, R15
+	MULQ  R10
+	MOVQ  AX, R13
+	MOVQ  DX, R14
+	MOVQ  (BP), AX
+	MULQ  R11
+	IMULQ R12, R15
+	ADDQ  AX, R14
+	ADCQ  DX, R15
+	MOVQ  8(BP), AX
+	MOVQ  AX, R8
+	MULQ  R10
+	ADDQ  AX, R14
+	ADCQ  $0x00, DX
+	MOVQ  DX, R10
+	MOVQ  8(BP), AX
+	MULQ  R11
+	ADDQ  AX, R15
+	ADCQ  $0x00, DX
+	IMULQ R12, R8
+	ADDQ  R10, R15
+	ADCQ  DX, R8
+	MOVQ  R13, R10
+	MOVQ  R14, R11
+	MOVQ  R15, R12
+	ANDQ  $0x03, R12
+	MOVQ  R15, R13
+	ANDQ  $-4, R13
+	MOVQ  R8, R14
+	SHRQ  $0x02, R8, R15
+	SHRQ  $0x02, R8
+	ADDQ  R13, R10
+	ADCQ  R14, R11
+	ADCQ  $0x00, R12
+	ADDQ  R15, R10
+	ADCQ  R8, R11
+	ADCQ  $0x00, R12
+
 hashADDone:
 	RET
 
-// ----------------------------------------------------------------------------
-// func chacha20Poly1305Open(dst, key, src, ad []byte) bool
-TEXT ·chacha20Poly1305Open(SB), 0, $288-97
+// func chacha20Poly1305Open(dst []byte, key []uint32, src []byte, ad []byte) bool
+// Requires: AVX, AVX2, BMI2, CMOV, SSE2
+TEXT ·chacha20Poly1305Open(SB), $288-97
 	// For aligned stack access
 	MOVQ SP, BP
-	ADDQ $32, BP
+	ADDQ $0x20, BP
 	ANDQ $-32, BP
-	MOVQ dst+0(FP), oup
-	MOVQ key+24(FP), keyp
-	MOVQ src+48(FP), inp
-	MOVQ src_len+56(FP), inl
-	MOVQ ad+72(FP), adp
+	MOVQ dst_base+0(FP), DI
+	MOVQ key_base+24(FP), R8
+	MOVQ src_base+48(FP), SI
+	MOVQ src_len+56(FP), BX
+	MOVQ ad_base+72(FP), CX
 
 	// Check for AVX2 support
-	CMPB ·useAVX2(SB), $1
+	CMPB ·useAVX2+0(SB), $0x01
 	JE   chacha20Poly1305Open_AVX2
 
 	// Special optimization, for very short buffers
-	CMPQ inl, $128
-	JBE  openSSE128 // About 16% faster
+	CMPQ BX, $0x80
+	JBE  openSSE128
 
 	// For long buffers, prepare the poly key first
-	MOVOU ·chacha20Constants<>(SB), A0
-	MOVOU (1*16)(keyp), B0
-	MOVOU (2*16)(keyp), C0
-	MOVOU (3*16)(keyp), D0
-	MOVO  D0, T1
+	MOVOU ·chacha20Constants<>+0(SB), X0
+	MOVOU 16(R8), X3
+	MOVOU 32(R8), X6
+	MOVOU 48(R8), X9
+	MOVO  X9, X13
 
 	// Store state on stack for future use
-	MOVO B0, state1Store
-	MOVO C0, state2Store
-	MOVO D0, ctr3Store
-	MOVQ $10, itr2
+	MOVO X3, 32(BP)
+	MOVO X6, 48(BP)
+	MOVO X9, 128(BP)
+	MOVQ $0x0000000a, R9
 
 openSSEPreparePolyKey:
-	chachaQR(A0, B0, C0, D0, T0)
-	shiftB0Left;  shiftC0Left; shiftD0Left
-	chachaQR(A0, B0, C0, D0, T0)
-	shiftB0Right; shiftC0Right; shiftD0Right
-	DECQ          itr2
-	JNE           openSSEPreparePolyKey
+	PADDD X3, X0
+	PXOR  X0, X9
+	ROL16(X9, X12)
+	PADDD X9, X6
+	PXOR  X6, X3
+	MOVO  X3, X12
+	PSLLL $0x0c, X12
+	PSRLL $0x14, X3
+	PXOR  X12, X3
+	PADDD X3, X0
+	PXOR  X0, X9
+	ROL8(X9, X12)
+	PADDD X9, X6
+	PXOR  X6, X3
+	MOVO  X3, X12
+	PSLLL $0x07, X12
+	PSRLL $0x19, X3
+	PXOR  X12, X3
+	BYTE  $0x66
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xdb
+	BYTE  $0x04
+	BYTE  $0x66
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xf6
+	BYTE  $0x08
+	BYTE  $0x66
+	BYTE  $0x45
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xc9
+	BYTE  $0x0c
+	PADDD X3, X0
+	PXOR  X0, X9
+	ROL16(X9, X12)
+	PADDD X9, X6
+	PXOR  X6, X3
+	MOVO  X3, X12
+	PSLLL $0x0c, X12
+	PSRLL $0x14, X3
+	PXOR  X12, X3
+	PADDD X3, X0
+	PXOR  X0, X9
+	ROL8(X9, X12)
+	PADDD X9, X6
+	PXOR  X6, X3
+	MOVO  X3, X12
+	PSLLL $0x07, X12
+	PSRLL $0x19, X3
+	PXOR  X12, X3
+	BYTE  $0x66
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xdb
+	BYTE  $0x0c
+	BYTE  $0x66
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xf6
+	BYTE  $0x08
+	BYTE  $0x66
+	BYTE  $0x45
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xc9
+	BYTE  $0x04
+	DECQ  R9
+	JNE   openSSEPreparePolyKey
 
 	// A0|B0 hold the Poly1305 32-byte key, C0,D0 can be discarded
-	PADDL ·chacha20Constants<>(SB), A0; PADDL state1Store, B0
+	PADDL ·chacha20Constants<>+0(SB), X0
+	PADDL 32(BP), X3
 
 	// Clamp and store the key
-	PAND ·polyClampMask<>(SB), A0
-	MOVO A0, rStore; MOVO B0, sStore
+	PAND ·polyClampMask<>+0(SB), X0
+	MOVO X0, (BP)
+	MOVO X3, 16(BP)
 
 	// Hash AAD
-	MOVQ ad_len+80(FP), itr2
+	MOVQ ad_len+80(FP), R9
 	CALL polyHashADInternal<>(SB)
 
 openSSEMainLoop:
-	CMPQ inl, $256
+	CMPQ BX, $0x00000100
 	JB   openSSEMainLoopDone
 
 	// Load state, increment counter blocks
-	MOVO ·chacha20Constants<>(SB), A0; MOVO state1Store, B0; MOVO state2Store, C0; MOVO ctr3Store, D0; PADDL ·sseIncMask<>(SB), D0
-	MOVO A0, A1; MOVO B0, B1; MOVO C0, C1; MOVO D0, D1; PADDL ·sseIncMask<>(SB), D1
-	MOVO A1, A2; MOVO B1, B2; MOVO C1, C2; MOVO D1, D2; PADDL ·sseIncMask<>(SB), D2
-	MOVO A2, A3; MOVO B2, B3; MOVO C2, C3; MOVO D2, D3; PADDL ·sseIncMask<>(SB), D3
+	MOVO  ·chacha20Constants<>+0(SB), X0
+	MOVO  32(BP), X3
+	MOVO  48(BP), X6
+	MOVO  128(BP), X9
+	PADDL ·sseIncMask<>+0(SB), X9
+	MOVO  X0, X1
+	MOVO  X3, X4
+	MOVO  X6, X7
+	MOVO  X9, X10
+	PADDL ·sseIncMask<>+0(SB), X10
+	MOVO  X1, X2
+	MOVO  X4, X5
+	MOVO  X7, X8
+	MOVO  X10, X11
+	PADDL ·sseIncMask<>+0(SB), X11
+	MOVO  X2, X12
+	MOVO  X5, X13
+	MOVO  X8, X14
+	MOVO  X11, X15
+	PADDL ·sseIncMask<>+0(SB), X15
 
 	// Store counters
-	MOVO D0, ctr0Store; MOVO D1, ctr1Store; MOVO D2, ctr2Store; MOVO D3, ctr3Store
+	MOVO X9, 80(BP)
+	MOVO X10, 96(BP)
+	MOVO X11, 112(BP)
+	MOVO X15, 128(BP)
 
-	// There are 10 ChaCha20 iterations of 2QR each, so for 6 iterations we hash 2 blocks, and for the remaining 4 only 1 block - for a total of 16
-	MOVQ $4, itr1
-	MOVQ inp, itr2
+	// There are 10 ChaCha20 iterations of 2QR each, so for 6 iterations we hash
+	// 2 blocks, and for the remaining 4 only 1 block - for a total of 16
+	MOVQ $0x00000004, CX
+	MOVQ SI, R9
 
 openSSEInternalLoop:
-	MOVO          C3, tmpStore
-	chachaQR(A0, B0, C0, D0, C3); chachaQR(A1, B1, C1, D1, C3); chachaQR(A2, B2, C2, D2, C3)
-	MOVO          tmpStore, C3
-	MOVO          C1, tmpStore
-	chachaQR(A3, B3, C3, D3, C1)
-	MOVO          tmpStore, C1
-	polyAdd(0(itr2))
-	shiftB0Left;  shiftB1Left; shiftB2Left; shiftB3Left
-	shiftC0Left;  shiftC1Left; shiftC2Left; shiftC3Left
-	shiftD0Left;  shiftD1Left; shiftD2Left; shiftD3Left
-	polyMulStage1
-	polyMulStage2
-	LEAQ          (2*8)(itr2), itr2
-	MOVO          C3, tmpStore
-	chachaQR(A0, B0, C0, D0, C3); chachaQR(A1, B1, C1, D1, C3); chachaQR(A2, B2, C2, D2, C3)
-	MOVO          tmpStore, C3
-	MOVO          C1, tmpStore
-	polyMulStage3
-	chachaQR(A3, B3, C3, D3, C1)
-	MOVO          tmpStore, C1
-	polyMulReduceStage
-	shiftB0Right; shiftB1Right; shiftB2Right; shiftB3Right
-	shiftC0Right; shiftC1Right; shiftC2Right; shiftC3Right
-	shiftD0Right; shiftD1Right; shiftD2Right; shiftD3Right
-	DECQ          itr1
-	JGE           openSSEInternalLoop
-
-	polyAdd(0(itr2))
-	polyMul
-	LEAQ (2*8)(itr2), itr2
-
-	CMPQ itr1, $-6
-	JG   openSSEInternalLoop
+	MOVO  X14, 64(BP)
+	PADDD X3, X0
+	PXOR  X0, X9
+	ROL16(X9, X14)
+	PADDD X9, X6
+	PXOR  X6, X3
+	MOVO  X3, X14
+	PSLLL $0x0c, X14
+	PSRLL $0x14, X3
+	PXOR  X14, X3
+	PADDD X3, X0
+	PXOR  X0, X9
+	ROL8(X9, X14)
+	PADDD X9, X6
+	PXOR  X6, X3
+	MOVO  X3, X14
+	PSLLL $0x07, X14
+	PSRLL $0x19, X3
+	PXOR  X14, X3
+	PADDD X4, X1
+	PXOR  X1, X10
+	ROL16(X10, X14)
+	PADDD X10, X7
+	PXOR  X7, X4
+	MOVO  X4, X14
+	PSLLL $0x0c, X14
+	PSRLL $0x14, X4
+	PXOR  X14, X4
+	PADDD X4, X1
+	PXOR  X1, X10
+	ROL8(X10, X14)
+	PADDD X10, X7
+	PXOR  X7, X4
+	MOVO  X4, X14
+	PSLLL $0x07, X14
+	PSRLL $0x19, X4
+	PXOR  X14, X4
+	PADDD X5, X2
+	PXOR  X2, X11
+	ROL16(X11, X14)
+	PADDD X11, X8
+	PXOR  X8, X5
+	MOVO  X5, X14
+	PSLLL $0x0c, X14
+	PSRLL $0x14, X5
+	PXOR  X14, X5
+	PADDD X5, X2
+	PXOR  X2, X11
+	ROL8(X11, X14)
+	PADDD X11, X8
+	PXOR  X8, X5
+	MOVO  X5, X14
+	PSLLL $0x07, X14
+	PSRLL $0x19, X5
+	PXOR  X14, X5
+	MOVO  64(BP), X14
+	MOVO  X7, 64(BP)
+	PADDD X13, X12
+	PXOR  X12, X15
+	ROL16(X15, X7)
+	PADDD X15, X14
+	PXOR  X14, X13
+	MOVO  X13, X7
+	PSLLL $0x0c, X7
+	PSRLL $0x14, X13
+	PXOR  X7, X13
+	PADDD X13, X12
+	PXOR  X12, X15
+	ROL8(X15, X7)
+	PADDD X15, X14
+	PXOR  X14, X13
+	MOVO  X13, X7
+	PSLLL $0x07, X7
+	PSRLL $0x19, X13
+	PXOR  X7, X13
+	MOVO  64(BP), X7
+	ADDQ  (R9), R10
+	ADCQ  8(R9), R11
+	ADCQ  $0x01, R12
+	BYTE  $0x66
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xdb
+	BYTE  $0x04
+	BYTE  $0x66
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xe4
+	BYTE  $0x04
+	BYTE  $0x66
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xed
+	BYTE  $0x04
+	BYTE  $0x66
+	BYTE  $0x45
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xed
+	BYTE  $0x04
+	BYTE  $0x66
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xf6
+	BYTE  $0x08
+	BYTE  $0x66
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xff
+	BYTE  $0x08
+	BYTE  $0x66
+	BYTE  $0x45
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xc0
+	BYTE  $0x08
+	BYTE  $0x66
+	BYTE  $0x45
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xf6
+	BYTE  $0x08
+	BYTE  $0x66
+	BYTE  $0x45
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xc9
+	BYTE  $0x0c
+	BYTE  $0x66
+	BYTE  $0x45
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xd2
+	BYTE  $0x0c
+	BYTE  $0x66
+	BYTE  $0x45
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xdb
+	BYTE  $0x0c
+	BYTE  $0x66
+	BYTE  $0x45
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xff
+	BYTE  $0x0c
+	MOVQ  (BP), AX
+	MOVQ  AX, R15
+	MULQ  R10
+	MOVQ  AX, R13
+	MOVQ  DX, R14
+	MOVQ  (BP), AX
+	MULQ  R11
+	IMULQ R12, R15
+	ADDQ  AX, R14
+	ADCQ  DX, R15
+	MOVQ  8(BP), AX
+	MOVQ  AX, R8
+	MULQ  R10
+	ADDQ  AX, R14
+	ADCQ  $0x00, DX
+	MOVQ  DX, R10
+	MOVQ  8(BP), AX
+	MULQ  R11
+	ADDQ  AX, R15
+	ADCQ  $0x00, DX
+	LEAQ  16(R9), R9
+	MOVO  X14, 64(BP)
+	PADDD X3, X0
+	PXOR  X0, X9
+	ROL16(X9, X14)
+	PADDD X9, X6
+	PXOR  X6, X3
+	MOVO  X3, X14
+	PSLLL $0x0c, X14
+	PSRLL $0x14, X3
+	PXOR  X14, X3
+	PADDD X3, X0
+	PXOR  X0, X9
+	ROL8(X9, X14)
+	PADDD X9, X6
+	PXOR  X6, X3
+	MOVO  X3, X14
+	PSLLL $0x07, X14
+	PSRLL $0x19, X3
+	PXOR  X14, X3
+	PADDD X4, X1
+	PXOR  X1, X10
+	ROL16(X10, X14)
+	PADDD X10, X7
+	PXOR  X7, X4
+	MOVO  X4, X14
+	PSLLL $0x0c, X14
+	PSRLL $0x14, X4
+	PXOR  X14, X4
+	PADDD X4, X1
+	PXOR  X1, X10
+	ROL8(X10, X14)
+	PADDD X10, X7
+	PXOR  X7, X4
+	MOVO  X4, X14
+	PSLLL $0x07, X14
+	PSRLL $0x19, X4
+	PXOR  X14, X4
+	PADDD X5, X2
+	PXOR  X2, X11
+	ROL16(X11, X14)
+	PADDD X11, X8
+	PXOR  X8, X5
+	MOVO  X5, X14
+	PSLLL $0x0c, X14
+	PSRLL $0x14, X5
+	PXOR  X14, X5
+	PADDD X5, X2
+	PXOR  X2, X11
+	ROL8(X11, X14)
+	PADDD X11, X8
+	PXOR  X8, X5
+	MOVO  X5, X14
+	PSLLL $0x07, X14
+	PSRLL $0x19, X5
+	PXOR  X14, X5
+	MOVO  64(BP), X14
+	MOVO  X7, 64(BP)
+	IMULQ R12, R8
+	ADDQ  R10, R15
+	ADCQ  DX, R8
+	PADDD X13, X12
+	PXOR  X12, X15
+	ROL16(X15, X7)
+	PADDD X15, X14
+	PXOR  X14, X13
+	MOVO  X13, X7
+	PSLLL $0x0c, X7
+	PSRLL $0x14, X13
+	PXOR  X7, X13
+	PADDD X13, X12
+	PXOR  X12, X15
+	ROL8(X15, X7)
+	PADDD X15, X14
+	PXOR  X14, X13
+	MOVO  X13, X7
+	PSLLL $0x07, X7
+	PSRLL $0x19, X13
+	PXOR  X7, X13
+	MOVO  64(BP), X7
+	MOVQ  R13, R10
+	MOVQ  R14, R11
+	MOVQ  R15, R12
+	ANDQ  $0x03, R12
+	MOVQ  R15, R13
+	ANDQ  $-4, R13
+	MOVQ  R8, R14
+	SHRQ  $0x02, R8, R15
+	SHRQ  $0x02, R8
+	ADDQ  R13, R10
+	ADCQ  R14, R11
+	ADCQ  $0x00, R12
+	ADDQ  R15, R10
+	ADCQ  R8, R11
+	ADCQ  $0x00, R12
+	BYTE  $0x66
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xdb
+	BYTE  $0x0c
+	BYTE  $0x66
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xe4
+	BYTE  $0x0c
+	BYTE  $0x66
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xed
+	BYTE  $0x0c
+	BYTE  $0x66
+	BYTE  $0x45
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xed
+	BYTE  $0x0c
+	BYTE  $0x66
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xf6
+	BYTE  $0x08
+	BYTE  $0x66
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xff
+	BYTE  $0x08
+	BYTE  $0x66
+	BYTE  $0x45
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xc0
+	BYTE  $0x08
+	BYTE  $0x66
+	BYTE  $0x45
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xf6
+	BYTE  $0x08
+	BYTE  $0x66
+	BYTE  $0x45
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xc9
+	BYTE  $0x04
+	BYTE  $0x66
+	BYTE  $0x45
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xd2
+	BYTE  $0x04
+	BYTE  $0x66
+	BYTE  $0x45
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xdb
+	BYTE  $0x04
+	BYTE  $0x66
+	BYTE  $0x45
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xff
+	BYTE  $0x04
+	DECQ  CX
+	JGE   openSSEInternalLoop
+	ADDQ  (R9), R10
+	ADCQ  8(R9), R11
+	ADCQ  $0x01, R12
+	MOVQ  (BP), AX
+	MOVQ  AX, R15
+	MULQ  R10
+	MOVQ  AX, R13
+	MOVQ  DX, R14
+	MOVQ  (BP), AX
+	MULQ  R11
+	IMULQ R12, R15
+	ADDQ  AX, R14
+	ADCQ  DX, R15
+	MOVQ  8(BP), AX
+	MOVQ  AX, R8
+	MULQ  R10
+	ADDQ  AX, R14
+	ADCQ  $0x00, DX
+	MOVQ  DX, R10
+	MOVQ  8(BP), AX
+	MULQ  R11
+	ADDQ  AX, R15
+	ADCQ  $0x00, DX
+	IMULQ R12, R8
+	ADDQ  R10, R15
+	ADCQ  DX, R8
+	MOVQ  R13, R10
+	MOVQ  R14, R11
+	MOVQ  R15, R12
+	ANDQ  $0x03, R12
+	MOVQ  R15, R13
+	ANDQ  $-4, R13
+	MOVQ  R8, R14
+	SHRQ  $0x02, R8, R15
+	SHRQ  $0x02, R8
+	ADDQ  R13, R10
+	ADCQ  R14, R11
+	ADCQ  $0x00, R12
+	ADDQ  R15, R10
+	ADCQ  R8, R11
+	ADCQ  $0x00, R12
+	LEAQ  16(R9), R9
+	CMPQ  CX, $-6
+	JG    openSSEInternalLoop
 
 	// Add in the state
-	PADDD ·chacha20Constants<>(SB), A0; PADDD ·chacha20Constants<>(SB), A1; PADDD ·chacha20Constants<>(SB), A2; PADDD ·chacha20Constants<>(SB), A3
-	PADDD state1Store, B0; PADDD state1Store, B1; PADDD state1Store, B2; PADDD state1Store, B3
-	PADDD state2Store, C0; PADDD state2Store, C1; PADDD state2Store, C2; PADDD state2Store, C3
-	PADDD ctr0Store, D0; PADDD ctr1Store, D1; PADDD ctr2Store, D2; PADDD ctr3Store, D3
+	PADDD ·chacha20Constants<>+0(SB), X0
+	PADDD ·chacha20Constants<>+0(SB), X1
+	PADDD ·chacha20Constants<>+0(SB), X2
+	PADDD ·chacha20Constants<>+0(SB), X12
+	PADDD 32(BP), X3
+	PADDD 32(BP), X4
+	PADDD 32(BP), X5
+	PADDD 32(BP), X13
+	PADDD 48(BP), X6
+	PADDD 48(BP), X7
+	PADDD 48(BP), X8
+	PADDD 48(BP), X14
+	PADDD 80(BP), X9
+	PADDD 96(BP), X10
+	PADDD 112(BP), X11
+	PADDD 128(BP), X15
 
 	// Load - xor - store
-	MOVO  D3, tmpStore
-	MOVOU (0*16)(inp), D3; PXOR D3, A0; MOVOU A0, (0*16)(oup)
-	MOVOU (1*16)(inp), D3; PXOR D3, B0; MOVOU B0, (1*16)(oup)
-	MOVOU (2*16)(inp), D3; PXOR D3, C0; MOVOU C0, (2*16)(oup)
-	MOVOU (3*16)(inp), D3; PXOR D3, D0; MOVOU D0, (3*16)(oup)
-	MOVOU (4*16)(inp), D0; PXOR D0, A1; MOVOU A1, (4*16)(oup)
-	MOVOU (5*16)(inp), D0; PXOR D0, B1; MOVOU B1, (5*16)(oup)
-	MOVOU (6*16)(inp), D0; PXOR D0, C1; MOVOU C1, (6*16)(oup)
-	MOVOU (7*16)(inp), D0; PXOR D0, D1; MOVOU D1, (7*16)(oup)
-	MOVOU (8*16)(inp), D0; PXOR D0, A2; MOVOU A2, (8*16)(oup)
-	MOVOU (9*16)(inp), D0; PXOR D0, B2; MOVOU B2, (9*16)(oup)
-	MOVOU (10*16)(inp), D0; PXOR D0, C2; MOVOU C2, (10*16)(oup)
-	MOVOU (11*16)(inp), D0; PXOR D0, D2; MOVOU D2, (11*16)(oup)
-	MOVOU (12*16)(inp), D0; PXOR D0, A3; MOVOU A3, (12*16)(oup)
-	MOVOU (13*16)(inp), D0; PXOR D0, B3; MOVOU B3, (13*16)(oup)
-	MOVOU (14*16)(inp), D0; PXOR D0, C3; MOVOU C3, (14*16)(oup)
-	MOVOU (15*16)(inp), D0; PXOR tmpStore, D0; MOVOU D0, (15*16)(oup)
-	LEAQ  256(inp), inp
-	LEAQ  256(oup), oup
-	SUBQ  $256, inl
+	MOVO  X15, 64(BP)
+	MOVOU (SI), X15
+	PXOR  X15, X0
+	MOVOU X0, (DI)
+	MOVOU 16(SI), X15
+	PXOR  X15, X3
+	MOVOU X3, 16(DI)
+	MOVOU 32(SI), X15
+	PXOR  X15, X6
+	MOVOU X6, 32(DI)
+	MOVOU 48(SI), X15
+	PXOR  X15, X9
+	MOVOU X9, 48(DI)
+	MOVOU 64(SI), X9
+	PXOR  X9, X1
+	MOVOU X1, 64(DI)
+	MOVOU 80(SI), X9
+	PXOR  X9, X4
+	MOVOU X4, 80(DI)
+	MOVOU 96(SI), X9
+	PXOR  X9, X7
+	MOVOU X7, 96(DI)
+	MOVOU 112(SI), X9
+	PXOR  X9, X10
+	MOVOU X10, 112(DI)
+	MOVOU 128(SI), X9
+	PXOR  X9, X2
+	MOVOU X2, 128(DI)
+	MOVOU 144(SI), X9
+	PXOR  X9, X5
+	MOVOU X5, 144(DI)
+	MOVOU 160(SI), X9
+	PXOR  X9, X8
+	MOVOU X8, 160(DI)
+	MOVOU 176(SI), X9
+	PXOR  X9, X11
+	MOVOU X11, 176(DI)
+	MOVOU 192(SI), X9
+	PXOR  X9, X12
+	MOVOU X12, 192(DI)
+	MOVOU 208(SI), X9
+	PXOR  X9, X13
+	MOVOU X13, 208(DI)
+	MOVOU 224(SI), X9
+	PXOR  X9, X14
+	MOVOU X14, 224(DI)
+	MOVOU 240(SI), X9
+	PXOR  64(BP), X9
+	MOVOU X9, 240(DI)
+	LEAQ  256(SI), SI
+	LEAQ  256(DI), DI
+	SUBQ  $0x00000100, BX
 	JMP   openSSEMainLoop
 
 openSSEMainLoopDone:
 	// Handle the various tail sizes efficiently
-	TESTQ inl, inl
+	TESTQ BX, BX
 	JE    openSSEFinalize
-	CMPQ  inl, $64
+	CMPQ  BX, $0x40
 	JBE   openSSETail64
-	CMPQ  inl, $128
+	CMPQ  BX, $0x80
 	JBE   openSSETail128
-	CMPQ  inl, $192
+	CMPQ  BX, $0xc0
 	JBE   openSSETail192
 	JMP   openSSETail256
 
 openSSEFinalize:
 	// Hash in the PT, AAD lengths
-	ADDQ ad_len+80(FP), acc0; ADCQ src_len+56(FP), acc1; ADCQ $1, acc2
-	polyMul
+	ADDQ  ad_len+80(FP), R10
+	ADCQ  src_len+56(FP), R11
+	ADCQ  $0x01, R12
+	MOVQ  (BP), AX
+	MOVQ  AX, R15
+	MULQ  R10
+	MOVQ  AX, R13
+	MOVQ  DX, R14
+	MOVQ  (BP), AX
+	MULQ  R11
+	IMULQ R12, R15
+	ADDQ  AX, R14
+	ADCQ  DX, R15
+	MOVQ  8(BP), AX
+	MOVQ  AX, R8
+	MULQ  R10
+	ADDQ  AX, R14
+	ADCQ  $0x00, DX
+	MOVQ  DX, R10
+	MOVQ  8(BP), AX
+	MULQ  R11
+	ADDQ  AX, R15
+	ADCQ  $0x00, DX
+	IMULQ R12, R8
+	ADDQ  R10, R15
+	ADCQ  DX, R8
+	MOVQ  R13, R10
+	MOVQ  R14, R11
+	MOVQ  R15, R12
+	ANDQ  $0x03, R12
+	MOVQ  R15, R13
+	ANDQ  $-4, R13
+	MOVQ  R8, R14
+	SHRQ  $0x02, R8, R15
+	SHRQ  $0x02, R8
+	ADDQ  R13, R10
+	ADCQ  R14, R11
+	ADCQ  $0x00, R12
+	ADDQ  R15, R10
+	ADCQ  R8, R11
+	ADCQ  $0x00, R12
 
 	// Final reduce
-	MOVQ    acc0, t0
-	MOVQ    acc1, t1
-	MOVQ    acc2, t2
-	SUBQ    $-5, acc0
-	SBBQ    $-1, acc1
-	SBBQ    $3, acc2
-	CMOVQCS t0, acc0
-	CMOVQCS t1, acc1
-	CMOVQCS t2, acc2
+	MOVQ    R10, R13
+	MOVQ    R11, R14
+	MOVQ    R12, R15
+	SUBQ    $-5, R10
+	SBBQ    $-1, R11
+	SBBQ    $0x03, R12
+	CMOVQCS R13, R10
+	CMOVQCS R14, R11
+	CMOVQCS R15, R12
 
 	// Add in the "s" part of the key
-	ADDQ 0+sStore, acc0
-	ADCQ 8+sStore, acc1
+	ADDQ 16(BP), R10
+	ADCQ 24(BP), R11
 
 	// Finally, constant time compare to the tag at the end of the message
 	XORQ    AX, AX
-	MOVQ    $1, DX
-	XORQ    (0*8)(inp), acc0
-	XORQ    (1*8)(inp), acc1
-	ORQ     acc1, acc0
+	MOVQ    $0x00000001, DX
+	XORQ    (SI), R10
+	XORQ    8(SI), R11
+	ORQ     R11, R10
 	CMOVQEQ DX, AX
 
 	// Return true iff tags are equal
 	MOVB AX, ret+96(FP)
 	RET
 
-// ----------------------------------------------------------------------------
-// Special optimization for buffers smaller than 129 bytes
 openSSE128:
-	// For up to 128 bytes of ciphertext and 64 bytes for the poly key, we require to process three blocks
-	MOVOU ·chacha20Constants<>(SB), A0; MOVOU (1*16)(keyp), B0; MOVOU (2*16)(keyp), C0; MOVOU (3*16)(keyp), D0
-	MOVO  A0, A1; MOVO B0, B1; MOVO C0, C1; MOVO D0, D1; PADDL ·sseIncMask<>(SB), D1
-	MOVO  A1, A2; MOVO B1, B2; MOVO C1, C2; MOVO D1, D2; PADDL ·sseIncMask<>(SB), D2
-	MOVO  B0, T1; MOVO C0, T2; MOVO D1, T3
-	MOVQ  $10, itr2
+	MOVOU ·chacha20Constants<>+0(SB), X0
+	MOVOU 16(R8), X3
+	MOVOU 32(R8), X6
+	MOVOU 48(R8), X9
+	MOVO  X0, X1
+	MOVO  X3, X4
+	MOVO  X6, X7
+	MOVO  X9, X10
+	PADDL ·sseIncMask<>+0(SB), X10
+	MOVO  X1, X2
+	MOVO  X4, X5
+	MOVO  X7, X8
+	MOVO  X10, X11
+	PADDL ·sseIncMask<>+0(SB), X11
+	MOVO  X3, X13
+	MOVO  X6, X14
+	MOVO  X10, X15
+	MOVQ  $0x0000000a, R9
 
 openSSE128InnerCipherLoop:
-	chachaQR(A0, B0, C0, D0, T0); chachaQR(A1, B1, C1, D1, T0); chachaQR(A2, B2, C2, D2, T0)
-	shiftB0Left;  shiftB1Left; shiftB2Left
-	shiftC0Left;  shiftC1Left; shiftC2Left
-	shiftD0Left;  shiftD1Left; shiftD2Left
-	chachaQR(A0, B0, C0, D0, T0); chachaQR(A1, B1, C1, D1, T0); chachaQR(A2, B2, C2, D2, T0)
-	shiftB0Right; shiftB1Right; shiftB2Right
-	shiftC0Right; shiftC1Right; shiftC2Right
-	shiftD0Right; shiftD1Right; shiftD2Right
-	DECQ          itr2
-	JNE           openSSE128InnerCipherLoop
+	PADDD X3, X0
+	PXOR  X0, X9
+	ROL16(X9, X12)
+	PADDD X9, X6
+	PXOR  X6, X3
+	MOVO  X3, X12
+	PSLLL $0x0c, X12
+	PSRLL $0x14, X3
+	PXOR  X12, X3
+	PADDD X3, X0
+	PXOR  X0, X9
+	ROL8(X9, X12)
+	PADDD X9, X6
+	PXOR  X6, X3
+	MOVO  X3, X12
+	PSLLL $0x07, X12
+	PSRLL $0x19, X3
+	PXOR  X12, X3
+	PADDD X4, X1
+	PXOR  X1, X10
+	ROL16(X10, X12)
+	PADDD X10, X7
+	PXOR  X7, X4
+	MOVO  X4, X12
+	PSLLL $0x0c, X12
+	PSRLL $0x14, X4
+	PXOR  X12, X4
+	PADDD X4, X1
+	PXOR  X1, X10
+	ROL8(X10, X12)
+	PADDD X10, X7
+	PXOR  X7, X4
+	MOVO  X4, X12
+	PSLLL $0x07, X12
+	PSRLL $0x19, X4
+	PXOR  X12, X4
+	PADDD X5, X2
+	PXOR  X2, X11
+	ROL16(X11, X12)
+	PADDD X11, X8
+	PXOR  X8, X5
+	MOVO  X5, X12
+	PSLLL $0x0c, X12
+	PSRLL $0x14, X5
+	PXOR  X12, X5
+	PADDD X5, X2
+	PXOR  X2, X11
+	ROL8(X11, X12)
+	PADDD X11, X8
+	PXOR  X8, X5
+	MOVO  X5, X12
+	PSLLL $0x07, X12
+	PSRLL $0x19, X5
+	PXOR  X12, X5
+	BYTE  $0x66
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xdb
+	BYTE  $0x04
+	BYTE  $0x66
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xe4
+	BYTE  $0x04
+	BYTE  $0x66
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xed
+	BYTE  $0x04
+	BYTE  $0x66
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xf6
+	BYTE  $0x08
+	BYTE  $0x66
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xff
+	BYTE  $0x08
+	BYTE  $0x66
+	BYTE  $0x45
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xc0
+	BYTE  $0x08
+	BYTE  $0x66
+	BYTE  $0x45
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xc9
+	BYTE  $0x0c
+	BYTE  $0x66
+	BYTE  $0x45
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xd2
+	BYTE  $0x0c
+	BYTE  $0x66
+	BYTE  $0x45
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xdb
+	BYTE  $0x0c
+	PADDD X3, X0
+	PXOR  X0, X9
+	ROL16(X9, X12)
+	PADDD X9, X6
+	PXOR  X6, X3
+	MOVO  X3, X12
+	PSLLL $0x0c, X12
+	PSRLL $0x14, X3
+	PXOR  X12, X3
+	PADDD X3, X0
+	PXOR  X0, X9
+	ROL8(X9, X12)
+	PADDD X9, X6
+	PXOR  X6, X3
+	MOVO  X3, X12
+	PSLLL $0x07, X12
+	PSRLL $0x19, X3
+	PXOR  X12, X3
+	PADDD X4, X1
+	PXOR  X1, X10
+	ROL16(X10, X12)
+	PADDD X10, X7
+	PXOR  X7, X4
+	MOVO  X4, X12
+	PSLLL $0x0c, X12
+	PSRLL $0x14, X4
+	PXOR  X12, X4
+	PADDD X4, X1
+	PXOR  X1, X10
+	ROL8(X10, X12)
+	PADDD X10, X7
+	PXOR  X7, X4
+	MOVO  X4, X12
+	PSLLL $0x07, X12
+	PSRLL $0x19, X4
+	PXOR  X12, X4
+	PADDD X5, X2
+	PXOR  X2, X11
+	ROL16(X11, X12)
+	PADDD X11, X8
+	PXOR  X8, X5
+	MOVO  X5, X12
+	PSLLL $0x0c, X12
+	PSRLL $0x14, X5
+	PXOR  X12, X5
+	PADDD X5, X2
+	PXOR  X2, X11
+	ROL8(X11, X12)
+	PADDD X11, X8
+	PXOR  X8, X5
+	MOVO  X5, X12
+	PSLLL $0x07, X12
+	PSRLL $0x19, X5
+	PXOR  X12, X5
+	BYTE  $0x66
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xdb
+	BYTE  $0x0c
+	BYTE  $0x66
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xe4
+	BYTE  $0x0c
+	BYTE  $0x66
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xed
+	BYTE  $0x0c
+	BYTE  $0x66
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xf6
+	BYTE  $0x08
+	BYTE  $0x66
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xff
+	BYTE  $0x08
+	BYTE  $0x66
+	BYTE  $0x45
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xc0
+	BYTE  $0x08
+	BYTE  $0x66
+	BYTE  $0x45
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xc9
+	BYTE  $0x04
+	BYTE  $0x66
+	BYTE  $0x45
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xd2
+	BYTE  $0x04
+	BYTE  $0x66
+	BYTE  $0x45
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xdb
+	BYTE  $0x04
+	DECQ  R9
+	JNE   openSSE128InnerCipherLoop
 
 	// A0|B0 hold the Poly1305 32-byte key, C0,D0 can be discarded
-	PADDL ·chacha20Constants<>(SB), A0; PADDL ·chacha20Constants<>(SB), A1; PADDL ·chacha20Constants<>(SB), A2
-	PADDL T1, B0; PADDL T1, B1; PADDL T1, B2
-	PADDL T2, C1; PADDL T2, C2
-	PADDL T3, D1; PADDL ·sseIncMask<>(SB), T3; PADDL T3, D2
+	PADDL ·chacha20Constants<>+0(SB), X0
+	PADDL ·chacha20Constants<>+0(SB), X1
+	PADDL ·chacha20Constants<>+0(SB), X2
+	PADDL X13, X3
+	PADDL X13, X4
+	PADDL X13, X5
+	PADDL X14, X7
+	PADDL X14, X8
+	PADDL X15, X10
+	PADDL ·sseIncMask<>+0(SB), X15
+	PADDL X15, X11
 
 	// Clamp and store the key
-	PAND  ·polyClampMask<>(SB), A0
-	MOVOU A0, rStore; MOVOU B0, sStore
+	PAND  ·polyClampMask<>+0(SB), X0
+	MOVOU X0, (BP)
+	MOVOU X3, 16(BP)
 
 	// Hash
-	MOVQ ad_len+80(FP), itr2
+	MOVQ ad_len+80(FP), R9
 	CALL polyHashADInternal<>(SB)
 
 openSSE128Open:
-	CMPQ inl, $16
+	CMPQ BX, $0x10
 	JB   openSSETail16
-	SUBQ $16, inl
+	SUBQ $0x10, BX
 
 	// Load for hashing
-	polyAdd(0(inp))
+	ADDQ (SI), R10
+	ADCQ 8(SI), R11
+	ADCQ $0x01, R12
 
 	// Load for decryption
-	MOVOU (inp), T0; PXOR T0, A1; MOVOU A1, (oup)
-	LEAQ  (1*16)(inp), inp
-	LEAQ  (1*16)(oup), oup
-	polyMul
+	MOVOU (SI), X12
+	PXOR  X12, X1
+	MOVOU X1, (DI)
+	LEAQ  16(SI), SI
+	LEAQ  16(DI), DI
+	MOVQ  (BP), AX
+	MOVQ  AX, R15
+	MULQ  R10
+	MOVQ  AX, R13
+	MOVQ  DX, R14
+	MOVQ  (BP), AX
+	MULQ  R11
+	IMULQ R12, R15
+	ADDQ  AX, R14
+	ADCQ  DX, R15
+	MOVQ  8(BP), AX
+	MOVQ  AX, R8
+	MULQ  R10
+	ADDQ  AX, R14
+	ADCQ  $0x00, DX
+	MOVQ  DX, R10
+	MOVQ  8(BP), AX
+	MULQ  R11
+	ADDQ  AX, R15
+	ADCQ  $0x00, DX
+	IMULQ R12, R8
+	ADDQ  R10, R15
+	ADCQ  DX, R8
+	MOVQ  R13, R10
+	MOVQ  R14, R11
+	MOVQ  R15, R12
+	ANDQ  $0x03, R12
+	MOVQ  R15, R13
+	ANDQ  $-4, R13
+	MOVQ  R8, R14
+	SHRQ  $0x02, R8, R15
+	SHRQ  $0x02, R8
+	ADDQ  R13, R10
+	ADCQ  R14, R11
+	ADCQ  $0x00, R12
+	ADDQ  R15, R10
+	ADCQ  R8, R11
+	ADCQ  $0x00, R12
 
 	// Shift the stream "left"
-	MOVO B1, A1
-	MOVO C1, B1
-	MOVO D1, C1
-	MOVO A2, D1
-	MOVO B2, A2
-	MOVO C2, B2
-	MOVO D2, C2
+	MOVO X4, X1
+	MOVO X7, X4
+	MOVO X10, X7
+	MOVO X2, X10
+	MOVO X5, X2
+	MOVO X8, X5
+	MOVO X11, X8
 	JMP  openSSE128Open
 
 openSSETail16:
-	TESTQ inl, inl
+	TESTQ BX, BX
 	JE    openSSEFinalize
 
 	// We can safely load the CT from the end, because it is padded with the MAC
-	MOVQ   inl, itr2
-	SHLQ   $4, itr2
-	LEAQ   ·andMask<>(SB), t0
-	MOVOU  (inp), T0
-	ADDQ   inl, inp
-	PAND   -16(t0)(itr2*1), T0
-	MOVO   T0, 0+tmpStore
-	MOVQ   T0, t0
-	MOVQ   8+tmpStore, t1
-	PXOR   A1, T0
+	MOVQ  BX, R9
+	SHLQ  $0x04, R9
+	LEAQ  ·andMask<>+0(SB), R13
+	MOVOU (SI), X12
+	ADDQ  BX, SI
+	PAND  -16(R13)(R9*1), X12
+	MOVO  X12, 64(BP)
+	MOVQ  X12, R13
+	MOVQ  72(BP), R14
+	PXOR  X1, X12
 
 	// We can only store one byte at a time, since plaintext can be shorter than 16 bytes
 openSSETail16Store:
-	MOVQ T0, t3
-	MOVB t3, (oup)
-	PSRLDQ $1, T0
-	INCQ   oup
-	DECQ   inl
+	MOVQ   X12, R8
+	MOVB   R8, (DI)
+	PSRLDQ $0x01, X12
+	INCQ   DI
+	DECQ   BX
 	JNE    openSSETail16Store
-	ADDQ   t0, acc0; ADCQ t1, acc1; ADCQ $1, acc2
-	polyMul
+	ADDQ   R13, R10
+	ADCQ   R14, R11
+	ADCQ   $0x01, R12
+	MOVQ   (BP), AX
+	MOVQ   AX, R15
+	MULQ   R10
+	MOVQ   AX, R13
+	MOVQ   DX, R14
+	MOVQ   (BP), AX
+	MULQ   R11
+	IMULQ  R12, R15
+	ADDQ   AX, R14
+	ADCQ   DX, R15
+	MOVQ   8(BP), AX
+	MOVQ   AX, R8
+	MULQ   R10
+	ADDQ   AX, R14
+	ADCQ   $0x00, DX
+	MOVQ   DX, R10
+	MOVQ   8(BP), AX
+	MULQ   R11
+	ADDQ   AX, R15
+	ADCQ   $0x00, DX
+	IMULQ  R12, R8
+	ADDQ   R10, R15
+	ADCQ   DX, R8
+	MOVQ   R13, R10
+	MOVQ   R14, R11
+	MOVQ   R15, R12
+	ANDQ   $0x03, R12
+	MOVQ   R15, R13
+	ANDQ   $-4, R13
+	MOVQ   R8, R14
+	SHRQ   $0x02, R8, R15
+	SHRQ   $0x02, R8
+	ADDQ   R13, R10
+	ADCQ   R14, R11
+	ADCQ   $0x00, R12
+	ADDQ   R15, R10
+	ADCQ   R8, R11
+	ADCQ   $0x00, R12
 	JMP    openSSEFinalize
 
-// ----------------------------------------------------------------------------
-// Special optimization for the last 64 bytes of ciphertext
 openSSETail64:
-	// Need to decrypt up to 64 bytes - prepare single block
-	MOVO ·chacha20Constants<>(SB), A0; MOVO state1Store, B0; MOVO state2Store, C0; MOVO ctr3Store, D0; PADDL ·sseIncMask<>(SB), D0; MOVO D0, ctr0Store
-	XORQ itr2, itr2
-	MOVQ inl, itr1
-	CMPQ itr1, $16
-	JB   openSSETail64LoopB
+	MOVO  ·chacha20Constants<>+0(SB), X0
+	MOVO  32(BP), X3
+	MOVO  48(BP), X6
+	MOVO  128(BP), X9
+	PADDL ·sseIncMask<>+0(SB), X9
+	MOVO  X9, 80(BP)
+	XORQ  R9, R9
+	MOVQ  BX, CX
+	CMPQ  CX, $0x10
+	JB    openSSETail64LoopB
 
 openSSETail64LoopA:
-	// Perform ChaCha rounds, while hashing the remaining input
-	polyAdd(0(inp)(itr2*1))
-	polyMul
-	SUBQ $16, itr1
+	ADDQ  (SI)(R9*1), R10
+	ADCQ  8(SI)(R9*1), R11
+	ADCQ  $0x01, R12
+	MOVQ  (BP), AX
+	MOVQ  AX, R15
+	MULQ  R10
+	MOVQ  AX, R13
+	MOVQ  DX, R14
+	MOVQ  (BP), AX
+	MULQ  R11
+	IMULQ R12, R15
+	ADDQ  AX, R14
+	ADCQ  DX, R15
+	MOVQ  8(BP), AX
+	MOVQ  AX, R8
+	MULQ  R10
+	ADDQ  AX, R14
+	ADCQ  $0x00, DX
+	MOVQ  DX, R10
+	MOVQ  8(BP), AX
+	MULQ  R11
+	ADDQ  AX, R15
+	ADCQ  $0x00, DX
+	IMULQ R12, R8
+	ADDQ  R10, R15
+	ADCQ  DX, R8
+	MOVQ  R13, R10
+	MOVQ  R14, R11
+	MOVQ  R15, R12
+	ANDQ  $0x03, R12
+	MOVQ  R15, R13
+	ANDQ  $-4, R13
+	MOVQ  R8, R14
+	SHRQ  $0x02, R8, R15
+	SHRQ  $0x02, R8
+	ADDQ  R13, R10
+	ADCQ  R14, R11
+	ADCQ  $0x00, R12
+	ADDQ  R15, R10
+	ADCQ  R8, R11
+	ADCQ  $0x00, R12
+	SUBQ  $0x10, CX
 
 openSSETail64LoopB:
-	ADDQ          $16, itr2
-	chachaQR(A0, B0, C0, D0, T0)
-	shiftB0Left;  shiftC0Left; shiftD0Left
-	chachaQR(A0, B0, C0, D0, T0)
-	shiftB0Right; shiftC0Right; shiftD0Right
-
-	CMPQ itr1, $16
-	JAE  openSSETail64LoopA
-
-	CMPQ itr2, $160
-	JNE  openSSETail64LoopB
-
-	PADDL ·chacha20Constants<>(SB), A0; PADDL state1Store, B0; PADDL state2Store, C0; PADDL ctr0Store, D0
+	ADDQ  $0x10, R9
+	PADDD X3, X0
+	PXOR  X0, X9
+	ROL16(X9, X12)
+	PADDD X9, X6
+	PXOR  X6, X3
+	MOVO  X3, X12
+	PSLLL $0x0c, X12
+	PSRLL $0x14, X3
+	PXOR  X12, X3
+	PADDD X3, X0
+	PXOR  X0, X9
+	ROL8(X9, X12)
+	PADDD X9, X6
+	PXOR  X6, X3
+	MOVO  X3, X12
+	PSLLL $0x07, X12
+	PSRLL $0x19, X3
+	PXOR  X12, X3
+	BYTE  $0x66
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xdb
+	BYTE  $0x04
+	BYTE  $0x66
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xf6
+	BYTE  $0x08
+	BYTE  $0x66
+	BYTE  $0x45
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xc9
+	BYTE  $0x0c
+	PADDD X3, X0
+	PXOR  X0, X9
+	ROL16(X9, X12)
+	PADDD X9, X6
+	PXOR  X6, X3
+	MOVO  X3, X12
+	PSLLL $0x0c, X12
+	PSRLL $0x14, X3
+	PXOR  X12, X3
+	PADDD X3, X0
+	PXOR  X0, X9
+	ROL8(X9, X12)
+	PADDD X9, X6
+	PXOR  X6, X3
+	MOVO  X3, X12
+	PSLLL $0x07, X12
+	PSRLL $0x19, X3
+	PXOR  X12, X3
+	BYTE  $0x66
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xdb
+	BYTE  $0x0c
+	BYTE  $0x66
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xf6
+	BYTE  $0x08
+	BYTE  $0x66
+	BYTE  $0x45
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xc9
+	BYTE  $0x04
+	CMPQ  CX, $0x10
+	JAE   openSSETail64LoopA
+	CMPQ  R9, $0xa0
+	JNE   openSSETail64LoopB
+	PADDL ·chacha20Constants<>+0(SB), X0
+	PADDL 32(BP), X3
+	PADDL 48(BP), X6
+	PADDL 80(BP), X9
 
 openSSETail64DecLoop:
-	CMPQ  inl, $16
+	CMPQ  BX, $0x10
 	JB    openSSETail64DecLoopDone
-	SUBQ  $16, inl
-	MOVOU (inp), T0
-	PXOR  T0, A0
-	MOVOU A0, (oup)
-	LEAQ  16(inp), inp
-	LEAQ  16(oup), oup
-	MOVO  B0, A0
-	MOVO  C0, B0
-	MOVO  D0, C0
+	SUBQ  $0x10, BX
+	MOVOU (SI), X12
+	PXOR  X12, X0
+	MOVOU X0, (DI)
+	LEAQ  16(SI), SI
+	LEAQ  16(DI), DI
+	MOVO  X3, X0
+	MOVO  X6, X3
+	MOVO  X9, X6
 	JMP   openSSETail64DecLoop
 
 openSSETail64DecLoopDone:
-	MOVO A0, A1
+	MOVO X0, X1
 	JMP  openSSETail16
 
-// ----------------------------------------------------------------------------
-// Special optimization for the last 128 bytes of ciphertext
 openSSETail128:
-	// Need to decrypt up to 128 bytes - prepare two blocks
-	MOVO ·chacha20Constants<>(SB), A1; MOVO state1Store, B1; MOVO state2Store, C1; MOVO ctr3Store, D1; PADDL ·sseIncMask<>(SB), D1; MOVO D1, ctr0Store
-	MOVO A1, A0; MOVO B1, B0; MOVO C1, C0; MOVO D1, D0; PADDL ·sseIncMask<>(SB), D0; MOVO D0, ctr1Store
-	XORQ itr2, itr2
-	MOVQ inl, itr1
-	ANDQ $-16, itr1
+	MOVO  ·chacha20Constants<>+0(SB), X1
+	MOVO  32(BP), X4
+	MOVO  48(BP), X7
+	MOVO  128(BP), X10
+	PADDL ·sseIncMask<>+0(SB), X10
+	MOVO  X10, 80(BP)
+	MOVO  X1, X0
+	MOVO  X4, X3
+	MOVO  X7, X6
+	MOVO  X10, X9
+	PADDL ·sseIncMask<>+0(SB), X9
+	MOVO  X9, 96(BP)
+	XORQ  R9, R9
+	MOVQ  BX, CX
+	ANDQ  $-16, CX
 
 openSSETail128LoopA:
-	// Perform ChaCha rounds, while hashing the remaining input
-	polyAdd(0(inp)(itr2*1))
-	polyMul
+	ADDQ  (SI)(R9*1), R10
+	ADCQ  8(SI)(R9*1), R11
+	ADCQ  $0x01, R12
+	MOVQ  (BP), AX
+	MOVQ  AX, R15
+	MULQ  R10
+	MOVQ  AX, R13
+	MOVQ  DX, R14
+	MOVQ  (BP), AX
+	MULQ  R11
+	IMULQ R12, R15
+	ADDQ  AX, R14
+	ADCQ  DX, R15
+	MOVQ  8(BP), AX
+	MOVQ  AX, R8
+	MULQ  R10
+	ADDQ  AX, R14
+	ADCQ  $0x00, DX
+	MOVQ  DX, R10
+	MOVQ  8(BP), AX
+	MULQ  R11
+	ADDQ  AX, R15
+	ADCQ  $0x00, DX
+	IMULQ R12, R8
+	ADDQ  R10, R15
+	ADCQ  DX, R8
+	MOVQ  R13, R10
+	MOVQ  R14, R11
+	MOVQ  R15, R12
+	ANDQ  $0x03, R12
+	MOVQ  R15, R13
+	ANDQ  $-4, R13
+	MOVQ  R8, R14
+	SHRQ  $0x02, R8, R15
+	SHRQ  $0x02, R8
+	ADDQ  R13, R10
+	ADCQ  R14, R11
+	ADCQ  $0x00, R12
+	ADDQ  R15, R10
+	ADCQ  R8, R11
+	ADCQ  $0x00, R12
 
 openSSETail128LoopB:
-	ADDQ          $16, itr2
-	chachaQR(A0, B0, C0, D0, T0); chachaQR(A1, B1, C1, D1, T0)
-	shiftB0Left;  shiftC0Left; shiftD0Left
-	shiftB1Left;  shiftC1Left; shiftD1Left
-	chachaQR(A0, B0, C0, D0, T0); chachaQR(A1, B1, C1, D1, T0)
-	shiftB0Right; shiftC0Right; shiftD0Right
-	shiftB1Right; shiftC1Right; shiftD1Right
-
-	CMPQ itr2, itr1
-	JB   openSSETail128LoopA
-
-	CMPQ itr2, $160
-	JNE  openSSETail128LoopB
-
-	PADDL ·chacha20Constants<>(SB), A0; PADDL ·chacha20Constants<>(SB), A1
-	PADDL state1Store, B0; PADDL state1Store, B1
-	PADDL state2Store, C0; PADDL state2Store, C1
-	PADDL ctr1Store, D0; PADDL ctr0Store, D1
-
-	MOVOU (0*16)(inp), T0; MOVOU (1*16)(inp), T1; MOVOU (2*16)(inp), T2; MOVOU (3*16)(inp), T3
-	PXOR  T0, A1; PXOR T1, B1; PXOR T2, C1; PXOR T3, D1
-	MOVOU A1, (0*16)(oup); MOVOU B1, (1*16)(oup); MOVOU C1, (2*16)(oup); MOVOU D1, (3*16)(oup)
-
-	SUBQ $64, inl
-	LEAQ 64(inp), inp
-	LEAQ 64(oup), oup
-	JMP  openSSETail64DecLoop
-
-// ----------------------------------------------------------------------------
-// Special optimization for the last 192 bytes of ciphertext
+	ADDQ  $0x10, R9
+	PADDD X3, X0
+	PXOR  X0, X9
+	ROL16(X9, X12)
+	PADDD X9, X6
+	PXOR  X6, X3
+	MOVO  X3, X12
+	PSLLL $0x0c, X12
+	PSRLL $0x14, X3
+	PXOR  X12, X3
+	PADDD X3, X0
+	PXOR  X0, X9
+	ROL8(X9, X12)
+	PADDD X9, X6
+	PXOR  X6, X3
+	MOVO  X3, X12
+	PSLLL $0x07, X12
+	PSRLL $0x19, X3
+	PXOR  X12, X3
+	PADDD X4, X1
+	PXOR  X1, X10
+	ROL16(X10, X12)
+	PADDD X10, X7
+	PXOR  X7, X4
+	MOVO  X4, X12
+	PSLLL $0x0c, X12
+	PSRLL $0x14, X4
+	PXOR  X12, X4
+	PADDD X4, X1
+	PXOR  X1, X10
+	ROL8(X10, X12)
+	PADDD X10, X7
+	PXOR  X7, X4
+	MOVO  X4, X12
+	PSLLL $0x07, X12
+	PSRLL $0x19, X4
+	PXOR  X12, X4
+	BYTE  $0x66
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xdb
+	BYTE  $0x04
+	BYTE  $0x66
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xf6
+	BYTE  $0x08
+	BYTE  $0x66
+	BYTE  $0x45
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xc9
+	BYTE  $0x0c
+	BYTE  $0x66
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xe4
+	BYTE  $0x04
+	BYTE  $0x66
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xff
+	BYTE  $0x08
+	BYTE  $0x66
+	BYTE  $0x45
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xd2
+	BYTE  $0x0c
+	PADDD X3, X0
+	PXOR  X0, X9
+	ROL16(X9, X12)
+	PADDD X9, X6
+	PXOR  X6, X3
+	MOVO  X3, X12
+	PSLLL $0x0c, X12
+	PSRLL $0x14, X3
+	PXOR  X12, X3
+	PADDD X3, X0
+	PXOR  X0, X9
+	ROL8(X9, X12)
+	PADDD X9, X6
+	PXOR  X6, X3
+	MOVO  X3, X12
+	PSLLL $0x07, X12
+	PSRLL $0x19, X3
+	PXOR  X12, X3
+	PADDD X4, X1
+	PXOR  X1, X10
+	ROL16(X10, X12)
+	PADDD X10, X7
+	PXOR  X7, X4
+	MOVO  X4, X12
+	PSLLL $0x0c, X12
+	PSRLL $0x14, X4
+	PXOR  X12, X4
+	PADDD X4, X1
+	PXOR  X1, X10
+	ROL8(X10, X12)
+	PADDD X10, X7
+	PXOR  X7, X4
+	MOVO  X4, X12
+	PSLLL $0x07, X12
+	PSRLL $0x19, X4
+	PXOR  X12, X4
+	BYTE  $0x66
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xdb
+	BYTE  $0x0c
+	BYTE  $0x66
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xf6
+	BYTE  $0x08
+	BYTE  $0x66
+	BYTE  $0x45
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xc9
+	BYTE  $0x04
+	BYTE  $0x66
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xe4
+	BYTE  $0x0c
+	BYTE  $0x66
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xff
+	BYTE  $0x08
+	BYTE  $0x66
+	BYTE  $0x45
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xd2
+	BYTE  $0x04
+	CMPQ  R9, CX
+	JB    openSSETail128LoopA
+	CMPQ  R9, $0xa0
+	JNE   openSSETail128LoopB
+	PADDL ·chacha20Constants<>+0(SB), X0
+	PADDL ·chacha20Constants<>+0(SB), X1
+	PADDL 32(BP), X3
+	PADDL 32(BP), X4
+	PADDL 48(BP), X6
+	PADDL 48(BP), X7
+	PADDL 96(BP), X9
+	PADDL 80(BP), X10
+	MOVOU (SI), X12
+	MOVOU 16(SI), X13
+	MOVOU 32(SI), X14
+	MOVOU 48(SI), X15
+	PXOR  X12, X1
+	PXOR  X13, X4
+	PXOR  X14, X7
+	PXOR  X15, X10
+	MOVOU X1, (DI)
+	MOVOU X4, 16(DI)
+	MOVOU X7, 32(DI)
+	MOVOU X10, 48(DI)
+	SUBQ  $0x40, BX
+	LEAQ  64(SI), SI
+	LEAQ  64(DI), DI
+	JMP   openSSETail64DecLoop
+
 openSSETail192:
-	// Need to decrypt up to 192 bytes - prepare three blocks
-	MOVO ·chacha20Constants<>(SB), A2; MOVO state1Store, B2; MOVO state2Store, C2; MOVO ctr3Store, D2; PADDL ·sseIncMask<>(SB), D2; MOVO D2, ctr0Store
-	MOVO A2, A1; MOVO B2, B1; MOVO C2, C1; MOVO D2, D1; PADDL ·sseIncMask<>(SB), D1; MOVO D1, ctr1Store
-	MOVO A1, A0; MOVO B1, B0; MOVO C1, C0; MOVO D1, D0; PADDL ·sseIncMask<>(SB), D0; MOVO D0, ctr2Store
-
-	MOVQ    inl, itr1
-	MOVQ    $160, itr2
-	CMPQ    itr1, $160
-	CMOVQGT itr2, itr1
-	ANDQ    $-16, itr1
-	XORQ    itr2, itr2
+	MOVO    ·chacha20Constants<>+0(SB), X2
+	MOVO    32(BP), X5
+	MOVO    48(BP), X8
+	MOVO    128(BP), X11
+	PADDL   ·sseIncMask<>+0(SB), X11
+	MOVO    X11, 80(BP)
+	MOVO    X2, X1
+	MOVO    X5, X4
+	MOVO    X8, X7
+	MOVO    X11, X10
+	PADDL   ·sseIncMask<>+0(SB), X10
+	MOVO    X10, 96(BP)
+	MOVO    X1, X0
+	MOVO    X4, X3
+	MOVO    X7, X6
+	MOVO    X10, X9
+	PADDL   ·sseIncMask<>+0(SB), X9
+	MOVO    X9, 112(BP)
+	MOVQ    BX, CX
+	MOVQ    $0x000000a0, R9
+	CMPQ    CX, $0xa0
+	CMOVQGT R9, CX
+	ANDQ    $-16, CX
+	XORQ    R9, R9
 
 openSSLTail192LoopA:
-	// Perform ChaCha rounds, while hashing the remaining input
-	polyAdd(0(inp)(itr2*1))
-	polyMul
+	ADDQ  (SI)(R9*1), R10
+	ADCQ  8(SI)(R9*1), R11
+	ADCQ  $0x01, R12
+	MOVQ  (BP), AX
+	MOVQ  AX, R15
+	MULQ  R10
+	MOVQ  AX, R13
+	MOVQ  DX, R14
+	MOVQ  (BP), AX
+	MULQ  R11
+	IMULQ R12, R15
+	ADDQ  AX, R14
+	ADCQ  DX, R15
+	MOVQ  8(BP), AX
+	MOVQ  AX, R8
+	MULQ  R10
+	ADDQ  AX, R14
+	ADCQ  $0x00, DX
+	MOVQ  DX, R10
+	MOVQ  8(BP), AX
+	MULQ  R11
+	ADDQ  AX, R15
+	ADCQ  $0x00, DX
+	IMULQ R12, R8
+	ADDQ  R10, R15
+	ADCQ  DX, R8
+	MOVQ  R13, R10
+	MOVQ  R14, R11
+	MOVQ  R15, R12
+	ANDQ  $0x03, R12
+	MOVQ  R15, R13
+	ANDQ  $-4, R13
+	MOVQ  R8, R14
+	SHRQ  $0x02, R8, R15
+	SHRQ  $0x02, R8
+	ADDQ  R13, R10
+	ADCQ  R14, R11
+	ADCQ  $0x00, R12
+	ADDQ  R15, R10
+	ADCQ  R8, R11
+	ADCQ  $0x00, R12
 
 openSSLTail192LoopB:
-	ADDQ         $16, itr2
-	chachaQR(A0, B0, C0, D0, T0); chachaQR(A1, B1, C1, D1, T0); chachaQR(A2, B2, C2, D2, T0)
-	shiftB0Left; shiftC0Left; shiftD0Left
-	shiftB1Left; shiftC1Left; shiftD1Left
-	shiftB2Left; shiftC2Left; shiftD2Left
-
-	chachaQR(A0, B0, C0, D0, T0); chachaQR(A1, B1, C1, D1, T0); chachaQR(A2, B2, C2, D2, T0)
-	shiftB0Right; shiftC0Right; shiftD0Right
-	shiftB1Right; shiftC1Right; shiftD1Right
-	shiftB2Right; shiftC2Right; shiftD2Right
-
-	CMPQ itr2, itr1
-	JB   openSSLTail192LoopA
-
-	CMPQ itr2, $160
-	JNE  openSSLTail192LoopB
-
-	CMPQ inl, $176
-	JB   openSSLTail192Store
-
-	polyAdd(160(inp))
-	polyMul
-
-	CMPQ inl, $192
-	JB   openSSLTail192Store
-
-	polyAdd(176(inp))
-	polyMul
+	ADDQ  $0x10, R9
+	PADDD X3, X0
+	PXOR  X0, X9
+	ROL16(X9, X12)
+	PADDD X9, X6
+	PXOR  X6, X3
+	MOVO  X3, X12
+	PSLLL $0x0c, X12
+	PSRLL $0x14, X3
+	PXOR  X12, X3
+	PADDD X3, X0
+	PXOR  X0, X9
+	ROL8(X9, X12)
+	PADDD X9, X6
+	PXOR  X6, X3
+	MOVO  X3, X12
+	PSLLL $0x07, X12
+	PSRLL $0x19, X3
+	PXOR  X12, X3
+	PADDD X4, X1
+	PXOR  X1, X10
+	ROL16(X10, X12)
+	PADDD X10, X7
+	PXOR  X7, X4
+	MOVO  X4, X12
+	PSLLL $0x0c, X12
+	PSRLL $0x14, X4
+	PXOR  X12, X4
+	PADDD X4, X1
+	PXOR  X1, X10
+	ROL8(X10, X12)
+	PADDD X10, X7
+	PXOR  X7, X4
+	MOVO  X4, X12
+	PSLLL $0x07, X12
+	PSRLL $0x19, X4
+	PXOR  X12, X4
+	PADDD X5, X2
+	PXOR  X2, X11
+	ROL16(X11, X12)
+	PADDD X11, X8
+	PXOR  X8, X5
+	MOVO  X5, X12
+	PSLLL $0x0c, X12
+	PSRLL $0x14, X5
+	PXOR  X12, X5
+	PADDD X5, X2
+	PXOR  X2, X11
+	ROL8(X11, X12)
+	PADDD X11, X8
+	PXOR  X8, X5
+	MOVO  X5, X12
+	PSLLL $0x07, X12
+	PSRLL $0x19, X5
+	PXOR  X12, X5
+	BYTE  $0x66
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xdb
+	BYTE  $0x04
+	BYTE  $0x66
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xf6
+	BYTE  $0x08
+	BYTE  $0x66
+	BYTE  $0x45
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xc9
+	BYTE  $0x0c
+	BYTE  $0x66
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xe4
+	BYTE  $0x04
+	BYTE  $0x66
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xff
+	BYTE  $0x08
+	BYTE  $0x66
+	BYTE  $0x45
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xd2
+	BYTE  $0x0c
+	BYTE  $0x66
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xed
+	BYTE  $0x04
+	BYTE  $0x66
+	BYTE  $0x45
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xc0
+	BYTE  $0x08
+	BYTE  $0x66
+	BYTE  $0x45
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xdb
+	BYTE  $0x0c
+	PADDD X3, X0
+	PXOR  X0, X9
+	ROL16(X9, X12)
+	PADDD X9, X6
+	PXOR  X6, X3
+	MOVO  X3, X12
+	PSLLL $0x0c, X12
+	PSRLL $0x14, X3
+	PXOR  X12, X3
+	PADDD X3, X0
+	PXOR  X0, X9
+	ROL8(X9, X12)
+	PADDD X9, X6
+	PXOR  X6, X3
+	MOVO  X3, X12
+	PSLLL $0x07, X12
+	PSRLL $0x19, X3
+	PXOR  X12, X3
+	PADDD X4, X1
+	PXOR  X1, X10
+	ROL16(X10, X12)
+	PADDD X10, X7
+	PXOR  X7, X4
+	MOVO  X4, X12
+	PSLLL $0x0c, X12
+	PSRLL $0x14, X4
+	PXOR  X12, X4
+	PADDD X4, X1
+	PXOR  X1, X10
+	ROL8(X10, X12)
+	PADDD X10, X7
+	PXOR  X7, X4
+	MOVO  X4, X12
+	PSLLL $0x07, X12
+	PSRLL $0x19, X4
+	PXOR  X12, X4
+	PADDD X5, X2
+	PXOR  X2, X11
+	ROL16(X11, X12)
+	PADDD X11, X8
+	PXOR  X8, X5
+	MOVO  X5, X12
+	PSLLL $0x0c, X12
+	PSRLL $0x14, X5
+	PXOR  X12, X5
+	PADDD X5, X2
+	PXOR  X2, X11
+	ROL8(X11, X12)
+	PADDD X11, X8
+	PXOR  X8, X5
+	MOVO  X5, X12
+	PSLLL $0x07, X12
+	PSRLL $0x19, X5
+	PXOR  X12, X5
+	BYTE  $0x66
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xdb
+	BYTE  $0x0c
+	BYTE  $0x66
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xf6
+	BYTE  $0x08
+	BYTE  $0x66
+	BYTE  $0x45
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xc9
+	BYTE  $0x04
+	BYTE  $0x66
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xe4
+	BYTE  $0x0c
+	BYTE  $0x66
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xff
+	BYTE  $0x08
+	BYTE  $0x66
+	BYTE  $0x45
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xd2
+	BYTE  $0x04
+	BYTE  $0x66
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xed
+	BYTE  $0x0c
+	BYTE  $0x66
+	BYTE  $0x45
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xc0
+	BYTE  $0x08
+	BYTE  $0x66
+	BYTE  $0x45
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xdb
+	BYTE  $0x04
+	CMPQ  R9, CX
+	JB    openSSLTail192LoopA
+	CMPQ  R9, $0xa0
+	JNE   openSSLTail192LoopB
+	CMPQ  BX, $0xb0
+	JB    openSSLTail192Store
+	ADDQ  160(SI), R10
+	ADCQ  168(SI), R11
+	ADCQ  $0x01, R12
+	MOVQ  (BP), AX
+	MOVQ  AX, R15
+	MULQ  R10
+	MOVQ  AX, R13
+	MOVQ  DX, R14
+	MOVQ  (BP), AX
+	MULQ  R11
+	IMULQ R12, R15
+	ADDQ  AX, R14
+	ADCQ  DX, R15
+	MOVQ  8(BP), AX
+	MOVQ  AX, R8
+	MULQ  R10
+	ADDQ  AX, R14
+	ADCQ  $0x00, DX
+	MOVQ  DX, R10
+	MOVQ  8(BP), AX
+	MULQ  R11
+	ADDQ  AX, R15
+	ADCQ  $0x00, DX
+	IMULQ R12, R8
+	ADDQ  R10, R15
+	ADCQ  DX, R8
+	MOVQ  R13, R10
+	MOVQ  R14, R11
+	MOVQ  R15, R12
+	ANDQ  $0x03, R12
+	MOVQ  R15, R13
+	ANDQ  $-4, R13
+	MOVQ  R8, R14
+	SHRQ  $0x02, R8, R15
+	SHRQ  $0x02, R8
+	ADDQ  R13, R10
+	ADCQ  R14, R11
+	ADCQ  $0x00, R12
+	ADDQ  R15, R10
+	ADCQ  R8, R11
+	ADCQ  $0x00, R12
+	CMPQ  BX, $0xc0
+	JB    openSSLTail192Store
+	ADDQ  176(SI), R10
+	ADCQ  184(SI), R11
+	ADCQ  $0x01, R12
+	MOVQ  (BP), AX
+	MOVQ  AX, R15
+	MULQ  R10
+	MOVQ  AX, R13
+	MOVQ  DX, R14
+	MOVQ  (BP), AX
+	MULQ  R11
+	IMULQ R12, R15
+	ADDQ  AX, R14
+	ADCQ  DX, R15
+	MOVQ  8(BP), AX
+	MOVQ  AX, R8
+	MULQ  R10
+	ADDQ  AX, R14
+	ADCQ  $0x00, DX
+	MOVQ  DX, R10
+	MOVQ  8(BP), AX
+	MULQ  R11
+	ADDQ  AX, R15
+	ADCQ  $0x00, DX
+	IMULQ R12, R8
+	ADDQ  R10, R15
+	ADCQ  DX, R8
+	MOVQ  R13, R10
+	MOVQ  R14, R11
+	MOVQ  R15, R12
+	ANDQ  $0x03, R12
+	MOVQ  R15, R13
+	ANDQ  $-4, R13
+	MOVQ  R8, R14
+	SHRQ  $0x02, R8, R15
+	SHRQ  $0x02, R8
+	ADDQ  R13, R10
+	ADCQ  R14, R11
+	ADCQ  $0x00, R12
+	ADDQ  R15, R10
+	ADCQ  R8, R11
+	ADCQ  $0x00, R12
 
 openSSLTail192Store:
-	PADDL ·chacha20Constants<>(SB), A0; PADDL ·chacha20Constants<>(SB), A1; PADDL ·chacha20Constants<>(SB), A2
-	PADDL state1Store, B0; PADDL state1Store, B1; PADDL state1Store, B2
-	PADDL state2Store, C0; PADDL state2Store, C1; PADDL state2Store, C2
-	PADDL ctr2Store, D0; PADDL ctr1Store, D1; PADDL ctr0Store, D2
-
-	MOVOU (0*16)(inp), T0; MOVOU (1*16)(inp), T1; MOVOU (2*16)(inp), T2; MOVOU (3*16)(inp), T3
-	PXOR  T0, A2; PXOR T1, B2; PXOR T2, C2; PXOR T3, D2
-	MOVOU A2, (0*16)(oup); MOVOU B2, (1*16)(oup); MOVOU C2, (2*16)(oup); MOVOU D2, (3*16)(oup)
-
-	MOVOU (4*16)(inp), T0; MOVOU (5*16)(inp), T1; MOVOU (6*16)(inp), T2; MOVOU (7*16)(inp), T3
-	PXOR  T0, A1; PXOR T1, B1; PXOR T2, C1; PXOR T3, D1
-	MOVOU A1, (4*16)(oup); MOVOU B1, (5*16)(oup); MOVOU C1, (6*16)(oup); MOVOU D1, (7*16)(oup)
-
-	SUBQ $128, inl
-	LEAQ 128(inp), inp
-	LEAQ 128(oup), oup
-	JMP  openSSETail64DecLoop
-
-// ----------------------------------------------------------------------------
-// Special optimization for the last 256 bytes of ciphertext
+	PADDL ·chacha20Constants<>+0(SB), X0
+	PADDL ·chacha20Constants<>+0(SB), X1
+	PADDL ·chacha20Constants<>+0(SB), X2
+	PADDL 32(BP), X3
+	PADDL 32(BP), X4
+	PADDL 32(BP), X5
+	PADDL 48(BP), X6
+	PADDL 48(BP), X7
+	PADDL 48(BP), X8
+	PADDL 112(BP), X9
+	PADDL 96(BP), X10
+	PADDL 80(BP), X11
+	MOVOU (SI), X12
+	MOVOU 16(SI), X13
+	MOVOU 32(SI), X14
+	MOVOU 48(SI), X15
+	PXOR  X12, X2
+	PXOR  X13, X5
+	PXOR  X14, X8
+	PXOR  X15, X11
+	MOVOU X2, (DI)
+	MOVOU X5, 16(DI)
+	MOVOU X8, 32(DI)
+	MOVOU X11, 48(DI)
+	MOVOU 64(SI), X12
+	MOVOU 80(SI), X13
+	MOVOU 96(SI), X14
+	MOVOU 112(SI), X15
+	PXOR  X12, X1
+	PXOR  X13, X4
+	PXOR  X14, X7
+	PXOR  X15, X10
+	MOVOU X1, 64(DI)
+	MOVOU X4, 80(DI)
+	MOVOU X7, 96(DI)
+	MOVOU X10, 112(DI)
+	SUBQ  $0x80, BX
+	LEAQ  128(SI), SI
+	LEAQ  128(DI), DI
+	JMP   openSSETail64DecLoop
+
 openSSETail256:
-	// Need to decrypt up to 256 bytes - prepare four blocks
-	MOVO ·chacha20Constants<>(SB), A0; MOVO state1Store, B0; MOVO state2Store, C0; MOVO ctr3Store, D0; PADDL ·sseIncMask<>(SB), D0
-	MOVO A0, A1; MOVO B0, B1; MOVO C0, C1; MOVO D0, D1; PADDL ·sseIncMask<>(SB), D1
-	MOVO A1, A2; MOVO B1, B2; MOVO C1, C2; MOVO D1, D2; PADDL ·sseIncMask<>(SB), D2
-	MOVO A2, A3; MOVO B2, B3; MOVO C2, C3; MOVO D2, D3; PADDL ·sseIncMask<>(SB), D3
+	MOVO  ·chacha20Constants<>+0(SB), X0
+	MOVO  32(BP), X3
+	MOVO  48(BP), X6
+	MOVO  128(BP), X9
+	PADDL ·sseIncMask<>+0(SB), X9
+	MOVO  X0, X1
+	MOVO  X3, X4
+	MOVO  X6, X7
+	MOVO  X9, X10
+	PADDL ·sseIncMask<>+0(SB), X10
+	MOVO  X1, X2
+	MOVO  X4, X5
+	MOVO  X7, X8
+	MOVO  X10, X11
+	PADDL ·sseIncMask<>+0(SB), X11
+	MOVO  X2, X12
+	MOVO  X5, X13
+	MOVO  X8, X14
+	MOVO  X11, X15
+	PADDL ·sseIncMask<>+0(SB), X15
 
 	// Store counters
-	MOVO D0, ctr0Store; MOVO D1, ctr1Store; MOVO D2, ctr2Store; MOVO D3, ctr3Store
-	XORQ itr2, itr2
+	MOVO X9, 80(BP)
+	MOVO X10, 96(BP)
+	MOVO X11, 112(BP)
+	MOVO X15, 128(BP)
+	XORQ R9, R9
 
 openSSETail256Loop:
-	// This loop inteleaves 8 ChaCha quarter rounds with 1 poly multiplication
-	polyAdd(0(inp)(itr2*1))
-	MOVO          C3, tmpStore
-	chachaQR(A0, B0, C0, D0, C3); chachaQR(A1, B1, C1, D1, C3); chachaQR(A2, B2, C2, D2, C3)
-	MOVO          tmpStore, C3
-	MOVO          C1, tmpStore
-	chachaQR(A3, B3, C3, D3, C1)
-	MOVO          tmpStore, C1
-	shiftB0Left;  shiftB1Left; shiftB2Left; shiftB3Left
-	shiftC0Left;  shiftC1Left; shiftC2Left; shiftC3Left
-	shiftD0Left;  shiftD1Left; shiftD2Left; shiftD3Left
-	polyMulStage1
-	polyMulStage2
-	MOVO          C3, tmpStore
-	chachaQR(A0, B0, C0, D0, C3); chachaQR(A1, B1, C1, D1, C3); chachaQR(A2, B2, C2, D2, C3)
-	MOVO          tmpStore, C3
-	MOVO          C1, tmpStore
-	chachaQR(A3, B3, C3, D3, C1)
-	MOVO          tmpStore, C1
-	polyMulStage3
-	polyMulReduceStage
-	shiftB0Right; shiftB1Right; shiftB2Right; shiftB3Right
-	shiftC0Right; shiftC1Right; shiftC2Right; shiftC3Right
-	shiftD0Right; shiftD1Right; shiftD2Right; shiftD3Right
-	ADDQ          $2*8, itr2
-	CMPQ          itr2, $160
-	JB            openSSETail256Loop
-	MOVQ          inl, itr1
-	ANDQ          $-16, itr1
+	ADDQ  (SI)(R9*1), R10
+	ADCQ  8(SI)(R9*1), R11
+	ADCQ  $0x01, R12
+	MOVO  X14, 64(BP)
+	PADDD X3, X0
+	PXOR  X0, X9
+	ROL16(X9, X14)
+	PADDD X9, X6
+	PXOR  X6, X3
+	MOVO  X3, X14
+	PSLLL $0x0c, X14
+	PSRLL $0x14, X3
+	PXOR  X14, X3
+	PADDD X3, X0
+	PXOR  X0, X9
+	ROL8(X9, X14)
+	PADDD X9, X6
+	PXOR  X6, X3
+	MOVO  X3, X14
+	PSLLL $0x07, X14
+	PSRLL $0x19, X3
+	PXOR  X14, X3
+	PADDD X4, X1
+	PXOR  X1, X10
+	ROL16(X10, X14)
+	PADDD X10, X7
+	PXOR  X7, X4
+	MOVO  X4, X14
+	PSLLL $0x0c, X14
+	PSRLL $0x14, X4
+	PXOR  X14, X4
+	PADDD X4, X1
+	PXOR  X1, X10
+	ROL8(X10, X14)
+	PADDD X10, X7
+	PXOR  X7, X4
+	MOVO  X4, X14
+	PSLLL $0x07, X14
+	PSRLL $0x19, X4
+	PXOR  X14, X4
+	PADDD X5, X2
+	PXOR  X2, X11
+	ROL16(X11, X14)
+	PADDD X11, X8
+	PXOR  X8, X5
+	MOVO  X5, X14
+	PSLLL $0x0c, X14
+	PSRLL $0x14, X5
+	PXOR  X14, X5
+	PADDD X5, X2
+	PXOR  X2, X11
+	ROL8(X11, X14)
+	PADDD X11, X8
+	PXOR  X8, X5
+	MOVO  X5, X14
+	PSLLL $0x07, X14
+	PSRLL $0x19, X5
+	PXOR  X14, X5
+	MOVO  64(BP), X14
+	MOVO  X7, 64(BP)
+	PADDD X13, X12
+	PXOR  X12, X15
+	ROL16(X15, X7)
+	PADDD X15, X14
+	PXOR  X14, X13
+	MOVO  X13, X7
+	PSLLL $0x0c, X7
+	PSRLL $0x14, X13
+	PXOR  X7, X13
+	PADDD X13, X12
+	PXOR  X12, X15
+	ROL8(X15, X7)
+	PADDD X15, X14
+	PXOR  X14, X13
+	MOVO  X13, X7
+	PSLLL $0x07, X7
+	PSRLL $0x19, X13
+	PXOR  X7, X13
+	MOVO  64(BP), X7
+	BYTE  $0x66
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xdb
+	BYTE  $0x04
+	BYTE  $0x66
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xe4
+	BYTE  $0x04
+	BYTE  $0x66
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xed
+	BYTE  $0x04
+	BYTE  $0x66
+	BYTE  $0x45
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xed
+	BYTE  $0x04
+	BYTE  $0x66
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xf6
+	BYTE  $0x08
+	BYTE  $0x66
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xff
+	BYTE  $0x08
+	BYTE  $0x66
+	BYTE  $0x45
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xc0
+	BYTE  $0x08
+	BYTE  $0x66
+	BYTE  $0x45
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xf6
+	BYTE  $0x08
+	BYTE  $0x66
+	BYTE  $0x45
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xc9
+	BYTE  $0x0c
+	BYTE  $0x66
+	BYTE  $0x45
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xd2
+	BYTE  $0x0c
+	BYTE  $0x66
+	BYTE  $0x45
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xdb
+	BYTE  $0x0c
+	BYTE  $0x66
+	BYTE  $0x45
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xff
+	BYTE  $0x0c
+	MOVQ  (BP), AX
+	MOVQ  AX, R15
+	MULQ  R10
+	MOVQ  AX, R13
+	MOVQ  DX, R14
+	MOVQ  (BP), AX
+	MULQ  R11
+	IMULQ R12, R15
+	ADDQ  AX, R14
+	ADCQ  DX, R15
+	MOVQ  8(BP), AX
+	MOVQ  AX, R8
+	MULQ  R10
+	ADDQ  AX, R14
+	ADCQ  $0x00, DX
+	MOVQ  DX, R10
+	MOVQ  8(BP), AX
+	MULQ  R11
+	ADDQ  AX, R15
+	ADCQ  $0x00, DX
+	MOVO  X14, 64(BP)
+	PADDD X3, X0
+	PXOR  X0, X9
+	ROL16(X9, X14)
+	PADDD X9, X6
+	PXOR  X6, X3
+	MOVO  X3, X14
+	PSLLL $0x0c, X14
+	PSRLL $0x14, X3
+	PXOR  X14, X3
+	PADDD X3, X0
+	PXOR  X0, X9
+	ROL8(X9, X14)
+	PADDD X9, X6
+	PXOR  X6, X3
+	MOVO  X3, X14
+	PSLLL $0x07, X14
+	PSRLL $0x19, X3
+	PXOR  X14, X3
+	PADDD X4, X1
+	PXOR  X1, X10
+	ROL16(X10, X14)
+	PADDD X10, X7
+	PXOR  X7, X4
+	MOVO  X4, X14
+	PSLLL $0x0c, X14
+	PSRLL $0x14, X4
+	PXOR  X14, X4
+	PADDD X4, X1
+	PXOR  X1, X10
+	ROL8(X10, X14)
+	PADDD X10, X7
+	PXOR  X7, X4
+	MOVO  X4, X14
+	PSLLL $0x07, X14
+	PSRLL $0x19, X4
+	PXOR  X14, X4
+	PADDD X5, X2
+	PXOR  X2, X11
+	ROL16(X11, X14)
+	PADDD X11, X8
+	PXOR  X8, X5
+	MOVO  X5, X14
+	PSLLL $0x0c, X14
+	PSRLL $0x14, X5
+	PXOR  X14, X5
+	PADDD X5, X2
+	PXOR  X2, X11
+	ROL8(X11, X14)
+	PADDD X11, X8
+	PXOR  X8, X5
+	MOVO  X5, X14
+	PSLLL $0x07, X14
+	PSRLL $0x19, X5
+	PXOR  X14, X5
+	MOVO  64(BP), X14
+	MOVO  X7, 64(BP)
+	PADDD X13, X12
+	PXOR  X12, X15
+	ROL16(X15, X7)
+	PADDD X15, X14
+	PXOR  X14, X13
+	MOVO  X13, X7
+	PSLLL $0x0c, X7
+	PSRLL $0x14, X13
+	PXOR  X7, X13
+	PADDD X13, X12
+	PXOR  X12, X15
+	ROL8(X15, X7)
+	PADDD X15, X14
+	PXOR  X14, X13
+	MOVO  X13, X7
+	PSLLL $0x07, X7
+	PSRLL $0x19, X13
+	PXOR  X7, X13
+	MOVO  64(BP), X7
+	IMULQ R12, R8
+	ADDQ  R10, R15
+	ADCQ  DX, R8
+	MOVQ  R13, R10
+	MOVQ  R14, R11
+	MOVQ  R15, R12
+	ANDQ  $0x03, R12
+	MOVQ  R15, R13
+	ANDQ  $-4, R13
+	MOVQ  R8, R14
+	SHRQ  $0x02, R8, R15
+	SHRQ  $0x02, R8
+	ADDQ  R13, R10
+	ADCQ  R14, R11
+	ADCQ  $0x00, R12
+	ADDQ  R15, R10
+	ADCQ  R8, R11
+	ADCQ  $0x00, R12
+	BYTE  $0x66
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xdb
+	BYTE  $0x0c
+	BYTE  $0x66
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xe4
+	BYTE  $0x0c
+	BYTE  $0x66
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xed
+	BYTE  $0x0c
+	BYTE  $0x66
+	BYTE  $0x45
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xed
+	BYTE  $0x0c
+	BYTE  $0x66
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xf6
+	BYTE  $0x08
+	BYTE  $0x66
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xff
+	BYTE  $0x08
+	BYTE  $0x66
+	BYTE  $0x45
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xc0
+	BYTE  $0x08
+	BYTE  $0x66
+	BYTE  $0x45
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xf6
+	BYTE  $0x08
+	BYTE  $0x66
+	BYTE  $0x45
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xc9
+	BYTE  $0x04
+	BYTE  $0x66
+	BYTE  $0x45
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xd2
+	BYTE  $0x04
+	BYTE  $0x66
+	BYTE  $0x45
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xdb
+	BYTE  $0x04
+	BYTE  $0x66
+	BYTE  $0x45
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xff
+	BYTE  $0x04
+	ADDQ  $0x10, R9
+	CMPQ  R9, $0xa0
+	JB    openSSETail256Loop
+	MOVQ  BX, CX
+	ANDQ  $-16, CX
 
 openSSETail256HashLoop:
-	polyAdd(0(inp)(itr2*1))
-	polyMul
-	ADDQ $2*8, itr2
-	CMPQ itr2, itr1
-	JB   openSSETail256HashLoop
+	ADDQ  (SI)(R9*1), R10
+	ADCQ  8(SI)(R9*1), R11
+	ADCQ  $0x01, R12
+	MOVQ  (BP), AX
+	MOVQ  AX, R15
+	MULQ  R10
+	MOVQ  AX, R13
+	MOVQ  DX, R14
+	MOVQ  (BP), AX
+	MULQ  R11
+	IMULQ R12, R15
+	ADDQ  AX, R14
+	ADCQ  DX, R15
+	MOVQ  8(BP), AX
+	MOVQ  AX, R8
+	MULQ  R10
+	ADDQ  AX, R14
+	ADCQ  $0x00, DX
+	MOVQ  DX, R10
+	MOVQ  8(BP), AX
+	MULQ  R11
+	ADDQ  AX, R15
+	ADCQ  $0x00, DX
+	IMULQ R12, R8
+	ADDQ  R10, R15
+	ADCQ  DX, R8
+	MOVQ  R13, R10
+	MOVQ  R14, R11
+	MOVQ  R15, R12
+	ANDQ  $0x03, R12
+	MOVQ  R15, R13
+	ANDQ  $-4, R13
+	MOVQ  R8, R14
+	SHRQ  $0x02, R8, R15
+	SHRQ  $0x02, R8
+	ADDQ  R13, R10
+	ADCQ  R14, R11
+	ADCQ  $0x00, R12
+	ADDQ  R15, R10
+	ADCQ  R8, R11
+	ADCQ  $0x00, R12
+	ADDQ  $0x10, R9
+	CMPQ  R9, CX
+	JB    openSSETail256HashLoop
 
 	// Add in the state
-	PADDD ·chacha20Constants<>(SB), A0; PADDD ·chacha20Constants<>(SB), A1; PADDD ·chacha20Constants<>(SB), A2; PADDD ·chacha20Constants<>(SB), A3
-	PADDD state1Store, B0; PADDD state1Store, B1; PADDD state1Store, B2; PADDD state1Store, B3
-	PADDD state2Store, C0; PADDD state2Store, C1; PADDD state2Store, C2; PADDD state2Store, C3
-	PADDD ctr0Store, D0; PADDD ctr1Store, D1; PADDD ctr2Store, D2; PADDD ctr3Store, D3
-	MOVO  D3, tmpStore
+	PADDD ·chacha20Constants<>+0(SB), X0
+	PADDD ·chacha20Constants<>+0(SB), X1
+	PADDD ·chacha20Constants<>+0(SB), X2
+	PADDD ·chacha20Constants<>+0(SB), X12
+	PADDD 32(BP), X3
+	PADDD 32(BP), X4
+	PADDD 32(BP), X5
+	PADDD 32(BP), X13
+	PADDD 48(BP), X6
+	PADDD 48(BP), X7
+	PADDD 48(BP), X8
+	PADDD 48(BP), X14
+	PADDD 80(BP), X9
+	PADDD 96(BP), X10
+	PADDD 112(BP), X11
+	PADDD 128(BP), X15
+	MOVO  X15, 64(BP)
 
 	// Load - xor - store
-	MOVOU (0*16)(inp), D3; PXOR D3, A0
-	MOVOU (1*16)(inp), D3; PXOR D3, B0
-	MOVOU (2*16)(inp), D3; PXOR D3, C0
-	MOVOU (3*16)(inp), D3; PXOR D3, D0
-	MOVOU A0, (0*16)(oup)
-	MOVOU B0, (1*16)(oup)
-	MOVOU C0, (2*16)(oup)
-	MOVOU D0, (3*16)(oup)
-	MOVOU (4*16)(inp), A0; MOVOU (5*16)(inp), B0; MOVOU (6*16)(inp), C0; MOVOU (7*16)(inp), D0
-	PXOR  A0, A1; PXOR B0, B1; PXOR C0, C1; PXOR D0, D1
-	MOVOU A1, (4*16)(oup); MOVOU B1, (5*16)(oup); MOVOU C1, (6*16)(oup); MOVOU D1, (7*16)(oup)
-	MOVOU (8*16)(inp), A0; MOVOU (9*16)(inp), B0; MOVOU (10*16)(inp), C0; MOVOU (11*16)(inp), D0
-	PXOR  A0, A2; PXOR B0, B2; PXOR C0, C2; PXOR D0, D2
-	MOVOU A2, (8*16)(oup); MOVOU B2, (9*16)(oup); MOVOU C2, (10*16)(oup); MOVOU D2, (11*16)(oup)
-	LEAQ  192(inp), inp
-	LEAQ  192(oup), oup
-	SUBQ  $192, inl
-	MOVO  A3, A0
-	MOVO  B3, B0
-	MOVO  C3, C0
-	MOVO  tmpStore, D0
-
-	JMP openSSETail64DecLoop
-
-// ----------------------------------------------------------------------------
-// ------------------------- AVX2 Code ----------------------------------------
+	MOVOU (SI), X15
+	PXOR  X15, X0
+	MOVOU 16(SI), X15
+	PXOR  X15, X3
+	MOVOU 32(SI), X15
+	PXOR  X15, X6
+	MOVOU 48(SI), X15
+	PXOR  X15, X9
+	MOVOU X0, (DI)
+	MOVOU X3, 16(DI)
+	MOVOU X6, 32(DI)
+	MOVOU X9, 48(DI)
+	MOVOU 64(SI), X0
+	MOVOU 80(SI), X3
+	MOVOU 96(SI), X6
+	MOVOU 112(SI), X9
+	PXOR  X0, X1
+	PXOR  X3, X4
+	PXOR  X6, X7
+	PXOR  X9, X10
+	MOVOU X1, 64(DI)
+	MOVOU X4, 80(DI)
+	MOVOU X7, 96(DI)
+	MOVOU X10, 112(DI)
+	MOVOU 128(SI), X0
+	MOVOU 144(SI), X3
+	MOVOU 160(SI), X6
+	MOVOU 176(SI), X9
+	PXOR  X0, X2
+	PXOR  X3, X5
+	PXOR  X6, X8
+	PXOR  X9, X11
+	MOVOU X2, 128(DI)
+	MOVOU X5, 144(DI)
+	MOVOU X8, 160(DI)
+	MOVOU X11, 176(DI)
+	LEAQ  192(SI), SI
+	LEAQ  192(DI), DI
+	SUBQ  $0xc0, BX
+	MOVO  X12, X0
+	MOVO  X13, X3
+	MOVO  X14, X6
+	MOVO  64(BP), X9
+	JMP   openSSETail64DecLoop
+
 chacha20Poly1305Open_AVX2:
 	VZEROUPPER
-	VMOVDQU ·chacha20Constants<>(SB), AA0
-	BYTE    $0xc4; BYTE $0x42; BYTE $0x7d; BYTE $0x5a; BYTE $0x70; BYTE $0x10 // broadcasti128 16(r8), ymm14
-	BYTE    $0xc4; BYTE $0x42; BYTE $0x7d; BYTE $0x5a; BYTE $0x60; BYTE $0x20 // broadcasti128 32(r8), ymm12
-	BYTE    $0xc4; BYTE $0xc2; BYTE $0x7d; BYTE $0x5a; BYTE $0x60; BYTE $0x30 // broadcasti128 48(r8), ymm4
-	VPADDD  ·avx2InitMask<>(SB), DD0, DD0
+	VMOVDQU ·chacha20Constants<>+0(SB), Y0
+	BYTE    $0xc4
+	BYTE    $0x42
+	BYTE    $0x7d
+	BYTE    $0x5a
+	BYTE    $0x70
+	BYTE    $0x10
+	BYTE    $0xc4
+	BYTE    $0x42
+	BYTE    $0x7d
+	BYTE    $0x5a
+	BYTE    $0x60
+	BYTE    $0x20
+	BYTE    $0xc4
+	BYTE    $0xc2
+	BYTE    $0x7d
+	BYTE    $0x5a
+	BYTE    $0x60
+	BYTE    $0x30
+	VPADDD  ·avx2InitMask<>+0(SB), Y4, Y4
 
 	// Special optimization, for very short buffers
-	CMPQ inl, $192
+	CMPQ BX, $0xc0
 	JBE  openAVX2192
-	CMPQ inl, $320
+	CMPQ BX, $0x00000140
 	JBE  openAVX2320
 
 	// For the general key prepare the key first - as a byproduct we have 64 bytes of cipher stream
-	VMOVDQA BB0, state1StoreAVX2
-	VMOVDQA CC0, state2StoreAVX2
-	VMOVDQA DD0, ctr3StoreAVX2
-	MOVQ    $10, itr2
+	VMOVDQA Y14, 32(BP)
+	VMOVDQA Y12, 64(BP)
+	VMOVDQA Y4, 192(BP)
+	MOVQ    $0x0000000a, R9
 
 openAVX2PreparePolyKey:
-	chachaQR_AVX2(AA0, BB0, CC0, DD0, TT0)
-	VPALIGNR $4, BB0, BB0, BB0; VPALIGNR $8, CC0, CC0, CC0; VPALIGNR $12, DD0, DD0, DD0
-	chachaQR_AVX2(AA0, BB0, CC0, DD0, TT0)
-	VPALIGNR $12, BB0, BB0, BB0; VPALIGNR $8, CC0, CC0, CC0; VPALIGNR $4, DD0, DD0, DD0
-	DECQ     itr2
-	JNE      openAVX2PreparePolyKey
-
-	VPADDD ·chacha20Constants<>(SB), AA0, AA0
-	VPADDD state1StoreAVX2, BB0, BB0
-	VPADDD state2StoreAVX2, CC0, CC0
-	VPADDD ctr3StoreAVX2, DD0, DD0
-
-	VPERM2I128 $0x02, AA0, BB0, TT0
+	VPADDD     Y14, Y0, Y0
+	VPXOR      Y0, Y4, Y4
+	VPSHUFB    ·rol16<>+0(SB), Y4, Y4
+	VPADDD     Y4, Y12, Y12
+	VPXOR      Y12, Y14, Y14
+	VPSLLD     $0x0c, Y14, Y3
+	VPSRLD     $0x14, Y14, Y14
+	VPXOR      Y3, Y14, Y14
+	VPADDD     Y14, Y0, Y0
+	VPXOR      Y0, Y4, Y4
+	VPSHUFB    ·rol8<>+0(SB), Y4, Y4
+	VPADDD     Y4, Y12, Y12
+	VPXOR      Y12, Y14, Y14
+	VPSLLD     $0x07, Y14, Y3
+	VPSRLD     $0x19, Y14, Y14
+	VPXOR      Y3, Y14, Y14
+	VPALIGNR   $0x04, Y14, Y14, Y14
+	VPALIGNR   $0x08, Y12, Y12, Y12
+	VPALIGNR   $0x0c, Y4, Y4, Y4
+	VPADDD     Y14, Y0, Y0
+	VPXOR      Y0, Y4, Y4
+	VPSHUFB    ·rol16<>+0(SB), Y4, Y4
+	VPADDD     Y4, Y12, Y12
+	VPXOR      Y12, Y14, Y14
+	VPSLLD     $0x0c, Y14, Y3
+	VPSRLD     $0x14, Y14, Y14
+	VPXOR      Y3, Y14, Y14
+	VPADDD     Y14, Y0, Y0
+	VPXOR      Y0, Y4, Y4
+	VPSHUFB    ·rol8<>+0(SB), Y4, Y4
+	VPADDD     Y4, Y12, Y12
+	VPXOR      Y12, Y14, Y14
+	VPSLLD     $0x07, Y14, Y3
+	VPSRLD     $0x19, Y14, Y14
+	VPXOR      Y3, Y14, Y14
+	VPALIGNR   $0x0c, Y14, Y14, Y14
+	VPALIGNR   $0x08, Y12, Y12, Y12
+	VPALIGNR   $0x04, Y4, Y4, Y4
+	DECQ       R9
+	JNE        openAVX2PreparePolyKey
+	VPADDD     ·chacha20Constants<>+0(SB), Y0, Y0
+	VPADDD     32(BP), Y14, Y14
+	VPADDD     64(BP), Y12, Y12
+	VPADDD     192(BP), Y4, Y4
+	VPERM2I128 $0x02, Y0, Y14, Y3
 
 	// Clamp and store poly key
-	VPAND   ·polyClampMask<>(SB), TT0, TT0
-	VMOVDQA TT0, rsStoreAVX2
+	VPAND   ·polyClampMask<>+0(SB), Y3, Y3
+	VMOVDQA Y3, (BP)
 
 	// Stream for the first 64 bytes
-	VPERM2I128 $0x13, AA0, BB0, AA0
-	VPERM2I128 $0x13, CC0, DD0, BB0
+	VPERM2I128 $0x13, Y0, Y14, Y0
+	VPERM2I128 $0x13, Y12, Y4, Y14
 
 	// Hash AD + first 64 bytes
-	MOVQ ad_len+80(FP), itr2
+	MOVQ ad_len+80(FP), R9
 	CALL polyHashADInternal<>(SB)
-	XORQ itr1, itr1
+	XORQ CX, CX
 
 openAVX2InitialHash64:
-	polyAdd(0(inp)(itr1*1))
-	polyMulAVX2
-	ADDQ $16, itr1
-	CMPQ itr1, $64
-	JNE  openAVX2InitialHash64
+	ADDQ  (SI)(CX*1), R10
+	ADCQ  8(SI)(CX*1), R11
+	ADCQ  $0x01, R12
+	MOVQ  (BP), DX
+	MOVQ  DX, R15
+	MULXQ R10, R13, R14
+	IMULQ R12, R15
+	MULXQ R11, AX, DX
+	ADDQ  AX, R14
+	ADCQ  DX, R15
+	MOVQ  8(BP), DX
+	MULXQ R10, R10, AX
+	ADDQ  R10, R14
+	MULXQ R11, R11, R8
+	ADCQ  R11, R15
+	ADCQ  $0x00, R8
+	IMULQ R12, DX
+	ADDQ  AX, R15
+	ADCQ  DX, R8
+	MOVQ  R13, R10
+	MOVQ  R14, R11
+	MOVQ  R15, R12
+	ANDQ  $0x03, R12
+	MOVQ  R15, R13
+	ANDQ  $-4, R13
+	MOVQ  R8, R14
+	SHRQ  $0x02, R8, R15
+	SHRQ  $0x02, R8
+	ADDQ  R13, R10
+	ADCQ  R14, R11
+	ADCQ  $0x00, R12
+	ADDQ  R15, R10
+	ADCQ  R8, R11
+	ADCQ  $0x00, R12
+	ADDQ  $0x10, CX
+	CMPQ  CX, $0x40
+	JNE   openAVX2InitialHash64
 
 	// Decrypt the first 64 bytes
-	VPXOR   (0*32)(inp), AA0, AA0
-	VPXOR   (1*32)(inp), BB0, BB0
-	VMOVDQU AA0, (0*32)(oup)
-	VMOVDQU BB0, (1*32)(oup)
-	LEAQ    (2*32)(inp), inp
-	LEAQ    (2*32)(oup), oup
-	SUBQ    $64, inl
+	VPXOR   (SI), Y0, Y0
+	VPXOR   32(SI), Y14, Y14
+	VMOVDQU Y0, (DI)
+	VMOVDQU Y14, 32(DI)
+	LEAQ    64(SI), SI
+	LEAQ    64(DI), DI
+	SUBQ    $0x40, BX
 
 openAVX2MainLoop:
-	CMPQ inl, $512
+	CMPQ BX, $0x00000200
 	JB   openAVX2MainLoopDone
 
 	// Load state, increment counter blocks, store the incremented counters
-	VMOVDQU ·chacha20Constants<>(SB), AA0; VMOVDQA AA0, AA1; VMOVDQA AA0, AA2; VMOVDQA AA0, AA3
-	VMOVDQA state1StoreAVX2, BB0; VMOVDQA BB0, BB1; VMOVDQA BB0, BB2; VMOVDQA BB0, BB3
-	VMOVDQA state2StoreAVX2, CC0; VMOVDQA CC0, CC1; VMOVDQA CC0, CC2; VMOVDQA CC0, CC3
-	VMOVDQA ctr3StoreAVX2, DD0; VPADDD ·avx2IncMask<>(SB), DD0, DD0; VPADDD ·avx2IncMask<>(SB), DD0, DD1; VPADDD ·avx2IncMask<>(SB), DD1, DD2; VPADDD ·avx2IncMask<>(SB), DD2, DD3
-	VMOVDQA DD0, ctr0StoreAVX2; VMOVDQA DD1, ctr1StoreAVX2; VMOVDQA DD2, ctr2StoreAVX2; VMOVDQA DD3, ctr3StoreAVX2
-	XORQ    itr1, itr1
+	VMOVDQU ·chacha20Constants<>+0(SB), Y0
+	VMOVDQA Y0, Y5
+	VMOVDQA Y0, Y6
+	VMOVDQA Y0, Y7
+	VMOVDQA 32(BP), Y14
+	VMOVDQA Y14, Y9
+	VMOVDQA Y14, Y10
+	VMOVDQA Y14, Y11
+	VMOVDQA 64(BP), Y12
+	VMOVDQA Y12, Y13
+	VMOVDQA Y12, Y8
+	VMOVDQA Y12, Y15
+	VMOVDQA 192(BP), Y4
+	VPADDD  ·avx2IncMask<>+0(SB), Y4, Y4
+	VPADDD  ·avx2IncMask<>+0(SB), Y4, Y1
+	VPADDD  ·avx2IncMask<>+0(SB), Y1, Y2
+	VPADDD  ·avx2IncMask<>+0(SB), Y2, Y3
+	VMOVDQA Y4, 96(BP)
+	VMOVDQA Y1, 128(BP)
+	VMOVDQA Y2, 160(BP)
+	VMOVDQA Y3, 192(BP)
+	XORQ    CX, CX
 
 openAVX2InternalLoop:
-	// Lets just say this spaghetti loop interleaves 2 quarter rounds with 3 poly multiplications
-	// Effectively per 512 bytes of stream we hash 480 bytes of ciphertext
-	polyAdd(0*8(inp)(itr1*1))
-	VPADDD   BB0, AA0, AA0; VPADDD BB1, AA1, AA1; VPADDD BB2, AA2, AA2; VPADDD BB3, AA3, AA3
-	polyMulStage1_AVX2
-	VPXOR    AA0, DD0, DD0; VPXOR AA1, DD1, DD1; VPXOR AA2, DD2, DD2; VPXOR AA3, DD3, DD3
-	VPSHUFB  ·rol16<>(SB), DD0, DD0; VPSHUFB ·rol16<>(SB), DD1, DD1; VPSHUFB ·rol16<>(SB), DD2, DD2; VPSHUFB ·rol16<>(SB), DD3, DD3
-	polyMulStage2_AVX2
-	VPADDD   DD0, CC0, CC0; VPADDD DD1, CC1, CC1; VPADDD DD2, CC2, CC2; VPADDD DD3, CC3, CC3
-	VPXOR    CC0, BB0, BB0; VPXOR CC1, BB1, BB1; VPXOR CC2, BB2, BB2; VPXOR CC3, BB3, BB3
-	polyMulStage3_AVX2
-	VMOVDQA  CC3, tmpStoreAVX2
-	VPSLLD   $12, BB0, CC3; VPSRLD $20, BB0, BB0; VPXOR CC3, BB0, BB0
-	VPSLLD   $12, BB1, CC3; VPSRLD $20, BB1, BB1; VPXOR CC3, BB1, BB1
-	VPSLLD   $12, BB2, CC3; VPSRLD $20, BB2, BB2; VPXOR CC3, BB2, BB2
-	VPSLLD   $12, BB3, CC3; VPSRLD $20, BB3, BB3; VPXOR CC3, BB3, BB3
-	VMOVDQA  tmpStoreAVX2, CC3
-	polyMulReduceStage
-	VPADDD   BB0, AA0, AA0; VPADDD BB1, AA1, AA1; VPADDD BB2, AA2, AA2; VPADDD BB3, AA3, AA3
-	VPXOR    AA0, DD0, DD0; VPXOR AA1, DD1, DD1; VPXOR AA2, DD2, DD2; VPXOR AA3, DD3, DD3
-	VPSHUFB  ·rol8<>(SB), DD0, DD0; VPSHUFB ·rol8<>(SB), DD1, DD1; VPSHUFB ·rol8<>(SB), DD2, DD2; VPSHUFB ·rol8<>(SB), DD3, DD3
-	polyAdd(2*8(inp)(itr1*1))
-	VPADDD   DD0, CC0, CC0; VPADDD DD1, CC1, CC1; VPADDD DD2, CC2, CC2; VPADDD DD3, CC3, CC3
-	polyMulStage1_AVX2
-	VPXOR    CC0, BB0, BB0; VPXOR CC1, BB1, BB1; VPXOR CC2, BB2, BB2; VPXOR CC3, BB3, BB3
-	VMOVDQA  CC3, tmpStoreAVX2
-	VPSLLD   $7, BB0, CC3; VPSRLD $25, BB0, BB0; VPXOR CC3, BB0, BB0
-	VPSLLD   $7, BB1, CC3; VPSRLD $25, BB1, BB1; VPXOR CC3, BB1, BB1
-	VPSLLD   $7, BB2, CC3; VPSRLD $25, BB2, BB2; VPXOR CC3, BB2, BB2
-	VPSLLD   $7, BB3, CC3; VPSRLD $25, BB3, BB3; VPXOR CC3, BB3, BB3
-	VMOVDQA  tmpStoreAVX2, CC3
-	polyMulStage2_AVX2
-	VPALIGNR $4, BB0, BB0, BB0; VPALIGNR $4, BB1, BB1, BB1; VPALIGNR $4, BB2, BB2, BB2; VPALIGNR $4, BB3, BB3, BB3
-	VPALIGNR $8, CC0, CC0, CC0; VPALIGNR $8, CC1, CC1, CC1; VPALIGNR $8, CC2, CC2, CC2; VPALIGNR $8, CC3, CC3, CC3
-	VPALIGNR $12, DD0, DD0, DD0; VPALIGNR $12, DD1, DD1, DD1; VPALIGNR $12, DD2, DD2, DD2; VPALIGNR $12, DD3, DD3, DD3
-	VPADDD   BB0, AA0, AA0; VPADDD BB1, AA1, AA1; VPADDD BB2, AA2, AA2; VPADDD BB3, AA3, AA3
-	polyMulStage3_AVX2
-	VPXOR    AA0, DD0, DD0; VPXOR AA1, DD1, DD1; VPXOR AA2, DD2, DD2; VPXOR AA3, DD3, DD3
-	VPSHUFB  ·rol16<>(SB), DD0, DD0; VPSHUFB ·rol16<>(SB), DD1, DD1; VPSHUFB ·rol16<>(SB), DD2, DD2; VPSHUFB ·rol16<>(SB), DD3, DD3
-	polyMulReduceStage
-	VPADDD   DD0, CC0, CC0; VPADDD DD1, CC1, CC1; VPADDD DD2, CC2, CC2; VPADDD DD3, CC3, CC3
-	VPXOR    CC0, BB0, BB0; VPXOR CC1, BB1, BB1; VPXOR CC2, BB2, BB2; VPXOR CC3, BB3, BB3
-	polyAdd(4*8(inp)(itr1*1))
-	LEAQ     (6*8)(itr1), itr1
-	VMOVDQA  CC3, tmpStoreAVX2
-	VPSLLD   $12, BB0, CC3; VPSRLD $20, BB0, BB0; VPXOR CC3, BB0, BB0
-	VPSLLD   $12, BB1, CC3; VPSRLD $20, BB1, BB1; VPXOR CC3, BB1, BB1
-	VPSLLD   $12, BB2, CC3; VPSRLD $20, BB2, BB2; VPXOR CC3, BB2, BB2
-	VPSLLD   $12, BB3, CC3; VPSRLD $20, BB3, BB3; VPXOR CC3, BB3, BB3
-	VMOVDQA  tmpStoreAVX2, CC3
-	polyMulStage1_AVX2
-	VPADDD   BB0, AA0, AA0; VPADDD BB1, AA1, AA1; VPADDD BB2, AA2, AA2; VPADDD BB3, AA3, AA3
-	VPXOR    AA0, DD0, DD0; VPXOR AA1, DD1, DD1; VPXOR AA2, DD2, DD2; VPXOR AA3, DD3, DD3
-	polyMulStage2_AVX2
-	VPSHUFB  ·rol8<>(SB), DD0, DD0; VPSHUFB ·rol8<>(SB), DD1, DD1; VPSHUFB ·rol8<>(SB), DD2, DD2; VPSHUFB ·rol8<>(SB), DD3, DD3
-	VPADDD   DD0, CC0, CC0; VPADDD DD1, CC1, CC1; VPADDD DD2, CC2, CC2; VPADDD DD3, CC3, CC3
-	polyMulStage3_AVX2
-	VPXOR    CC0, BB0, BB0; VPXOR CC1, BB1, BB1; VPXOR CC2, BB2, BB2; VPXOR CC3, BB3, BB3
-	VMOVDQA  CC3, tmpStoreAVX2
-	VPSLLD   $7, BB0, CC3; VPSRLD $25, BB0, BB0; VPXOR CC3, BB0, BB0
-	VPSLLD   $7, BB1, CC3; VPSRLD $25, BB1, BB1; VPXOR CC3, BB1, BB1
-	VPSLLD   $7, BB2, CC3; VPSRLD $25, BB2, BB2; VPXOR CC3, BB2, BB2
-	VPSLLD   $7, BB3, CC3; VPSRLD $25, BB3, BB3; VPXOR CC3, BB3, BB3
-	VMOVDQA  tmpStoreAVX2, CC3
-	polyMulReduceStage
-	VPALIGNR $12, BB0, BB0, BB0; VPALIGNR $12, BB1, BB1, BB1; VPALIGNR $12, BB2, BB2, BB2; VPALIGNR $12, BB3, BB3, BB3
-	VPALIGNR $8, CC0, CC0, CC0; VPALIGNR $8, CC1, CC1, CC1; VPALIGNR $8, CC2, CC2, CC2; VPALIGNR $8, CC3, CC3, CC3
-	VPALIGNR $4, DD0, DD0, DD0; VPALIGNR $4, DD1, DD1, DD1; VPALIGNR $4, DD2, DD2, DD2; VPALIGNR $4, DD3, DD3, DD3
-	CMPQ     itr1, $480
+	ADDQ     (SI)(CX*1), R10
+	ADCQ     8(SI)(CX*1), R11
+	ADCQ     $0x01, R12
+	VPADDD   Y14, Y0, Y0
+	VPADDD   Y9, Y5, Y5
+	VPADDD   Y10, Y6, Y6
+	VPADDD   Y11, Y7, Y7
+	MOVQ     (BP), DX
+	MOVQ     DX, R15
+	MULXQ    R10, R13, R14
+	IMULQ    R12, R15
+	MULXQ    R11, AX, DX
+	ADDQ     AX, R14
+	ADCQ     DX, R15
+	VPXOR    Y0, Y4, Y4
+	VPXOR    Y5, Y1, Y1
+	VPXOR    Y6, Y2, Y2
+	VPXOR    Y7, Y3, Y3
+	VPSHUFB  ·rol16<>+0(SB), Y4, Y4
+	VPSHUFB  ·rol16<>+0(SB), Y1, Y1
+	VPSHUFB  ·rol16<>+0(SB), Y2, Y2
+	VPSHUFB  ·rol16<>+0(SB), Y3, Y3
+	MOVQ     8(BP), DX
+	MULXQ    R10, R10, AX
+	ADDQ     R10, R14
+	MULXQ    R11, R11, R8
+	ADCQ     R11, R15
+	ADCQ     $0x00, R8
+	VPADDD   Y4, Y12, Y12
+	VPADDD   Y1, Y13, Y13
+	VPADDD   Y2, Y8, Y8
+	VPADDD   Y3, Y15, Y15
+	VPXOR    Y12, Y14, Y14
+	VPXOR    Y13, Y9, Y9
+	VPXOR    Y8, Y10, Y10
+	VPXOR    Y15, Y11, Y11
+	IMULQ    R12, DX
+	ADDQ     AX, R15
+	ADCQ     DX, R8
+	VMOVDQA  Y15, 224(BP)
+	VPSLLD   $0x0c, Y14, Y15
+	VPSRLD   $0x14, Y14, Y14
+	VPXOR    Y15, Y14, Y14
+	VPSLLD   $0x0c, Y9, Y15
+	VPSRLD   $0x14, Y9, Y9
+	VPXOR    Y15, Y9, Y9
+	VPSLLD   $0x0c, Y10, Y15
+	VPSRLD   $0x14, Y10, Y10
+	VPXOR    Y15, Y10, Y10
+	VPSLLD   $0x0c, Y11, Y15
+	VPSRLD   $0x14, Y11, Y11
+	VPXOR    Y15, Y11, Y11
+	VMOVDQA  224(BP), Y15
+	MOVQ     R13, R10
+	MOVQ     R14, R11
+	MOVQ     R15, R12
+	ANDQ     $0x03, R12
+	MOVQ     R15, R13
+	ANDQ     $-4, R13
+	MOVQ     R8, R14
+	SHRQ     $0x02, R8, R15
+	SHRQ     $0x02, R8
+	ADDQ     R13, R10
+	ADCQ     R14, R11
+	ADCQ     $0x00, R12
+	ADDQ     R15, R10
+	ADCQ     R8, R11
+	ADCQ     $0x00, R12
+	VPADDD   Y14, Y0, Y0
+	VPADDD   Y9, Y5, Y5
+	VPADDD   Y10, Y6, Y6
+	VPADDD   Y11, Y7, Y7
+	VPXOR    Y0, Y4, Y4
+	VPXOR    Y5, Y1, Y1
+	VPXOR    Y6, Y2, Y2
+	VPXOR    Y7, Y3, Y3
+	VPSHUFB  ·rol8<>+0(SB), Y4, Y4
+	VPSHUFB  ·rol8<>+0(SB), Y1, Y1
+	VPSHUFB  ·rol8<>+0(SB), Y2, Y2
+	VPSHUFB  ·rol8<>+0(SB), Y3, Y3
+	ADDQ     16(SI)(CX*1), R10
+	ADCQ     24(SI)(CX*1), R11
+	ADCQ     $0x01, R12
+	VPADDD   Y4, Y12, Y12
+	VPADDD   Y1, Y13, Y13
+	VPADDD   Y2, Y8, Y8
+	VPADDD   Y3, Y15, Y15
+	MOVQ     (BP), DX
+	MOVQ     DX, R15
+	MULXQ    R10, R13, R14
+	IMULQ    R12, R15
+	MULXQ    R11, AX, DX
+	ADDQ     AX, R14
+	ADCQ     DX, R15
+	VPXOR    Y12, Y14, Y14
+	VPXOR    Y13, Y9, Y9
+	VPXOR    Y8, Y10, Y10
+	VPXOR    Y15, Y11, Y11
+	VMOVDQA  Y15, 224(BP)
+	VPSLLD   $0x07, Y14, Y15
+	VPSRLD   $0x19, Y14, Y14
+	VPXOR    Y15, Y14, Y14
+	VPSLLD   $0x07, Y9, Y15
+	VPSRLD   $0x19, Y9, Y9
+	VPXOR    Y15, Y9, Y9
+	VPSLLD   $0x07, Y10, Y15
+	VPSRLD   $0x19, Y10, Y10
+	VPXOR    Y15, Y10, Y10
+	VPSLLD   $0x07, Y11, Y15
+	VPSRLD   $0x19, Y11, Y11
+	VPXOR    Y15, Y11, Y11
+	VMOVDQA  224(BP), Y15
+	MOVQ     8(BP), DX
+	MULXQ    R10, R10, AX
+	ADDQ     R10, R14
+	MULXQ    R11, R11, R8
+	ADCQ     R11, R15
+	ADCQ     $0x00, R8
+	VPALIGNR $0x04, Y14, Y14, Y14
+	VPALIGNR $0x04, Y9, Y9, Y9
+	VPALIGNR $0x04, Y10, Y10, Y10
+	VPALIGNR $0x04, Y11, Y11, Y11
+	VPALIGNR $0x08, Y12, Y12, Y12
+	VPALIGNR $0x08, Y13, Y13, Y13
+	VPALIGNR $0x08, Y8, Y8, Y8
+	VPALIGNR $0x08, Y15, Y15, Y15
+	VPALIGNR $0x0c, Y4, Y4, Y4
+	VPALIGNR $0x0c, Y1, Y1, Y1
+	VPALIGNR $0x0c, Y2, Y2, Y2
+	VPALIGNR $0x0c, Y3, Y3, Y3
+	VPADDD   Y14, Y0, Y0
+	VPADDD   Y9, Y5, Y5
+	VPADDD   Y10, Y6, Y6
+	VPADDD   Y11, Y7, Y7
+	IMULQ    R12, DX
+	ADDQ     AX, R15
+	ADCQ     DX, R8
+	VPXOR    Y0, Y4, Y4
+	VPXOR    Y5, Y1, Y1
+	VPXOR    Y6, Y2, Y2
+	VPXOR    Y7, Y3, Y3
+	VPSHUFB  ·rol16<>+0(SB), Y4, Y4
+	VPSHUFB  ·rol16<>+0(SB), Y1, Y1
+	VPSHUFB  ·rol16<>+0(SB), Y2, Y2
+	VPSHUFB  ·rol16<>+0(SB), Y3, Y3
+	MOVQ     R13, R10
+	MOVQ     R14, R11
+	MOVQ     R15, R12
+	ANDQ     $0x03, R12
+	MOVQ     R15, R13
+	ANDQ     $-4, R13
+	MOVQ     R8, R14
+	SHRQ     $0x02, R8, R15
+	SHRQ     $0x02, R8
+	ADDQ     R13, R10
+	ADCQ     R14, R11
+	ADCQ     $0x00, R12
+	ADDQ     R15, R10
+	ADCQ     R8, R11
+	ADCQ     $0x00, R12
+	VPADDD   Y4, Y12, Y12
+	VPADDD   Y1, Y13, Y13
+	VPADDD   Y2, Y8, Y8
+	VPADDD   Y3, Y15, Y15
+	VPXOR    Y12, Y14, Y14
+	VPXOR    Y13, Y9, Y9
+	VPXOR    Y8, Y10, Y10
+	VPXOR    Y15, Y11, Y11
+	ADDQ     32(SI)(CX*1), R10
+	ADCQ     40(SI)(CX*1), R11
+	ADCQ     $0x01, R12
+	LEAQ     48(CX), CX
+	VMOVDQA  Y15, 224(BP)
+	VPSLLD   $0x0c, Y14, Y15
+	VPSRLD   $0x14, Y14, Y14
+	VPXOR    Y15, Y14, Y14
+	VPSLLD   $0x0c, Y9, Y15
+	VPSRLD   $0x14, Y9, Y9
+	VPXOR    Y15, Y9, Y9
+	VPSLLD   $0x0c, Y10, Y15
+	VPSRLD   $0x14, Y10, Y10
+	VPXOR    Y15, Y10, Y10
+	VPSLLD   $0x0c, Y11, Y15
+	VPSRLD   $0x14, Y11, Y11
+	VPXOR    Y15, Y11, Y11
+	VMOVDQA  224(BP), Y15
+	MOVQ     (BP), DX
+	MOVQ     DX, R15
+	MULXQ    R10, R13, R14
+	IMULQ    R12, R15
+	MULXQ    R11, AX, DX
+	ADDQ     AX, R14
+	ADCQ     DX, R15
+	VPADDD   Y14, Y0, Y0
+	VPADDD   Y9, Y5, Y5
+	VPADDD   Y10, Y6, Y6
+	VPADDD   Y11, Y7, Y7
+	VPXOR    Y0, Y4, Y4
+	VPXOR    Y5, Y1, Y1
+	VPXOR    Y6, Y2, Y2
+	VPXOR    Y7, Y3, Y3
+	MOVQ     8(BP), DX
+	MULXQ    R10, R10, AX
+	ADDQ     R10, R14
+	MULXQ    R11, R11, R8
+	ADCQ     R11, R15
+	ADCQ     $0x00, R8
+	VPSHUFB  ·rol8<>+0(SB), Y4, Y4
+	VPSHUFB  ·rol8<>+0(SB), Y1, Y1
+	VPSHUFB  ·rol8<>+0(SB), Y2, Y2
+	VPSHUFB  ·rol8<>+0(SB), Y3, Y3
+	VPADDD   Y4, Y12, Y12
+	VPADDD   Y1, Y13, Y13
+	VPADDD   Y2, Y8, Y8
+	VPADDD   Y3, Y15, Y15
+	IMULQ    R12, DX
+	ADDQ     AX, R15
+	ADCQ     DX, R8
+	VPXOR    Y12, Y14, Y14
+	VPXOR    Y13, Y9, Y9
+	VPXOR    Y8, Y10, Y10
+	VPXOR    Y15, Y11, Y11
+	VMOVDQA  Y15, 224(BP)
+	VPSLLD   $0x07, Y14, Y15
+	VPSRLD   $0x19, Y14, Y14
+	VPXOR    Y15, Y14, Y14
+	VPSLLD   $0x07, Y9, Y15
+	VPSRLD   $0x19, Y9, Y9
+	VPXOR    Y15, Y9, Y9
+	VPSLLD   $0x07, Y10, Y15
+	VPSRLD   $0x19, Y10, Y10
+	VPXOR    Y15, Y10, Y10
+	VPSLLD   $0x07, Y11, Y15
+	VPSRLD   $0x19, Y11, Y11
+	VPXOR    Y15, Y11, Y11
+	VMOVDQA  224(BP), Y15
+	MOVQ     R13, R10
+	MOVQ     R14, R11
+	MOVQ     R15, R12
+	ANDQ     $0x03, R12
+	MOVQ     R15, R13
+	ANDQ     $-4, R13
+	MOVQ     R8, R14
+	SHRQ     $0x02, R8, R15
+	SHRQ     $0x02, R8
+	ADDQ     R13, R10
+	ADCQ     R14, R11
+	ADCQ     $0x00, R12
+	ADDQ     R15, R10
+	ADCQ     R8, R11
+	ADCQ     $0x00, R12
+	VPALIGNR $0x0c, Y14, Y14, Y14
+	VPALIGNR $0x0c, Y9, Y9, Y9
+	VPALIGNR $0x0c, Y10, Y10, Y10
+	VPALIGNR $0x0c, Y11, Y11, Y11
+	VPALIGNR $0x08, Y12, Y12, Y12
+	VPALIGNR $0x08, Y13, Y13, Y13
+	VPALIGNR $0x08, Y8, Y8, Y8
+	VPALIGNR $0x08, Y15, Y15, Y15
+	VPALIGNR $0x04, Y4, Y4, Y4
+	VPALIGNR $0x04, Y1, Y1, Y1
+	VPALIGNR $0x04, Y2, Y2, Y2
+	VPALIGNR $0x04, Y3, Y3, Y3
+	CMPQ     CX, $0x000001e0
 	JNE      openAVX2InternalLoop
-
-	VPADDD  ·chacha20Constants<>(SB), AA0, AA0; VPADDD ·chacha20Constants<>(SB), AA1, AA1; VPADDD ·chacha20Constants<>(SB), AA2, AA2; VPADDD ·chacha20Constants<>(SB), AA3, AA3
-	VPADDD  state1StoreAVX2, BB0, BB0; VPADDD state1StoreAVX2, BB1, BB1; VPADDD state1StoreAVX2, BB2, BB2; VPADDD state1StoreAVX2, BB3, BB3
-	VPADDD  state2StoreAVX2, CC0, CC0; VPADDD state2StoreAVX2, CC1, CC1; VPADDD state2StoreAVX2, CC2, CC2; VPADDD state2StoreAVX2, CC3, CC3
-	VPADDD  ctr0StoreAVX2, DD0, DD0; VPADDD ctr1StoreAVX2, DD1, DD1; VPADDD ctr2StoreAVX2, DD2, DD2; VPADDD ctr3StoreAVX2, DD3, DD3
-	VMOVDQA CC3, tmpStoreAVX2
+	VPADDD   ·chacha20Constants<>+0(SB), Y0, Y0
+	VPADDD   ·chacha20Constants<>+0(SB), Y5, Y5
+	VPADDD   ·chacha20Constants<>+0(SB), Y6, Y6
+	VPADDD   ·chacha20Constants<>+0(SB), Y7, Y7
+	VPADDD   32(BP), Y14, Y14
+	VPADDD   32(BP), Y9, Y9
+	VPADDD   32(BP), Y10, Y10
+	VPADDD   32(BP), Y11, Y11
+	VPADDD   64(BP), Y12, Y12
+	VPADDD   64(BP), Y13, Y13
+	VPADDD   64(BP), Y8, Y8
+	VPADDD   64(BP), Y15, Y15
+	VPADDD   96(BP), Y4, Y4
+	VPADDD   128(BP), Y1, Y1
+	VPADDD   160(BP), Y2, Y2
+	VPADDD   192(BP), Y3, Y3
+	VMOVDQA  Y15, 224(BP)
 
 	// We only hashed 480 of the 512 bytes available - hash the remaining 32 here
-	polyAdd(480(inp))
-	polyMulAVX2
-	VPERM2I128 $0x02, AA0, BB0, CC3; VPERM2I128 $0x13, AA0, BB0, BB0; VPERM2I128 $0x02, CC0, DD0, AA0; VPERM2I128 $0x13, CC0, DD0, CC0
-	VPXOR      (0*32)(inp), CC3, CC3; VPXOR (1*32)(inp), AA0, AA0; VPXOR (2*32)(inp), BB0, BB0; VPXOR (3*32)(inp), CC0, CC0
-	VMOVDQU    CC3, (0*32)(oup); VMOVDQU AA0, (1*32)(oup); VMOVDQU BB0, (2*32)(oup); VMOVDQU CC0, (3*32)(oup)
-	VPERM2I128 $0x02, AA1, BB1, AA0; VPERM2I128 $0x02, CC1, DD1, BB0; VPERM2I128 $0x13, AA1, BB1, CC0; VPERM2I128 $0x13, CC1, DD1, DD0
-	VPXOR      (4*32)(inp), AA0, AA0; VPXOR (5*32)(inp), BB0, BB0; VPXOR (6*32)(inp), CC0, CC0; VPXOR (7*32)(inp), DD0, DD0
-	VMOVDQU    AA0, (4*32)(oup); VMOVDQU BB0, (5*32)(oup); VMOVDQU CC0, (6*32)(oup); VMOVDQU DD0, (7*32)(oup)
+	ADDQ       480(SI), R10
+	ADCQ       488(SI), R11
+	ADCQ       $0x01, R12
+	MOVQ       (BP), DX
+	MOVQ       DX, R15
+	MULXQ      R10, R13, R14
+	IMULQ      R12, R15
+	MULXQ      R11, AX, DX
+	ADDQ       AX, R14
+	ADCQ       DX, R15
+	MOVQ       8(BP), DX
+	MULXQ      R10, R10, AX
+	ADDQ       R10, R14
+	MULXQ      R11, R11, R8
+	ADCQ       R11, R15
+	ADCQ       $0x00, R8
+	IMULQ      R12, DX
+	ADDQ       AX, R15
+	ADCQ       DX, R8
+	MOVQ       R13, R10
+	MOVQ       R14, R11
+	MOVQ       R15, R12
+	ANDQ       $0x03, R12
+	MOVQ       R15, R13
+	ANDQ       $-4, R13
+	MOVQ       R8, R14
+	SHRQ       $0x02, R8, R15
+	SHRQ       $0x02, R8
+	ADDQ       R13, R10
+	ADCQ       R14, R11
+	ADCQ       $0x00, R12
+	ADDQ       R15, R10
+	ADCQ       R8, R11
+	ADCQ       $0x00, R12
+	VPERM2I128 $0x02, Y0, Y14, Y15
+	VPERM2I128 $0x13, Y0, Y14, Y14
+	VPERM2I128 $0x02, Y12, Y4, Y0
+	VPERM2I128 $0x13, Y12, Y4, Y12
+	VPXOR      (SI), Y15, Y15
+	VPXOR      32(SI), Y0, Y0
+	VPXOR      64(SI), Y14, Y14
+	VPXOR      96(SI), Y12, Y12
+	VMOVDQU    Y15, (DI)
+	VMOVDQU    Y0, 32(DI)
+	VMOVDQU    Y14, 64(DI)
+	VMOVDQU    Y12, 96(DI)
+	VPERM2I128 $0x02, Y5, Y9, Y0
+	VPERM2I128 $0x02, Y13, Y1, Y14
+	VPERM2I128 $0x13, Y5, Y9, Y12
+	VPERM2I128 $0x13, Y13, Y1, Y4
+	VPXOR      128(SI), Y0, Y0
+	VPXOR      160(SI), Y14, Y14
+	VPXOR      192(SI), Y12, Y12
+	VPXOR      224(SI), Y4, Y4
+	VMOVDQU    Y0, 128(DI)
+	VMOVDQU    Y14, 160(DI)
+	VMOVDQU    Y12, 192(DI)
+	VMOVDQU    Y4, 224(DI)
 
 	// and here
-	polyAdd(496(inp))
-	polyMulAVX2
-	VPERM2I128 $0x02, AA2, BB2, AA0; VPERM2I128 $0x02, CC2, DD2, BB0; VPERM2I128 $0x13, AA2, BB2, CC0; VPERM2I128 $0x13, CC2, DD2, DD0
-	VPXOR      (8*32)(inp), AA0, AA0; VPXOR (9*32)(inp), BB0, BB0; VPXOR (10*32)(inp), CC0, CC0; VPXOR (11*32)(inp), DD0, DD0
-	VMOVDQU    AA0, (8*32)(oup); VMOVDQU BB0, (9*32)(oup); VMOVDQU CC0, (10*32)(oup); VMOVDQU DD0, (11*32)(oup)
-	VPERM2I128 $0x02, AA3, BB3, AA0; VPERM2I128 $0x02, tmpStoreAVX2, DD3, BB0; VPERM2I128 $0x13, AA3, BB3, CC0; VPERM2I128 $0x13, tmpStoreAVX2, DD3, DD0
-	VPXOR      (12*32)(inp), AA0, AA0; VPXOR (13*32)(inp), BB0, BB0; VPXOR (14*32)(inp), CC0, CC0; VPXOR (15*32)(inp), DD0, DD0
-	VMOVDQU    AA0, (12*32)(oup); VMOVDQU BB0, (13*32)(oup); VMOVDQU CC0, (14*32)(oup); VMOVDQU DD0, (15*32)(oup)
-	LEAQ       (32*16)(inp), inp
-	LEAQ       (32*16)(oup), oup
-	SUBQ       $(32*16), inl
+	ADDQ       496(SI), R10
+	ADCQ       504(SI), R11
+	ADCQ       $0x01, R12
+	MOVQ       (BP), DX
+	MOVQ       DX, R15
+	MULXQ      R10, R13, R14
+	IMULQ      R12, R15
+	MULXQ      R11, AX, DX
+	ADDQ       AX, R14
+	ADCQ       DX, R15
+	MOVQ       8(BP), DX
+	MULXQ      R10, R10, AX
+	ADDQ       R10, R14
+	MULXQ      R11, R11, R8
+	ADCQ       R11, R15
+	ADCQ       $0x00, R8
+	IMULQ      R12, DX
+	ADDQ       AX, R15
+	ADCQ       DX, R8
+	MOVQ       R13, R10
+	MOVQ       R14, R11
+	MOVQ       R15, R12
+	ANDQ       $0x03, R12
+	MOVQ       R15, R13
+	ANDQ       $-4, R13
+	MOVQ       R8, R14
+	SHRQ       $0x02, R8, R15
+	SHRQ       $0x02, R8
+	ADDQ       R13, R10
+	ADCQ       R14, R11
+	ADCQ       $0x00, R12
+	ADDQ       R15, R10
+	ADCQ       R8, R11
+	ADCQ       $0x00, R12
+	VPERM2I128 $0x02, Y6, Y10, Y0
+	VPERM2I128 $0x02, Y8, Y2, Y14
+	VPERM2I128 $0x13, Y6, Y10, Y12
+	VPERM2I128 $0x13, Y8, Y2, Y4
+	VPXOR      256(SI), Y0, Y0
+	VPXOR      288(SI), Y14, Y14
+	VPXOR      320(SI), Y12, Y12
+	VPXOR      352(SI), Y4, Y4
+	VMOVDQU    Y0, 256(DI)
+	VMOVDQU    Y14, 288(DI)
+	VMOVDQU    Y12, 320(DI)
+	VMOVDQU    Y4, 352(DI)
+	VPERM2I128 $0x02, Y7, Y11, Y0
+	VPERM2I128 $0x02, 224(BP), Y3, Y14
+	VPERM2I128 $0x13, Y7, Y11, Y12
+	VPERM2I128 $0x13, 224(BP), Y3, Y4
+	VPXOR      384(SI), Y0, Y0
+	VPXOR      416(SI), Y14, Y14
+	VPXOR      448(SI), Y12, Y12
+	VPXOR      480(SI), Y4, Y4
+	VMOVDQU    Y0, 384(DI)
+	VMOVDQU    Y14, 416(DI)
+	VMOVDQU    Y12, 448(DI)
+	VMOVDQU    Y4, 480(DI)
+	LEAQ       512(SI), SI
+	LEAQ       512(DI), DI
+	SUBQ       $0x00000200, BX
 	JMP        openAVX2MainLoop
 
 openAVX2MainLoopDone:
 	// Handle the various tail sizes efficiently
-	TESTQ inl, inl
+	TESTQ BX, BX
 	JE    openSSEFinalize
-	CMPQ  inl, $128
+	CMPQ  BX, $0x80
 	JBE   openAVX2Tail128
-	CMPQ  inl, $256
+	CMPQ  BX, $0x00000100
 	JBE   openAVX2Tail256
-	CMPQ  inl, $384
+	CMPQ  BX, $0x00000180
 	JBE   openAVX2Tail384
 	JMP   openAVX2Tail512
 
-// ----------------------------------------------------------------------------
-// Special optimization for buffers smaller than 193 bytes
 openAVX2192:
-	// For up to 192 bytes of ciphertext and 64 bytes for the poly key, we process four blocks
-	VMOVDQA AA0, AA1
-	VMOVDQA BB0, BB1
-	VMOVDQA CC0, CC1
-	VPADDD  ·avx2IncMask<>(SB), DD0, DD1
-	VMOVDQA AA0, AA2
-	VMOVDQA BB0, BB2
-	VMOVDQA CC0, CC2
-	VMOVDQA DD0, DD2
-	VMOVDQA DD1, TT3
-	MOVQ    $10, itr2
+	VMOVDQA Y0, Y5
+	VMOVDQA Y14, Y9
+	VMOVDQA Y12, Y13
+	VPADDD  ·avx2IncMask<>+0(SB), Y4, Y1
+	VMOVDQA Y0, Y6
+	VMOVDQA Y14, Y10
+	VMOVDQA Y12, Y8
+	VMOVDQA Y4, Y2
+	VMOVDQA Y1, Y15
+	MOVQ    $0x0000000a, R9
 
 openAVX2192InnerCipherLoop:
-	chachaQR_AVX2(AA0, BB0, CC0, DD0, TT0); chachaQR_AVX2(AA1, BB1, CC1, DD1, TT0)
-	VPALIGNR   $4, BB0, BB0, BB0; VPALIGNR $4, BB1, BB1, BB1
-	VPALIGNR   $8, CC0, CC0, CC0; VPALIGNR $8, CC1, CC1, CC1
-	VPALIGNR   $12, DD0, DD0, DD0; VPALIGNR $12, DD1, DD1, DD1
-	chachaQR_AVX2(AA0, BB0, CC0, DD0, TT0); chachaQR_AVX2(AA1, BB1, CC1, DD1, TT0)
-	VPALIGNR   $12, BB0, BB0, BB0; VPALIGNR $12, BB1, BB1, BB1
-	VPALIGNR   $8, CC0, CC0, CC0; VPALIGNR $8, CC1, CC1, CC1
-	VPALIGNR   $4, DD0, DD0, DD0; VPALIGNR $4, DD1, DD1, DD1
-	DECQ       itr2
+	VPADDD     Y14, Y0, Y0
+	VPXOR      Y0, Y4, Y4
+	VPSHUFB    ·rol16<>+0(SB), Y4, Y4
+	VPADDD     Y4, Y12, Y12
+	VPXOR      Y12, Y14, Y14
+	VPSLLD     $0x0c, Y14, Y3
+	VPSRLD     $0x14, Y14, Y14
+	VPXOR      Y3, Y14, Y14
+	VPADDD     Y14, Y0, Y0
+	VPXOR      Y0, Y4, Y4
+	VPSHUFB    ·rol8<>+0(SB), Y4, Y4
+	VPADDD     Y4, Y12, Y12
+	VPXOR      Y12, Y14, Y14
+	VPSLLD     $0x07, Y14, Y3
+	VPSRLD     $0x19, Y14, Y14
+	VPXOR      Y3, Y14, Y14
+	VPADDD     Y9, Y5, Y5
+	VPXOR      Y5, Y1, Y1
+	VPSHUFB    ·rol16<>+0(SB), Y1, Y1
+	VPADDD     Y1, Y13, Y13
+	VPXOR      Y13, Y9, Y9
+	VPSLLD     $0x0c, Y9, Y3
+	VPSRLD     $0x14, Y9, Y9
+	VPXOR      Y3, Y9, Y9
+	VPADDD     Y9, Y5, Y5
+	VPXOR      Y5, Y1, Y1
+	VPSHUFB    ·rol8<>+0(SB), Y1, Y1
+	VPADDD     Y1, Y13, Y13
+	VPXOR      Y13, Y9, Y9
+	VPSLLD     $0x07, Y9, Y3
+	VPSRLD     $0x19, Y9, Y9
+	VPXOR      Y3, Y9, Y9
+	VPALIGNR   $0x04, Y14, Y14, Y14
+	VPALIGNR   $0x04, Y9, Y9, Y9
+	VPALIGNR   $0x08, Y12, Y12, Y12
+	VPALIGNR   $0x08, Y13, Y13, Y13
+	VPALIGNR   $0x0c, Y4, Y4, Y4
+	VPALIGNR   $0x0c, Y1, Y1, Y1
+	VPADDD     Y14, Y0, Y0
+	VPXOR      Y0, Y4, Y4
+	VPSHUFB    ·rol16<>+0(SB), Y4, Y4
+	VPADDD     Y4, Y12, Y12
+	VPXOR      Y12, Y14, Y14
+	VPSLLD     $0x0c, Y14, Y3
+	VPSRLD     $0x14, Y14, Y14
+	VPXOR      Y3, Y14, Y14
+	VPADDD     Y14, Y0, Y0
+	VPXOR      Y0, Y4, Y4
+	VPSHUFB    ·rol8<>+0(SB), Y4, Y4
+	VPADDD     Y4, Y12, Y12
+	VPXOR      Y12, Y14, Y14
+	VPSLLD     $0x07, Y14, Y3
+	VPSRLD     $0x19, Y14, Y14
+	VPXOR      Y3, Y14, Y14
+	VPADDD     Y9, Y5, Y5
+	VPXOR      Y5, Y1, Y1
+	VPSHUFB    ·rol16<>+0(SB), Y1, Y1
+	VPADDD     Y1, Y13, Y13
+	VPXOR      Y13, Y9, Y9
+	VPSLLD     $0x0c, Y9, Y3
+	VPSRLD     $0x14, Y9, Y9
+	VPXOR      Y3, Y9, Y9
+	VPADDD     Y9, Y5, Y5
+	VPXOR      Y5, Y1, Y1
+	VPSHUFB    ·rol8<>+0(SB), Y1, Y1
+	VPADDD     Y1, Y13, Y13
+	VPXOR      Y13, Y9, Y9
+	VPSLLD     $0x07, Y9, Y3
+	VPSRLD     $0x19, Y9, Y9
+	VPXOR      Y3, Y9, Y9
+	VPALIGNR   $0x0c, Y14, Y14, Y14
+	VPALIGNR   $0x0c, Y9, Y9, Y9
+	VPALIGNR   $0x08, Y12, Y12, Y12
+	VPALIGNR   $0x08, Y13, Y13, Y13
+	VPALIGNR   $0x04, Y4, Y4, Y4
+	VPALIGNR   $0x04, Y1, Y1, Y1
+	DECQ       R9
 	JNE        openAVX2192InnerCipherLoop
-	VPADDD     AA2, AA0, AA0; VPADDD AA2, AA1, AA1
-	VPADDD     BB2, BB0, BB0; VPADDD BB2, BB1, BB1
-	VPADDD     CC2, CC0, CC0; VPADDD CC2, CC1, CC1
-	VPADDD     DD2, DD0, DD0; VPADDD TT3, DD1, DD1
-	VPERM2I128 $0x02, AA0, BB0, TT0
+	VPADDD     Y6, Y0, Y0
+	VPADDD     Y6, Y5, Y5
+	VPADDD     Y10, Y14, Y14
+	VPADDD     Y10, Y9, Y9
+	VPADDD     Y8, Y12, Y12
+	VPADDD     Y8, Y13, Y13
+	VPADDD     Y2, Y4, Y4
+	VPADDD     Y15, Y1, Y1
+	VPERM2I128 $0x02, Y0, Y14, Y3
 
 	// Clamp and store poly key
-	VPAND   ·polyClampMask<>(SB), TT0, TT0
-	VMOVDQA TT0, rsStoreAVX2
+	VPAND   ·polyClampMask<>+0(SB), Y3, Y3
+	VMOVDQA Y3, (BP)
 
 	// Stream for up to 192 bytes
-	VPERM2I128 $0x13, AA0, BB0, AA0
-	VPERM2I128 $0x13, CC0, DD0, BB0
-	VPERM2I128 $0x02, AA1, BB1, CC0
-	VPERM2I128 $0x02, CC1, DD1, DD0
-	VPERM2I128 $0x13, AA1, BB1, AA1
-	VPERM2I128 $0x13, CC1, DD1, BB1
+	VPERM2I128 $0x13, Y0, Y14, Y0
+	VPERM2I128 $0x13, Y12, Y4, Y14
+	VPERM2I128 $0x02, Y5, Y9, Y12
+	VPERM2I128 $0x02, Y13, Y1, Y4
+	VPERM2I128 $0x13, Y5, Y9, Y5
+	VPERM2I128 $0x13, Y13, Y1, Y9
 
 openAVX2ShortOpen:
 	// Hash
-	MOVQ ad_len+80(FP), itr2
+	MOVQ ad_len+80(FP), R9
 	CALL polyHashADInternal<>(SB)
 
 openAVX2ShortOpenLoop:
-	CMPQ inl, $32
+	CMPQ BX, $0x20
 	JB   openAVX2ShortTail32
-	SUBQ $32, inl
+	SUBQ $0x20, BX
 
 	// Load for hashing
-	polyAdd(0*8(inp))
-	polyMulAVX2
-	polyAdd(2*8(inp))
-	polyMulAVX2
+	ADDQ  (SI), R10
+	ADCQ  8(SI), R11
+	ADCQ  $0x01, R12
+	MOVQ  (BP), DX
+	MOVQ  DX, R15
+	MULXQ R10, R13, R14
+	IMULQ R12, R15
+	MULXQ R11, AX, DX
+	ADDQ  AX, R14
+	ADCQ  DX, R15
+	MOVQ  8(BP), DX
+	MULXQ R10, R10, AX
+	ADDQ  R10, R14
+	MULXQ R11, R11, R8
+	ADCQ  R11, R15
+	ADCQ  $0x00, R8
+	IMULQ R12, DX
+	ADDQ  AX, R15
+	ADCQ  DX, R8
+	MOVQ  R13, R10
+	MOVQ  R14, R11
+	MOVQ  R15, R12
+	ANDQ  $0x03, R12
+	MOVQ  R15, R13
+	ANDQ  $-4, R13
+	MOVQ  R8, R14
+	SHRQ  $0x02, R8, R15
+	SHRQ  $0x02, R8
+	ADDQ  R13, R10
+	ADCQ  R14, R11
+	ADCQ  $0x00, R12
+	ADDQ  R15, R10
+	ADCQ  R8, R11
+	ADCQ  $0x00, R12
+	ADDQ  16(SI), R10
+	ADCQ  24(SI), R11
+	ADCQ  $0x01, R12
+	MOVQ  (BP), DX
+	MOVQ  DX, R15
+	MULXQ R10, R13, R14
+	IMULQ R12, R15
+	MULXQ R11, AX, DX
+	ADDQ  AX, R14
+	ADCQ  DX, R15
+	MOVQ  8(BP), DX
+	MULXQ R10, R10, AX
+	ADDQ  R10, R14
+	MULXQ R11, R11, R8
+	ADCQ  R11, R15
+	ADCQ  $0x00, R8
+	IMULQ R12, DX
+	ADDQ  AX, R15
+	ADCQ  DX, R8
+	MOVQ  R13, R10
+	MOVQ  R14, R11
+	MOVQ  R15, R12
+	ANDQ  $0x03, R12
+	MOVQ  R15, R13
+	ANDQ  $-4, R13
+	MOVQ  R8, R14
+	SHRQ  $0x02, R8, R15
+	SHRQ  $0x02, R8
+	ADDQ  R13, R10
+	ADCQ  R14, R11
+	ADCQ  $0x00, R12
+	ADDQ  R15, R10
+	ADCQ  R8, R11
+	ADCQ  $0x00, R12
 
 	// Load for decryption
-	VPXOR   (inp), AA0, AA0
-	VMOVDQU AA0, (oup)
-	LEAQ    (1*32)(inp), inp
-	LEAQ    (1*32)(oup), oup
+	VPXOR   (SI), Y0, Y0
+	VMOVDQU Y0, (DI)
+	LEAQ    32(SI), SI
+	LEAQ    32(DI), DI
 
 	// Shift stream left
-	VMOVDQA BB0, AA0
-	VMOVDQA CC0, BB0
-	VMOVDQA DD0, CC0
-	VMOVDQA AA1, DD0
-	VMOVDQA BB1, AA1
-	VMOVDQA CC1, BB1
-	VMOVDQA DD1, CC1
-	VMOVDQA AA2, DD1
-	VMOVDQA BB2, AA2
+	VMOVDQA Y14, Y0
+	VMOVDQA Y12, Y14
+	VMOVDQA Y4, Y12
+	VMOVDQA Y5, Y4
+	VMOVDQA Y9, Y5
+	VMOVDQA Y13, Y9
+	VMOVDQA Y1, Y13
+	VMOVDQA Y6, Y1
+	VMOVDQA Y10, Y6
 	JMP     openAVX2ShortOpenLoop
 
 openAVX2ShortTail32:
-	CMPQ    inl, $16
-	VMOVDQA A0, A1
+	CMPQ    BX, $0x10
+	VMOVDQA X0, X1
 	JB      openAVX2ShortDone
-
-	SUBQ $16, inl
+	SUBQ    $0x10, BX
 
 	// Load for hashing
-	polyAdd(0*8(inp))
-	polyMulAVX2
+	ADDQ  (SI), R10
+	ADCQ  8(SI), R11
+	ADCQ  $0x01, R12
+	MOVQ  (BP), DX
+	MOVQ  DX, R15
+	MULXQ R10, R13, R14
+	IMULQ R12, R15
+	MULXQ R11, AX, DX
+	ADDQ  AX, R14
+	ADCQ  DX, R15
+	MOVQ  8(BP), DX
+	MULXQ R10, R10, AX
+	ADDQ  R10, R14
+	MULXQ R11, R11, R8
+	ADCQ  R11, R15
+	ADCQ  $0x00, R8
+	IMULQ R12, DX
+	ADDQ  AX, R15
+	ADCQ  DX, R8
+	MOVQ  R13, R10
+	MOVQ  R14, R11
+	MOVQ  R15, R12
+	ANDQ  $0x03, R12
+	MOVQ  R15, R13
+	ANDQ  $-4, R13
+	MOVQ  R8, R14
+	SHRQ  $0x02, R8, R15
+	SHRQ  $0x02, R8
+	ADDQ  R13, R10
+	ADCQ  R14, R11
+	ADCQ  $0x00, R12
+	ADDQ  R15, R10
+	ADCQ  R8, R11
+	ADCQ  $0x00, R12
 
 	// Load for decryption
-	VPXOR      (inp), A0, T0
-	VMOVDQU    T0, (oup)
-	LEAQ       (1*16)(inp), inp
-	LEAQ       (1*16)(oup), oup
-	VPERM2I128 $0x11, AA0, AA0, AA0
-	VMOVDQA    A0, A1
+	VPXOR      (SI), X0, X12
+	VMOVDQU    X12, (DI)
+	LEAQ       16(SI), SI
+	LEAQ       16(DI), DI
+	VPERM2I128 $0x11, Y0, Y0, Y0
+	VMOVDQA    X0, X1
 
 openAVX2ShortDone:
 	VZEROUPPER
 	JMP openSSETail16
 
-// ----------------------------------------------------------------------------
-// Special optimization for buffers smaller than 321 bytes
 openAVX2320:
-	// For up to 320 bytes of ciphertext and 64 bytes for the poly key, we process six blocks
-	VMOVDQA AA0, AA1; VMOVDQA BB0, BB1; VMOVDQA CC0, CC1; VPADDD ·avx2IncMask<>(SB), DD0, DD1
-	VMOVDQA AA0, AA2; VMOVDQA BB0, BB2; VMOVDQA CC0, CC2; VPADDD ·avx2IncMask<>(SB), DD1, DD2
-	VMOVDQA BB0, TT1; VMOVDQA CC0, TT2; VMOVDQA DD0, TT3
-	MOVQ    $10, itr2
+	VMOVDQA Y0, Y5
+	VMOVDQA Y14, Y9
+	VMOVDQA Y12, Y13
+	VPADDD  ·avx2IncMask<>+0(SB), Y4, Y1
+	VMOVDQA Y0, Y6
+	VMOVDQA Y14, Y10
+	VMOVDQA Y12, Y8
+	VPADDD  ·avx2IncMask<>+0(SB), Y1, Y2
+	VMOVDQA Y14, Y7
+	VMOVDQA Y12, Y11
+	VMOVDQA Y4, Y15
+	MOVQ    $0x0000000a, R9
 
 openAVX2320InnerCipherLoop:
-	chachaQR_AVX2(AA0, BB0, CC0, DD0, TT0); chachaQR_AVX2(AA1, BB1, CC1, DD1, TT0); chachaQR_AVX2(AA2, BB2, CC2, DD2, TT0)
-	VPALIGNR $4, BB0, BB0, BB0; VPALIGNR $4, BB1, BB1, BB1; VPALIGNR $4, BB2, BB2, BB2
-	VPALIGNR $8, CC0, CC0, CC0; VPALIGNR $8, CC1, CC1, CC1; VPALIGNR $8, CC2, CC2, CC2
-	VPALIGNR $12, DD0, DD0, DD0; VPALIGNR $12, DD1, DD1, DD1; VPALIGNR $12, DD2, DD2, DD2
-	chachaQR_AVX2(AA0, BB0, CC0, DD0, TT0); chachaQR_AVX2(AA1, BB1, CC1, DD1, TT0); chachaQR_AVX2(AA2, BB2, CC2, DD2, TT0)
-	VPALIGNR $12, BB0, BB0, BB0; VPALIGNR $12, BB1, BB1, BB1; VPALIGNR $12, BB2, BB2, BB2
-	VPALIGNR $8, CC0, CC0, CC0; VPALIGNR $8, CC1, CC1, CC1; VPALIGNR $8, CC2, CC2, CC2
-	VPALIGNR $4, DD0, DD0, DD0; VPALIGNR $4, DD1, DD1, DD1; VPALIGNR $4, DD2, DD2, DD2
-	DECQ     itr2
+	VPADDD   Y14, Y0, Y0
+	VPXOR    Y0, Y4, Y4
+	VPSHUFB  ·rol16<>+0(SB), Y4, Y4
+	VPADDD   Y4, Y12, Y12
+	VPXOR    Y12, Y14, Y14
+	VPSLLD   $0x0c, Y14, Y3
+	VPSRLD   $0x14, Y14, Y14
+	VPXOR    Y3, Y14, Y14
+	VPADDD   Y14, Y0, Y0
+	VPXOR    Y0, Y4, Y4
+	VPSHUFB  ·rol8<>+0(SB), Y4, Y4
+	VPADDD   Y4, Y12, Y12
+	VPXOR    Y12, Y14, Y14
+	VPSLLD   $0x07, Y14, Y3
+	VPSRLD   $0x19, Y14, Y14
+	VPXOR    Y3, Y14, Y14
+	VPADDD   Y9, Y5, Y5
+	VPXOR    Y5, Y1, Y1
+	VPSHUFB  ·rol16<>+0(SB), Y1, Y1
+	VPADDD   Y1, Y13, Y13
+	VPXOR    Y13, Y9, Y9
+	VPSLLD   $0x0c, Y9, Y3
+	VPSRLD   $0x14, Y9, Y9
+	VPXOR    Y3, Y9, Y9
+	VPADDD   Y9, Y5, Y5
+	VPXOR    Y5, Y1, Y1
+	VPSHUFB  ·rol8<>+0(SB), Y1, Y1
+	VPADDD   Y1, Y13, Y13
+	VPXOR    Y13, Y9, Y9
+	VPSLLD   $0x07, Y9, Y3
+	VPSRLD   $0x19, Y9, Y9
+	VPXOR    Y3, Y9, Y9
+	VPADDD   Y10, Y6, Y6
+	VPXOR    Y6, Y2, Y2
+	VPSHUFB  ·rol16<>+0(SB), Y2, Y2
+	VPADDD   Y2, Y8, Y8
+	VPXOR    Y8, Y10, Y10
+	VPSLLD   $0x0c, Y10, Y3
+	VPSRLD   $0x14, Y10, Y10
+	VPXOR    Y3, Y10, Y10
+	VPADDD   Y10, Y6, Y6
+	VPXOR    Y6, Y2, Y2
+	VPSHUFB  ·rol8<>+0(SB), Y2, Y2
+	VPADDD   Y2, Y8, Y8
+	VPXOR    Y8, Y10, Y10
+	VPSLLD   $0x07, Y10, Y3
+	VPSRLD   $0x19, Y10, Y10
+	VPXOR    Y3, Y10, Y10
+	VPALIGNR $0x04, Y14, Y14, Y14
+	VPALIGNR $0x04, Y9, Y9, Y9
+	VPALIGNR $0x04, Y10, Y10, Y10
+	VPALIGNR $0x08, Y12, Y12, Y12
+	VPALIGNR $0x08, Y13, Y13, Y13
+	VPALIGNR $0x08, Y8, Y8, Y8
+	VPALIGNR $0x0c, Y4, Y4, Y4
+	VPALIGNR $0x0c, Y1, Y1, Y1
+	VPALIGNR $0x0c, Y2, Y2, Y2
+	VPADDD   Y14, Y0, Y0
+	VPXOR    Y0, Y4, Y4
+	VPSHUFB  ·rol16<>+0(SB), Y4, Y4
+	VPADDD   Y4, Y12, Y12
+	VPXOR    Y12, Y14, Y14
+	VPSLLD   $0x0c, Y14, Y3
+	VPSRLD   $0x14, Y14, Y14
+	VPXOR    Y3, Y14, Y14
+	VPADDD   Y14, Y0, Y0
+	VPXOR    Y0, Y4, Y4
+	VPSHUFB  ·rol8<>+0(SB), Y4, Y4
+	VPADDD   Y4, Y12, Y12
+	VPXOR    Y12, Y14, Y14
+	VPSLLD   $0x07, Y14, Y3
+	VPSRLD   $0x19, Y14, Y14
+	VPXOR    Y3, Y14, Y14
+	VPADDD   Y9, Y5, Y5
+	VPXOR    Y5, Y1, Y1
+	VPSHUFB  ·rol16<>+0(SB), Y1, Y1
+	VPADDD   Y1, Y13, Y13
+	VPXOR    Y13, Y9, Y9
+	VPSLLD   $0x0c, Y9, Y3
+	VPSRLD   $0x14, Y9, Y9
+	VPXOR    Y3, Y9, Y9
+	VPADDD   Y9, Y5, Y5
+	VPXOR    Y5, Y1, Y1
+	VPSHUFB  ·rol8<>+0(SB), Y1, Y1
+	VPADDD   Y1, Y13, Y13
+	VPXOR    Y13, Y9, Y9
+	VPSLLD   $0x07, Y9, Y3
+	VPSRLD   $0x19, Y9, Y9
+	VPXOR    Y3, Y9, Y9
+	VPADDD   Y10, Y6, Y6
+	VPXOR    Y6, Y2, Y2
+	VPSHUFB  ·rol16<>+0(SB), Y2, Y2
+	VPADDD   Y2, Y8, Y8
+	VPXOR    Y8, Y10, Y10
+	VPSLLD   $0x0c, Y10, Y3
+	VPSRLD   $0x14, Y10, Y10
+	VPXOR    Y3, Y10, Y10
+	VPADDD   Y10, Y6, Y6
+	VPXOR    Y6, Y2, Y2
+	VPSHUFB  ·rol8<>+0(SB), Y2, Y2
+	VPADDD   Y2, Y8, Y8
+	VPXOR    Y8, Y10, Y10
+	VPSLLD   $0x07, Y10, Y3
+	VPSRLD   $0x19, Y10, Y10
+	VPXOR    Y3, Y10, Y10
+	VPALIGNR $0x0c, Y14, Y14, Y14
+	VPALIGNR $0x0c, Y9, Y9, Y9
+	VPALIGNR $0x0c, Y10, Y10, Y10
+	VPALIGNR $0x08, Y12, Y12, Y12
+	VPALIGNR $0x08, Y13, Y13, Y13
+	VPALIGNR $0x08, Y8, Y8, Y8
+	VPALIGNR $0x04, Y4, Y4, Y4
+	VPALIGNR $0x04, Y1, Y1, Y1
+	VPALIGNR $0x04, Y2, Y2, Y2
+	DECQ     R9
 	JNE      openAVX2320InnerCipherLoop
-
-	VMOVDQA ·chacha20Constants<>(SB), TT0
-	VPADDD  TT0, AA0, AA0; VPADDD TT0, AA1, AA1; VPADDD TT0, AA2, AA2
-	VPADDD  TT1, BB0, BB0; VPADDD TT1, BB1, BB1; VPADDD TT1, BB2, BB2
-	VPADDD  TT2, CC0, CC0; VPADDD TT2, CC1, CC1; VPADDD TT2, CC2, CC2
-	VMOVDQA ·avx2IncMask<>(SB), TT0
-	VPADDD  TT3, DD0, DD0; VPADDD TT0, TT3, TT3
-	VPADDD  TT3, DD1, DD1; VPADDD TT0, TT3, TT3
-	VPADDD  TT3, DD2, DD2
+	VMOVDQA  ·chacha20Constants<>+0(SB), Y3
+	VPADDD   Y3, Y0, Y0
+	VPADDD   Y3, Y5, Y5
+	VPADDD   Y3, Y6, Y6
+	VPADDD   Y7, Y14, Y14
+	VPADDD   Y7, Y9, Y9
+	VPADDD   Y7, Y10, Y10
+	VPADDD   Y11, Y12, Y12
+	VPADDD   Y11, Y13, Y13
+	VPADDD   Y11, Y8, Y8
+	VMOVDQA  ·avx2IncMask<>+0(SB), Y3
+	VPADDD   Y15, Y4, Y4
+	VPADDD   Y3, Y15, Y15
+	VPADDD   Y15, Y1, Y1
+	VPADDD   Y3, Y15, Y15
+	VPADDD   Y15, Y2, Y2
 
 	// Clamp and store poly key
-	VPERM2I128 $0x02, AA0, BB0, TT0
-	VPAND      ·polyClampMask<>(SB), TT0, TT0
-	VMOVDQA    TT0, rsStoreAVX2
+	VPERM2I128 $0x02, Y0, Y14, Y3
+	VPAND      ·polyClampMask<>+0(SB), Y3, Y3
+	VMOVDQA    Y3, (BP)
 
 	// Stream for up to 320 bytes
-	VPERM2I128 $0x13, AA0, BB0, AA0
-	VPERM2I128 $0x13, CC0, DD0, BB0
-	VPERM2I128 $0x02, AA1, BB1, CC0
-	VPERM2I128 $0x02, CC1, DD1, DD0
-	VPERM2I128 $0x13, AA1, BB1, AA1
-	VPERM2I128 $0x13, CC1, DD1, BB1
-	VPERM2I128 $0x02, AA2, BB2, CC1
-	VPERM2I128 $0x02, CC2, DD2, DD1
-	VPERM2I128 $0x13, AA2, BB2, AA2
-	VPERM2I128 $0x13, CC2, DD2, BB2
+	VPERM2I128 $0x13, Y0, Y14, Y0
+	VPERM2I128 $0x13, Y12, Y4, Y14
+	VPERM2I128 $0x02, Y5, Y9, Y12
+	VPERM2I128 $0x02, Y13, Y1, Y4
+	VPERM2I128 $0x13, Y5, Y9, Y5
+	VPERM2I128 $0x13, Y13, Y1, Y9
+	VPERM2I128 $0x02, Y6, Y10, Y13
+	VPERM2I128 $0x02, Y8, Y2, Y1
+	VPERM2I128 $0x13, Y6, Y10, Y6
+	VPERM2I128 $0x13, Y8, Y2, Y10
 	JMP        openAVX2ShortOpen
 
-// ----------------------------------------------------------------------------
-// Special optimization for the last 128 bytes of ciphertext
 openAVX2Tail128:
 	// Need to decrypt up to 128 bytes - prepare two blocks
-	VMOVDQA ·chacha20Constants<>(SB), AA1
-	VMOVDQA state1StoreAVX2, BB1
-	VMOVDQA state2StoreAVX2, CC1
-	VMOVDQA ctr3StoreAVX2, DD1
-	VPADDD  ·avx2IncMask<>(SB), DD1, DD1
-	VMOVDQA DD1, DD0
-
-	XORQ  itr2, itr2
-	MOVQ  inl, itr1
-	ANDQ  $-16, itr1
-	TESTQ itr1, itr1
-	JE    openAVX2Tail128LoopB
+	VMOVDQA ·chacha20Constants<>+0(SB), Y5
+	VMOVDQA 32(BP), Y9
+	VMOVDQA 64(BP), Y13
+	VMOVDQA 192(BP), Y1
+	VPADDD  ·avx2IncMask<>+0(SB), Y1, Y1
+	VMOVDQA Y1, Y4
+	XORQ    R9, R9
+	MOVQ    BX, CX
+	ANDQ    $-16, CX
+	TESTQ   CX, CX
+	JE      openAVX2Tail128LoopB
 
 openAVX2Tail128LoopA:
-	// Perform ChaCha rounds, while hashing the remaining input
-	polyAdd(0(inp)(itr2*1))
-	polyMulAVX2
+	ADDQ  (SI)(R9*1), R10
+	ADCQ  8(SI)(R9*1), R11
+	ADCQ  $0x01, R12
+	MOVQ  (BP), DX
+	MOVQ  DX, R15
+	MULXQ R10, R13, R14
+	IMULQ R12, R15
+	MULXQ R11, AX, DX
+	ADDQ  AX, R14
+	ADCQ  DX, R15
+	MOVQ  8(BP), DX
+	MULXQ R10, R10, AX
+	ADDQ  R10, R14
+	MULXQ R11, R11, R8
+	ADCQ  R11, R15
+	ADCQ  $0x00, R8
+	IMULQ R12, DX
+	ADDQ  AX, R15
+	ADCQ  DX, R8
+	MOVQ  R13, R10
+	MOVQ  R14, R11
+	MOVQ  R15, R12
+	ANDQ  $0x03, R12
+	MOVQ  R15, R13
+	ANDQ  $-4, R13
+	MOVQ  R8, R14
+	SHRQ  $0x02, R8, R15
+	SHRQ  $0x02, R8
+	ADDQ  R13, R10
+	ADCQ  R14, R11
+	ADCQ  $0x00, R12
+	ADDQ  R15, R10
+	ADCQ  R8, R11
+	ADCQ  $0x00, R12
 
 openAVX2Tail128LoopB:
-	ADDQ     $16, itr2
-	chachaQR_AVX2(AA1, BB1, CC1, DD1, TT0)
-	VPALIGNR $4, BB1, BB1, BB1
-	VPALIGNR $8, CC1, CC1, CC1
-	VPALIGNR $12, DD1, DD1, DD1
-	chachaQR_AVX2(AA1, BB1, CC1, DD1, TT0)
-	VPALIGNR $12, BB1, BB1, BB1
-	VPALIGNR $8, CC1, CC1, CC1
-	VPALIGNR $4, DD1, DD1, DD1
-	CMPQ     itr2, itr1
-	JB       openAVX2Tail128LoopA
-	CMPQ     itr2, $160
-	JNE      openAVX2Tail128LoopB
-
-	VPADDD     ·chacha20Constants<>(SB), AA1, AA1
-	VPADDD     state1StoreAVX2, BB1, BB1
-	VPADDD     state2StoreAVX2, CC1, CC1
-	VPADDD     DD0, DD1, DD1
-	VPERM2I128 $0x02, AA1, BB1, AA0; VPERM2I128 $0x02, CC1, DD1, BB0; VPERM2I128 $0x13, AA1, BB1, CC0; VPERM2I128 $0x13, CC1, DD1, DD0
+	ADDQ       $0x10, R9
+	VPADDD     Y9, Y5, Y5
+	VPXOR      Y5, Y1, Y1
+	VPSHUFB    ·rol16<>+0(SB), Y1, Y1
+	VPADDD     Y1, Y13, Y13
+	VPXOR      Y13, Y9, Y9
+	VPSLLD     $0x0c, Y9, Y3
+	VPSRLD     $0x14, Y9, Y9
+	VPXOR      Y3, Y9, Y9
+	VPADDD     Y9, Y5, Y5
+	VPXOR      Y5, Y1, Y1
+	VPSHUFB    ·rol8<>+0(SB), Y1, Y1
+	VPADDD     Y1, Y13, Y13
+	VPXOR      Y13, Y9, Y9
+	VPSLLD     $0x07, Y9, Y3
+	VPSRLD     $0x19, Y9, Y9
+	VPXOR      Y3, Y9, Y9
+	VPALIGNR   $0x04, Y9, Y9, Y9
+	VPALIGNR   $0x08, Y13, Y13, Y13
+	VPALIGNR   $0x0c, Y1, Y1, Y1
+	VPADDD     Y9, Y5, Y5
+	VPXOR      Y5, Y1, Y1
+	VPSHUFB    ·rol16<>+0(SB), Y1, Y1
+	VPADDD     Y1, Y13, Y13
+	VPXOR      Y13, Y9, Y9
+	VPSLLD     $0x0c, Y9, Y3
+	VPSRLD     $0x14, Y9, Y9
+	VPXOR      Y3, Y9, Y9
+	VPADDD     Y9, Y5, Y5
+	VPXOR      Y5, Y1, Y1
+	VPSHUFB    ·rol8<>+0(SB), Y1, Y1
+	VPADDD     Y1, Y13, Y13
+	VPXOR      Y13, Y9, Y9
+	VPSLLD     $0x07, Y9, Y3
+	VPSRLD     $0x19, Y9, Y9
+	VPXOR      Y3, Y9, Y9
+	VPALIGNR   $0x0c, Y9, Y9, Y9
+	VPALIGNR   $0x08, Y13, Y13, Y13
+	VPALIGNR   $0x04, Y1, Y1, Y1
+	CMPQ       R9, CX
+	JB         openAVX2Tail128LoopA
+	CMPQ       R9, $0xa0
+	JNE        openAVX2Tail128LoopB
+	VPADDD     ·chacha20Constants<>+0(SB), Y5, Y5
+	VPADDD     32(BP), Y9, Y9
+	VPADDD     64(BP), Y13, Y13
+	VPADDD     Y4, Y1, Y1
+	VPERM2I128 $0x02, Y5, Y9, Y0
+	VPERM2I128 $0x02, Y13, Y1, Y14
+	VPERM2I128 $0x13, Y5, Y9, Y12
+	VPERM2I128 $0x13, Y13, Y1, Y4
 
 openAVX2TailLoop:
-	CMPQ inl, $32
+	CMPQ BX, $0x20
 	JB   openAVX2Tail
-	SUBQ $32, inl
+	SUBQ $0x20, BX
 
 	// Load for decryption
-	VPXOR   (inp), AA0, AA0
-	VMOVDQU AA0, (oup)
-	LEAQ    (1*32)(inp), inp
-	LEAQ    (1*32)(oup), oup
-	VMOVDQA BB0, AA0
-	VMOVDQA CC0, BB0
-	VMOVDQA DD0, CC0
+	VPXOR   (SI), Y0, Y0
+	VMOVDQU Y0, (DI)
+	LEAQ    32(SI), SI
+	LEAQ    32(DI), DI
+	VMOVDQA Y14, Y0
+	VMOVDQA Y12, Y14
+	VMOVDQA Y4, Y12
 	JMP     openAVX2TailLoop
 
 openAVX2Tail:
-	CMPQ    inl, $16
-	VMOVDQA A0, A1
+	CMPQ    BX, $0x10
+	VMOVDQA X0, X1
 	JB      openAVX2TailDone
-	SUBQ    $16, inl
+	SUBQ    $0x10, BX
 
 	// Load for decryption
-	VPXOR      (inp), A0, T0
-	VMOVDQU    T0, (oup)
-	LEAQ       (1*16)(inp), inp
-	LEAQ       (1*16)(oup), oup
-	VPERM2I128 $0x11, AA0, AA0, AA0
-	VMOVDQA    A0, A1
+	VPXOR      (SI), X0, X12
+	VMOVDQU    X12, (DI)
+	LEAQ       16(SI), SI
+	LEAQ       16(DI), DI
+	VPERM2I128 $0x11, Y0, Y0, Y0
+	VMOVDQA    X0, X1
 
 openAVX2TailDone:
 	VZEROUPPER
 	JMP openSSETail16
 
-// ----------------------------------------------------------------------------
-// Special optimization for the last 256 bytes of ciphertext
 openAVX2Tail256:
-	// Need to decrypt up to 256 bytes - prepare four blocks
-	VMOVDQA ·chacha20Constants<>(SB), AA0; VMOVDQA AA0, AA1
-	VMOVDQA state1StoreAVX2, BB0; VMOVDQA BB0, BB1
-	VMOVDQA state2StoreAVX2, CC0; VMOVDQA CC0, CC1
-	VMOVDQA ctr3StoreAVX2, DD0
-	VPADDD  ·avx2IncMask<>(SB), DD0, DD0
-	VPADDD  ·avx2IncMask<>(SB), DD0, DD1
-	VMOVDQA DD0, TT1
-	VMOVDQA DD1, TT2
+	VMOVDQA ·chacha20Constants<>+0(SB), Y0
+	VMOVDQA Y0, Y5
+	VMOVDQA 32(BP), Y14
+	VMOVDQA Y14, Y9
+	VMOVDQA 64(BP), Y12
+	VMOVDQA Y12, Y13
+	VMOVDQA 192(BP), Y4
+	VPADDD  ·avx2IncMask<>+0(SB), Y4, Y4
+	VPADDD  ·avx2IncMask<>+0(SB), Y4, Y1
+	VMOVDQA Y4, Y7
+	VMOVDQA Y1, Y11
 
 	// Compute the number of iterations that will hash data
-	MOVQ    inl, tmpStoreAVX2
-	MOVQ    inl, itr1
-	SUBQ    $128, itr1
-	SHRQ    $4, itr1
-	MOVQ    $10, itr2
-	CMPQ    itr1, $10
-	CMOVQGT itr2, itr1
-	MOVQ    inp, inl
-	XORQ    itr2, itr2
+	MOVQ    BX, 224(BP)
+	MOVQ    BX, CX
+	SUBQ    $0x80, CX
+	SHRQ    $0x04, CX
+	MOVQ    $0x0000000a, R9
+	CMPQ    CX, $0x0a
+	CMOVQGT R9, CX
+	MOVQ    SI, BX
+	XORQ    R9, R9
 
 openAVX2Tail256LoopA:
-	polyAdd(0(inl))
-	polyMulAVX2
-	LEAQ 16(inl), inl
+	ADDQ  (BX), R10
+	ADCQ  8(BX), R11
+	ADCQ  $0x01, R12
+	MOVQ  (BP), DX
+	MOVQ  DX, R15
+	MULXQ R10, R13, R14
+	IMULQ R12, R15
+	MULXQ R11, AX, DX
+	ADDQ  AX, R14
+	ADCQ  DX, R15
+	MOVQ  8(BP), DX
+	MULXQ R10, R10, AX
+	ADDQ  R10, R14
+	MULXQ R11, R11, R8
+	ADCQ  R11, R15
+	ADCQ  $0x00, R8
+	IMULQ R12, DX
+	ADDQ  AX, R15
+	ADCQ  DX, R8
+	MOVQ  R13, R10
+	MOVQ  R14, R11
+	MOVQ  R15, R12
+	ANDQ  $0x03, R12
+	MOVQ  R15, R13
+	ANDQ  $-4, R13
+	MOVQ  R8, R14
+	SHRQ  $0x02, R8, R15
+	SHRQ  $0x02, R8
+	ADDQ  R13, R10
+	ADCQ  R14, R11
+	ADCQ  $0x00, R12
+	ADDQ  R15, R10
+	ADCQ  R8, R11
+	ADCQ  $0x00, R12
+	LEAQ  16(BX), BX
 
-	// Perform ChaCha rounds, while hashing the remaining input
 openAVX2Tail256LoopB:
-	chachaQR_AVX2(AA0, BB0, CC0, DD0, TT0); chachaQR_AVX2(AA1, BB1, CC1, DD1, TT0)
-	VPALIGNR $4, BB0, BB0, BB0; VPALIGNR $4, BB1, BB1, BB1
-	VPALIGNR $8, CC0, CC0, CC0; VPALIGNR $8, CC1, CC1, CC1
-	VPALIGNR $12, DD0, DD0, DD0; VPALIGNR $12, DD1, DD1, DD1
-	INCQ     itr2
-	chachaQR_AVX2(AA0, BB0, CC0, DD0, TT0); chachaQR_AVX2(AA1, BB1, CC1, DD1, TT0)
-	VPALIGNR $12, BB0, BB0, BB0; VPALIGNR $12, BB1, BB1, BB1
-	VPALIGNR $8, CC0, CC0, CC0; VPALIGNR $8, CC1, CC1, CC1
-	VPALIGNR $4, DD0, DD0, DD0; VPALIGNR $4, DD1, DD1, DD1
-	CMPQ     itr2, itr1
+	VPADDD   Y14, Y0, Y0
+	VPXOR    Y0, Y4, Y4
+	VPSHUFB  ·rol16<>+0(SB), Y4, Y4
+	VPADDD   Y4, Y12, Y12
+	VPXOR    Y12, Y14, Y14
+	VPSLLD   $0x0c, Y14, Y3
+	VPSRLD   $0x14, Y14, Y14
+	VPXOR    Y3, Y14, Y14
+	VPADDD   Y14, Y0, Y0
+	VPXOR    Y0, Y4, Y4
+	VPSHUFB  ·rol8<>+0(SB), Y4, Y4
+	VPADDD   Y4, Y12, Y12
+	VPXOR    Y12, Y14, Y14
+	VPSLLD   $0x07, Y14, Y3
+	VPSRLD   $0x19, Y14, Y14
+	VPXOR    Y3, Y14, Y14
+	VPADDD   Y9, Y5, Y5
+	VPXOR    Y5, Y1, Y1
+	VPSHUFB  ·rol16<>+0(SB), Y1, Y1
+	VPADDD   Y1, Y13, Y13
+	VPXOR    Y13, Y9, Y9
+	VPSLLD   $0x0c, Y9, Y3
+	VPSRLD   $0x14, Y9, Y9
+	VPXOR    Y3, Y9, Y9
+	VPADDD   Y9, Y5, Y5
+	VPXOR    Y5, Y1, Y1
+	VPSHUFB  ·rol8<>+0(SB), Y1, Y1
+	VPADDD   Y1, Y13, Y13
+	VPXOR    Y13, Y9, Y9
+	VPSLLD   $0x07, Y9, Y3
+	VPSRLD   $0x19, Y9, Y9
+	VPXOR    Y3, Y9, Y9
+	VPALIGNR $0x04, Y14, Y14, Y14
+	VPALIGNR $0x04, Y9, Y9, Y9
+	VPALIGNR $0x08, Y12, Y12, Y12
+	VPALIGNR $0x08, Y13, Y13, Y13
+	VPALIGNR $0x0c, Y4, Y4, Y4
+	VPALIGNR $0x0c, Y1, Y1, Y1
+	INCQ     R9
+	VPADDD   Y14, Y0, Y0
+	VPXOR    Y0, Y4, Y4
+	VPSHUFB  ·rol16<>+0(SB), Y4, Y4
+	VPADDD   Y4, Y12, Y12
+	VPXOR    Y12, Y14, Y14
+	VPSLLD   $0x0c, Y14, Y3
+	VPSRLD   $0x14, Y14, Y14
+	VPXOR    Y3, Y14, Y14
+	VPADDD   Y14, Y0, Y0
+	VPXOR    Y0, Y4, Y4
+	VPSHUFB  ·rol8<>+0(SB), Y4, Y4
+	VPADDD   Y4, Y12, Y12
+	VPXOR    Y12, Y14, Y14
+	VPSLLD   $0x07, Y14, Y3
+	VPSRLD   $0x19, Y14, Y14
+	VPXOR    Y3, Y14, Y14
+	VPADDD   Y9, Y5, Y5
+	VPXOR    Y5, Y1, Y1
+	VPSHUFB  ·rol16<>+0(SB), Y1, Y1
+	VPADDD   Y1, Y13, Y13
+	VPXOR    Y13, Y9, Y9
+	VPSLLD   $0x0c, Y9, Y3
+	VPSRLD   $0x14, Y9, Y9
+	VPXOR    Y3, Y9, Y9
+	VPADDD   Y9, Y5, Y5
+	VPXOR    Y5, Y1, Y1
+	VPSHUFB  ·rol8<>+0(SB), Y1, Y1
+	VPADDD   Y1, Y13, Y13
+	VPXOR    Y13, Y9, Y9
+	VPSLLD   $0x07, Y9, Y3
+	VPSRLD   $0x19, Y9, Y9
+	VPXOR    Y3, Y9, Y9
+	VPALIGNR $0x0c, Y14, Y14, Y14
+	VPALIGNR $0x0c, Y9, Y9, Y9
+	VPALIGNR $0x08, Y12, Y12, Y12
+	VPALIGNR $0x08, Y13, Y13, Y13
+	VPALIGNR $0x04, Y4, Y4, Y4
+	VPALIGNR $0x04, Y1, Y1, Y1
+	CMPQ     R9, CX
 	JB       openAVX2Tail256LoopA
+	CMPQ     R9, $0x0a
+	JNE      openAVX2Tail256LoopB
+	MOVQ     BX, R9
+	SUBQ     SI, BX
+	MOVQ     BX, CX
+	MOVQ     224(BP), BX
 
-	CMPQ itr2, $10
-	JNE  openAVX2Tail256LoopB
-
-	MOVQ inl, itr2
-	SUBQ inp, inl
-	MOVQ inl, itr1
-	MOVQ tmpStoreAVX2, inl
-
-	// Hash the remainder of data (if any)
 openAVX2Tail256Hash:
-	ADDQ $16, itr1
-	CMPQ itr1, inl
-	JGT  openAVX2Tail256HashEnd
-	polyAdd (0(itr2))
-	polyMulAVX2
-	LEAQ 16(itr2), itr2
-	JMP  openAVX2Tail256Hash
-
-// Store 128 bytes safely, then go to store loop
+	ADDQ  $0x10, CX
+	CMPQ  CX, BX
+	JGT   openAVX2Tail256HashEnd
+	ADDQ  (R9), R10
+	ADCQ  8(R9), R11
+	ADCQ  $0x01, R12
+	MOVQ  (BP), DX
+	MOVQ  DX, R15
+	MULXQ R10, R13, R14
+	IMULQ R12, R15
+	MULXQ R11, AX, DX
+	ADDQ  AX, R14
+	ADCQ  DX, R15
+	MOVQ  8(BP), DX
+	MULXQ R10, R10, AX
+	ADDQ  R10, R14
+	MULXQ R11, R11, R8
+	ADCQ  R11, R15
+	ADCQ  $0x00, R8
+	IMULQ R12, DX
+	ADDQ  AX, R15
+	ADCQ  DX, R8
+	MOVQ  R13, R10
+	MOVQ  R14, R11
+	MOVQ  R15, R12
+	ANDQ  $0x03, R12
+	MOVQ  R15, R13
+	ANDQ  $-4, R13
+	MOVQ  R8, R14
+	SHRQ  $0x02, R8, R15
+	SHRQ  $0x02, R8
+	ADDQ  R13, R10
+	ADCQ  R14, R11
+	ADCQ  $0x00, R12
+	ADDQ  R15, R10
+	ADCQ  R8, R11
+	ADCQ  $0x00, R12
+	LEAQ  16(R9), R9
+	JMP   openAVX2Tail256Hash
+
 openAVX2Tail256HashEnd:
-	VPADDD     ·chacha20Constants<>(SB), AA0, AA0; VPADDD ·chacha20Constants<>(SB), AA1, AA1
-	VPADDD     state1StoreAVX2, BB0, BB0; VPADDD state1StoreAVX2, BB1, BB1
-	VPADDD     state2StoreAVX2, CC0, CC0; VPADDD state2StoreAVX2, CC1, CC1
-	VPADDD     TT1, DD0, DD0; VPADDD TT2, DD1, DD1
-	VPERM2I128 $0x02, AA0, BB0, AA2; VPERM2I128 $0x02, CC0, DD0, BB2; VPERM2I128 $0x13, AA0, BB0, CC2; VPERM2I128 $0x13, CC0, DD0, DD2
-	VPERM2I128 $0x02, AA1, BB1, AA0; VPERM2I128 $0x02, CC1, DD1, BB0; VPERM2I128 $0x13, AA1, BB1, CC0; VPERM2I128 $0x13, CC1, DD1, DD0
-
-	VPXOR   (0*32)(inp), AA2, AA2; VPXOR (1*32)(inp), BB2, BB2; VPXOR (2*32)(inp), CC2, CC2; VPXOR (3*32)(inp), DD2, DD2
-	VMOVDQU AA2, (0*32)(oup); VMOVDQU BB2, (1*32)(oup); VMOVDQU CC2, (2*32)(oup); VMOVDQU DD2, (3*32)(oup)
-	LEAQ    (4*32)(inp), inp
-	LEAQ    (4*32)(oup), oup
-	SUBQ    $4*32, inl
-
-	JMP openAVX2TailLoop
-
-// ----------------------------------------------------------------------------
-// Special optimization for the last 384 bytes of ciphertext
+	VPADDD     ·chacha20Constants<>+0(SB), Y0, Y0
+	VPADDD     ·chacha20Constants<>+0(SB), Y5, Y5
+	VPADDD     32(BP), Y14, Y14
+	VPADDD     32(BP), Y9, Y9
+	VPADDD     64(BP), Y12, Y12
+	VPADDD     64(BP), Y13, Y13
+	VPADDD     Y7, Y4, Y4
+	VPADDD     Y11, Y1, Y1
+	VPERM2I128 $0x02, Y0, Y14, Y6
+	VPERM2I128 $0x02, Y12, Y4, Y10
+	VPERM2I128 $0x13, Y0, Y14, Y8
+	VPERM2I128 $0x13, Y12, Y4, Y2
+	VPERM2I128 $0x02, Y5, Y9, Y0
+	VPERM2I128 $0x02, Y13, Y1, Y14
+	VPERM2I128 $0x13, Y5, Y9, Y12
+	VPERM2I128 $0x13, Y13, Y1, Y4
+	VPXOR      (SI), Y6, Y6
+	VPXOR      32(SI), Y10, Y10
+	VPXOR      64(SI), Y8, Y8
+	VPXOR      96(SI), Y2, Y2
+	VMOVDQU    Y6, (DI)
+	VMOVDQU    Y10, 32(DI)
+	VMOVDQU    Y8, 64(DI)
+	VMOVDQU    Y2, 96(DI)
+	LEAQ       128(SI), SI
+	LEAQ       128(DI), DI
+	SUBQ       $0x80, BX
+	JMP        openAVX2TailLoop
+
 openAVX2Tail384:
 	// Need to decrypt up to 384 bytes - prepare six blocks
-	VMOVDQA ·chacha20Constants<>(SB), AA0; VMOVDQA AA0, AA1; VMOVDQA AA0, AA2
-	VMOVDQA state1StoreAVX2, BB0; VMOVDQA BB0, BB1; VMOVDQA BB0, BB2
-	VMOVDQA state2StoreAVX2, CC0; VMOVDQA CC0, CC1; VMOVDQA CC0, CC2
-	VMOVDQA ctr3StoreAVX2, DD0
-	VPADDD  ·avx2IncMask<>(SB), DD0, DD0
-	VPADDD  ·avx2IncMask<>(SB), DD0, DD1
-	VPADDD  ·avx2IncMask<>(SB), DD1, DD2
-	VMOVDQA DD0, ctr0StoreAVX2
-	VMOVDQA DD1, ctr1StoreAVX2
-	VMOVDQA DD2, ctr2StoreAVX2
+	VMOVDQA ·chacha20Constants<>+0(SB), Y0
+	VMOVDQA Y0, Y5
+	VMOVDQA Y0, Y6
+	VMOVDQA 32(BP), Y14
+	VMOVDQA Y14, Y9
+	VMOVDQA Y14, Y10
+	VMOVDQA 64(BP), Y12
+	VMOVDQA Y12, Y13
+	VMOVDQA Y12, Y8
+	VMOVDQA 192(BP), Y4
+	VPADDD  ·avx2IncMask<>+0(SB), Y4, Y4
+	VPADDD  ·avx2IncMask<>+0(SB), Y4, Y1
+	VPADDD  ·avx2IncMask<>+0(SB), Y1, Y2
+	VMOVDQA Y4, 96(BP)
+	VMOVDQA Y1, 128(BP)
+	VMOVDQA Y2, 160(BP)
 
 	// Compute the number of iterations that will hash two blocks of data
-	MOVQ    inl, tmpStoreAVX2
-	MOVQ    inl, itr1
-	SUBQ    $256, itr1
-	SHRQ    $4, itr1
-	ADDQ    $6, itr1
-	MOVQ    $10, itr2
-	CMPQ    itr1, $10
-	CMOVQGT itr2, itr1
-	MOVQ    inp, inl
-	XORQ    itr2, itr2
-
-	// Perform ChaCha rounds, while hashing the remaining input
+	MOVQ    BX, 224(BP)
+	MOVQ    BX, CX
+	SUBQ    $0x00000100, CX
+	SHRQ    $0x04, CX
+	ADDQ    $0x06, CX
+	MOVQ    $0x0000000a, R9
+	CMPQ    CX, $0x0a
+	CMOVQGT R9, CX
+	MOVQ    SI, BX
+	XORQ    R9, R9
+
 openAVX2Tail384LoopB:
-	polyAdd(0(inl))
-	polyMulAVX2
-	LEAQ 16(inl), inl
+	ADDQ  (BX), R10
+	ADCQ  8(BX), R11
+	ADCQ  $0x01, R12
+	MOVQ  (BP), DX
+	MOVQ  DX, R15
+	MULXQ R10, R13, R14
+	IMULQ R12, R15
+	MULXQ R11, AX, DX
+	ADDQ  AX, R14
+	ADCQ  DX, R15
+	MOVQ  8(BP), DX
+	MULXQ R10, R10, AX
+	ADDQ  R10, R14
+	MULXQ R11, R11, R8
+	ADCQ  R11, R15
+	ADCQ  $0x00, R8
+	IMULQ R12, DX
+	ADDQ  AX, R15
+	ADCQ  DX, R8
+	MOVQ  R13, R10
+	MOVQ  R14, R11
+	MOVQ  R15, R12
+	ANDQ  $0x03, R12
+	MOVQ  R15, R13
+	ANDQ  $-4, R13
+	MOVQ  R8, R14
+	SHRQ  $0x02, R8, R15
+	SHRQ  $0x02, R8
+	ADDQ  R13, R10
+	ADCQ  R14, R11
+	ADCQ  $0x00, R12
+	ADDQ  R15, R10
+	ADCQ  R8, R11
+	ADCQ  $0x00, R12
+	LEAQ  16(BX), BX
 
 openAVX2Tail384LoopA:
-	chachaQR_AVX2(AA0, BB0, CC0, DD0, TT0); chachaQR_AVX2(AA1, BB1, CC1, DD1, TT0); chachaQR_AVX2(AA2, BB2, CC2, DD2, TT0)
-	VPALIGNR $4, BB0, BB0, BB0; VPALIGNR $4, BB1, BB1, BB1; VPALIGNR $4, BB2, BB2, BB2
-	VPALIGNR $8, CC0, CC0, CC0; VPALIGNR $8, CC1, CC1, CC1; VPALIGNR $8, CC2, CC2, CC2
-	VPALIGNR $12, DD0, DD0, DD0; VPALIGNR $12, DD1, DD1, DD1; VPALIGNR $12, DD2, DD2, DD2
-	polyAdd(0(inl))
-	polyMulAVX2
-	LEAQ     16(inl), inl
-	INCQ     itr2
-	chachaQR_AVX2(AA0, BB0, CC0, DD0, TT0); chachaQR_AVX2(AA1, BB1, CC1, DD1, TT0); chachaQR_AVX2(AA2, BB2, CC2, DD2, TT0)
-	VPALIGNR $12, BB0, BB0, BB0; VPALIGNR $12, BB1, BB1, BB1; VPALIGNR $12, BB2, BB2, BB2
-	VPALIGNR $8, CC0, CC0, CC0; VPALIGNR $8, CC1, CC1, CC1; VPALIGNR $8, CC2, CC2, CC2
-	VPALIGNR $4, DD0, DD0, DD0; VPALIGNR $4, DD1, DD1, DD1; VPALIGNR $4, DD2, DD2, DD2
-
-	CMPQ itr2, itr1
-	JB   openAVX2Tail384LoopB
-
-	CMPQ itr2, $10
-	JNE  openAVX2Tail384LoopA
-
-	MOVQ inl, itr2
-	SUBQ inp, inl
-	MOVQ inl, itr1
-	MOVQ tmpStoreAVX2, inl
+	VPADDD   Y14, Y0, Y0
+	VPXOR    Y0, Y4, Y4
+	VPSHUFB  ·rol16<>+0(SB), Y4, Y4
+	VPADDD   Y4, Y12, Y12
+	VPXOR    Y12, Y14, Y14
+	VPSLLD   $0x0c, Y14, Y3
+	VPSRLD   $0x14, Y14, Y14
+	VPXOR    Y3, Y14, Y14
+	VPADDD   Y14, Y0, Y0
+	VPXOR    Y0, Y4, Y4
+	VPSHUFB  ·rol8<>+0(SB), Y4, Y4
+	VPADDD   Y4, Y12, Y12
+	VPXOR    Y12, Y14, Y14
+	VPSLLD   $0x07, Y14, Y3
+	VPSRLD   $0x19, Y14, Y14
+	VPXOR    Y3, Y14, Y14
+	VPADDD   Y9, Y5, Y5
+	VPXOR    Y5, Y1, Y1
+	VPSHUFB  ·rol16<>+0(SB), Y1, Y1
+	VPADDD   Y1, Y13, Y13
+	VPXOR    Y13, Y9, Y9
+	VPSLLD   $0x0c, Y9, Y3
+	VPSRLD   $0x14, Y9, Y9
+	VPXOR    Y3, Y9, Y9
+	VPADDD   Y9, Y5, Y5
+	VPXOR    Y5, Y1, Y1
+	VPSHUFB  ·rol8<>+0(SB), Y1, Y1
+	VPADDD   Y1, Y13, Y13
+	VPXOR    Y13, Y9, Y9
+	VPSLLD   $0x07, Y9, Y3
+	VPSRLD   $0x19, Y9, Y9
+	VPXOR    Y3, Y9, Y9
+	VPADDD   Y10, Y6, Y6
+	VPXOR    Y6, Y2, Y2
+	VPSHUFB  ·rol16<>+0(SB), Y2, Y2
+	VPADDD   Y2, Y8, Y8
+	VPXOR    Y8, Y10, Y10
+	VPSLLD   $0x0c, Y10, Y3
+	VPSRLD   $0x14, Y10, Y10
+	VPXOR    Y3, Y10, Y10
+	VPADDD   Y10, Y6, Y6
+	VPXOR    Y6, Y2, Y2
+	VPSHUFB  ·rol8<>+0(SB), Y2, Y2
+	VPADDD   Y2, Y8, Y8
+	VPXOR    Y8, Y10, Y10
+	VPSLLD   $0x07, Y10, Y3
+	VPSRLD   $0x19, Y10, Y10
+	VPXOR    Y3, Y10, Y10
+	VPALIGNR $0x04, Y14, Y14, Y14
+	VPALIGNR $0x04, Y9, Y9, Y9
+	VPALIGNR $0x04, Y10, Y10, Y10
+	VPALIGNR $0x08, Y12, Y12, Y12
+	VPALIGNR $0x08, Y13, Y13, Y13
+	VPALIGNR $0x08, Y8, Y8, Y8
+	VPALIGNR $0x0c, Y4, Y4, Y4
+	VPALIGNR $0x0c, Y1, Y1, Y1
+	VPALIGNR $0x0c, Y2, Y2, Y2
+	ADDQ     (BX), R10
+	ADCQ     8(BX), R11
+	ADCQ     $0x01, R12
+	MOVQ     (BP), DX
+	MOVQ     DX, R15
+	MULXQ    R10, R13, R14
+	IMULQ    R12, R15
+	MULXQ    R11, AX, DX
+	ADDQ     AX, R14
+	ADCQ     DX, R15
+	MOVQ     8(BP), DX
+	MULXQ    R10, R10, AX
+	ADDQ     R10, R14
+	MULXQ    R11, R11, R8
+	ADCQ     R11, R15
+	ADCQ     $0x00, R8
+	IMULQ    R12, DX
+	ADDQ     AX, R15
+	ADCQ     DX, R8
+	MOVQ     R13, R10
+	MOVQ     R14, R11
+	MOVQ     R15, R12
+	ANDQ     $0x03, R12
+	MOVQ     R15, R13
+	ANDQ     $-4, R13
+	MOVQ     R8, R14
+	SHRQ     $0x02, R8, R15
+	SHRQ     $0x02, R8
+	ADDQ     R13, R10
+	ADCQ     R14, R11
+	ADCQ     $0x00, R12
+	ADDQ     R15, R10
+	ADCQ     R8, R11
+	ADCQ     $0x00, R12
+	LEAQ     16(BX), BX
+	INCQ     R9
+	VPADDD   Y14, Y0, Y0
+	VPXOR    Y0, Y4, Y4
+	VPSHUFB  ·rol16<>+0(SB), Y4, Y4
+	VPADDD   Y4, Y12, Y12
+	VPXOR    Y12, Y14, Y14
+	VPSLLD   $0x0c, Y14, Y3
+	VPSRLD   $0x14, Y14, Y14
+	VPXOR    Y3, Y14, Y14
+	VPADDD   Y14, Y0, Y0
+	VPXOR    Y0, Y4, Y4
+	VPSHUFB  ·rol8<>+0(SB), Y4, Y4
+	VPADDD   Y4, Y12, Y12
+	VPXOR    Y12, Y14, Y14
+	VPSLLD   $0x07, Y14, Y3
+	VPSRLD   $0x19, Y14, Y14
+	VPXOR    Y3, Y14, Y14
+	VPADDD   Y9, Y5, Y5
+	VPXOR    Y5, Y1, Y1
+	VPSHUFB  ·rol16<>+0(SB), Y1, Y1
+	VPADDD   Y1, Y13, Y13
+	VPXOR    Y13, Y9, Y9
+	VPSLLD   $0x0c, Y9, Y3
+	VPSRLD   $0x14, Y9, Y9
+	VPXOR    Y3, Y9, Y9
+	VPADDD   Y9, Y5, Y5
+	VPXOR    Y5, Y1, Y1
+	VPSHUFB  ·rol8<>+0(SB), Y1, Y1
+	VPADDD   Y1, Y13, Y13
+	VPXOR    Y13, Y9, Y9
+	VPSLLD   $0x07, Y9, Y3
+	VPSRLD   $0x19, Y9, Y9
+	VPXOR    Y3, Y9, Y9
+	VPADDD   Y10, Y6, Y6
+	VPXOR    Y6, Y2, Y2
+	VPSHUFB  ·rol16<>+0(SB), Y2, Y2
+	VPADDD   Y2, Y8, Y8
+	VPXOR    Y8, Y10, Y10
+	VPSLLD   $0x0c, Y10, Y3
+	VPSRLD   $0x14, Y10, Y10
+	VPXOR    Y3, Y10, Y10
+	VPADDD   Y10, Y6, Y6
+	VPXOR    Y6, Y2, Y2
+	VPSHUFB  ·rol8<>+0(SB), Y2, Y2
+	VPADDD   Y2, Y8, Y8
+	VPXOR    Y8, Y10, Y10
+	VPSLLD   $0x07, Y10, Y3
+	VPSRLD   $0x19, Y10, Y10
+	VPXOR    Y3, Y10, Y10
+	VPALIGNR $0x0c, Y14, Y14, Y14
+	VPALIGNR $0x0c, Y9, Y9, Y9
+	VPALIGNR $0x0c, Y10, Y10, Y10
+	VPALIGNR $0x08, Y12, Y12, Y12
+	VPALIGNR $0x08, Y13, Y13, Y13
+	VPALIGNR $0x08, Y8, Y8, Y8
+	VPALIGNR $0x04, Y4, Y4, Y4
+	VPALIGNR $0x04, Y1, Y1, Y1
+	VPALIGNR $0x04, Y2, Y2, Y2
+	CMPQ     R9, CX
+	JB       openAVX2Tail384LoopB
+	CMPQ     R9, $0x0a
+	JNE      openAVX2Tail384LoopA
+	MOVQ     BX, R9
+	SUBQ     SI, BX
+	MOVQ     BX, CX
+	MOVQ     224(BP), BX
 
 openAVX2Tail384Hash:
-	ADDQ $16, itr1
-	CMPQ itr1, inl
-	JGT  openAVX2Tail384HashEnd
-	polyAdd(0(itr2))
-	polyMulAVX2
-	LEAQ 16(itr2), itr2
-	JMP  openAVX2Tail384Hash
-
-// Store 256 bytes safely, then go to store loop
+	ADDQ  $0x10, CX
+	CMPQ  CX, BX
+	JGT   openAVX2Tail384HashEnd
+	ADDQ  (R9), R10
+	ADCQ  8(R9), R11
+	ADCQ  $0x01, R12
+	MOVQ  (BP), DX
+	MOVQ  DX, R15
+	MULXQ R10, R13, R14
+	IMULQ R12, R15
+	MULXQ R11, AX, DX
+	ADDQ  AX, R14
+	ADCQ  DX, R15
+	MOVQ  8(BP), DX
+	MULXQ R10, R10, AX
+	ADDQ  R10, R14
+	MULXQ R11, R11, R8
+	ADCQ  R11, R15
+	ADCQ  $0x00, R8
+	IMULQ R12, DX
+	ADDQ  AX, R15
+	ADCQ  DX, R8
+	MOVQ  R13, R10
+	MOVQ  R14, R11
+	MOVQ  R15, R12
+	ANDQ  $0x03, R12
+	MOVQ  R15, R13
+	ANDQ  $-4, R13
+	MOVQ  R8, R14
+	SHRQ  $0x02, R8, R15
+	SHRQ  $0x02, R8
+	ADDQ  R13, R10
+	ADCQ  R14, R11
+	ADCQ  $0x00, R12
+	ADDQ  R15, R10
+	ADCQ  R8, R11
+	ADCQ  $0x00, R12
+	LEAQ  16(R9), R9
+	JMP   openAVX2Tail384Hash
+
 openAVX2Tail384HashEnd:
-	VPADDD     ·chacha20Constants<>(SB), AA0, AA0; VPADDD ·chacha20Constants<>(SB), AA1, AA1; VPADDD ·chacha20Constants<>(SB), AA2, AA2
-	VPADDD     state1StoreAVX2, BB0, BB0; VPADDD state1StoreAVX2, BB1, BB1; VPADDD state1StoreAVX2, BB2, BB2
-	VPADDD     state2StoreAVX2, CC0, CC0; VPADDD state2StoreAVX2, CC1, CC1; VPADDD state2StoreAVX2, CC2, CC2
-	VPADDD     ctr0StoreAVX2, DD0, DD0; VPADDD ctr1StoreAVX2, DD1, DD1; VPADDD ctr2StoreAVX2, DD2, DD2
-	VPERM2I128 $0x02, AA0, BB0, TT0; VPERM2I128 $0x02, CC0, DD0, TT1; VPERM2I128 $0x13, AA0, BB0, TT2; VPERM2I128 $0x13, CC0, DD0, TT3
-	VPXOR      (0*32)(inp), TT0, TT0; VPXOR (1*32)(inp), TT1, TT1; VPXOR (2*32)(inp), TT2, TT2; VPXOR (3*32)(inp), TT3, TT3
-	VMOVDQU    TT0, (0*32)(oup); VMOVDQU TT1, (1*32)(oup); VMOVDQU TT2, (2*32)(oup); VMOVDQU TT3, (3*32)(oup)
-	VPERM2I128 $0x02, AA1, BB1, TT0; VPERM2I128 $0x02, CC1, DD1, TT1; VPERM2I128 $0x13, AA1, BB1, TT2; VPERM2I128 $0x13, CC1, DD1, TT3
-	VPXOR      (4*32)(inp), TT0, TT0; VPXOR (5*32)(inp), TT1, TT1; VPXOR (6*32)(inp), TT2, TT2; VPXOR (7*32)(inp), TT3, TT3
-	VMOVDQU    TT0, (4*32)(oup); VMOVDQU TT1, (5*32)(oup); VMOVDQU TT2, (6*32)(oup); VMOVDQU TT3, (7*32)(oup)
-	VPERM2I128 $0x02, AA2, BB2, AA0; VPERM2I128 $0x02, CC2, DD2, BB0; VPERM2I128 $0x13, AA2, BB2, CC0; VPERM2I128 $0x13, CC2, DD2, DD0
-	LEAQ       (8*32)(inp), inp
-	LEAQ       (8*32)(oup), oup
-	SUBQ       $8*32, inl
+	VPADDD     ·chacha20Constants<>+0(SB), Y0, Y0
+	VPADDD     ·chacha20Constants<>+0(SB), Y5, Y5
+	VPADDD     ·chacha20Constants<>+0(SB), Y6, Y6
+	VPADDD     32(BP), Y14, Y14
+	VPADDD     32(BP), Y9, Y9
+	VPADDD     32(BP), Y10, Y10
+	VPADDD     64(BP), Y12, Y12
+	VPADDD     64(BP), Y13, Y13
+	VPADDD     64(BP), Y8, Y8
+	VPADDD     96(BP), Y4, Y4
+	VPADDD     128(BP), Y1, Y1
+	VPADDD     160(BP), Y2, Y2
+	VPERM2I128 $0x02, Y0, Y14, Y3
+	VPERM2I128 $0x02, Y12, Y4, Y7
+	VPERM2I128 $0x13, Y0, Y14, Y11
+	VPERM2I128 $0x13, Y12, Y4, Y15
+	VPXOR      (SI), Y3, Y3
+	VPXOR      32(SI), Y7, Y7
+	VPXOR      64(SI), Y11, Y11
+	VPXOR      96(SI), Y15, Y15
+	VMOVDQU    Y3, (DI)
+	VMOVDQU    Y7, 32(DI)
+	VMOVDQU    Y11, 64(DI)
+	VMOVDQU    Y15, 96(DI)
+	VPERM2I128 $0x02, Y5, Y9, Y3
+	VPERM2I128 $0x02, Y13, Y1, Y7
+	VPERM2I128 $0x13, Y5, Y9, Y11
+	VPERM2I128 $0x13, Y13, Y1, Y15
+	VPXOR      128(SI), Y3, Y3
+	VPXOR      160(SI), Y7, Y7
+	VPXOR      192(SI), Y11, Y11
+	VPXOR      224(SI), Y15, Y15
+	VMOVDQU    Y3, 128(DI)
+	VMOVDQU    Y7, 160(DI)
+	VMOVDQU    Y11, 192(DI)
+	VMOVDQU    Y15, 224(DI)
+	VPERM2I128 $0x02, Y6, Y10, Y0
+	VPERM2I128 $0x02, Y8, Y2, Y14
+	VPERM2I128 $0x13, Y6, Y10, Y12
+	VPERM2I128 $0x13, Y8, Y2, Y4
+	LEAQ       256(SI), SI
+	LEAQ       256(DI), DI
+	SUBQ       $0x00000100, BX
 	JMP        openAVX2TailLoop
 
-// ----------------------------------------------------------------------------
-// Special optimization for the last 512 bytes of ciphertext
 openAVX2Tail512:
-	VMOVDQU ·chacha20Constants<>(SB), AA0; VMOVDQA AA0, AA1; VMOVDQA AA0, AA2; VMOVDQA AA0, AA3
-	VMOVDQA state1StoreAVX2, BB0; VMOVDQA BB0, BB1; VMOVDQA BB0, BB2; VMOVDQA BB0, BB3
-	VMOVDQA state2StoreAVX2, CC0; VMOVDQA CC0, CC1; VMOVDQA CC0, CC2; VMOVDQA CC0, CC3
-	VMOVDQA ctr3StoreAVX2, DD0; VPADDD ·avx2IncMask<>(SB), DD0, DD0; VPADDD ·avx2IncMask<>(SB), DD0, DD1; VPADDD ·avx2IncMask<>(SB), DD1, DD2; VPADDD ·avx2IncMask<>(SB), DD2, DD3
-	VMOVDQA DD0, ctr0StoreAVX2; VMOVDQA DD1, ctr1StoreAVX2; VMOVDQA DD2, ctr2StoreAVX2; VMOVDQA DD3, ctr3StoreAVX2
-	XORQ    itr1, itr1
-	MOVQ    inp, itr2
+	VMOVDQU ·chacha20Constants<>+0(SB), Y0
+	VMOVDQA Y0, Y5
+	VMOVDQA Y0, Y6
+	VMOVDQA Y0, Y7
+	VMOVDQA 32(BP), Y14
+	VMOVDQA Y14, Y9
+	VMOVDQA Y14, Y10
+	VMOVDQA Y14, Y11
+	VMOVDQA 64(BP), Y12
+	VMOVDQA Y12, Y13
+	VMOVDQA Y12, Y8
+	VMOVDQA Y12, Y15
+	VMOVDQA 192(BP), Y4
+	VPADDD  ·avx2IncMask<>+0(SB), Y4, Y4
+	VPADDD  ·avx2IncMask<>+0(SB), Y4, Y1
+	VPADDD  ·avx2IncMask<>+0(SB), Y1, Y2
+	VPADDD  ·avx2IncMask<>+0(SB), Y2, Y3
+	VMOVDQA Y4, 96(BP)
+	VMOVDQA Y1, 128(BP)
+	VMOVDQA Y2, 160(BP)
+	VMOVDQA Y3, 192(BP)
+	XORQ    CX, CX
+	MOVQ    SI, R9
 
 openAVX2Tail512LoopB:
-	polyAdd(0(itr2))
-	polyMulAVX2
-	LEAQ (2*8)(itr2), itr2
+	ADDQ  (R9), R10
+	ADCQ  8(R9), R11
+	ADCQ  $0x01, R12
+	MOVQ  (BP), DX
+	MOVQ  DX, R15
+	MULXQ R10, R13, R14
+	IMULQ R12, R15
+	MULXQ R11, AX, DX
+	ADDQ  AX, R14
+	ADCQ  DX, R15
+	MOVQ  8(BP), DX
+	MULXQ R10, R10, AX
+	ADDQ  R10, R14
+	MULXQ R11, R11, R8
+	ADCQ  R11, R15
+	ADCQ  $0x00, R8
+	IMULQ R12, DX
+	ADDQ  AX, R15
+	ADCQ  DX, R8
+	MOVQ  R13, R10
+	MOVQ  R14, R11
+	MOVQ  R15, R12
+	ANDQ  $0x03, R12
+	MOVQ  R15, R13
+	ANDQ  $-4, R13
+	MOVQ  R8, R14
+	SHRQ  $0x02, R8, R15
+	SHRQ  $0x02, R8
+	ADDQ  R13, R10
+	ADCQ  R14, R11
+	ADCQ  $0x00, R12
+	ADDQ  R15, R10
+	ADCQ  R8, R11
+	ADCQ  $0x00, R12
+	LEAQ  16(R9), R9
 
 openAVX2Tail512LoopA:
-	VPADDD   BB0, AA0, AA0; VPADDD BB1, AA1, AA1; VPADDD BB2, AA2, AA2; VPADDD BB3, AA3, AA3
-	VPXOR    AA0, DD0, DD0; VPXOR AA1, DD1, DD1; VPXOR AA2, DD2, DD2; VPXOR AA3, DD3, DD3
-	VPSHUFB  ·rol16<>(SB), DD0, DD0; VPSHUFB ·rol16<>(SB), DD1, DD1; VPSHUFB ·rol16<>(SB), DD2, DD2; VPSHUFB ·rol16<>(SB), DD3, DD3
-	VPADDD   DD0, CC0, CC0; VPADDD DD1, CC1, CC1; VPADDD DD2, CC2, CC2; VPADDD DD3, CC3, CC3
-	VPXOR    CC0, BB0, BB0; VPXOR CC1, BB1, BB1; VPXOR CC2, BB2, BB2; VPXOR CC3, BB3, BB3
-	VMOVDQA  CC3, tmpStoreAVX2
-	VPSLLD   $12, BB0, CC3; VPSRLD $20, BB0, BB0; VPXOR CC3, BB0, BB0
-	VPSLLD   $12, BB1, CC3; VPSRLD $20, BB1, BB1; VPXOR CC3, BB1, BB1
-	VPSLLD   $12, BB2, CC3; VPSRLD $20, BB2, BB2; VPXOR CC3, BB2, BB2
-	VPSLLD   $12, BB3, CC3; VPSRLD $20, BB3, BB3; VPXOR CC3, BB3, BB3
-	VMOVDQA  tmpStoreAVX2, CC3
-	polyAdd(0*8(itr2))
-	polyMulAVX2
-	VPADDD   BB0, AA0, AA0; VPADDD BB1, AA1, AA1; VPADDD BB2, AA2, AA2; VPADDD BB3, AA3, AA3
-	VPXOR    AA0, DD0, DD0; VPXOR AA1, DD1, DD1; VPXOR AA2, DD2, DD2; VPXOR AA3, DD3, DD3
-	VPSHUFB  ·rol8<>(SB), DD0, DD0; VPSHUFB ·rol8<>(SB), DD1, DD1; VPSHUFB ·rol8<>(SB), DD2, DD2; VPSHUFB ·rol8<>(SB), DD3, DD3
-	VPADDD   DD0, CC0, CC0; VPADDD DD1, CC1, CC1; VPADDD DD2, CC2, CC2; VPADDD DD3, CC3, CC3
-	VPXOR    CC0, BB0, BB0; VPXOR CC1, BB1, BB1; VPXOR CC2, BB2, BB2; VPXOR CC3, BB3, BB3
-	VMOVDQA  CC3, tmpStoreAVX2
-	VPSLLD   $7, BB0, CC3; VPSRLD $25, BB0, BB0; VPXOR CC3, BB0, BB0
-	VPSLLD   $7, BB1, CC3; VPSRLD $25, BB1, BB1; VPXOR CC3, BB1, BB1
-	VPSLLD   $7, BB2, CC3; VPSRLD $25, BB2, BB2; VPXOR CC3, BB2, BB2
-	VPSLLD   $7, BB3, CC3; VPSRLD $25, BB3, BB3; VPXOR CC3, BB3, BB3
-	VMOVDQA  tmpStoreAVX2, CC3
-	VPALIGNR $4, BB0, BB0, BB0; VPALIGNR $4, BB1, BB1, BB1; VPALIGNR $4, BB2, BB2, BB2; VPALIGNR $4, BB3, BB3, BB3
-	VPALIGNR $8, CC0, CC0, CC0; VPALIGNR $8, CC1, CC1, CC1; VPALIGNR $8, CC2, CC2, CC2; VPALIGNR $8, CC3, CC3, CC3
-	VPALIGNR $12, DD0, DD0, DD0; VPALIGNR $12, DD1, DD1, DD1; VPALIGNR $12, DD2, DD2, DD2; VPALIGNR $12, DD3, DD3, DD3
-	VPADDD   BB0, AA0, AA0; VPADDD BB1, AA1, AA1; VPADDD BB2, AA2, AA2; VPADDD BB3, AA3, AA3
-	VPXOR    AA0, DD0, DD0; VPXOR AA1, DD1, DD1; VPXOR AA2, DD2, DD2; VPXOR AA3, DD3, DD3
-	VPSHUFB  ·rol16<>(SB), DD0, DD0; VPSHUFB ·rol16<>(SB), DD1, DD1; VPSHUFB ·rol16<>(SB), DD2, DD2; VPSHUFB ·rol16<>(SB), DD3, DD3
-	VPADDD   DD0, CC0, CC0; VPADDD DD1, CC1, CC1; VPADDD DD2, CC2, CC2; VPADDD DD3, CC3, CC3
-	VPXOR    CC0, BB0, BB0; VPXOR CC1, BB1, BB1; VPXOR CC2, BB2, BB2; VPXOR CC3, BB3, BB3
-	polyAdd(2*8(itr2))
-	polyMulAVX2
-	LEAQ     (4*8)(itr2), itr2
-	VMOVDQA  CC3, tmpStoreAVX2
-	VPSLLD   $12, BB0, CC3; VPSRLD $20, BB0, BB0; VPXOR CC3, BB0, BB0
-	VPSLLD   $12, BB1, CC3; VPSRLD $20, BB1, BB1; VPXOR CC3, BB1, BB1
-	VPSLLD   $12, BB2, CC3; VPSRLD $20, BB2, BB2; VPXOR CC3, BB2, BB2
-	VPSLLD   $12, BB3, CC3; VPSRLD $20, BB3, BB3; VPXOR CC3, BB3, BB3
-	VMOVDQA  tmpStoreAVX2, CC3
-	VPADDD   BB0, AA0, AA0; VPADDD BB1, AA1, AA1; VPADDD BB2, AA2, AA2; VPADDD BB3, AA3, AA3
-	VPXOR    AA0, DD0, DD0; VPXOR AA1, DD1, DD1; VPXOR AA2, DD2, DD2; VPXOR AA3, DD3, DD3
-	VPSHUFB  ·rol8<>(SB), DD0, DD0; VPSHUFB ·rol8<>(SB), DD1, DD1; VPSHUFB ·rol8<>(SB), DD2, DD2; VPSHUFB ·rol8<>(SB), DD3, DD3
-	VPADDD   DD0, CC0, CC0; VPADDD DD1, CC1, CC1; VPADDD DD2, CC2, CC2; VPADDD DD3, CC3, CC3
-	VPXOR    CC0, BB0, BB0; VPXOR CC1, BB1, BB1; VPXOR CC2, BB2, BB2; VPXOR CC3, BB3, BB3
-	VMOVDQA  CC3, tmpStoreAVX2
-	VPSLLD   $7, BB0, CC3; VPSRLD $25, BB0, BB0; VPXOR CC3, BB0, BB0
-	VPSLLD   $7, BB1, CC3; VPSRLD $25, BB1, BB1; VPXOR CC3, BB1, BB1
-	VPSLLD   $7, BB2, CC3; VPSRLD $25, BB2, BB2; VPXOR CC3, BB2, BB2
-	VPSLLD   $7, BB3, CC3; VPSRLD $25, BB3, BB3; VPXOR CC3, BB3, BB3
-	VMOVDQA  tmpStoreAVX2, CC3
-	VPALIGNR $12, BB0, BB0, BB0; VPALIGNR $12, BB1, BB1, BB1; VPALIGNR $12, BB2, BB2, BB2; VPALIGNR $12, BB3, BB3, BB3
-	VPALIGNR $8, CC0, CC0, CC0; VPALIGNR $8, CC1, CC1, CC1; VPALIGNR $8, CC2, CC2, CC2; VPALIGNR $8, CC3, CC3, CC3
-	VPALIGNR $4, DD0, DD0, DD0; VPALIGNR $4, DD1, DD1, DD1; VPALIGNR $4, DD2, DD2, DD2; VPALIGNR $4, DD3, DD3, DD3
-	INCQ     itr1
-	CMPQ     itr1, $4
+	VPADDD   Y14, Y0, Y0
+	VPADDD   Y9, Y5, Y5
+	VPADDD   Y10, Y6, Y6
+	VPADDD   Y11, Y7, Y7
+	VPXOR    Y0, Y4, Y4
+	VPXOR    Y5, Y1, Y1
+	VPXOR    Y6, Y2, Y2
+	VPXOR    Y7, Y3, Y3
+	VPSHUFB  ·rol16<>+0(SB), Y4, Y4
+	VPSHUFB  ·rol16<>+0(SB), Y1, Y1
+	VPSHUFB  ·rol16<>+0(SB), Y2, Y2
+	VPSHUFB  ·rol16<>+0(SB), Y3, Y3
+	VPADDD   Y4, Y12, Y12
+	VPADDD   Y1, Y13, Y13
+	VPADDD   Y2, Y8, Y8
+	VPADDD   Y3, Y15, Y15
+	VPXOR    Y12, Y14, Y14
+	VPXOR    Y13, Y9, Y9
+	VPXOR    Y8, Y10, Y10
+	VPXOR    Y15, Y11, Y11
+	VMOVDQA  Y15, 224(BP)
+	VPSLLD   $0x0c, Y14, Y15
+	VPSRLD   $0x14, Y14, Y14
+	VPXOR    Y15, Y14, Y14
+	VPSLLD   $0x0c, Y9, Y15
+	VPSRLD   $0x14, Y9, Y9
+	VPXOR    Y15, Y9, Y9
+	VPSLLD   $0x0c, Y10, Y15
+	VPSRLD   $0x14, Y10, Y10
+	VPXOR    Y15, Y10, Y10
+	VPSLLD   $0x0c, Y11, Y15
+	VPSRLD   $0x14, Y11, Y11
+	VPXOR    Y15, Y11, Y11
+	VMOVDQA  224(BP), Y15
+	ADDQ     (R9), R10
+	ADCQ     8(R9), R11
+	ADCQ     $0x01, R12
+	MOVQ     (BP), DX
+	MOVQ     DX, R15
+	MULXQ    R10, R13, R14
+	IMULQ    R12, R15
+	MULXQ    R11, AX, DX
+	ADDQ     AX, R14
+	ADCQ     DX, R15
+	MOVQ     8(BP), DX
+	MULXQ    R10, R10, AX
+	ADDQ     R10, R14
+	MULXQ    R11, R11, R8
+	ADCQ     R11, R15
+	ADCQ     $0x00, R8
+	IMULQ    R12, DX
+	ADDQ     AX, R15
+	ADCQ     DX, R8
+	MOVQ     R13, R10
+	MOVQ     R14, R11
+	MOVQ     R15, R12
+	ANDQ     $0x03, R12
+	MOVQ     R15, R13
+	ANDQ     $-4, R13
+	MOVQ     R8, R14
+	SHRQ     $0x02, R8, R15
+	SHRQ     $0x02, R8
+	ADDQ     R13, R10
+	ADCQ     R14, R11
+	ADCQ     $0x00, R12
+	ADDQ     R15, R10
+	ADCQ     R8, R11
+	ADCQ     $0x00, R12
+	VPADDD   Y14, Y0, Y0
+	VPADDD   Y9, Y5, Y5
+	VPADDD   Y10, Y6, Y6
+	VPADDD   Y11, Y7, Y7
+	VPXOR    Y0, Y4, Y4
+	VPXOR    Y5, Y1, Y1
+	VPXOR    Y6, Y2, Y2
+	VPXOR    Y7, Y3, Y3
+	VPSHUFB  ·rol8<>+0(SB), Y4, Y4
+	VPSHUFB  ·rol8<>+0(SB), Y1, Y1
+	VPSHUFB  ·rol8<>+0(SB), Y2, Y2
+	VPSHUFB  ·rol8<>+0(SB), Y3, Y3
+	VPADDD   Y4, Y12, Y12
+	VPADDD   Y1, Y13, Y13
+	VPADDD   Y2, Y8, Y8
+	VPADDD   Y3, Y15, Y15
+	VPXOR    Y12, Y14, Y14
+	VPXOR    Y13, Y9, Y9
+	VPXOR    Y8, Y10, Y10
+	VPXOR    Y15, Y11, Y11
+	VMOVDQA  Y15, 224(BP)
+	VPSLLD   $0x07, Y14, Y15
+	VPSRLD   $0x19, Y14, Y14
+	VPXOR    Y15, Y14, Y14
+	VPSLLD   $0x07, Y9, Y15
+	VPSRLD   $0x19, Y9, Y9
+	VPXOR    Y15, Y9, Y9
+	VPSLLD   $0x07, Y10, Y15
+	VPSRLD   $0x19, Y10, Y10
+	VPXOR    Y15, Y10, Y10
+	VPSLLD   $0x07, Y11, Y15
+	VPSRLD   $0x19, Y11, Y11
+	VPXOR    Y15, Y11, Y11
+	VMOVDQA  224(BP), Y15
+	VPALIGNR $0x04, Y14, Y14, Y14
+	VPALIGNR $0x04, Y9, Y9, Y9
+	VPALIGNR $0x04, Y10, Y10, Y10
+	VPALIGNR $0x04, Y11, Y11, Y11
+	VPALIGNR $0x08, Y12, Y12, Y12
+	VPALIGNR $0x08, Y13, Y13, Y13
+	VPALIGNR $0x08, Y8, Y8, Y8
+	VPALIGNR $0x08, Y15, Y15, Y15
+	VPALIGNR $0x0c, Y4, Y4, Y4
+	VPALIGNR $0x0c, Y1, Y1, Y1
+	VPALIGNR $0x0c, Y2, Y2, Y2
+	VPALIGNR $0x0c, Y3, Y3, Y3
+	VPADDD   Y14, Y0, Y0
+	VPADDD   Y9, Y5, Y5
+	VPADDD   Y10, Y6, Y6
+	VPADDD   Y11, Y7, Y7
+	VPXOR    Y0, Y4, Y4
+	VPXOR    Y5, Y1, Y1
+	VPXOR    Y6, Y2, Y2
+	VPXOR    Y7, Y3, Y3
+	VPSHUFB  ·rol16<>+0(SB), Y4, Y4
+	VPSHUFB  ·rol16<>+0(SB), Y1, Y1
+	VPSHUFB  ·rol16<>+0(SB), Y2, Y2
+	VPSHUFB  ·rol16<>+0(SB), Y3, Y3
+	VPADDD   Y4, Y12, Y12
+	VPADDD   Y1, Y13, Y13
+	VPADDD   Y2, Y8, Y8
+	VPADDD   Y3, Y15, Y15
+	VPXOR    Y12, Y14, Y14
+	VPXOR    Y13, Y9, Y9
+	VPXOR    Y8, Y10, Y10
+	VPXOR    Y15, Y11, Y11
+	ADDQ     16(R9), R10
+	ADCQ     24(R9), R11
+	ADCQ     $0x01, R12
+	MOVQ     (BP), DX
+	MOVQ     DX, R15
+	MULXQ    R10, R13, R14
+	IMULQ    R12, R15
+	MULXQ    R11, AX, DX
+	ADDQ     AX, R14
+	ADCQ     DX, R15
+	MOVQ     8(BP), DX
+	MULXQ    R10, R10, AX
+	ADDQ     R10, R14
+	MULXQ    R11, R11, R8
+	ADCQ     R11, R15
+	ADCQ     $0x00, R8
+	IMULQ    R12, DX
+	ADDQ     AX, R15
+	ADCQ     DX, R8
+	MOVQ     R13, R10
+	MOVQ     R14, R11
+	MOVQ     R15, R12
+	ANDQ     $0x03, R12
+	MOVQ     R15, R13
+	ANDQ     $-4, R13
+	MOVQ     R8, R14
+	SHRQ     $0x02, R8, R15
+	SHRQ     $0x02, R8
+	ADDQ     R13, R10
+	ADCQ     R14, R11
+	ADCQ     $0x00, R12
+	ADDQ     R15, R10
+	ADCQ     R8, R11
+	ADCQ     $0x00, R12
+	LEAQ     32(R9), R9
+	VMOVDQA  Y15, 224(BP)
+	VPSLLD   $0x0c, Y14, Y15
+	VPSRLD   $0x14, Y14, Y14
+	VPXOR    Y15, Y14, Y14
+	VPSLLD   $0x0c, Y9, Y15
+	VPSRLD   $0x14, Y9, Y9
+	VPXOR    Y15, Y9, Y9
+	VPSLLD   $0x0c, Y10, Y15
+	VPSRLD   $0x14, Y10, Y10
+	VPXOR    Y15, Y10, Y10
+	VPSLLD   $0x0c, Y11, Y15
+	VPSRLD   $0x14, Y11, Y11
+	VPXOR    Y15, Y11, Y11
+	VMOVDQA  224(BP), Y15
+	VPADDD   Y14, Y0, Y0
+	VPADDD   Y9, Y5, Y5
+	VPADDD   Y10, Y6, Y6
+	VPADDD   Y11, Y7, Y7
+	VPXOR    Y0, Y4, Y4
+	VPXOR    Y5, Y1, Y1
+	VPXOR    Y6, Y2, Y2
+	VPXOR    Y7, Y3, Y3
+	VPSHUFB  ·rol8<>+0(SB), Y4, Y4
+	VPSHUFB  ·rol8<>+0(SB), Y1, Y1
+	VPSHUFB  ·rol8<>+0(SB), Y2, Y2
+	VPSHUFB  ·rol8<>+0(SB), Y3, Y3
+	VPADDD   Y4, Y12, Y12
+	VPADDD   Y1, Y13, Y13
+	VPADDD   Y2, Y8, Y8
+	VPADDD   Y3, Y15, Y15
+	VPXOR    Y12, Y14, Y14
+	VPXOR    Y13, Y9, Y9
+	VPXOR    Y8, Y10, Y10
+	VPXOR    Y15, Y11, Y11
+	VMOVDQA  Y15, 224(BP)
+	VPSLLD   $0x07, Y14, Y15
+	VPSRLD   $0x19, Y14, Y14
+	VPXOR    Y15, Y14, Y14
+	VPSLLD   $0x07, Y9, Y15
+	VPSRLD   $0x19, Y9, Y9
+	VPXOR    Y15, Y9, Y9
+	VPSLLD   $0x07, Y10, Y15
+	VPSRLD   $0x19, Y10, Y10
+	VPXOR    Y15, Y10, Y10
+	VPSLLD   $0x07, Y11, Y15
+	VPSRLD   $0x19, Y11, Y11
+	VPXOR    Y15, Y11, Y11
+	VMOVDQA  224(BP), Y15
+	VPALIGNR $0x0c, Y14, Y14, Y14
+	VPALIGNR $0x0c, Y9, Y9, Y9
+	VPALIGNR $0x0c, Y10, Y10, Y10
+	VPALIGNR $0x0c, Y11, Y11, Y11
+	VPALIGNR $0x08, Y12, Y12, Y12
+	VPALIGNR $0x08, Y13, Y13, Y13
+	VPALIGNR $0x08, Y8, Y8, Y8
+	VPALIGNR $0x08, Y15, Y15, Y15
+	VPALIGNR $0x04, Y4, Y4, Y4
+	VPALIGNR $0x04, Y1, Y1, Y1
+	VPALIGNR $0x04, Y2, Y2, Y2
+	VPALIGNR $0x04, Y3, Y3, Y3
+	INCQ     CX
+	CMPQ     CX, $0x04
 	JLT      openAVX2Tail512LoopB
-
-	CMPQ itr1, $10
-	JNE  openAVX2Tail512LoopA
-
-	MOVQ inl, itr1
-	SUBQ $384, itr1
-	ANDQ $-16, itr1
+	CMPQ     CX, $0x0a
+	JNE      openAVX2Tail512LoopA
+	MOVQ     BX, CX
+	SUBQ     $0x00000180, CX
+	ANDQ     $-16, CX
 
 openAVX2Tail512HashLoop:
-	TESTQ itr1, itr1
+	TESTQ CX, CX
 	JE    openAVX2Tail512HashEnd
-	polyAdd(0(itr2))
-	polyMulAVX2
-	LEAQ  16(itr2), itr2
-	SUBQ  $16, itr1
+	ADDQ  (R9), R10
+	ADCQ  8(R9), R11
+	ADCQ  $0x01, R12
+	MOVQ  (BP), DX
+	MOVQ  DX, R15
+	MULXQ R10, R13, R14
+	IMULQ R12, R15
+	MULXQ R11, AX, DX
+	ADDQ  AX, R14
+	ADCQ  DX, R15
+	MOVQ  8(BP), DX
+	MULXQ R10, R10, AX
+	ADDQ  R10, R14
+	MULXQ R11, R11, R8
+	ADCQ  R11, R15
+	ADCQ  $0x00, R8
+	IMULQ R12, DX
+	ADDQ  AX, R15
+	ADCQ  DX, R8
+	MOVQ  R13, R10
+	MOVQ  R14, R11
+	MOVQ  R15, R12
+	ANDQ  $0x03, R12
+	MOVQ  R15, R13
+	ANDQ  $-4, R13
+	MOVQ  R8, R14
+	SHRQ  $0x02, R8, R15
+	SHRQ  $0x02, R8
+	ADDQ  R13, R10
+	ADCQ  R14, R11
+	ADCQ  $0x00, R12
+	ADDQ  R15, R10
+	ADCQ  R8, R11
+	ADCQ  $0x00, R12
+	LEAQ  16(R9), R9
+	SUBQ  $0x10, CX
 	JMP   openAVX2Tail512HashLoop
 
 openAVX2Tail512HashEnd:
-	VPADDD     ·chacha20Constants<>(SB), AA0, AA0; VPADDD ·chacha20Constants<>(SB), AA1, AA1; VPADDD ·chacha20Constants<>(SB), AA2, AA2; VPADDD ·chacha20Constants<>(SB), AA3, AA3
-	VPADDD     state1StoreAVX2, BB0, BB0; VPADDD state1StoreAVX2, BB1, BB1; VPADDD state1StoreAVX2, BB2, BB2; VPADDD state1StoreAVX2, BB3, BB3
-	VPADDD     state2StoreAVX2, CC0, CC0; VPADDD state2StoreAVX2, CC1, CC1; VPADDD state2StoreAVX2, CC2, CC2; VPADDD state2StoreAVX2, CC3, CC3
-	VPADDD     ctr0StoreAVX2, DD0, DD0; VPADDD ctr1StoreAVX2, DD1, DD1; VPADDD ctr2StoreAVX2, DD2, DD2; VPADDD ctr3StoreAVX2, DD3, DD3
-	VMOVDQA    CC3, tmpStoreAVX2
-	VPERM2I128 $0x02, AA0, BB0, CC3; VPERM2I128 $0x13, AA0, BB0, BB0; VPERM2I128 $0x02, CC0, DD0, AA0; VPERM2I128 $0x13, CC0, DD0, CC0
-	VPXOR      (0*32)(inp), CC3, CC3; VPXOR (1*32)(inp), AA0, AA0; VPXOR (2*32)(inp), BB0, BB0; VPXOR (3*32)(inp), CC0, CC0
-	VMOVDQU    CC3, (0*32)(oup); VMOVDQU AA0, (1*32)(oup); VMOVDQU BB0, (2*32)(oup); VMOVDQU CC0, (3*32)(oup)
-	VPERM2I128 $0x02, AA1, BB1, AA0; VPERM2I128 $0x02, CC1, DD1, BB0; VPERM2I128 $0x13, AA1, BB1, CC0; VPERM2I128 $0x13, CC1, DD1, DD0
-	VPXOR      (4*32)(inp), AA0, AA0; VPXOR (5*32)(inp), BB0, BB0; VPXOR (6*32)(inp), CC0, CC0; VPXOR (7*32)(inp), DD0, DD0
-	VMOVDQU    AA0, (4*32)(oup); VMOVDQU BB0, (5*32)(oup); VMOVDQU CC0, (6*32)(oup); VMOVDQU DD0, (7*32)(oup)
-	VPERM2I128 $0x02, AA2, BB2, AA0; VPERM2I128 $0x02, CC2, DD2, BB0; VPERM2I128 $0x13, AA2, BB2, CC0; VPERM2I128 $0x13, CC2, DD2, DD0
-	VPXOR      (8*32)(inp), AA0, AA0; VPXOR (9*32)(inp), BB0, BB0; VPXOR (10*32)(inp), CC0, CC0; VPXOR (11*32)(inp), DD0, DD0
-	VMOVDQU    AA0, (8*32)(oup); VMOVDQU BB0, (9*32)(oup); VMOVDQU CC0, (10*32)(oup); VMOVDQU DD0, (11*32)(oup)
-	VPERM2I128 $0x02, AA3, BB3, AA0; VPERM2I128 $0x02, tmpStoreAVX2, DD3, BB0; VPERM2I128 $0x13, AA3, BB3, CC0; VPERM2I128 $0x13, tmpStoreAVX2, DD3, DD0
-
-	LEAQ (12*32)(inp), inp
-	LEAQ (12*32)(oup), oup
-	SUBQ $12*32, inl
-
-	JMP openAVX2TailLoop
-
-// ----------------------------------------------------------------------------
-// ----------------------------------------------------------------------------
-// func chacha20Poly1305Seal(dst, key, src, ad []byte)
-TEXT ·chacha20Poly1305Seal(SB), 0, $288-96
-	// For aligned stack access
+	VPADDD     ·chacha20Constants<>+0(SB), Y0, Y0
+	VPADDD     ·chacha20Constants<>+0(SB), Y5, Y5
+	VPADDD     ·chacha20Constants<>+0(SB), Y6, Y6
+	VPADDD     ·chacha20Constants<>+0(SB), Y7, Y7
+	VPADDD     32(BP), Y14, Y14
+	VPADDD     32(BP), Y9, Y9
+	VPADDD     32(BP), Y10, Y10
+	VPADDD     32(BP), Y11, Y11
+	VPADDD     64(BP), Y12, Y12
+	VPADDD     64(BP), Y13, Y13
+	VPADDD     64(BP), Y8, Y8
+	VPADDD     64(BP), Y15, Y15
+	VPADDD     96(BP), Y4, Y4
+	VPADDD     128(BP), Y1, Y1
+	VPADDD     160(BP), Y2, Y2
+	VPADDD     192(BP), Y3, Y3
+	VMOVDQA    Y15, 224(BP)
+	VPERM2I128 $0x02, Y0, Y14, Y15
+	VPERM2I128 $0x13, Y0, Y14, Y14
+	VPERM2I128 $0x02, Y12, Y4, Y0
+	VPERM2I128 $0x13, Y12, Y4, Y12
+	VPXOR      (SI), Y15, Y15
+	VPXOR      32(SI), Y0, Y0
+	VPXOR      64(SI), Y14, Y14
+	VPXOR      96(SI), Y12, Y12
+	VMOVDQU    Y15, (DI)
+	VMOVDQU    Y0, 32(DI)
+	VMOVDQU    Y14, 64(DI)
+	VMOVDQU    Y12, 96(DI)
+	VPERM2I128 $0x02, Y5, Y9, Y0
+	VPERM2I128 $0x02, Y13, Y1, Y14
+	VPERM2I128 $0x13, Y5, Y9, Y12
+	VPERM2I128 $0x13, Y13, Y1, Y4
+	VPXOR      128(SI), Y0, Y0
+	VPXOR      160(SI), Y14, Y14
+	VPXOR      192(SI), Y12, Y12
+	VPXOR      224(SI), Y4, Y4
+	VMOVDQU    Y0, 128(DI)
+	VMOVDQU    Y14, 160(DI)
+	VMOVDQU    Y12, 192(DI)
+	VMOVDQU    Y4, 224(DI)
+	VPERM2I128 $0x02, Y6, Y10, Y0
+	VPERM2I128 $0x02, Y8, Y2, Y14
+	VPERM2I128 $0x13, Y6, Y10, Y12
+	VPERM2I128 $0x13, Y8, Y2, Y4
+	VPXOR      256(SI), Y0, Y0
+	VPXOR      288(SI), Y14, Y14
+	VPXOR      320(SI), Y12, Y12
+	VPXOR      352(SI), Y4, Y4
+	VMOVDQU    Y0, 256(DI)
+	VMOVDQU    Y14, 288(DI)
+	VMOVDQU    Y12, 320(DI)
+	VMOVDQU    Y4, 352(DI)
+	VPERM2I128 $0x02, Y7, Y11, Y0
+	VPERM2I128 $0x02, 224(BP), Y3, Y14
+	VPERM2I128 $0x13, Y7, Y11, Y12
+	VPERM2I128 $0x13, 224(BP), Y3, Y4
+	LEAQ       384(SI), SI
+	LEAQ       384(DI), DI
+	SUBQ       $0x00000180, BX
+	JMP        openAVX2TailLoop
+
+DATA ·chacha20Constants<>+0(SB)/4, $0x61707865
+DATA ·chacha20Constants<>+4(SB)/4, $0x3320646e
+DATA ·chacha20Constants<>+8(SB)/4, $0x79622d32
+DATA ·chacha20Constants<>+12(SB)/4, $0x6b206574
+DATA ·chacha20Constants<>+16(SB)/4, $0x61707865
+DATA ·chacha20Constants<>+20(SB)/4, $0x3320646e
+DATA ·chacha20Constants<>+24(SB)/4, $0x79622d32
+DATA ·chacha20Constants<>+28(SB)/4, $0x6b206574
+GLOBL ·chacha20Constants<>(SB), RODATA|NOPTR, $32
+
+DATA ·polyClampMask<>+0(SB)/8, $0x0ffffffc0fffffff
+DATA ·polyClampMask<>+8(SB)/8, $0x0ffffffc0ffffffc
+DATA ·polyClampMask<>+16(SB)/8, $0xffffffffffffffff
+DATA ·polyClampMask<>+24(SB)/8, $0xffffffffffffffff
+GLOBL ·polyClampMask<>(SB), RODATA|NOPTR, $32
+
+DATA ·sseIncMask<>+0(SB)/8, $0x0000000000000001
+DATA ·sseIncMask<>+8(SB)/8, $0x0000000000000000
+GLOBL ·sseIncMask<>(SB), RODATA|NOPTR, $16
+
+DATA ·andMask<>+0(SB)/8, $0x00000000000000ff
+DATA ·andMask<>+8(SB)/8, $0x0000000000000000
+DATA ·andMask<>+16(SB)/8, $0x000000000000ffff
+DATA ·andMask<>+24(SB)/8, $0x0000000000000000
+DATA ·andMask<>+32(SB)/8, $0x0000000000ffffff
+DATA ·andMask<>+40(SB)/8, $0x0000000000000000
+DATA ·andMask<>+48(SB)/8, $0x00000000ffffffff
+DATA ·andMask<>+56(SB)/8, $0x0000000000000000
+DATA ·andMask<>+64(SB)/8, $0x000000ffffffffff
+DATA ·andMask<>+72(SB)/8, $0x0000000000000000
+DATA ·andMask<>+80(SB)/8, $0x0000ffffffffffff
+DATA ·andMask<>+88(SB)/8, $0x0000000000000000
+DATA ·andMask<>+96(SB)/8, $0x00ffffffffffffff
+DATA ·andMask<>+104(SB)/8, $0x0000000000000000
+DATA ·andMask<>+112(SB)/8, $0xffffffffffffffff
+DATA ·andMask<>+120(SB)/8, $0x0000000000000000
+DATA ·andMask<>+128(SB)/8, $0xffffffffffffffff
+DATA ·andMask<>+136(SB)/8, $0x00000000000000ff
+DATA ·andMask<>+144(SB)/8, $0xffffffffffffffff
+DATA ·andMask<>+152(SB)/8, $0x000000000000ffff
+DATA ·andMask<>+160(SB)/8, $0xffffffffffffffff
+DATA ·andMask<>+168(SB)/8, $0x0000000000ffffff
+DATA ·andMask<>+176(SB)/8, $0xffffffffffffffff
+DATA ·andMask<>+184(SB)/8, $0x00000000ffffffff
+DATA ·andMask<>+192(SB)/8, $0xffffffffffffffff
+DATA ·andMask<>+200(SB)/8, $0x000000ffffffffff
+DATA ·andMask<>+208(SB)/8, $0xffffffffffffffff
+DATA ·andMask<>+216(SB)/8, $0x0000ffffffffffff
+DATA ·andMask<>+224(SB)/8, $0xffffffffffffffff
+DATA ·andMask<>+232(SB)/8, $0x00ffffffffffffff
+GLOBL ·andMask<>(SB), RODATA|NOPTR, $240
+
+DATA ·avx2InitMask<>+0(SB)/8, $0x0000000000000000
+DATA ·avx2InitMask<>+8(SB)/8, $0x0000000000000000
+DATA ·avx2InitMask<>+16(SB)/8, $0x0000000000000001
+DATA ·avx2InitMask<>+24(SB)/8, $0x0000000000000000
+GLOBL ·avx2InitMask<>(SB), RODATA|NOPTR, $32
+
+DATA ·rol16<>+0(SB)/8, $0x0504070601000302
+DATA ·rol16<>+8(SB)/8, $0x0d0c0f0e09080b0a
+DATA ·rol16<>+16(SB)/8, $0x0504070601000302
+DATA ·rol16<>+24(SB)/8, $0x0d0c0f0e09080b0a
+GLOBL ·rol16<>(SB), RODATA|NOPTR, $32
+
+DATA ·rol8<>+0(SB)/8, $0x0605040702010003
+DATA ·rol8<>+8(SB)/8, $0x0e0d0c0f0a09080b
+DATA ·rol8<>+16(SB)/8, $0x0605040702010003
+DATA ·rol8<>+24(SB)/8, $0x0e0d0c0f0a09080b
+GLOBL ·rol8<>(SB), RODATA|NOPTR, $32
+
+DATA ·avx2IncMask<>+0(SB)/8, $0x0000000000000002
+DATA ·avx2IncMask<>+8(SB)/8, $0x0000000000000000
+DATA ·avx2IncMask<>+16(SB)/8, $0x0000000000000002
+DATA ·avx2IncMask<>+24(SB)/8, $0x0000000000000000
+GLOBL ·avx2IncMask<>(SB), RODATA|NOPTR, $32
+
+// func chacha20Poly1305Seal(dst []byte, key []uint32, src []byte, ad []byte)
+// Requires: AVX, AVX2, BMI2, CMOV, SSE2
+TEXT ·chacha20Poly1305Seal(SB), $288-96
 	MOVQ SP, BP
-	ADDQ $32, BP
+	ADDQ $0x20, BP
 	ANDQ $-32, BP
-	MOVQ dst+0(FP), oup
-	MOVQ key+24(FP), keyp
-	MOVQ src+48(FP), inp
-	MOVQ src_len+56(FP), inl
-	MOVQ ad+72(FP), adp
-
-	CMPB ·useAVX2(SB), $1
+	MOVQ dst_base+0(FP), DI
+	MOVQ key_base+24(FP), R8
+	MOVQ src_base+48(FP), SI
+	MOVQ src_len+56(FP), BX
+	MOVQ ad_base+72(FP), CX
+	CMPB ·useAVX2+0(SB), $0x01
 	JE   chacha20Poly1305Seal_AVX2
 
 	// Special optimization, for very short buffers
-	CMPQ inl, $128
-	JBE  sealSSE128 // About 15% faster
+	CMPQ BX, $0x80
+	JBE  sealSSE128
 
 	// In the seal case - prepare the poly key + 3 blocks of stream in the first iteration
-	MOVOU ·chacha20Constants<>(SB), A0
-	MOVOU (1*16)(keyp), B0
-	MOVOU (2*16)(keyp), C0
-	MOVOU (3*16)(keyp), D0
+	MOVOU ·chacha20Constants<>+0(SB), X0
+	MOVOU 16(R8), X3
+	MOVOU 32(R8), X6
+	MOVOU 48(R8), X9
 
 	// Store state on stack for future use
-	MOVO B0, state1Store
-	MOVO C0, state2Store
+	MOVO X3, 32(BP)
+	MOVO X6, 48(BP)
 
 	// Load state, increment counter blocks
-	MOVO A0, A1; MOVO B0, B1; MOVO C0, C1; MOVO D0, D1; PADDL ·sseIncMask<>(SB), D1
-	MOVO A1, A2; MOVO B1, B2; MOVO C1, C2; MOVO D1, D2; PADDL ·sseIncMask<>(SB), D2
-	MOVO A2, A3; MOVO B2, B3; MOVO C2, C3; MOVO D2, D3; PADDL ·sseIncMask<>(SB), D3
+	MOVO  X0, X1
+	MOVO  X3, X4
+	MOVO  X6, X7
+	MOVO  X9, X10
+	PADDL ·sseIncMask<>+0(SB), X10
+	MOVO  X1, X2
+	MOVO  X4, X5
+	MOVO  X7, X8
+	MOVO  X10, X11
+	PADDL ·sseIncMask<>+0(SB), X11
+	MOVO  X2, X12
+	MOVO  X5, X13
+	MOVO  X8, X14
+	MOVO  X11, X15
+	PADDL ·sseIncMask<>+0(SB), X15
 
 	// Store counters
-	MOVO D0, ctr0Store; MOVO D1, ctr1Store; MOVO D2, ctr2Store; MOVO D3, ctr3Store
-	MOVQ $10, itr2
+	MOVO X9, 80(BP)
+	MOVO X10, 96(BP)
+	MOVO X11, 112(BP)
+	MOVO X15, 128(BP)
+	MOVQ $0x0000000a, R9
 
 sealSSEIntroLoop:
-	MOVO         C3, tmpStore
-	chachaQR(A0, B0, C0, D0, C3); chachaQR(A1, B1, C1, D1, C3); chachaQR(A2, B2, C2, D2, C3)
-	MOVO         tmpStore, C3
-	MOVO         C1, tmpStore
-	chachaQR(A3, B3, C3, D3, C1)
-	MOVO         tmpStore, C1
-	shiftB0Left; shiftB1Left; shiftB2Left; shiftB3Left
-	shiftC0Left; shiftC1Left; shiftC2Left; shiftC3Left
-	shiftD0Left; shiftD1Left; shiftD2Left; shiftD3Left
-
-	MOVO          C3, tmpStore
-	chachaQR(A0, B0, C0, D0, C3); chachaQR(A1, B1, C1, D1, C3); chachaQR(A2, B2, C2, D2, C3)
-	MOVO          tmpStore, C3
-	MOVO          C1, tmpStore
-	chachaQR(A3, B3, C3, D3, C1)
-	MOVO          tmpStore, C1
-	shiftB0Right; shiftB1Right; shiftB2Right; shiftB3Right
-	shiftC0Right; shiftC1Right; shiftC2Right; shiftC3Right
-	shiftD0Right; shiftD1Right; shiftD2Right; shiftD3Right
-	DECQ          itr2
-	JNE           sealSSEIntroLoop
+	MOVO  X14, 64(BP)
+	PADDD X3, X0
+	PXOR  X0, X9
+	ROL16(X9, X14)
+	PADDD X9, X6
+	PXOR  X6, X3
+	MOVO  X3, X14
+	PSLLL $0x0c, X14
+	PSRLL $0x14, X3
+	PXOR  X14, X3
+	PADDD X3, X0
+	PXOR  X0, X9
+	ROL8(X9, X14)
+	PADDD X9, X6
+	PXOR  X6, X3
+	MOVO  X3, X14
+	PSLLL $0x07, X14
+	PSRLL $0x19, X3
+	PXOR  X14, X3
+	PADDD X4, X1
+	PXOR  X1, X10
+	ROL16(X10, X14)
+	PADDD X10, X7
+	PXOR  X7, X4
+	MOVO  X4, X14
+	PSLLL $0x0c, X14
+	PSRLL $0x14, X4
+	PXOR  X14, X4
+	PADDD X4, X1
+	PXOR  X1, X10
+	ROL8(X10, X14)
+	PADDD X10, X7
+	PXOR  X7, X4
+	MOVO  X4, X14
+	PSLLL $0x07, X14
+	PSRLL $0x19, X4
+	PXOR  X14, X4
+	PADDD X5, X2
+	PXOR  X2, X11
+	ROL16(X11, X14)
+	PADDD X11, X8
+	PXOR  X8, X5
+	MOVO  X5, X14
+	PSLLL $0x0c, X14
+	PSRLL $0x14, X5
+	PXOR  X14, X5
+	PADDD X5, X2
+	PXOR  X2, X11
+	ROL8(X11, X14)
+	PADDD X11, X8
+	PXOR  X8, X5
+	MOVO  X5, X14
+	PSLLL $0x07, X14
+	PSRLL $0x19, X5
+	PXOR  X14, X5
+	MOVO  64(BP), X14
+	MOVO  X7, 64(BP)
+	PADDD X13, X12
+	PXOR  X12, X15
+	ROL16(X15, X7)
+	PADDD X15, X14
+	PXOR  X14, X13
+	MOVO  X13, X7
+	PSLLL $0x0c, X7
+	PSRLL $0x14, X13
+	PXOR  X7, X13
+	PADDD X13, X12
+	PXOR  X12, X15
+	ROL8(X15, X7)
+	PADDD X15, X14
+	PXOR  X14, X13
+	MOVO  X13, X7
+	PSLLL $0x07, X7
+	PSRLL $0x19, X13
+	PXOR  X7, X13
+	MOVO  64(BP), X7
+	BYTE  $0x66
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xdb
+	BYTE  $0x04
+	BYTE  $0x66
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xe4
+	BYTE  $0x04
+	BYTE  $0x66
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xed
+	BYTE  $0x04
+	BYTE  $0x66
+	BYTE  $0x45
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xed
+	BYTE  $0x04
+	BYTE  $0x66
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xf6
+	BYTE  $0x08
+	BYTE  $0x66
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xff
+	BYTE  $0x08
+	BYTE  $0x66
+	BYTE  $0x45
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xc0
+	BYTE  $0x08
+	BYTE  $0x66
+	BYTE  $0x45
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xf6
+	BYTE  $0x08
+	BYTE  $0x66
+	BYTE  $0x45
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xc9
+	BYTE  $0x0c
+	BYTE  $0x66
+	BYTE  $0x45
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xd2
+	BYTE  $0x0c
+	BYTE  $0x66
+	BYTE  $0x45
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xdb
+	BYTE  $0x0c
+	BYTE  $0x66
+	BYTE  $0x45
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xff
+	BYTE  $0x0c
+	MOVO  X14, 64(BP)
+	PADDD X3, X0
+	PXOR  X0, X9
+	ROL16(X9, X14)
+	PADDD X9, X6
+	PXOR  X6, X3
+	MOVO  X3, X14
+	PSLLL $0x0c, X14
+	PSRLL $0x14, X3
+	PXOR  X14, X3
+	PADDD X3, X0
+	PXOR  X0, X9
+	ROL8(X9, X14)
+	PADDD X9, X6
+	PXOR  X6, X3
+	MOVO  X3, X14
+	PSLLL $0x07, X14
+	PSRLL $0x19, X3
+	PXOR  X14, X3
+	PADDD X4, X1
+	PXOR  X1, X10
+	ROL16(X10, X14)
+	PADDD X10, X7
+	PXOR  X7, X4
+	MOVO  X4, X14
+	PSLLL $0x0c, X14
+	PSRLL $0x14, X4
+	PXOR  X14, X4
+	PADDD X4, X1
+	PXOR  X1, X10
+	ROL8(X10, X14)
+	PADDD X10, X7
+	PXOR  X7, X4
+	MOVO  X4, X14
+	PSLLL $0x07, X14
+	PSRLL $0x19, X4
+	PXOR  X14, X4
+	PADDD X5, X2
+	PXOR  X2, X11
+	ROL16(X11, X14)
+	PADDD X11, X8
+	PXOR  X8, X5
+	MOVO  X5, X14
+	PSLLL $0x0c, X14
+	PSRLL $0x14, X5
+	PXOR  X14, X5
+	PADDD X5, X2
+	PXOR  X2, X11
+	ROL8(X11, X14)
+	PADDD X11, X8
+	PXOR  X8, X5
+	MOVO  X5, X14
+	PSLLL $0x07, X14
+	PSRLL $0x19, X5
+	PXOR  X14, X5
+	MOVO  64(BP), X14
+	MOVO  X7, 64(BP)
+	PADDD X13, X12
+	PXOR  X12, X15
+	ROL16(X15, X7)
+	PADDD X15, X14
+	PXOR  X14, X13
+	MOVO  X13, X7
+	PSLLL $0x0c, X7
+	PSRLL $0x14, X13
+	PXOR  X7, X13
+	PADDD X13, X12
+	PXOR  X12, X15
+	ROL8(X15, X7)
+	PADDD X15, X14
+	PXOR  X14, X13
+	MOVO  X13, X7
+	PSLLL $0x07, X7
+	PSRLL $0x19, X13
+	PXOR  X7, X13
+	MOVO  64(BP), X7
+	BYTE  $0x66
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xdb
+	BYTE  $0x0c
+	BYTE  $0x66
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xe4
+	BYTE  $0x0c
+	BYTE  $0x66
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xed
+	BYTE  $0x0c
+	BYTE  $0x66
+	BYTE  $0x45
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xed
+	BYTE  $0x0c
+	BYTE  $0x66
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xf6
+	BYTE  $0x08
+	BYTE  $0x66
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xff
+	BYTE  $0x08
+	BYTE  $0x66
+	BYTE  $0x45
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xc0
+	BYTE  $0x08
+	BYTE  $0x66
+	BYTE  $0x45
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xf6
+	BYTE  $0x08
+	BYTE  $0x66
+	BYTE  $0x45
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xc9
+	BYTE  $0x04
+	BYTE  $0x66
+	BYTE  $0x45
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xd2
+	BYTE  $0x04
+	BYTE  $0x66
+	BYTE  $0x45
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xdb
+	BYTE  $0x04
+	BYTE  $0x66
+	BYTE  $0x45
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xff
+	BYTE  $0x04
+	DECQ  R9
+	JNE   sealSSEIntroLoop
 
 	// Add in the state
-	PADDD ·chacha20Constants<>(SB), A0; PADDD ·chacha20Constants<>(SB), A1; PADDD ·chacha20Constants<>(SB), A2; PADDD ·chacha20Constants<>(SB), A3
-	PADDD state1Store, B0; PADDD state1Store, B1; PADDD state1Store, B2; PADDD state1Store, B3
-	PADDD state2Store, C1; PADDD state2Store, C2; PADDD state2Store, C3
-	PADDD ctr1Store, D1; PADDD ctr2Store, D2; PADDD ctr3Store, D3
+	PADDD ·chacha20Constants<>+0(SB), X0
+	PADDD ·chacha20Constants<>+0(SB), X1
+	PADDD ·chacha20Constants<>+0(SB), X2
+	PADDD ·chacha20Constants<>+0(SB), X12
+	PADDD 32(BP), X3
+	PADDD 32(BP), X4
+	PADDD 32(BP), X5
+	PADDD 32(BP), X13
+	PADDD 48(BP), X7
+	PADDD 48(BP), X8
+	PADDD 48(BP), X14
+	PADDD 96(BP), X10
+	PADDD 112(BP), X11
+	PADDD 128(BP), X15
 
 	// Clamp and store the key
-	PAND ·polyClampMask<>(SB), A0
-	MOVO A0, rStore
-	MOVO B0, sStore
+	PAND ·polyClampMask<>+0(SB), X0
+	MOVO X0, (BP)
+	MOVO X3, 16(BP)
 
 	// Hash AAD
-	MOVQ ad_len+80(FP), itr2
-	CALL polyHashADInternal<>(SB)
-
-	MOVOU (0*16)(inp), A0; MOVOU (1*16)(inp), B0; MOVOU (2*16)(inp), C0; MOVOU (3*16)(inp), D0
-	PXOR  A0, A1; PXOR B0, B1; PXOR C0, C1; PXOR D0, D1
-	MOVOU A1, (0*16)(oup); MOVOU B1, (1*16)(oup); MOVOU C1, (2*16)(oup); MOVOU D1, (3*16)(oup)
-	MOVOU (4*16)(inp), A0; MOVOU (5*16)(inp), B0; MOVOU (6*16)(inp), C0; MOVOU (7*16)(inp), D0
-	PXOR  A0, A2; PXOR B0, B2; PXOR C0, C2; PXOR D0, D2
-	MOVOU A2, (4*16)(oup); MOVOU B2, (5*16)(oup); MOVOU C2, (6*16)(oup); MOVOU D2, (7*16)(oup)
-
-	MOVQ $128, itr1
-	SUBQ $128, inl
-	LEAQ 128(inp), inp
-
-	MOVO A3, A1; MOVO B3, B1; MOVO C3, C1; MOVO D3, D1
-
-	CMPQ inl, $64
-	JBE  sealSSE128SealHash
-
-	MOVOU (0*16)(inp), A0; MOVOU (1*16)(inp), B0; MOVOU (2*16)(inp), C0; MOVOU (3*16)(inp), D0
-	PXOR  A0, A3; PXOR B0, B3; PXOR C0, C3; PXOR D0, D3
-	MOVOU A3, (8*16)(oup); MOVOU B3, (9*16)(oup); MOVOU C3, (10*16)(oup); MOVOU D3, (11*16)(oup)
-
-	ADDQ $64, itr1
-	SUBQ $64, inl
-	LEAQ 64(inp), inp
-
-	MOVQ $2, itr1
-	MOVQ $8, itr2
-
-	CMPQ inl, $64
-	JBE  sealSSETail64
-	CMPQ inl, $128
-	JBE  sealSSETail128
-	CMPQ inl, $192
-	JBE  sealSSETail192
+	MOVQ  ad_len+80(FP), R9
+	CALL  polyHashADInternal<>(SB)
+	MOVOU (SI), X0
+	MOVOU 16(SI), X3
+	MOVOU 32(SI), X6
+	MOVOU 48(SI), X9
+	PXOR  X0, X1
+	PXOR  X3, X4
+	PXOR  X6, X7
+	PXOR  X9, X10
+	MOVOU X1, (DI)
+	MOVOU X4, 16(DI)
+	MOVOU X7, 32(DI)
+	MOVOU X10, 48(DI)
+	MOVOU 64(SI), X0
+	MOVOU 80(SI), X3
+	MOVOU 96(SI), X6
+	MOVOU 112(SI), X9
+	PXOR  X0, X2
+	PXOR  X3, X5
+	PXOR  X6, X8
+	PXOR  X9, X11
+	MOVOU X2, 64(DI)
+	MOVOU X5, 80(DI)
+	MOVOU X8, 96(DI)
+	MOVOU X11, 112(DI)
+	MOVQ  $0x00000080, CX
+	SUBQ  $0x80, BX
+	LEAQ  128(SI), SI
+	MOVO  X12, X1
+	MOVO  X13, X4
+	MOVO  X14, X7
+	MOVO  X15, X10
+	CMPQ  BX, $0x40
+	JBE   sealSSE128SealHash
+	MOVOU (SI), X0
+	MOVOU 16(SI), X3
+	MOVOU 32(SI), X6
+	MOVOU 48(SI), X9
+	PXOR  X0, X12
+	PXOR  X3, X13
+	PXOR  X6, X14
+	PXOR  X9, X15
+	MOVOU X12, 128(DI)
+	MOVOU X13, 144(DI)
+	MOVOU X14, 160(DI)
+	MOVOU X15, 176(DI)
+	ADDQ  $0x40, CX
+	SUBQ  $0x40, BX
+	LEAQ  64(SI), SI
+	MOVQ  $0x00000002, CX
+	MOVQ  $0x00000008, R9
+	CMPQ  BX, $0x40
+	JBE   sealSSETail64
+	CMPQ  BX, $0x80
+	JBE   sealSSETail128
+	CMPQ  BX, $0xc0
+	JBE   sealSSETail192
 
 sealSSEMainLoop:
 	// Load state, increment counter blocks
-	MOVO ·chacha20Constants<>(SB), A0; MOVO state1Store, B0; MOVO state2Store, C0; MOVO ctr3Store, D0; PADDL ·sseIncMask<>(SB), D0
-	MOVO A0, A1; MOVO B0, B1; MOVO C0, C1; MOVO D0, D1; PADDL ·sseIncMask<>(SB), D1
-	MOVO A1, A2; MOVO B1, B2; MOVO C1, C2; MOVO D1, D2; PADDL ·sseIncMask<>(SB), D2
-	MOVO A2, A3; MOVO B2, B3; MOVO C2, C3; MOVO D2, D3; PADDL ·sseIncMask<>(SB), D3
+	MOVO  ·chacha20Constants<>+0(SB), X0
+	MOVO  32(BP), X3
+	MOVO  48(BP), X6
+	MOVO  128(BP), X9
+	PADDL ·sseIncMask<>+0(SB), X9
+	MOVO  X0, X1
+	MOVO  X3, X4
+	MOVO  X6, X7
+	MOVO  X9, X10
+	PADDL ·sseIncMask<>+0(SB), X10
+	MOVO  X1, X2
+	MOVO  X4, X5
+	MOVO  X7, X8
+	MOVO  X10, X11
+	PADDL ·sseIncMask<>+0(SB), X11
+	MOVO  X2, X12
+	MOVO  X5, X13
+	MOVO  X8, X14
+	MOVO  X11, X15
+	PADDL ·sseIncMask<>+0(SB), X15
 
 	// Store counters
-	MOVO D0, ctr0Store; MOVO D1, ctr1Store; MOVO D2, ctr2Store; MOVO D3, ctr3Store
+	MOVO X9, 80(BP)
+	MOVO X10, 96(BP)
+	MOVO X11, 112(BP)
+	MOVO X15, 128(BP)
 
 sealSSEInnerLoop:
-	MOVO          C3, tmpStore
-	chachaQR(A0, B0, C0, D0, C3); chachaQR(A1, B1, C1, D1, C3); chachaQR(A2, B2, C2, D2, C3)
-	MOVO          tmpStore, C3
-	MOVO          C1, tmpStore
-	chachaQR(A3, B3, C3, D3, C1)
-	MOVO          tmpStore, C1
-	polyAdd(0(oup))
-	shiftB0Left;  shiftB1Left; shiftB2Left; shiftB3Left
-	shiftC0Left;  shiftC1Left; shiftC2Left; shiftC3Left
-	shiftD0Left;  shiftD1Left; shiftD2Left; shiftD3Left
-	polyMulStage1
-	polyMulStage2
-	LEAQ          (2*8)(oup), oup
-	MOVO          C3, tmpStore
-	chachaQR(A0, B0, C0, D0, C3); chachaQR(A1, B1, C1, D1, C3); chachaQR(A2, B2, C2, D2, C3)
-	MOVO          tmpStore, C3
-	MOVO          C1, tmpStore
-	polyMulStage3
-	chachaQR(A3, B3, C3, D3, C1)
-	MOVO          tmpStore, C1
-	polyMulReduceStage
-	shiftB0Right; shiftB1Right; shiftB2Right; shiftB3Right
-	shiftC0Right; shiftC1Right; shiftC2Right; shiftC3Right
-	shiftD0Right; shiftD1Right; shiftD2Right; shiftD3Right
-	DECQ          itr2
-	JGE           sealSSEInnerLoop
-	polyAdd(0(oup))
-	polyMul
-	LEAQ          (2*8)(oup), oup
-	DECQ          itr1
-	JG            sealSSEInnerLoop
+	MOVO  X14, 64(BP)
+	PADDD X3, X0
+	PXOR  X0, X9
+	ROL16(X9, X14)
+	PADDD X9, X6
+	PXOR  X6, X3
+	MOVO  X3, X14
+	PSLLL $0x0c, X14
+	PSRLL $0x14, X3
+	PXOR  X14, X3
+	PADDD X3, X0
+	PXOR  X0, X9
+	ROL8(X9, X14)
+	PADDD X9, X6
+	PXOR  X6, X3
+	MOVO  X3, X14
+	PSLLL $0x07, X14
+	PSRLL $0x19, X3
+	PXOR  X14, X3
+	PADDD X4, X1
+	PXOR  X1, X10
+	ROL16(X10, X14)
+	PADDD X10, X7
+	PXOR  X7, X4
+	MOVO  X4, X14
+	PSLLL $0x0c, X14
+	PSRLL $0x14, X4
+	PXOR  X14, X4
+	PADDD X4, X1
+	PXOR  X1, X10
+	ROL8(X10, X14)
+	PADDD X10, X7
+	PXOR  X7, X4
+	MOVO  X4, X14
+	PSLLL $0x07, X14
+	PSRLL $0x19, X4
+	PXOR  X14, X4
+	PADDD X5, X2
+	PXOR  X2, X11
+	ROL16(X11, X14)
+	PADDD X11, X8
+	PXOR  X8, X5
+	MOVO  X5, X14
+	PSLLL $0x0c, X14
+	PSRLL $0x14, X5
+	PXOR  X14, X5
+	PADDD X5, X2
+	PXOR  X2, X11
+	ROL8(X11, X14)
+	PADDD X11, X8
+	PXOR  X8, X5
+	MOVO  X5, X14
+	PSLLL $0x07, X14
+	PSRLL $0x19, X5
+	PXOR  X14, X5
+	MOVO  64(BP), X14
+	MOVO  X7, 64(BP)
+	PADDD X13, X12
+	PXOR  X12, X15
+	ROL16(X15, X7)
+	PADDD X15, X14
+	PXOR  X14, X13
+	MOVO  X13, X7
+	PSLLL $0x0c, X7
+	PSRLL $0x14, X13
+	PXOR  X7, X13
+	PADDD X13, X12
+	PXOR  X12, X15
+	ROL8(X15, X7)
+	PADDD X15, X14
+	PXOR  X14, X13
+	MOVO  X13, X7
+	PSLLL $0x07, X7
+	PSRLL $0x19, X13
+	PXOR  X7, X13
+	MOVO  64(BP), X7
+	ADDQ  (DI), R10
+	ADCQ  8(DI), R11
+	ADCQ  $0x01, R12
+	BYTE  $0x66
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xdb
+	BYTE  $0x04
+	BYTE  $0x66
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xe4
+	BYTE  $0x04
+	BYTE  $0x66
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xed
+	BYTE  $0x04
+	BYTE  $0x66
+	BYTE  $0x45
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xed
+	BYTE  $0x04
+	BYTE  $0x66
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xf6
+	BYTE  $0x08
+	BYTE  $0x66
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xff
+	BYTE  $0x08
+	BYTE  $0x66
+	BYTE  $0x45
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xc0
+	BYTE  $0x08
+	BYTE  $0x66
+	BYTE  $0x45
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xf6
+	BYTE  $0x08
+	BYTE  $0x66
+	BYTE  $0x45
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xc9
+	BYTE  $0x0c
+	BYTE  $0x66
+	BYTE  $0x45
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xd2
+	BYTE  $0x0c
+	BYTE  $0x66
+	BYTE  $0x45
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xdb
+	BYTE  $0x0c
+	BYTE  $0x66
+	BYTE  $0x45
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xff
+	BYTE  $0x0c
+	MOVQ  (BP), AX
+	MOVQ  AX, R15
+	MULQ  R10
+	MOVQ  AX, R13
+	MOVQ  DX, R14
+	MOVQ  (BP), AX
+	MULQ  R11
+	IMULQ R12, R15
+	ADDQ  AX, R14
+	ADCQ  DX, R15
+	MOVQ  8(BP), AX
+	MOVQ  AX, R8
+	MULQ  R10
+	ADDQ  AX, R14
+	ADCQ  $0x00, DX
+	MOVQ  DX, R10
+	MOVQ  8(BP), AX
+	MULQ  R11
+	ADDQ  AX, R15
+	ADCQ  $0x00, DX
+	LEAQ  16(DI), DI
+	MOVO  X14, 64(BP)
+	PADDD X3, X0
+	PXOR  X0, X9
+	ROL16(X9, X14)
+	PADDD X9, X6
+	PXOR  X6, X3
+	MOVO  X3, X14
+	PSLLL $0x0c, X14
+	PSRLL $0x14, X3
+	PXOR  X14, X3
+	PADDD X3, X0
+	PXOR  X0, X9
+	ROL8(X9, X14)
+	PADDD X9, X6
+	PXOR  X6, X3
+	MOVO  X3, X14
+	PSLLL $0x07, X14
+	PSRLL $0x19, X3
+	PXOR  X14, X3
+	PADDD X4, X1
+	PXOR  X1, X10
+	ROL16(X10, X14)
+	PADDD X10, X7
+	PXOR  X7, X4
+	MOVO  X4, X14
+	PSLLL $0x0c, X14
+	PSRLL $0x14, X4
+	PXOR  X14, X4
+	PADDD X4, X1
+	PXOR  X1, X10
+	ROL8(X10, X14)
+	PADDD X10, X7
+	PXOR  X7, X4
+	MOVO  X4, X14
+	PSLLL $0x07, X14
+	PSRLL $0x19, X4
+	PXOR  X14, X4
+	PADDD X5, X2
+	PXOR  X2, X11
+	ROL16(X11, X14)
+	PADDD X11, X8
+	PXOR  X8, X5
+	MOVO  X5, X14
+	PSLLL $0x0c, X14
+	PSRLL $0x14, X5
+	PXOR  X14, X5
+	PADDD X5, X2
+	PXOR  X2, X11
+	ROL8(X11, X14)
+	PADDD X11, X8
+	PXOR  X8, X5
+	MOVO  X5, X14
+	PSLLL $0x07, X14
+	PSRLL $0x19, X5
+	PXOR  X14, X5
+	MOVO  64(BP), X14
+	MOVO  X7, 64(BP)
+	IMULQ R12, R8
+	ADDQ  R10, R15
+	ADCQ  DX, R8
+	PADDD X13, X12
+	PXOR  X12, X15
+	ROL16(X15, X7)
+	PADDD X15, X14
+	PXOR  X14, X13
+	MOVO  X13, X7
+	PSLLL $0x0c, X7
+	PSRLL $0x14, X13
+	PXOR  X7, X13
+	PADDD X13, X12
+	PXOR  X12, X15
+	ROL8(X15, X7)
+	PADDD X15, X14
+	PXOR  X14, X13
+	MOVO  X13, X7
+	PSLLL $0x07, X7
+	PSRLL $0x19, X13
+	PXOR  X7, X13
+	MOVO  64(BP), X7
+	MOVQ  R13, R10
+	MOVQ  R14, R11
+	MOVQ  R15, R12
+	ANDQ  $0x03, R12
+	MOVQ  R15, R13
+	ANDQ  $-4, R13
+	MOVQ  R8, R14
+	SHRQ  $0x02, R8, R15
+	SHRQ  $0x02, R8
+	ADDQ  R13, R10
+	ADCQ  R14, R11
+	ADCQ  $0x00, R12
+	ADDQ  R15, R10
+	ADCQ  R8, R11
+	ADCQ  $0x00, R12
+	BYTE  $0x66
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xdb
+	BYTE  $0x0c
+	BYTE  $0x66
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xe4
+	BYTE  $0x0c
+	BYTE  $0x66
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xed
+	BYTE  $0x0c
+	BYTE  $0x66
+	BYTE  $0x45
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xed
+	BYTE  $0x0c
+	BYTE  $0x66
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xf6
+	BYTE  $0x08
+	BYTE  $0x66
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xff
+	BYTE  $0x08
+	BYTE  $0x66
+	BYTE  $0x45
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xc0
+	BYTE  $0x08
+	BYTE  $0x66
+	BYTE  $0x45
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xf6
+	BYTE  $0x08
+	BYTE  $0x66
+	BYTE  $0x45
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xc9
+	BYTE  $0x04
+	BYTE  $0x66
+	BYTE  $0x45
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xd2
+	BYTE  $0x04
+	BYTE  $0x66
+	BYTE  $0x45
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xdb
+	BYTE  $0x04
+	BYTE  $0x66
+	BYTE  $0x45
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xff
+	BYTE  $0x04
+	DECQ  R9
+	JGE   sealSSEInnerLoop
+	ADDQ  (DI), R10
+	ADCQ  8(DI), R11
+	ADCQ  $0x01, R12
+	MOVQ  (BP), AX
+	MOVQ  AX, R15
+	MULQ  R10
+	MOVQ  AX, R13
+	MOVQ  DX, R14
+	MOVQ  (BP), AX
+	MULQ  R11
+	IMULQ R12, R15
+	ADDQ  AX, R14
+	ADCQ  DX, R15
+	MOVQ  8(BP), AX
+	MOVQ  AX, R8
+	MULQ  R10
+	ADDQ  AX, R14
+	ADCQ  $0x00, DX
+	MOVQ  DX, R10
+	MOVQ  8(BP), AX
+	MULQ  R11
+	ADDQ  AX, R15
+	ADCQ  $0x00, DX
+	IMULQ R12, R8
+	ADDQ  R10, R15
+	ADCQ  DX, R8
+	MOVQ  R13, R10
+	MOVQ  R14, R11
+	MOVQ  R15, R12
+	ANDQ  $0x03, R12
+	MOVQ  R15, R13
+	ANDQ  $-4, R13
+	MOVQ  R8, R14
+	SHRQ  $0x02, R8, R15
+	SHRQ  $0x02, R8
+	ADDQ  R13, R10
+	ADCQ  R14, R11
+	ADCQ  $0x00, R12
+	ADDQ  R15, R10
+	ADCQ  R8, R11
+	ADCQ  $0x00, R12
+	LEAQ  16(DI), DI
+	DECQ  CX
+	JG    sealSSEInnerLoop
 
 	// Add in the state
-	PADDD ·chacha20Constants<>(SB), A0; PADDD ·chacha20Constants<>(SB), A1; PADDD ·chacha20Constants<>(SB), A2; PADDD ·chacha20Constants<>(SB), A3
-	PADDD state1Store, B0; PADDD state1Store, B1; PADDD state1Store, B2; PADDD state1Store, B3
-	PADDD state2Store, C0; PADDD state2Store, C1; PADDD state2Store, C2; PADDD state2Store, C3
-	PADDD ctr0Store, D0; PADDD ctr1Store, D1; PADDD ctr2Store, D2; PADDD ctr3Store, D3
-	MOVO  D3, tmpStore
+	PADDD ·chacha20Constants<>+0(SB), X0
+	PADDD ·chacha20Constants<>+0(SB), X1
+	PADDD ·chacha20Constants<>+0(SB), X2
+	PADDD ·chacha20Constants<>+0(SB), X12
+	PADDD 32(BP), X3
+	PADDD 32(BP), X4
+	PADDD 32(BP), X5
+	PADDD 32(BP), X13
+	PADDD 48(BP), X6
+	PADDD 48(BP), X7
+	PADDD 48(BP), X8
+	PADDD 48(BP), X14
+	PADDD 80(BP), X9
+	PADDD 96(BP), X10
+	PADDD 112(BP), X11
+	PADDD 128(BP), X15
+	MOVO  X15, 64(BP)
 
 	// Load - xor - store
-	MOVOU (0*16)(inp), D3; PXOR D3, A0
-	MOVOU (1*16)(inp), D3; PXOR D3, B0
-	MOVOU (2*16)(inp), D3; PXOR D3, C0
-	MOVOU (3*16)(inp), D3; PXOR D3, D0
-	MOVOU A0, (0*16)(oup)
-	MOVOU B0, (1*16)(oup)
-	MOVOU C0, (2*16)(oup)
-	MOVOU D0, (3*16)(oup)
-	MOVO  tmpStore, D3
-
-	MOVOU (4*16)(inp), A0; MOVOU (5*16)(inp), B0; MOVOU (6*16)(inp), C0; MOVOU (7*16)(inp), D0
-	PXOR  A0, A1; PXOR B0, B1; PXOR C0, C1; PXOR D0, D1
-	MOVOU A1, (4*16)(oup); MOVOU B1, (5*16)(oup); MOVOU C1, (6*16)(oup); MOVOU D1, (7*16)(oup)
-	MOVOU (8*16)(inp), A0; MOVOU (9*16)(inp), B0; MOVOU (10*16)(inp), C0; MOVOU (11*16)(inp), D0
-	PXOR  A0, A2; PXOR B0, B2; PXOR C0, C2; PXOR D0, D2
-	MOVOU A2, (8*16)(oup); MOVOU B2, (9*16)(oup); MOVOU C2, (10*16)(oup); MOVOU D2, (11*16)(oup)
-	ADDQ  $192, inp
-	MOVQ  $192, itr1
-	SUBQ  $192, inl
-	MOVO  A3, A1
-	MOVO  B3, B1
-	MOVO  C3, C1
-	MOVO  D3, D1
-	CMPQ  inl, $64
+	MOVOU (SI), X15
+	PXOR  X15, X0
+	MOVOU 16(SI), X15
+	PXOR  X15, X3
+	MOVOU 32(SI), X15
+	PXOR  X15, X6
+	MOVOU 48(SI), X15
+	PXOR  X15, X9
+	MOVOU X0, (DI)
+	MOVOU X3, 16(DI)
+	MOVOU X6, 32(DI)
+	MOVOU X9, 48(DI)
+	MOVO  64(BP), X15
+	MOVOU 64(SI), X0
+	MOVOU 80(SI), X3
+	MOVOU 96(SI), X6
+	MOVOU 112(SI), X9
+	PXOR  X0, X1
+	PXOR  X3, X4
+	PXOR  X6, X7
+	PXOR  X9, X10
+	MOVOU X1, 64(DI)
+	MOVOU X4, 80(DI)
+	MOVOU X7, 96(DI)
+	MOVOU X10, 112(DI)
+	MOVOU 128(SI), X0
+	MOVOU 144(SI), X3
+	MOVOU 160(SI), X6
+	MOVOU 176(SI), X9
+	PXOR  X0, X2
+	PXOR  X3, X5
+	PXOR  X6, X8
+	PXOR  X9, X11
+	MOVOU X2, 128(DI)
+	MOVOU X5, 144(DI)
+	MOVOU X8, 160(DI)
+	MOVOU X11, 176(DI)
+	ADDQ  $0xc0, SI
+	MOVQ  $0x000000c0, CX
+	SUBQ  $0xc0, BX
+	MOVO  X12, X1
+	MOVO  X13, X4
+	MOVO  X14, X7
+	MOVO  X15, X10
+	CMPQ  BX, $0x40
 	JBE   sealSSE128SealHash
-	MOVOU (0*16)(inp), A0; MOVOU (1*16)(inp), B0; MOVOU (2*16)(inp), C0; MOVOU (3*16)(inp), D0
-	PXOR  A0, A3; PXOR B0, B3; PXOR C0, C3; PXOR D0, D3
-	MOVOU A3, (12*16)(oup); MOVOU B3, (13*16)(oup); MOVOU C3, (14*16)(oup); MOVOU D3, (15*16)(oup)
-	LEAQ  64(inp), inp
-	SUBQ  $64, inl
-	MOVQ  $6, itr1
-	MOVQ  $4, itr2
-	CMPQ  inl, $192
+	MOVOU (SI), X0
+	MOVOU 16(SI), X3
+	MOVOU 32(SI), X6
+	MOVOU 48(SI), X9
+	PXOR  X0, X12
+	PXOR  X3, X13
+	PXOR  X6, X14
+	PXOR  X9, X15
+	MOVOU X12, 192(DI)
+	MOVOU X13, 208(DI)
+	MOVOU X14, 224(DI)
+	MOVOU X15, 240(DI)
+	LEAQ  64(SI), SI
+	SUBQ  $0x40, BX
+	MOVQ  $0x00000006, CX
+	MOVQ  $0x00000004, R9
+	CMPQ  BX, $0xc0
 	JG    sealSSEMainLoop
-
-	MOVQ  inl, itr1
-	TESTQ inl, inl
+	MOVQ  BX, CX
+	TESTQ BX, BX
 	JE    sealSSE128SealHash
-	MOVQ  $6, itr1
-	CMPQ  inl, $64
+	MOVQ  $0x00000006, CX
+	CMPQ  BX, $0x40
 	JBE   sealSSETail64
-	CMPQ  inl, $128
+	CMPQ  BX, $0x80
 	JBE   sealSSETail128
 	JMP   sealSSETail192
 
-// ----------------------------------------------------------------------------
-// Special optimization for the last 64 bytes of plaintext
 sealSSETail64:
-	// Need to encrypt up to 64 bytes - prepare single block, hash 192 or 256 bytes
-	MOVO  ·chacha20Constants<>(SB), A1
-	MOVO  state1Store, B1
-	MOVO  state2Store, C1
-	MOVO  ctr3Store, D1
-	PADDL ·sseIncMask<>(SB), D1
-	MOVO  D1, ctr0Store
+	MOVO  ·chacha20Constants<>+0(SB), X1
+	MOVO  32(BP), X4
+	MOVO  48(BP), X7
+	MOVO  128(BP), X10
+	PADDL ·sseIncMask<>+0(SB), X10
+	MOVO  X10, 80(BP)
 
 sealSSETail64LoopA:
-	// Perform ChaCha rounds, while hashing the previously encrypted ciphertext
-	polyAdd(0(oup))
-	polyMul
-	LEAQ 16(oup), oup
+	ADDQ  (DI), R10
+	ADCQ  8(DI), R11
+	ADCQ  $0x01, R12
+	MOVQ  (BP), AX
+	MOVQ  AX, R15
+	MULQ  R10
+	MOVQ  AX, R13
+	MOVQ  DX, R14
+	MOVQ  (BP), AX
+	MULQ  R11
+	IMULQ R12, R15
+	ADDQ  AX, R14
+	ADCQ  DX, R15
+	MOVQ  8(BP), AX
+	MOVQ  AX, R8
+	MULQ  R10
+	ADDQ  AX, R14
+	ADCQ  $0x00, DX
+	MOVQ  DX, R10
+	MOVQ  8(BP), AX
+	MULQ  R11
+	ADDQ  AX, R15
+	ADCQ  $0x00, DX
+	IMULQ R12, R8
+	ADDQ  R10, R15
+	ADCQ  DX, R8
+	MOVQ  R13, R10
+	MOVQ  R14, R11
+	MOVQ  R15, R12
+	ANDQ  $0x03, R12
+	MOVQ  R15, R13
+	ANDQ  $-4, R13
+	MOVQ  R8, R14
+	SHRQ  $0x02, R8, R15
+	SHRQ  $0x02, R8
+	ADDQ  R13, R10
+	ADCQ  R14, R11
+	ADCQ  $0x00, R12
+	ADDQ  R15, R10
+	ADCQ  R8, R11
+	ADCQ  $0x00, R12
+	LEAQ  16(DI), DI
 
 sealSSETail64LoopB:
-	chachaQR(A1, B1, C1, D1, T1)
-	shiftB1Left;  shiftC1Left; shiftD1Left
-	chachaQR(A1, B1, C1, D1, T1)
-	shiftB1Right; shiftC1Right; shiftD1Right
-	polyAdd(0(oup))
-	polyMul
-	LEAQ          16(oup), oup
-
-	DECQ itr1
-	JG   sealSSETail64LoopA
-
-	DECQ  itr2
+	PADDD X4, X1
+	PXOR  X1, X10
+	ROL16(X10, X13)
+	PADDD X10, X7
+	PXOR  X7, X4
+	MOVO  X4, X13
+	PSLLL $0x0c, X13
+	PSRLL $0x14, X4
+	PXOR  X13, X4
+	PADDD X4, X1
+	PXOR  X1, X10
+	ROL8(X10, X13)
+	PADDD X10, X7
+	PXOR  X7, X4
+	MOVO  X4, X13
+	PSLLL $0x07, X13
+	PSRLL $0x19, X4
+	PXOR  X13, X4
+	BYTE  $0x66
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xe4
+	BYTE  $0x04
+	BYTE  $0x66
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xff
+	BYTE  $0x08
+	BYTE  $0x66
+	BYTE  $0x45
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xd2
+	BYTE  $0x0c
+	PADDD X4, X1
+	PXOR  X1, X10
+	ROL16(X10, X13)
+	PADDD X10, X7
+	PXOR  X7, X4
+	MOVO  X4, X13
+	PSLLL $0x0c, X13
+	PSRLL $0x14, X4
+	PXOR  X13, X4
+	PADDD X4, X1
+	PXOR  X1, X10
+	ROL8(X10, X13)
+	PADDD X10, X7
+	PXOR  X7, X4
+	MOVO  X4, X13
+	PSLLL $0x07, X13
+	PSRLL $0x19, X4
+	PXOR  X13, X4
+	BYTE  $0x66
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xe4
+	BYTE  $0x0c
+	BYTE  $0x66
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xff
+	BYTE  $0x08
+	BYTE  $0x66
+	BYTE  $0x45
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xd2
+	BYTE  $0x04
+	ADDQ  (DI), R10
+	ADCQ  8(DI), R11
+	ADCQ  $0x01, R12
+	MOVQ  (BP), AX
+	MOVQ  AX, R15
+	MULQ  R10
+	MOVQ  AX, R13
+	MOVQ  DX, R14
+	MOVQ  (BP), AX
+	MULQ  R11
+	IMULQ R12, R15
+	ADDQ  AX, R14
+	ADCQ  DX, R15
+	MOVQ  8(BP), AX
+	MOVQ  AX, R8
+	MULQ  R10
+	ADDQ  AX, R14
+	ADCQ  $0x00, DX
+	MOVQ  DX, R10
+	MOVQ  8(BP), AX
+	MULQ  R11
+	ADDQ  AX, R15
+	ADCQ  $0x00, DX
+	IMULQ R12, R8
+	ADDQ  R10, R15
+	ADCQ  DX, R8
+	MOVQ  R13, R10
+	MOVQ  R14, R11
+	MOVQ  R15, R12
+	ANDQ  $0x03, R12
+	MOVQ  R15, R13
+	ANDQ  $-4, R13
+	MOVQ  R8, R14
+	SHRQ  $0x02, R8, R15
+	SHRQ  $0x02, R8
+	ADDQ  R13, R10
+	ADCQ  R14, R11
+	ADCQ  $0x00, R12
+	ADDQ  R15, R10
+	ADCQ  R8, R11
+	ADCQ  $0x00, R12
+	LEAQ  16(DI), DI
+	DECQ  CX
+	JG    sealSSETail64LoopA
+	DECQ  R9
 	JGE   sealSSETail64LoopB
-	PADDL ·chacha20Constants<>(SB), A1
-	PADDL state1Store, B1
-	PADDL state2Store, C1
-	PADDL ctr0Store, D1
+	PADDL ·chacha20Constants<>+0(SB), X1
+	PADDL 32(BP), X4
+	PADDL 48(BP), X7
+	PADDL 80(BP), X10
+	JMP   sealSSE128Seal
 
-	JMP sealSSE128Seal
-
-// ----------------------------------------------------------------------------
-// Special optimization for the last 128 bytes of plaintext
 sealSSETail128:
-	// Need to encrypt up to 128 bytes - prepare two blocks, hash 192 or 256 bytes
-	MOVO ·chacha20Constants<>(SB), A0; MOVO state1Store, B0; MOVO state2Store, C0; MOVO ctr3Store, D0; PADDL ·sseIncMask<>(SB), D0; MOVO D0, ctr0Store
-	MOVO A0, A1; MOVO B0, B1; MOVO C0, C1; MOVO D0, D1; PADDL ·sseIncMask<>(SB), D1; MOVO D1, ctr1Store
+	MOVO  ·chacha20Constants<>+0(SB), X0
+	MOVO  32(BP), X3
+	MOVO  48(BP), X6
+	MOVO  128(BP), X9
+	PADDL ·sseIncMask<>+0(SB), X9
+	MOVO  X9, 80(BP)
+	MOVO  X0, X1
+	MOVO  X3, X4
+	MOVO  X6, X7
+	MOVO  X9, X10
+	PADDL ·sseIncMask<>+0(SB), X10
+	MOVO  X10, 96(BP)
 
 sealSSETail128LoopA:
-	// Perform ChaCha rounds, while hashing the previously encrypted ciphertext
-	polyAdd(0(oup))
-	polyMul
-	LEAQ 16(oup), oup
+	ADDQ  (DI), R10
+	ADCQ  8(DI), R11
+	ADCQ  $0x01, R12
+	MOVQ  (BP), AX
+	MOVQ  AX, R15
+	MULQ  R10
+	MOVQ  AX, R13
+	MOVQ  DX, R14
+	MOVQ  (BP), AX
+	MULQ  R11
+	IMULQ R12, R15
+	ADDQ  AX, R14
+	ADCQ  DX, R15
+	MOVQ  8(BP), AX
+	MOVQ  AX, R8
+	MULQ  R10
+	ADDQ  AX, R14
+	ADCQ  $0x00, DX
+	MOVQ  DX, R10
+	MOVQ  8(BP), AX
+	MULQ  R11
+	ADDQ  AX, R15
+	ADCQ  $0x00, DX
+	IMULQ R12, R8
+	ADDQ  R10, R15
+	ADCQ  DX, R8
+	MOVQ  R13, R10
+	MOVQ  R14, R11
+	MOVQ  R15, R12
+	ANDQ  $0x03, R12
+	MOVQ  R15, R13
+	ANDQ  $-4, R13
+	MOVQ  R8, R14
+	SHRQ  $0x02, R8, R15
+	SHRQ  $0x02, R8
+	ADDQ  R13, R10
+	ADCQ  R14, R11
+	ADCQ  $0x00, R12
+	ADDQ  R15, R10
+	ADCQ  R8, R11
+	ADCQ  $0x00, R12
+	LEAQ  16(DI), DI
 
 sealSSETail128LoopB:
-	chachaQR(A0, B0, C0, D0, T0); chachaQR(A1, B1, C1, D1, T0)
-	shiftB0Left;  shiftC0Left; shiftD0Left
-	shiftB1Left;  shiftC1Left; shiftD1Left
-	polyAdd(0(oup))
-	polyMul
-	LEAQ          16(oup), oup
-	chachaQR(A0, B0, C0, D0, T0); chachaQR(A1, B1, C1, D1, T0)
-	shiftB0Right; shiftC0Right; shiftD0Right
-	shiftB1Right; shiftC1Right; shiftD1Right
-
-	DECQ itr1
-	JG   sealSSETail128LoopA
-
-	DECQ itr2
-	JGE  sealSSETail128LoopB
-
-	PADDL ·chacha20Constants<>(SB), A0; PADDL ·chacha20Constants<>(SB), A1
-	PADDL state1Store, B0; PADDL state1Store, B1
-	PADDL state2Store, C0; PADDL state2Store, C1
-	PADDL ctr0Store, D0; PADDL ctr1Store, D1
-
-	MOVOU (0*16)(inp), T0; MOVOU (1*16)(inp), T1; MOVOU (2*16)(inp), T2; MOVOU (3*16)(inp), T3
-	PXOR  T0, A0; PXOR T1, B0; PXOR T2, C0; PXOR T3, D0
-	MOVOU A0, (0*16)(oup); MOVOU B0, (1*16)(oup); MOVOU C0, (2*16)(oup); MOVOU D0, (3*16)(oup)
-
-	MOVQ $64, itr1
-	LEAQ 64(inp), inp
-	SUBQ $64, inl
-
-	JMP sealSSE128SealHash
-
-// ----------------------------------------------------------------------------
-// Special optimization for the last 192 bytes of plaintext
+	PADDD X3, X0
+	PXOR  X0, X9
+	ROL16(X9, X12)
+	PADDD X9, X6
+	PXOR  X6, X3
+	MOVO  X3, X12
+	PSLLL $0x0c, X12
+	PSRLL $0x14, X3
+	PXOR  X12, X3
+	PADDD X3, X0
+	PXOR  X0, X9
+	ROL8(X9, X12)
+	PADDD X9, X6
+	PXOR  X6, X3
+	MOVO  X3, X12
+	PSLLL $0x07, X12
+	PSRLL $0x19, X3
+	PXOR  X12, X3
+	PADDD X4, X1
+	PXOR  X1, X10
+	ROL16(X10, X12)
+	PADDD X10, X7
+	PXOR  X7, X4
+	MOVO  X4, X12
+	PSLLL $0x0c, X12
+	PSRLL $0x14, X4
+	PXOR  X12, X4
+	PADDD X4, X1
+	PXOR  X1, X10
+	ROL8(X10, X12)
+	PADDD X10, X7
+	PXOR  X7, X4
+	MOVO  X4, X12
+	PSLLL $0x07, X12
+	PSRLL $0x19, X4
+	PXOR  X12, X4
+	BYTE  $0x66
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xdb
+	BYTE  $0x04
+	BYTE  $0x66
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xf6
+	BYTE  $0x08
+	BYTE  $0x66
+	BYTE  $0x45
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xc9
+	BYTE  $0x0c
+	BYTE  $0x66
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xe4
+	BYTE  $0x04
+	BYTE  $0x66
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xff
+	BYTE  $0x08
+	BYTE  $0x66
+	BYTE  $0x45
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xd2
+	BYTE  $0x0c
+	ADDQ  (DI), R10
+	ADCQ  8(DI), R11
+	ADCQ  $0x01, R12
+	MOVQ  (BP), AX
+	MOVQ  AX, R15
+	MULQ  R10
+	MOVQ  AX, R13
+	MOVQ  DX, R14
+	MOVQ  (BP), AX
+	MULQ  R11
+	IMULQ R12, R15
+	ADDQ  AX, R14
+	ADCQ  DX, R15
+	MOVQ  8(BP), AX
+	MOVQ  AX, R8
+	MULQ  R10
+	ADDQ  AX, R14
+	ADCQ  $0x00, DX
+	MOVQ  DX, R10
+	MOVQ  8(BP), AX
+	MULQ  R11
+	ADDQ  AX, R15
+	ADCQ  $0x00, DX
+	IMULQ R12, R8
+	ADDQ  R10, R15
+	ADCQ  DX, R8
+	MOVQ  R13, R10
+	MOVQ  R14, R11
+	MOVQ  R15, R12
+	ANDQ  $0x03, R12
+	MOVQ  R15, R13
+	ANDQ  $-4, R13
+	MOVQ  R8, R14
+	SHRQ  $0x02, R8, R15
+	SHRQ  $0x02, R8
+	ADDQ  R13, R10
+	ADCQ  R14, R11
+	ADCQ  $0x00, R12
+	ADDQ  R15, R10
+	ADCQ  R8, R11
+	ADCQ  $0x00, R12
+	LEAQ  16(DI), DI
+	PADDD X3, X0
+	PXOR  X0, X9
+	ROL16(X9, X12)
+	PADDD X9, X6
+	PXOR  X6, X3
+	MOVO  X3, X12
+	PSLLL $0x0c, X12
+	PSRLL $0x14, X3
+	PXOR  X12, X3
+	PADDD X3, X0
+	PXOR  X0, X9
+	ROL8(X9, X12)
+	PADDD X9, X6
+	PXOR  X6, X3
+	MOVO  X3, X12
+	PSLLL $0x07, X12
+	PSRLL $0x19, X3
+	PXOR  X12, X3
+	PADDD X4, X1
+	PXOR  X1, X10
+	ROL16(X10, X12)
+	PADDD X10, X7
+	PXOR  X7, X4
+	MOVO  X4, X12
+	PSLLL $0x0c, X12
+	PSRLL $0x14, X4
+	PXOR  X12, X4
+	PADDD X4, X1
+	PXOR  X1, X10
+	ROL8(X10, X12)
+	PADDD X10, X7
+	PXOR  X7, X4
+	MOVO  X4, X12
+	PSLLL $0x07, X12
+	PSRLL $0x19, X4
+	PXOR  X12, X4
+	BYTE  $0x66
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xdb
+	BYTE  $0x0c
+	BYTE  $0x66
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xf6
+	BYTE  $0x08
+	BYTE  $0x66
+	BYTE  $0x45
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xc9
+	BYTE  $0x04
+	BYTE  $0x66
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xe4
+	BYTE  $0x0c
+	BYTE  $0x66
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xff
+	BYTE  $0x08
+	BYTE  $0x66
+	BYTE  $0x45
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xd2
+	BYTE  $0x04
+	DECQ  CX
+	JG    sealSSETail128LoopA
+	DECQ  R9
+	JGE   sealSSETail128LoopB
+	PADDL ·chacha20Constants<>+0(SB), X0
+	PADDL ·chacha20Constants<>+0(SB), X1
+	PADDL 32(BP), X3
+	PADDL 32(BP), X4
+	PADDL 48(BP), X6
+	PADDL 48(BP), X7
+	PADDL 80(BP), X9
+	PADDL 96(BP), X10
+	MOVOU (SI), X12
+	MOVOU 16(SI), X13
+	MOVOU 32(SI), X14
+	MOVOU 48(SI), X15
+	PXOR  X12, X0
+	PXOR  X13, X3
+	PXOR  X14, X6
+	PXOR  X15, X9
+	MOVOU X0, (DI)
+	MOVOU X3, 16(DI)
+	MOVOU X6, 32(DI)
+	MOVOU X9, 48(DI)
+	MOVQ  $0x00000040, CX
+	LEAQ  64(SI), SI
+	SUBQ  $0x40, BX
+	JMP   sealSSE128SealHash
+
 sealSSETail192:
-	// Need to encrypt up to 192 bytes - prepare three blocks, hash 192 or 256 bytes
-	MOVO ·chacha20Constants<>(SB), A0; MOVO state1Store, B0; MOVO state2Store, C0; MOVO ctr3Store, D0; PADDL ·sseIncMask<>(SB), D0; MOVO D0, ctr0Store
-	MOVO A0, A1; MOVO B0, B1; MOVO C0, C1; MOVO D0, D1; PADDL ·sseIncMask<>(SB), D1; MOVO D1, ctr1Store
-	MOVO A1, A2; MOVO B1, B2; MOVO C1, C2; MOVO D1, D2; PADDL ·sseIncMask<>(SB), D2; MOVO D2, ctr2Store
+	MOVO  ·chacha20Constants<>+0(SB), X0
+	MOVO  32(BP), X3
+	MOVO  48(BP), X6
+	MOVO  128(BP), X9
+	PADDL ·sseIncMask<>+0(SB), X9
+	MOVO  X9, 80(BP)
+	MOVO  X0, X1
+	MOVO  X3, X4
+	MOVO  X6, X7
+	MOVO  X9, X10
+	PADDL ·sseIncMask<>+0(SB), X10
+	MOVO  X10, 96(BP)
+	MOVO  X1, X2
+	MOVO  X4, X5
+	MOVO  X7, X8
+	MOVO  X10, X11
+	PADDL ·sseIncMask<>+0(SB), X11
+	MOVO  X11, 112(BP)
 
 sealSSETail192LoopA:
-	// Perform ChaCha rounds, while hashing the previously encrypted ciphertext
-	polyAdd(0(oup))
-	polyMul
-	LEAQ 16(oup), oup
+	ADDQ  (DI), R10
+	ADCQ  8(DI), R11
+	ADCQ  $0x01, R12
+	MOVQ  (BP), AX
+	MOVQ  AX, R15
+	MULQ  R10
+	MOVQ  AX, R13
+	MOVQ  DX, R14
+	MOVQ  (BP), AX
+	MULQ  R11
+	IMULQ R12, R15
+	ADDQ  AX, R14
+	ADCQ  DX, R15
+	MOVQ  8(BP), AX
+	MOVQ  AX, R8
+	MULQ  R10
+	ADDQ  AX, R14
+	ADCQ  $0x00, DX
+	MOVQ  DX, R10
+	MOVQ  8(BP), AX
+	MULQ  R11
+	ADDQ  AX, R15
+	ADCQ  $0x00, DX
+	IMULQ R12, R8
+	ADDQ  R10, R15
+	ADCQ  DX, R8
+	MOVQ  R13, R10
+	MOVQ  R14, R11
+	MOVQ  R15, R12
+	ANDQ  $0x03, R12
+	MOVQ  R15, R13
+	ANDQ  $-4, R13
+	MOVQ  R8, R14
+	SHRQ  $0x02, R8, R15
+	SHRQ  $0x02, R8
+	ADDQ  R13, R10
+	ADCQ  R14, R11
+	ADCQ  $0x00, R12
+	ADDQ  R15, R10
+	ADCQ  R8, R11
+	ADCQ  $0x00, R12
+	LEAQ  16(DI), DI
 
 sealSSETail192LoopB:
-	chachaQR(A0, B0, C0, D0, T0); chachaQR(A1, B1, C1, D1, T0); chachaQR(A2, B2, C2, D2, T0)
-	shiftB0Left; shiftC0Left; shiftD0Left
-	shiftB1Left; shiftC1Left; shiftD1Left
-	shiftB2Left; shiftC2Left; shiftD2Left
-
-	polyAdd(0(oup))
-	polyMul
-	LEAQ 16(oup), oup
-
-	chachaQR(A0, B0, C0, D0, T0); chachaQR(A1, B1, C1, D1, T0); chachaQR(A2, B2, C2, D2, T0)
-	shiftB0Right; shiftC0Right; shiftD0Right
-	shiftB1Right; shiftC1Right; shiftD1Right
-	shiftB2Right; shiftC2Right; shiftD2Right
-
-	DECQ itr1
-	JG   sealSSETail192LoopA
-
-	DECQ itr2
-	JGE  sealSSETail192LoopB
-
-	PADDL ·chacha20Constants<>(SB), A0; PADDL ·chacha20Constants<>(SB), A1; PADDL ·chacha20Constants<>(SB), A2
-	PADDL state1Store, B0; PADDL state1Store, B1; PADDL state1Store, B2
-	PADDL state2Store, C0; PADDL state2Store, C1; PADDL state2Store, C2
-	PADDL ctr0Store, D0; PADDL ctr1Store, D1; PADDL ctr2Store, D2
-
-	MOVOU (0*16)(inp), T0; MOVOU (1*16)(inp), T1; MOVOU (2*16)(inp), T2; MOVOU (3*16)(inp), T3
-	PXOR  T0, A0; PXOR T1, B0; PXOR T2, C0; PXOR T3, D0
-	MOVOU A0, (0*16)(oup); MOVOU B0, (1*16)(oup); MOVOU C0, (2*16)(oup); MOVOU D0, (3*16)(oup)
-	MOVOU (4*16)(inp), T0; MOVOU (5*16)(inp), T1; MOVOU (6*16)(inp), T2; MOVOU (7*16)(inp), T3
-	PXOR  T0, A1; PXOR T1, B1; PXOR T2, C1; PXOR T3, D1
-	MOVOU A1, (4*16)(oup); MOVOU B1, (5*16)(oup); MOVOU C1, (6*16)(oup); MOVOU D1, (7*16)(oup)
-
-	MOVO A2, A1
-	MOVO B2, B1
-	MOVO C2, C1
-	MOVO D2, D1
-	MOVQ $128, itr1
-	LEAQ 128(inp), inp
-	SUBQ $128, inl
-
-	JMP sealSSE128SealHash
-
-// ----------------------------------------------------------------------------
-// Special seal optimization for buffers smaller than 129 bytes
+	PADDD X3, X0
+	PXOR  X0, X9
+	ROL16(X9, X12)
+	PADDD X9, X6
+	PXOR  X6, X3
+	MOVO  X3, X12
+	PSLLL $0x0c, X12
+	PSRLL $0x14, X3
+	PXOR  X12, X3
+	PADDD X3, X0
+	PXOR  X0, X9
+	ROL8(X9, X12)
+	PADDD X9, X6
+	PXOR  X6, X3
+	MOVO  X3, X12
+	PSLLL $0x07, X12
+	PSRLL $0x19, X3
+	PXOR  X12, X3
+	PADDD X4, X1
+	PXOR  X1, X10
+	ROL16(X10, X12)
+	PADDD X10, X7
+	PXOR  X7, X4
+	MOVO  X4, X12
+	PSLLL $0x0c, X12
+	PSRLL $0x14, X4
+	PXOR  X12, X4
+	PADDD X4, X1
+	PXOR  X1, X10
+	ROL8(X10, X12)
+	PADDD X10, X7
+	PXOR  X7, X4
+	MOVO  X4, X12
+	PSLLL $0x07, X12
+	PSRLL $0x19, X4
+	PXOR  X12, X4
+	PADDD X5, X2
+	PXOR  X2, X11
+	ROL16(X11, X12)
+	PADDD X11, X8
+	PXOR  X8, X5
+	MOVO  X5, X12
+	PSLLL $0x0c, X12
+	PSRLL $0x14, X5
+	PXOR  X12, X5
+	PADDD X5, X2
+	PXOR  X2, X11
+	ROL8(X11, X12)
+	PADDD X11, X8
+	PXOR  X8, X5
+	MOVO  X5, X12
+	PSLLL $0x07, X12
+	PSRLL $0x19, X5
+	PXOR  X12, X5
+	BYTE  $0x66
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xdb
+	BYTE  $0x04
+	BYTE  $0x66
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xf6
+	BYTE  $0x08
+	BYTE  $0x66
+	BYTE  $0x45
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xc9
+	BYTE  $0x0c
+	BYTE  $0x66
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xe4
+	BYTE  $0x04
+	BYTE  $0x66
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xff
+	BYTE  $0x08
+	BYTE  $0x66
+	BYTE  $0x45
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xd2
+	BYTE  $0x0c
+	BYTE  $0x66
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xed
+	BYTE  $0x04
+	BYTE  $0x66
+	BYTE  $0x45
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xc0
+	BYTE  $0x08
+	BYTE  $0x66
+	BYTE  $0x45
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xdb
+	BYTE  $0x0c
+	ADDQ  (DI), R10
+	ADCQ  8(DI), R11
+	ADCQ  $0x01, R12
+	MOVQ  (BP), AX
+	MOVQ  AX, R15
+	MULQ  R10
+	MOVQ  AX, R13
+	MOVQ  DX, R14
+	MOVQ  (BP), AX
+	MULQ  R11
+	IMULQ R12, R15
+	ADDQ  AX, R14
+	ADCQ  DX, R15
+	MOVQ  8(BP), AX
+	MOVQ  AX, R8
+	MULQ  R10
+	ADDQ  AX, R14
+	ADCQ  $0x00, DX
+	MOVQ  DX, R10
+	MOVQ  8(BP), AX
+	MULQ  R11
+	ADDQ  AX, R15
+	ADCQ  $0x00, DX
+	IMULQ R12, R8
+	ADDQ  R10, R15
+	ADCQ  DX, R8
+	MOVQ  R13, R10
+	MOVQ  R14, R11
+	MOVQ  R15, R12
+	ANDQ  $0x03, R12
+	MOVQ  R15, R13
+	ANDQ  $-4, R13
+	MOVQ  R8, R14
+	SHRQ  $0x02, R8, R15
+	SHRQ  $0x02, R8
+	ADDQ  R13, R10
+	ADCQ  R14, R11
+	ADCQ  $0x00, R12
+	ADDQ  R15, R10
+	ADCQ  R8, R11
+	ADCQ  $0x00, R12
+	LEAQ  16(DI), DI
+	PADDD X3, X0
+	PXOR  X0, X9
+	ROL16(X9, X12)
+	PADDD X9, X6
+	PXOR  X6, X3
+	MOVO  X3, X12
+	PSLLL $0x0c, X12
+	PSRLL $0x14, X3
+	PXOR  X12, X3
+	PADDD X3, X0
+	PXOR  X0, X9
+	ROL8(X9, X12)
+	PADDD X9, X6
+	PXOR  X6, X3
+	MOVO  X3, X12
+	PSLLL $0x07, X12
+	PSRLL $0x19, X3
+	PXOR  X12, X3
+	PADDD X4, X1
+	PXOR  X1, X10
+	ROL16(X10, X12)
+	PADDD X10, X7
+	PXOR  X7, X4
+	MOVO  X4, X12
+	PSLLL $0x0c, X12
+	PSRLL $0x14, X4
+	PXOR  X12, X4
+	PADDD X4, X1
+	PXOR  X1, X10
+	ROL8(X10, X12)
+	PADDD X10, X7
+	PXOR  X7, X4
+	MOVO  X4, X12
+	PSLLL $0x07, X12
+	PSRLL $0x19, X4
+	PXOR  X12, X4
+	PADDD X5, X2
+	PXOR  X2, X11
+	ROL16(X11, X12)
+	PADDD X11, X8
+	PXOR  X8, X5
+	MOVO  X5, X12
+	PSLLL $0x0c, X12
+	PSRLL $0x14, X5
+	PXOR  X12, X5
+	PADDD X5, X2
+	PXOR  X2, X11
+	ROL8(X11, X12)
+	PADDD X11, X8
+	PXOR  X8, X5
+	MOVO  X5, X12
+	PSLLL $0x07, X12
+	PSRLL $0x19, X5
+	PXOR  X12, X5
+	BYTE  $0x66
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xdb
+	BYTE  $0x0c
+	BYTE  $0x66
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xf6
+	BYTE  $0x08
+	BYTE  $0x66
+	BYTE  $0x45
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xc9
+	BYTE  $0x04
+	BYTE  $0x66
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xe4
+	BYTE  $0x0c
+	BYTE  $0x66
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xff
+	BYTE  $0x08
+	BYTE  $0x66
+	BYTE  $0x45
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xd2
+	BYTE  $0x04
+	BYTE  $0x66
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xed
+	BYTE  $0x0c
+	BYTE  $0x66
+	BYTE  $0x45
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xc0
+	BYTE  $0x08
+	BYTE  $0x66
+	BYTE  $0x45
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xdb
+	BYTE  $0x04
+	DECQ  CX
+	JG    sealSSETail192LoopA
+	DECQ  R9
+	JGE   sealSSETail192LoopB
+	PADDL ·chacha20Constants<>+0(SB), X0
+	PADDL ·chacha20Constants<>+0(SB), X1
+	PADDL ·chacha20Constants<>+0(SB), X2
+	PADDL 32(BP), X3
+	PADDL 32(BP), X4
+	PADDL 32(BP), X5
+	PADDL 48(BP), X6
+	PADDL 48(BP), X7
+	PADDL 48(BP), X8
+	PADDL 80(BP), X9
+	PADDL 96(BP), X10
+	PADDL 112(BP), X11
+	MOVOU (SI), X12
+	MOVOU 16(SI), X13
+	MOVOU 32(SI), X14
+	MOVOU 48(SI), X15
+	PXOR  X12, X0
+	PXOR  X13, X3
+	PXOR  X14, X6
+	PXOR  X15, X9
+	MOVOU X0, (DI)
+	MOVOU X3, 16(DI)
+	MOVOU X6, 32(DI)
+	MOVOU X9, 48(DI)
+	MOVOU 64(SI), X12
+	MOVOU 80(SI), X13
+	MOVOU 96(SI), X14
+	MOVOU 112(SI), X15
+	PXOR  X12, X1
+	PXOR  X13, X4
+	PXOR  X14, X7
+	PXOR  X15, X10
+	MOVOU X1, 64(DI)
+	MOVOU X4, 80(DI)
+	MOVOU X7, 96(DI)
+	MOVOU X10, 112(DI)
+	MOVO  X2, X1
+	MOVO  X5, X4
+	MOVO  X8, X7
+	MOVO  X11, X10
+	MOVQ  $0x00000080, CX
+	LEAQ  128(SI), SI
+	SUBQ  $0x80, BX
+	JMP   sealSSE128SealHash
+
 sealSSE128:
-	// For up to 128 bytes of ciphertext and 64 bytes for the poly key, we require to process three blocks
-	MOVOU ·chacha20Constants<>(SB), A0; MOVOU (1*16)(keyp), B0; MOVOU (2*16)(keyp), C0; MOVOU (3*16)(keyp), D0
-	MOVO  A0, A1; MOVO B0, B1; MOVO C0, C1; MOVO D0, D1; PADDL ·sseIncMask<>(SB), D1
-	MOVO  A1, A2; MOVO B1, B2; MOVO C1, C2; MOVO D1, D2; PADDL ·sseIncMask<>(SB), D2
-	MOVO  B0, T1; MOVO C0, T2; MOVO D1, T3
-	MOVQ  $10, itr2
+	MOVOU ·chacha20Constants<>+0(SB), X0
+	MOVOU 16(R8), X3
+	MOVOU 32(R8), X6
+	MOVOU 48(R8), X9
+	MOVO  X0, X1
+	MOVO  X3, X4
+	MOVO  X6, X7
+	MOVO  X9, X10
+	PADDL ·sseIncMask<>+0(SB), X10
+	MOVO  X1, X2
+	MOVO  X4, X5
+	MOVO  X7, X8
+	MOVO  X10, X11
+	PADDL ·sseIncMask<>+0(SB), X11
+	MOVO  X3, X13
+	MOVO  X6, X14
+	MOVO  X10, X15
+	MOVQ  $0x0000000a, R9
 
 sealSSE128InnerCipherLoop:
-	chachaQR(A0, B0, C0, D0, T0); chachaQR(A1, B1, C1, D1, T0); chachaQR(A2, B2, C2, D2, T0)
-	shiftB0Left;  shiftB1Left; shiftB2Left
-	shiftC0Left;  shiftC1Left; shiftC2Left
-	shiftD0Left;  shiftD1Left; shiftD2Left
-	chachaQR(A0, B0, C0, D0, T0); chachaQR(A1, B1, C1, D1, T0); chachaQR(A2, B2, C2, D2, T0)
-	shiftB0Right; shiftB1Right; shiftB2Right
-	shiftC0Right; shiftC1Right; shiftC2Right
-	shiftD0Right; shiftD1Right; shiftD2Right
-	DECQ          itr2
-	JNE           sealSSE128InnerCipherLoop
+	PADDD X3, X0
+	PXOR  X0, X9
+	ROL16(X9, X12)
+	PADDD X9, X6
+	PXOR  X6, X3
+	MOVO  X3, X12
+	PSLLL $0x0c, X12
+	PSRLL $0x14, X3
+	PXOR  X12, X3
+	PADDD X3, X0
+	PXOR  X0, X9
+	ROL8(X9, X12)
+	PADDD X9, X6
+	PXOR  X6, X3
+	MOVO  X3, X12
+	PSLLL $0x07, X12
+	PSRLL $0x19, X3
+	PXOR  X12, X3
+	PADDD X4, X1
+	PXOR  X1, X10
+	ROL16(X10, X12)
+	PADDD X10, X7
+	PXOR  X7, X4
+	MOVO  X4, X12
+	PSLLL $0x0c, X12
+	PSRLL $0x14, X4
+	PXOR  X12, X4
+	PADDD X4, X1
+	PXOR  X1, X10
+	ROL8(X10, X12)
+	PADDD X10, X7
+	PXOR  X7, X4
+	MOVO  X4, X12
+	PSLLL $0x07, X12
+	PSRLL $0x19, X4
+	PXOR  X12, X4
+	PADDD X5, X2
+	PXOR  X2, X11
+	ROL16(X11, X12)
+	PADDD X11, X8
+	PXOR  X8, X5
+	MOVO  X5, X12
+	PSLLL $0x0c, X12
+	PSRLL $0x14, X5
+	PXOR  X12, X5
+	PADDD X5, X2
+	PXOR  X2, X11
+	ROL8(X11, X12)
+	PADDD X11, X8
+	PXOR  X8, X5
+	MOVO  X5, X12
+	PSLLL $0x07, X12
+	PSRLL $0x19, X5
+	PXOR  X12, X5
+	BYTE  $0x66
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xdb
+	BYTE  $0x04
+	BYTE  $0x66
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xe4
+	BYTE  $0x04
+	BYTE  $0x66
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xed
+	BYTE  $0x04
+	BYTE  $0x66
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xf6
+	BYTE  $0x08
+	BYTE  $0x66
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xff
+	BYTE  $0x08
+	BYTE  $0x66
+	BYTE  $0x45
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xc0
+	BYTE  $0x08
+	BYTE  $0x66
+	BYTE  $0x45
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xc9
+	BYTE  $0x0c
+	BYTE  $0x66
+	BYTE  $0x45
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xd2
+	BYTE  $0x0c
+	BYTE  $0x66
+	BYTE  $0x45
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xdb
+	BYTE  $0x0c
+	PADDD X3, X0
+	PXOR  X0, X9
+	ROL16(X9, X12)
+	PADDD X9, X6
+	PXOR  X6, X3
+	MOVO  X3, X12
+	PSLLL $0x0c, X12
+	PSRLL $0x14, X3
+	PXOR  X12, X3
+	PADDD X3, X0
+	PXOR  X0, X9
+	ROL8(X9, X12)
+	PADDD X9, X6
+	PXOR  X6, X3
+	MOVO  X3, X12
+	PSLLL $0x07, X12
+	PSRLL $0x19, X3
+	PXOR  X12, X3
+	PADDD X4, X1
+	PXOR  X1, X10
+	ROL16(X10, X12)
+	PADDD X10, X7
+	PXOR  X7, X4
+	MOVO  X4, X12
+	PSLLL $0x0c, X12
+	PSRLL $0x14, X4
+	PXOR  X12, X4
+	PADDD X4, X1
+	PXOR  X1, X10
+	ROL8(X10, X12)
+	PADDD X10, X7
+	PXOR  X7, X4
+	MOVO  X4, X12
+	PSLLL $0x07, X12
+	PSRLL $0x19, X4
+	PXOR  X12, X4
+	PADDD X5, X2
+	PXOR  X2, X11
+	ROL16(X11, X12)
+	PADDD X11, X8
+	PXOR  X8, X5
+	MOVO  X5, X12
+	PSLLL $0x0c, X12
+	PSRLL $0x14, X5
+	PXOR  X12, X5
+	PADDD X5, X2
+	PXOR  X2, X11
+	ROL8(X11, X12)
+	PADDD X11, X8
+	PXOR  X8, X5
+	MOVO  X5, X12
+	PSLLL $0x07, X12
+	PSRLL $0x19, X5
+	PXOR  X12, X5
+	BYTE  $0x66
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xdb
+	BYTE  $0x0c
+	BYTE  $0x66
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xe4
+	BYTE  $0x0c
+	BYTE  $0x66
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xed
+	BYTE  $0x0c
+	BYTE  $0x66
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xf6
+	BYTE  $0x08
+	BYTE  $0x66
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xff
+	BYTE  $0x08
+	BYTE  $0x66
+	BYTE  $0x45
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xc0
+	BYTE  $0x08
+	BYTE  $0x66
+	BYTE  $0x45
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xc9
+	BYTE  $0x04
+	BYTE  $0x66
+	BYTE  $0x45
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xd2
+	BYTE  $0x04
+	BYTE  $0x66
+	BYTE  $0x45
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xdb
+	BYTE  $0x04
+	DECQ  R9
+	JNE   sealSSE128InnerCipherLoop
 
 	// A0|B0 hold the Poly1305 32-byte key, C0,D0 can be discarded
-	PADDL ·chacha20Constants<>(SB), A0; PADDL ·chacha20Constants<>(SB), A1; PADDL ·chacha20Constants<>(SB), A2
-	PADDL T1, B0; PADDL T1, B1; PADDL T1, B2
-	PADDL T2, C1; PADDL T2, C2
-	PADDL T3, D1; PADDL ·sseIncMask<>(SB), T3; PADDL T3, D2
-	PAND  ·polyClampMask<>(SB), A0
-	MOVOU A0, rStore
-	MOVOU B0, sStore
+	PADDL ·chacha20Constants<>+0(SB), X0
+	PADDL ·chacha20Constants<>+0(SB), X1
+	PADDL ·chacha20Constants<>+0(SB), X2
+	PADDL X13, X3
+	PADDL X13, X4
+	PADDL X13, X5
+	PADDL X14, X7
+	PADDL X14, X8
+	PADDL X15, X10
+	PADDL ·sseIncMask<>+0(SB), X15
+	PADDL X15, X11
+	PAND  ·polyClampMask<>+0(SB), X0
+	MOVOU X0, (BP)
+	MOVOU X3, 16(BP)
 
 	// Hash
-	MOVQ ad_len+80(FP), itr2
+	MOVQ ad_len+80(FP), R9
 	CALL polyHashADInternal<>(SB)
-	XORQ itr1, itr1
+	XORQ CX, CX
 
 sealSSE128SealHash:
-	// itr1 holds the number of bytes encrypted but not yet hashed
-	CMPQ itr1, $16
-	JB   sealSSE128Seal
-	polyAdd(0(oup))
-	polyMul
-
-	SUBQ $16, itr1
-	ADDQ $16, oup
-
-	JMP sealSSE128SealHash
+	CMPQ  CX, $0x10
+	JB    sealSSE128Seal
+	ADDQ  (DI), R10
+	ADCQ  8(DI), R11
+	ADCQ  $0x01, R12
+	MOVQ  (BP), AX
+	MOVQ  AX, R15
+	MULQ  R10
+	MOVQ  AX, R13
+	MOVQ  DX, R14
+	MOVQ  (BP), AX
+	MULQ  R11
+	IMULQ R12, R15
+	ADDQ  AX, R14
+	ADCQ  DX, R15
+	MOVQ  8(BP), AX
+	MOVQ  AX, R8
+	MULQ  R10
+	ADDQ  AX, R14
+	ADCQ  $0x00, DX
+	MOVQ  DX, R10
+	MOVQ  8(BP), AX
+	MULQ  R11
+	ADDQ  AX, R15
+	ADCQ  $0x00, DX
+	IMULQ R12, R8
+	ADDQ  R10, R15
+	ADCQ  DX, R8
+	MOVQ  R13, R10
+	MOVQ  R14, R11
+	MOVQ  R15, R12
+	ANDQ  $0x03, R12
+	MOVQ  R15, R13
+	ANDQ  $-4, R13
+	MOVQ  R8, R14
+	SHRQ  $0x02, R8, R15
+	SHRQ  $0x02, R8
+	ADDQ  R13, R10
+	ADCQ  R14, R11
+	ADCQ  $0x00, R12
+	ADDQ  R15, R10
+	ADCQ  R8, R11
+	ADCQ  $0x00, R12
+	SUBQ  $0x10, CX
+	ADDQ  $0x10, DI
+	JMP   sealSSE128SealHash
 
 sealSSE128Seal:
-	CMPQ inl, $16
+	CMPQ BX, $0x10
 	JB   sealSSETail
-	SUBQ $16, inl
+	SUBQ $0x10, BX
 
 	// Load for decryption
-	MOVOU (inp), T0
-	PXOR  T0, A1
-	MOVOU A1, (oup)
-	LEAQ  (1*16)(inp), inp
-	LEAQ  (1*16)(oup), oup
+	MOVOU (SI), X12
+	PXOR  X12, X1
+	MOVOU X1, (DI)
+	LEAQ  16(SI), SI
+	LEAQ  16(DI), DI
 
 	// Extract for hashing
-	MOVQ   A1, t0
-	PSRLDQ $8, A1
-	MOVQ A1, t1
-	ADDQ   t0, acc0; ADCQ t1, acc1; ADCQ $1, acc2
-	polyMul
+	MOVQ   X1, R13
+	PSRLDQ $0x08, X1
+	MOVQ   X1, R14
+	ADDQ   R13, R10
+	ADCQ   R14, R11
+	ADCQ   $0x01, R12
+	MOVQ   (BP), AX
+	MOVQ   AX, R15
+	MULQ   R10
+	MOVQ   AX, R13
+	MOVQ   DX, R14
+	MOVQ   (BP), AX
+	MULQ   R11
+	IMULQ  R12, R15
+	ADDQ   AX, R14
+	ADCQ   DX, R15
+	MOVQ   8(BP), AX
+	MOVQ   AX, R8
+	MULQ   R10
+	ADDQ   AX, R14
+	ADCQ   $0x00, DX
+	MOVQ   DX, R10
+	MOVQ   8(BP), AX
+	MULQ   R11
+	ADDQ   AX, R15
+	ADCQ   $0x00, DX
+	IMULQ  R12, R8
+	ADDQ   R10, R15
+	ADCQ   DX, R8
+	MOVQ   R13, R10
+	MOVQ   R14, R11
+	MOVQ   R15, R12
+	ANDQ   $0x03, R12
+	MOVQ   R15, R13
+	ANDQ   $-4, R13
+	MOVQ   R8, R14
+	SHRQ   $0x02, R8, R15
+	SHRQ   $0x02, R8
+	ADDQ   R13, R10
+	ADCQ   R14, R11
+	ADCQ   $0x00, R12
+	ADDQ   R15, R10
+	ADCQ   R8, R11
+	ADCQ   $0x00, R12
 
 	// Shift the stream "left"
-	MOVO B1, A1
-	MOVO C1, B1
-	MOVO D1, C1
-	MOVO A2, D1
-	MOVO B2, A2
-	MOVO C2, B2
-	MOVO D2, C2
+	MOVO X4, X1
+	MOVO X7, X4
+	MOVO X10, X7
+	MOVO X2, X10
+	MOVO X5, X2
+	MOVO X8, X5
+	MOVO X11, X8
 	JMP  sealSSE128Seal
 
 sealSSETail:
-	TESTQ inl, inl
+	TESTQ BX, BX
 	JE    sealSSEFinalize
 
 	// We can only load the PT one byte at a time to avoid read after end of buffer
-	MOVQ inl, itr2
-	SHLQ $4, itr2
-	LEAQ ·andMask<>(SB), t0
-	MOVQ inl, itr1
-	LEAQ -1(inp)(inl*1), inp
-	XORQ t2, t2
-	XORQ t3, t3
+	MOVQ BX, R9
+	SHLQ $0x04, R9
+	LEAQ ·andMask<>+0(SB), R13
+	MOVQ BX, CX
+	LEAQ -1(SI)(BX*1), SI
+	XORQ R15, R15
+	XORQ R8, R8
 	XORQ AX, AX
 
 sealSSETailLoadLoop:
-	SHLQ $8, t2, t3
-	SHLQ $8, t2
-	MOVB (inp), AX
-	XORQ AX, t2
-	LEAQ   -1(inp), inp
-	DECQ   itr1
+	SHLQ   $0x08, R15, R8
+	SHLQ   $0x08, R15
+	MOVB   (SI), AX
+	XORQ   AX, R15
+	LEAQ   -1(SI), SI
+	DECQ   CX
 	JNE    sealSSETailLoadLoop
-	MOVQ t2, 0+tmpStore
-	MOVQ t3, 8+tmpStore
-	PXOR 0+tmpStore, A1
-	MOVOU  A1, (oup)
-	MOVOU  -16(t0)(itr2*1), T0
-	PAND   T0, A1
-	MOVQ   A1, t0
-	PSRLDQ $8, A1
-	MOVQ   A1, t1
-	ADDQ   t0, acc0; ADCQ t1, acc1; ADCQ $1, acc2
-	polyMul
-
-	ADDQ inl, oup
+	MOVQ   R15, 64(BP)
+	MOVQ   R8, 72(BP)
+	PXOR   64(BP), X1
+	MOVOU  X1, (DI)
+	MOVOU  -16(R13)(R9*1), X12
+	PAND   X12, X1
+	MOVQ   X1, R13
+	PSRLDQ $0x08, X1
+	MOVQ   X1, R14
+	ADDQ   R13, R10
+	ADCQ   R14, R11
+	ADCQ   $0x01, R12
+	MOVQ   (BP), AX
+	MOVQ   AX, R15
+	MULQ   R10
+	MOVQ   AX, R13
+	MOVQ   DX, R14
+	MOVQ   (BP), AX
+	MULQ   R11
+	IMULQ  R12, R15
+	ADDQ   AX, R14
+	ADCQ   DX, R15
+	MOVQ   8(BP), AX
+	MOVQ   AX, R8
+	MULQ   R10
+	ADDQ   AX, R14
+	ADCQ   $0x00, DX
+	MOVQ   DX, R10
+	MOVQ   8(BP), AX
+	MULQ   R11
+	ADDQ   AX, R15
+	ADCQ   $0x00, DX
+	IMULQ  R12, R8
+	ADDQ   R10, R15
+	ADCQ   DX, R8
+	MOVQ   R13, R10
+	MOVQ   R14, R11
+	MOVQ   R15, R12
+	ANDQ   $0x03, R12
+	MOVQ   R15, R13
+	ANDQ   $-4, R13
+	MOVQ   R8, R14
+	SHRQ   $0x02, R8, R15
+	SHRQ   $0x02, R8
+	ADDQ   R13, R10
+	ADCQ   R14, R11
+	ADCQ   $0x00, R12
+	ADDQ   R15, R10
+	ADCQ   R8, R11
+	ADCQ   $0x00, R12
+	ADDQ   BX, DI
 
 sealSSEFinalize:
 	// Hash in the buffer lengths
-	ADDQ ad_len+80(FP), acc0
-	ADCQ src_len+56(FP), acc1
-	ADCQ $1, acc2
-	polyMul
+	ADDQ  ad_len+80(FP), R10
+	ADCQ  src_len+56(FP), R11
+	ADCQ  $0x01, R12
+	MOVQ  (BP), AX
+	MOVQ  AX, R15
+	MULQ  R10
+	MOVQ  AX, R13
+	MOVQ  DX, R14
+	MOVQ  (BP), AX
+	MULQ  R11
+	IMULQ R12, R15
+	ADDQ  AX, R14
+	ADCQ  DX, R15
+	MOVQ  8(BP), AX
+	MOVQ  AX, R8
+	MULQ  R10
+	ADDQ  AX, R14
+	ADCQ  $0x00, DX
+	MOVQ  DX, R10
+	MOVQ  8(BP), AX
+	MULQ  R11
+	ADDQ  AX, R15
+	ADCQ  $0x00, DX
+	IMULQ R12, R8
+	ADDQ  R10, R15
+	ADCQ  DX, R8
+	MOVQ  R13, R10
+	MOVQ  R14, R11
+	MOVQ  R15, R12
+	ANDQ  $0x03, R12
+	MOVQ  R15, R13
+	ANDQ  $-4, R13
+	MOVQ  R8, R14
+	SHRQ  $0x02, R8, R15
+	SHRQ  $0x02, R8
+	ADDQ  R13, R10
+	ADCQ  R14, R11
+	ADCQ  $0x00, R12
+	ADDQ  R15, R10
+	ADCQ  R8, R11
+	ADCQ  $0x00, R12
 
 	// Final reduce
-	MOVQ    acc0, t0
-	MOVQ    acc1, t1
-	MOVQ    acc2, t2
-	SUBQ    $-5, acc0
-	SBBQ    $-1, acc1
-	SBBQ    $3, acc2
-	CMOVQCS t0, acc0
-	CMOVQCS t1, acc1
-	CMOVQCS t2, acc2
+	MOVQ    R10, R13
+	MOVQ    R11, R14
+	MOVQ    R12, R15
+	SUBQ    $-5, R10
+	SBBQ    $-1, R11
+	SBBQ    $0x03, R12
+	CMOVQCS R13, R10
+	CMOVQCS R14, R11
+	CMOVQCS R15, R12
 
 	// Add in the "s" part of the key
-	ADDQ 0+sStore, acc0
-	ADCQ 8+sStore, acc1
+	ADDQ 16(BP), R10
+	ADCQ 24(BP), R11
 
 	// Finally store the tag at the end of the message
-	MOVQ acc0, (0*8)(oup)
-	MOVQ acc1, (1*8)(oup)
+	MOVQ R10, (DI)
+	MOVQ R11, 8(DI)
 	RET
 
-// ----------------------------------------------------------------------------
-// ------------------------- AVX2 Code ----------------------------------------
 chacha20Poly1305Seal_AVX2:
 	VZEROUPPER
-	VMOVDQU ·chacha20Constants<>(SB), AA0
-	BYTE    $0xc4; BYTE $0x42; BYTE $0x7d; BYTE $0x5a; BYTE $0x70; BYTE $0x10 // broadcasti128 16(r8), ymm14
-	BYTE    $0xc4; BYTE $0x42; BYTE $0x7d; BYTE $0x5a; BYTE $0x60; BYTE $0x20 // broadcasti128 32(r8), ymm12
-	BYTE    $0xc4; BYTE $0xc2; BYTE $0x7d; BYTE $0x5a; BYTE $0x60; BYTE $0x30 // broadcasti128 48(r8), ymm4
-	VPADDD  ·avx2InitMask<>(SB), DD0, DD0
+	VMOVDQU ·chacha20Constants<>+0(SB), Y0
+	BYTE    $0xc4
+	BYTE    $0x42
+	BYTE    $0x7d
+	BYTE    $0x5a
+	BYTE    $0x70
+	BYTE    $0x10
+	BYTE    $0xc4
+	BYTE    $0x42
+	BYTE    $0x7d
+	BYTE    $0x5a
+	BYTE    $0x60
+	BYTE    $0x20
+	BYTE    $0xc4
+	BYTE    $0xc2
+	BYTE    $0x7d
+	BYTE    $0x5a
+	BYTE    $0x60
+	BYTE    $0x30
+	VPADDD  ·avx2InitMask<>+0(SB), Y4, Y4
 
 	// Special optimizations, for very short buffers
-	CMPQ inl, $192
-	JBE  seal192AVX2 // 33% faster
-	CMPQ inl, $320
-	JBE  seal320AVX2 // 17% faster
+	CMPQ BX, $0x000000c0
+	JBE  seal192AVX2
+	CMPQ BX, $0x00000140
+	JBE  seal320AVX2
 
 	// For the general key prepare the key first - as a byproduct we have 64 bytes of cipher stream
-	VMOVDQA AA0, AA1; VMOVDQA AA0, AA2; VMOVDQA AA0, AA3
-	VMOVDQA BB0, BB1; VMOVDQA BB0, BB2; VMOVDQA BB0, BB3; VMOVDQA BB0, state1StoreAVX2
-	VMOVDQA CC0, CC1; VMOVDQA CC0, CC2; VMOVDQA CC0, CC3; VMOVDQA CC0, state2StoreAVX2
-	VPADDD  ·avx2IncMask<>(SB), DD0, DD1; VMOVDQA DD0, ctr0StoreAVX2
-	VPADDD  ·avx2IncMask<>(SB), DD1, DD2; VMOVDQA DD1, ctr1StoreAVX2
-	VPADDD  ·avx2IncMask<>(SB), DD2, DD3; VMOVDQA DD2, ctr2StoreAVX2
-	VMOVDQA DD3, ctr3StoreAVX2
-	MOVQ    $10, itr2
+	VMOVDQA Y0, Y5
+	VMOVDQA Y0, Y6
+	VMOVDQA Y0, Y7
+	VMOVDQA Y14, Y9
+	VMOVDQA Y14, Y10
+	VMOVDQA Y14, Y11
+	VMOVDQA Y14, 32(BP)
+	VMOVDQA Y12, Y13
+	VMOVDQA Y12, Y8
+	VMOVDQA Y12, Y15
+	VMOVDQA Y12, 64(BP)
+	VPADDD  ·avx2IncMask<>+0(SB), Y4, Y1
+	VMOVDQA Y4, 96(BP)
+	VPADDD  ·avx2IncMask<>+0(SB), Y1, Y2
+	VMOVDQA Y1, 128(BP)
+	VPADDD  ·avx2IncMask<>+0(SB), Y2, Y3
+	VMOVDQA Y2, 160(BP)
+	VMOVDQA Y3, 192(BP)
+	MOVQ    $0x0000000a, R9
 
 sealAVX2IntroLoop:
-	VMOVDQA CC3, tmpStoreAVX2
-	chachaQR_AVX2(AA0, BB0, CC0, DD0, CC3); chachaQR_AVX2(AA1, BB1, CC1, DD1, CC3); chachaQR_AVX2(AA2, BB2, CC2, DD2, CC3)
-	VMOVDQA tmpStoreAVX2, CC3
-	VMOVDQA CC1, tmpStoreAVX2
-	chachaQR_AVX2(AA3, BB3, CC3, DD3, CC1)
-	VMOVDQA tmpStoreAVX2, CC1
-
-	VPALIGNR $4, BB0, BB0, BB0; VPALIGNR $8, CC0, CC0, CC0; VPALIGNR $12, DD0, DD0, DD0
-	VPALIGNR $4, BB1, BB1, BB1; VPALIGNR $8, CC1, CC1, CC1; VPALIGNR $12, DD1, DD1, DD1
-	VPALIGNR $4, BB2, BB2, BB2; VPALIGNR $8, CC2, CC2, CC2; VPALIGNR $12, DD2, DD2, DD2
-	VPALIGNR $4, BB3, BB3, BB3; VPALIGNR $8, CC3, CC3, CC3; VPALIGNR $12, DD3, DD3, DD3
-
-	VMOVDQA CC3, tmpStoreAVX2
-	chachaQR_AVX2(AA0, BB0, CC0, DD0, CC3); chachaQR_AVX2(AA1, BB1, CC1, DD1, CC3); chachaQR_AVX2(AA2, BB2, CC2, DD2, CC3)
-	VMOVDQA tmpStoreAVX2, CC3
-	VMOVDQA CC1, tmpStoreAVX2
-	chachaQR_AVX2(AA3, BB3, CC3, DD3, CC1)
-	VMOVDQA tmpStoreAVX2, CC1
-
-	VPALIGNR $12, BB0, BB0, BB0; VPALIGNR $8, CC0, CC0, CC0; VPALIGNR $4, DD0, DD0, DD0
-	VPALIGNR $12, BB1, BB1, BB1; VPALIGNR $8, CC1, CC1, CC1; VPALIGNR $4, DD1, DD1, DD1
-	VPALIGNR $12, BB2, BB2, BB2; VPALIGNR $8, CC2, CC2, CC2; VPALIGNR $4, DD2, DD2, DD2
-	VPALIGNR $12, BB3, BB3, BB3; VPALIGNR $8, CC3, CC3, CC3; VPALIGNR $4, DD3, DD3, DD3
-	DECQ     itr2
-	JNE      sealAVX2IntroLoop
-
-	VPADDD ·chacha20Constants<>(SB), AA0, AA0; VPADDD ·chacha20Constants<>(SB), AA1, AA1; VPADDD ·chacha20Constants<>(SB), AA2, AA2; VPADDD ·chacha20Constants<>(SB), AA3, AA3
-	VPADDD state1StoreAVX2, BB0, BB0; VPADDD state1StoreAVX2, BB1, BB1; VPADDD state1StoreAVX2, BB2, BB2; VPADDD state1StoreAVX2, BB3, BB3
-	VPADDD state2StoreAVX2, CC0, CC0; VPADDD state2StoreAVX2, CC1, CC1; VPADDD state2StoreAVX2, CC2, CC2; VPADDD state2StoreAVX2, CC3, CC3
-	VPADDD ctr0StoreAVX2, DD0, DD0; VPADDD ctr1StoreAVX2, DD1, DD1; VPADDD ctr2StoreAVX2, DD2, DD2; VPADDD ctr3StoreAVX2, DD3, DD3
-
-	VPERM2I128 $0x13, CC0, DD0, CC0 // Stream bytes 96 - 127
-	VPERM2I128 $0x02, AA0, BB0, DD0 // The Poly1305 key
-	VPERM2I128 $0x13, AA0, BB0, AA0 // Stream bytes 64 - 95
+	VMOVDQA    Y15, 224(BP)
+	VPADDD     Y14, Y0, Y0
+	VPXOR      Y0, Y4, Y4
+	VPSHUFB    ·rol16<>+0(SB), Y4, Y4
+	VPADDD     Y4, Y12, Y12
+	VPXOR      Y12, Y14, Y14
+	VPSLLD     $0x0c, Y14, Y15
+	VPSRLD     $0x14, Y14, Y14
+	VPXOR      Y15, Y14, Y14
+	VPADDD     Y14, Y0, Y0
+	VPXOR      Y0, Y4, Y4
+	VPSHUFB    ·rol8<>+0(SB), Y4, Y4
+	VPADDD     Y4, Y12, Y12
+	VPXOR      Y12, Y14, Y14
+	VPSLLD     $0x07, Y14, Y15
+	VPSRLD     $0x19, Y14, Y14
+	VPXOR      Y15, Y14, Y14
+	VPADDD     Y9, Y5, Y5
+	VPXOR      Y5, Y1, Y1
+	VPSHUFB    ·rol16<>+0(SB), Y1, Y1
+	VPADDD     Y1, Y13, Y13
+	VPXOR      Y13, Y9, Y9
+	VPSLLD     $0x0c, Y9, Y15
+	VPSRLD     $0x14, Y9, Y9
+	VPXOR      Y15, Y9, Y9
+	VPADDD     Y9, Y5, Y5
+	VPXOR      Y5, Y1, Y1
+	VPSHUFB    ·rol8<>+0(SB), Y1, Y1
+	VPADDD     Y1, Y13, Y13
+	VPXOR      Y13, Y9, Y9
+	VPSLLD     $0x07, Y9, Y15
+	VPSRLD     $0x19, Y9, Y9
+	VPXOR      Y15, Y9, Y9
+	VPADDD     Y10, Y6, Y6
+	VPXOR      Y6, Y2, Y2
+	VPSHUFB    ·rol16<>+0(SB), Y2, Y2
+	VPADDD     Y2, Y8, Y8
+	VPXOR      Y8, Y10, Y10
+	VPSLLD     $0x0c, Y10, Y15
+	VPSRLD     $0x14, Y10, Y10
+	VPXOR      Y15, Y10, Y10
+	VPADDD     Y10, Y6, Y6
+	VPXOR      Y6, Y2, Y2
+	VPSHUFB    ·rol8<>+0(SB), Y2, Y2
+	VPADDD     Y2, Y8, Y8
+	VPXOR      Y8, Y10, Y10
+	VPSLLD     $0x07, Y10, Y15
+	VPSRLD     $0x19, Y10, Y10
+	VPXOR      Y15, Y10, Y10
+	VMOVDQA    224(BP), Y15
+	VMOVDQA    Y13, 224(BP)
+	VPADDD     Y11, Y7, Y7
+	VPXOR      Y7, Y3, Y3
+	VPSHUFB    ·rol16<>+0(SB), Y3, Y3
+	VPADDD     Y3, Y15, Y15
+	VPXOR      Y15, Y11, Y11
+	VPSLLD     $0x0c, Y11, Y13
+	VPSRLD     $0x14, Y11, Y11
+	VPXOR      Y13, Y11, Y11
+	VPADDD     Y11, Y7, Y7
+	VPXOR      Y7, Y3, Y3
+	VPSHUFB    ·rol8<>+0(SB), Y3, Y3
+	VPADDD     Y3, Y15, Y15
+	VPXOR      Y15, Y11, Y11
+	VPSLLD     $0x07, Y11, Y13
+	VPSRLD     $0x19, Y11, Y11
+	VPXOR      Y13, Y11, Y11
+	VMOVDQA    224(BP), Y13
+	VPALIGNR   $0x04, Y14, Y14, Y14
+	VPALIGNR   $0x08, Y12, Y12, Y12
+	VPALIGNR   $0x0c, Y4, Y4, Y4
+	VPALIGNR   $0x04, Y9, Y9, Y9
+	VPALIGNR   $0x08, Y13, Y13, Y13
+	VPALIGNR   $0x0c, Y1, Y1, Y1
+	VPALIGNR   $0x04, Y10, Y10, Y10
+	VPALIGNR   $0x08, Y8, Y8, Y8
+	VPALIGNR   $0x0c, Y2, Y2, Y2
+	VPALIGNR   $0x04, Y11, Y11, Y11
+	VPALIGNR   $0x08, Y15, Y15, Y15
+	VPALIGNR   $0x0c, Y3, Y3, Y3
+	VMOVDQA    Y15, 224(BP)
+	VPADDD     Y14, Y0, Y0
+	VPXOR      Y0, Y4, Y4
+	VPSHUFB    ·rol16<>+0(SB), Y4, Y4
+	VPADDD     Y4, Y12, Y12
+	VPXOR      Y12, Y14, Y14
+	VPSLLD     $0x0c, Y14, Y15
+	VPSRLD     $0x14, Y14, Y14
+	VPXOR      Y15, Y14, Y14
+	VPADDD     Y14, Y0, Y0
+	VPXOR      Y0, Y4, Y4
+	VPSHUFB    ·rol8<>+0(SB), Y4, Y4
+	VPADDD     Y4, Y12, Y12
+	VPXOR      Y12, Y14, Y14
+	VPSLLD     $0x07, Y14, Y15
+	VPSRLD     $0x19, Y14, Y14
+	VPXOR      Y15, Y14, Y14
+	VPADDD     Y9, Y5, Y5
+	VPXOR      Y5, Y1, Y1
+	VPSHUFB    ·rol16<>+0(SB), Y1, Y1
+	VPADDD     Y1, Y13, Y13
+	VPXOR      Y13, Y9, Y9
+	VPSLLD     $0x0c, Y9, Y15
+	VPSRLD     $0x14, Y9, Y9
+	VPXOR      Y15, Y9, Y9
+	VPADDD     Y9, Y5, Y5
+	VPXOR      Y5, Y1, Y1
+	VPSHUFB    ·rol8<>+0(SB), Y1, Y1
+	VPADDD     Y1, Y13, Y13
+	VPXOR      Y13, Y9, Y9
+	VPSLLD     $0x07, Y9, Y15
+	VPSRLD     $0x19, Y9, Y9
+	VPXOR      Y15, Y9, Y9
+	VPADDD     Y10, Y6, Y6
+	VPXOR      Y6, Y2, Y2
+	VPSHUFB    ·rol16<>+0(SB), Y2, Y2
+	VPADDD     Y2, Y8, Y8
+	VPXOR      Y8, Y10, Y10
+	VPSLLD     $0x0c, Y10, Y15
+	VPSRLD     $0x14, Y10, Y10
+	VPXOR      Y15, Y10, Y10
+	VPADDD     Y10, Y6, Y6
+	VPXOR      Y6, Y2, Y2
+	VPSHUFB    ·rol8<>+0(SB), Y2, Y2
+	VPADDD     Y2, Y8, Y8
+	VPXOR      Y8, Y10, Y10
+	VPSLLD     $0x07, Y10, Y15
+	VPSRLD     $0x19, Y10, Y10
+	VPXOR      Y15, Y10, Y10
+	VMOVDQA    224(BP), Y15
+	VMOVDQA    Y13, 224(BP)
+	VPADDD     Y11, Y7, Y7
+	VPXOR      Y7, Y3, Y3
+	VPSHUFB    ·rol16<>+0(SB), Y3, Y3
+	VPADDD     Y3, Y15, Y15
+	VPXOR      Y15, Y11, Y11
+	VPSLLD     $0x0c, Y11, Y13
+	VPSRLD     $0x14, Y11, Y11
+	VPXOR      Y13, Y11, Y11
+	VPADDD     Y11, Y7, Y7
+	VPXOR      Y7, Y3, Y3
+	VPSHUFB    ·rol8<>+0(SB), Y3, Y3
+	VPADDD     Y3, Y15, Y15
+	VPXOR      Y15, Y11, Y11
+	VPSLLD     $0x07, Y11, Y13
+	VPSRLD     $0x19, Y11, Y11
+	VPXOR      Y13, Y11, Y11
+	VMOVDQA    224(BP), Y13
+	VPALIGNR   $0x0c, Y14, Y14, Y14
+	VPALIGNR   $0x08, Y12, Y12, Y12
+	VPALIGNR   $0x04, Y4, Y4, Y4
+	VPALIGNR   $0x0c, Y9, Y9, Y9
+	VPALIGNR   $0x08, Y13, Y13, Y13
+	VPALIGNR   $0x04, Y1, Y1, Y1
+	VPALIGNR   $0x0c, Y10, Y10, Y10
+	VPALIGNR   $0x08, Y8, Y8, Y8
+	VPALIGNR   $0x04, Y2, Y2, Y2
+	VPALIGNR   $0x0c, Y11, Y11, Y11
+	VPALIGNR   $0x08, Y15, Y15, Y15
+	VPALIGNR   $0x04, Y3, Y3, Y3
+	DECQ       R9
+	JNE        sealAVX2IntroLoop
+	VPADDD     ·chacha20Constants<>+0(SB), Y0, Y0
+	VPADDD     ·chacha20Constants<>+0(SB), Y5, Y5
+	VPADDD     ·chacha20Constants<>+0(SB), Y6, Y6
+	VPADDD     ·chacha20Constants<>+0(SB), Y7, Y7
+	VPADDD     32(BP), Y14, Y14
+	VPADDD     32(BP), Y9, Y9
+	VPADDD     32(BP), Y10, Y10
+	VPADDD     32(BP), Y11, Y11
+	VPADDD     64(BP), Y12, Y12
+	VPADDD     64(BP), Y13, Y13
+	VPADDD     64(BP), Y8, Y8
+	VPADDD     64(BP), Y15, Y15
+	VPADDD     96(BP), Y4, Y4
+	VPADDD     128(BP), Y1, Y1
+	VPADDD     160(BP), Y2, Y2
+	VPADDD     192(BP), Y3, Y3
+	VPERM2I128 $0x13, Y12, Y4, Y12
+	VPERM2I128 $0x02, Y0, Y14, Y4
+	VPERM2I128 $0x13, Y0, Y14, Y0
 
 	// Clamp and store poly key
-	VPAND   ·polyClampMask<>(SB), DD0, DD0
-	VMOVDQA DD0, rsStoreAVX2
+	VPAND   ·polyClampMask<>+0(SB), Y4, Y4
+	VMOVDQA Y4, (BP)
 
 	// Hash AD
-	MOVQ ad_len+80(FP), itr2
+	MOVQ ad_len+80(FP), R9
 	CALL polyHashADInternal<>(SB)
 
 	// Can store at least 320 bytes
-	VPXOR   (0*32)(inp), AA0, AA0
-	VPXOR   (1*32)(inp), CC0, CC0
-	VMOVDQU AA0, (0*32)(oup)
-	VMOVDQU CC0, (1*32)(oup)
-
-	VPERM2I128 $0x02, AA1, BB1, AA0; VPERM2I128 $0x02, CC1, DD1, BB0; VPERM2I128 $0x13, AA1, BB1, CC0; VPERM2I128 $0x13, CC1, DD1, DD0
-	VPXOR      (2*32)(inp), AA0, AA0; VPXOR (3*32)(inp), BB0, BB0; VPXOR (4*32)(inp), CC0, CC0; VPXOR (5*32)(inp), DD0, DD0
-	VMOVDQU    AA0, (2*32)(oup); VMOVDQU BB0, (3*32)(oup); VMOVDQU CC0, (4*32)(oup); VMOVDQU DD0, (5*32)(oup)
-	VPERM2I128 $0x02, AA2, BB2, AA0; VPERM2I128 $0x02, CC2, DD2, BB0; VPERM2I128 $0x13, AA2, BB2, CC0; VPERM2I128 $0x13, CC2, DD2, DD0
-	VPXOR      (6*32)(inp), AA0, AA0; VPXOR (7*32)(inp), BB0, BB0; VPXOR (8*32)(inp), CC0, CC0; VPXOR (9*32)(inp), DD0, DD0
-	VMOVDQU    AA0, (6*32)(oup); VMOVDQU BB0, (7*32)(oup); VMOVDQU CC0, (8*32)(oup); VMOVDQU DD0, (9*32)(oup)
-
-	MOVQ $320, itr1
-	SUBQ $320, inl
-	LEAQ 320(inp), inp
-
-	VPERM2I128 $0x02, AA3, BB3, AA0; VPERM2I128 $0x02, CC3, DD3, BB0; VPERM2I128 $0x13, AA3, BB3, CC0; VPERM2I128 $0x13, CC3, DD3, DD0
-	CMPQ       inl, $128
+	VPXOR      (SI), Y0, Y0
+	VPXOR      32(SI), Y12, Y12
+	VMOVDQU    Y0, (DI)
+	VMOVDQU    Y12, 32(DI)
+	VPERM2I128 $0x02, Y5, Y9, Y0
+	VPERM2I128 $0x02, Y13, Y1, Y14
+	VPERM2I128 $0x13, Y5, Y9, Y12
+	VPERM2I128 $0x13, Y13, Y1, Y4
+	VPXOR      64(SI), Y0, Y0
+	VPXOR      96(SI), Y14, Y14
+	VPXOR      128(SI), Y12, Y12
+	VPXOR      160(SI), Y4, Y4
+	VMOVDQU    Y0, 64(DI)
+	VMOVDQU    Y14, 96(DI)
+	VMOVDQU    Y12, 128(DI)
+	VMOVDQU    Y4, 160(DI)
+	VPERM2I128 $0x02, Y6, Y10, Y0
+	VPERM2I128 $0x02, Y8, Y2, Y14
+	VPERM2I128 $0x13, Y6, Y10, Y12
+	VPERM2I128 $0x13, Y8, Y2, Y4
+	VPXOR      192(SI), Y0, Y0
+	VPXOR      224(SI), Y14, Y14
+	VPXOR      256(SI), Y12, Y12
+	VPXOR      288(SI), Y4, Y4
+	VMOVDQU    Y0, 192(DI)
+	VMOVDQU    Y14, 224(DI)
+	VMOVDQU    Y12, 256(DI)
+	VMOVDQU    Y4, 288(DI)
+	MOVQ       $0x00000140, CX
+	SUBQ       $0x00000140, BX
+	LEAQ       320(SI), SI
+	VPERM2I128 $0x02, Y7, Y11, Y0
+	VPERM2I128 $0x02, Y15, Y3, Y14
+	VPERM2I128 $0x13, Y7, Y11, Y12
+	VPERM2I128 $0x13, Y15, Y3, Y4
+	CMPQ       BX, $0x80
 	JBE        sealAVX2SealHash
-
-	VPXOR   (0*32)(inp), AA0, AA0; VPXOR (1*32)(inp), BB0, BB0; VPXOR (2*32)(inp), CC0, CC0; VPXOR (3*32)(inp), DD0, DD0
-	VMOVDQU AA0, (10*32)(oup); VMOVDQU BB0, (11*32)(oup); VMOVDQU CC0, (12*32)(oup); VMOVDQU DD0, (13*32)(oup)
-	SUBQ    $128, inl
-	LEAQ    128(inp), inp
-
-	MOVQ $8, itr1
-	MOVQ $2, itr2
-
-	CMPQ inl, $128
-	JBE  sealAVX2Tail128
-	CMPQ inl, $256
-	JBE  sealAVX2Tail256
-	CMPQ inl, $384
-	JBE  sealAVX2Tail384
-	CMPQ inl, $512
-	JBE  sealAVX2Tail512
+	VPXOR      (SI), Y0, Y0
+	VPXOR      32(SI), Y14, Y14
+	VPXOR      64(SI), Y12, Y12
+	VPXOR      96(SI), Y4, Y4
+	VMOVDQU    Y0, 320(DI)
+	VMOVDQU    Y14, 352(DI)
+	VMOVDQU    Y12, 384(DI)
+	VMOVDQU    Y4, 416(DI)
+	SUBQ       $0x80, BX
+	LEAQ       128(SI), SI
+	MOVQ       $0x00000008, CX
+	MOVQ       $0x00000002, R9
+	CMPQ       BX, $0x80
+	JBE        sealAVX2Tail128
+	CMPQ       BX, $0x00000100
+	JBE        sealAVX2Tail256
+	CMPQ       BX, $0x00000180
+	JBE        sealAVX2Tail384
+	CMPQ       BX, $0x00000200
+	JBE        sealAVX2Tail512
 
 	// We have 448 bytes to hash, but main loop hashes 512 bytes at a time - perform some rounds, before the main loop
-	VMOVDQA ·chacha20Constants<>(SB), AA0; VMOVDQA AA0, AA1; VMOVDQA AA0, AA2; VMOVDQA AA0, AA3
-	VMOVDQA state1StoreAVX2, BB0; VMOVDQA BB0, BB1; VMOVDQA BB0, BB2; VMOVDQA BB0, BB3
-	VMOVDQA state2StoreAVX2, CC0; VMOVDQA CC0, CC1; VMOVDQA CC0, CC2; VMOVDQA CC0, CC3
-	VMOVDQA ctr3StoreAVX2, DD0
-	VPADDD  ·avx2IncMask<>(SB), DD0, DD0; VPADDD ·avx2IncMask<>(SB), DD0, DD1; VPADDD ·avx2IncMask<>(SB), DD1, DD2; VPADDD ·avx2IncMask<>(SB), DD2, DD3
-	VMOVDQA DD0, ctr0StoreAVX2; VMOVDQA DD1, ctr1StoreAVX2; VMOVDQA DD2, ctr2StoreAVX2; VMOVDQA DD3, ctr3StoreAVX2
-
-	VMOVDQA CC3, tmpStoreAVX2
-	chachaQR_AVX2(AA0, BB0, CC0, DD0, CC3); chachaQR_AVX2(AA1, BB1, CC1, DD1, CC3); chachaQR_AVX2(AA2, BB2, CC2, DD2, CC3)
-	VMOVDQA tmpStoreAVX2, CC3
-	VMOVDQA CC1, tmpStoreAVX2
-	chachaQR_AVX2(AA3, BB3, CC3, DD3, CC1)
-	VMOVDQA tmpStoreAVX2, CC1
-
-	VPALIGNR $4, BB0, BB0, BB0; VPALIGNR $8, CC0, CC0, CC0; VPALIGNR $12, DD0, DD0, DD0
-	VPALIGNR $4, BB1, BB1, BB1; VPALIGNR $8, CC1, CC1, CC1; VPALIGNR $12, DD1, DD1, DD1
-	VPALIGNR $4, BB2, BB2, BB2; VPALIGNR $8, CC2, CC2, CC2; VPALIGNR $12, DD2, DD2, DD2
-	VPALIGNR $4, BB3, BB3, BB3; VPALIGNR $8, CC3, CC3, CC3; VPALIGNR $12, DD3, DD3, DD3
-
-	VMOVDQA CC3, tmpStoreAVX2
-	chachaQR_AVX2(AA0, BB0, CC0, DD0, CC3); chachaQR_AVX2(AA1, BB1, CC1, DD1, CC3); chachaQR_AVX2(AA2, BB2, CC2, DD2, CC3)
-	VMOVDQA tmpStoreAVX2, CC3
-	VMOVDQA CC1, tmpStoreAVX2
-	chachaQR_AVX2(AA3, BB3, CC3, DD3, CC1)
-	VMOVDQA tmpStoreAVX2, CC1
-
-	VPALIGNR $12, BB0, BB0, BB0; VPALIGNR $8, CC0, CC0, CC0; VPALIGNR $4, DD0, DD0, DD0
-	VPALIGNR $12, BB1, BB1, BB1; VPALIGNR $8, CC1, CC1, CC1; VPALIGNR $4, DD1, DD1, DD1
-	VPALIGNR $12, BB2, BB2, BB2; VPALIGNR $8, CC2, CC2, CC2; VPALIGNR $4, DD2, DD2, DD2
-	VPALIGNR $12, BB3, BB3, BB3; VPALIGNR $8, CC3, CC3, CC3; VPALIGNR $4, DD3, DD3, DD3
-	VPADDD   BB0, AA0, AA0; VPADDD BB1, AA1, AA1; VPADDD BB2, AA2, AA2; VPADDD BB3, AA3, AA3
-	VPXOR    AA0, DD0, DD0; VPXOR AA1, DD1, DD1; VPXOR AA2, DD2, DD2; VPXOR AA3, DD3, DD3
-	VPSHUFB  ·rol16<>(SB), DD0, DD0; VPSHUFB ·rol16<>(SB), DD1, DD1; VPSHUFB ·rol16<>(SB), DD2, DD2; VPSHUFB ·rol16<>(SB), DD3, DD3
-	VPADDD   DD0, CC0, CC0; VPADDD DD1, CC1, CC1; VPADDD DD2, CC2, CC2; VPADDD DD3, CC3, CC3
-	VPXOR    CC0, BB0, BB0; VPXOR CC1, BB1, BB1; VPXOR CC2, BB2, BB2; VPXOR CC3, BB3, BB3
-	VMOVDQA  CC3, tmpStoreAVX2
-	VPSLLD   $12, BB0, CC3; VPSRLD $20, BB0, BB0; VPXOR CC3, BB0, BB0
-	VPSLLD   $12, BB1, CC3; VPSRLD $20, BB1, BB1; VPXOR CC3, BB1, BB1
-	VPSLLD   $12, BB2, CC3; VPSRLD $20, BB2, BB2; VPXOR CC3, BB2, BB2
-	VPSLLD   $12, BB3, CC3; VPSRLD $20, BB3, BB3; VPXOR CC3, BB3, BB3
-	VMOVDQA  tmpStoreAVX2, CC3
-
-	SUBQ $16, oup                  // Adjust the pointer
-	MOVQ $9, itr1
-	JMP  sealAVX2InternalLoopStart
+	VMOVDQA  ·chacha20Constants<>+0(SB), Y0
+	VMOVDQA  Y0, Y5
+	VMOVDQA  Y0, Y6
+	VMOVDQA  Y0, Y7
+	VMOVDQA  32(BP), Y14
+	VMOVDQA  Y14, Y9
+	VMOVDQA  Y14, Y10
+	VMOVDQA  Y14, Y11
+	VMOVDQA  64(BP), Y12
+	VMOVDQA  Y12, Y13
+	VMOVDQA  Y12, Y8
+	VMOVDQA  Y12, Y15
+	VMOVDQA  192(BP), Y4
+	VPADDD   ·avx2IncMask<>+0(SB), Y4, Y4
+	VPADDD   ·avx2IncMask<>+0(SB), Y4, Y1
+	VPADDD   ·avx2IncMask<>+0(SB), Y1, Y2
+	VPADDD   ·avx2IncMask<>+0(SB), Y2, Y3
+	VMOVDQA  Y4, 96(BP)
+	VMOVDQA  Y1, 128(BP)
+	VMOVDQA  Y2, 160(BP)
+	VMOVDQA  Y3, 192(BP)
+	VMOVDQA  Y15, 224(BP)
+	VPADDD   Y14, Y0, Y0
+	VPXOR    Y0, Y4, Y4
+	VPSHUFB  ·rol16<>+0(SB), Y4, Y4
+	VPADDD   Y4, Y12, Y12
+	VPXOR    Y12, Y14, Y14
+	VPSLLD   $0x0c, Y14, Y15
+	VPSRLD   $0x14, Y14, Y14
+	VPXOR    Y15, Y14, Y14
+	VPADDD   Y14, Y0, Y0
+	VPXOR    Y0, Y4, Y4
+	VPSHUFB  ·rol8<>+0(SB), Y4, Y4
+	VPADDD   Y4, Y12, Y12
+	VPXOR    Y12, Y14, Y14
+	VPSLLD   $0x07, Y14, Y15
+	VPSRLD   $0x19, Y14, Y14
+	VPXOR    Y15, Y14, Y14
+	VPADDD   Y9, Y5, Y5
+	VPXOR    Y5, Y1, Y1
+	VPSHUFB  ·rol16<>+0(SB), Y1, Y1
+	VPADDD   Y1, Y13, Y13
+	VPXOR    Y13, Y9, Y9
+	VPSLLD   $0x0c, Y9, Y15
+	VPSRLD   $0x14, Y9, Y9
+	VPXOR    Y15, Y9, Y9
+	VPADDD   Y9, Y5, Y5
+	VPXOR    Y5, Y1, Y1
+	VPSHUFB  ·rol8<>+0(SB), Y1, Y1
+	VPADDD   Y1, Y13, Y13
+	VPXOR    Y13, Y9, Y9
+	VPSLLD   $0x07, Y9, Y15
+	VPSRLD   $0x19, Y9, Y9
+	VPXOR    Y15, Y9, Y9
+	VPADDD   Y10, Y6, Y6
+	VPXOR    Y6, Y2, Y2
+	VPSHUFB  ·rol16<>+0(SB), Y2, Y2
+	VPADDD   Y2, Y8, Y8
+	VPXOR    Y8, Y10, Y10
+	VPSLLD   $0x0c, Y10, Y15
+	VPSRLD   $0x14, Y10, Y10
+	VPXOR    Y15, Y10, Y10
+	VPADDD   Y10, Y6, Y6
+	VPXOR    Y6, Y2, Y2
+	VPSHUFB  ·rol8<>+0(SB), Y2, Y2
+	VPADDD   Y2, Y8, Y8
+	VPXOR    Y8, Y10, Y10
+	VPSLLD   $0x07, Y10, Y15
+	VPSRLD   $0x19, Y10, Y10
+	VPXOR    Y15, Y10, Y10
+	VMOVDQA  224(BP), Y15
+	VMOVDQA  Y13, 224(BP)
+	VPADDD   Y11, Y7, Y7
+	VPXOR    Y7, Y3, Y3
+	VPSHUFB  ·rol16<>+0(SB), Y3, Y3
+	VPADDD   Y3, Y15, Y15
+	VPXOR    Y15, Y11, Y11
+	VPSLLD   $0x0c, Y11, Y13
+	VPSRLD   $0x14, Y11, Y11
+	VPXOR    Y13, Y11, Y11
+	VPADDD   Y11, Y7, Y7
+	VPXOR    Y7, Y3, Y3
+	VPSHUFB  ·rol8<>+0(SB), Y3, Y3
+	VPADDD   Y3, Y15, Y15
+	VPXOR    Y15, Y11, Y11
+	VPSLLD   $0x07, Y11, Y13
+	VPSRLD   $0x19, Y11, Y11
+	VPXOR    Y13, Y11, Y11
+	VMOVDQA  224(BP), Y13
+	VPALIGNR $0x04, Y14, Y14, Y14
+	VPALIGNR $0x08, Y12, Y12, Y12
+	VPALIGNR $0x0c, Y4, Y4, Y4
+	VPALIGNR $0x04, Y9, Y9, Y9
+	VPALIGNR $0x08, Y13, Y13, Y13
+	VPALIGNR $0x0c, Y1, Y1, Y1
+	VPALIGNR $0x04, Y10, Y10, Y10
+	VPALIGNR $0x08, Y8, Y8, Y8
+	VPALIGNR $0x0c, Y2, Y2, Y2
+	VPALIGNR $0x04, Y11, Y11, Y11
+	VPALIGNR $0x08, Y15, Y15, Y15
+	VPALIGNR $0x0c, Y3, Y3, Y3
+	VMOVDQA  Y15, 224(BP)
+	VPADDD   Y14, Y0, Y0
+	VPXOR    Y0, Y4, Y4
+	VPSHUFB  ·rol16<>+0(SB), Y4, Y4
+	VPADDD   Y4, Y12, Y12
+	VPXOR    Y12, Y14, Y14
+	VPSLLD   $0x0c, Y14, Y15
+	VPSRLD   $0x14, Y14, Y14
+	VPXOR    Y15, Y14, Y14
+	VPADDD   Y14, Y0, Y0
+	VPXOR    Y0, Y4, Y4
+	VPSHUFB  ·rol8<>+0(SB), Y4, Y4
+	VPADDD   Y4, Y12, Y12
+	VPXOR    Y12, Y14, Y14
+	VPSLLD   $0x07, Y14, Y15
+	VPSRLD   $0x19, Y14, Y14
+	VPXOR    Y15, Y14, Y14
+	VPADDD   Y9, Y5, Y5
+	VPXOR    Y5, Y1, Y1
+	VPSHUFB  ·rol16<>+0(SB), Y1, Y1
+	VPADDD   Y1, Y13, Y13
+	VPXOR    Y13, Y9, Y9
+	VPSLLD   $0x0c, Y9, Y15
+	VPSRLD   $0x14, Y9, Y9
+	VPXOR    Y15, Y9, Y9
+	VPADDD   Y9, Y5, Y5
+	VPXOR    Y5, Y1, Y1
+	VPSHUFB  ·rol8<>+0(SB), Y1, Y1
+	VPADDD   Y1, Y13, Y13
+	VPXOR    Y13, Y9, Y9
+	VPSLLD   $0x07, Y9, Y15
+	VPSRLD   $0x19, Y9, Y9
+	VPXOR    Y15, Y9, Y9
+	VPADDD   Y10, Y6, Y6
+	VPXOR    Y6, Y2, Y2
+	VPSHUFB  ·rol16<>+0(SB), Y2, Y2
+	VPADDD   Y2, Y8, Y8
+	VPXOR    Y8, Y10, Y10
+	VPSLLD   $0x0c, Y10, Y15
+	VPSRLD   $0x14, Y10, Y10
+	VPXOR    Y15, Y10, Y10
+	VPADDD   Y10, Y6, Y6
+	VPXOR    Y6, Y2, Y2
+	VPSHUFB  ·rol8<>+0(SB), Y2, Y2
+	VPADDD   Y2, Y8, Y8
+	VPXOR    Y8, Y10, Y10
+	VPSLLD   $0x07, Y10, Y15
+	VPSRLD   $0x19, Y10, Y10
+	VPXOR    Y15, Y10, Y10
+	VMOVDQA  224(BP), Y15
+	VMOVDQA  Y13, 224(BP)
+	VPADDD   Y11, Y7, Y7
+	VPXOR    Y7, Y3, Y3
+	VPSHUFB  ·rol16<>+0(SB), Y3, Y3
+	VPADDD   Y3, Y15, Y15
+	VPXOR    Y15, Y11, Y11
+	VPSLLD   $0x0c, Y11, Y13
+	VPSRLD   $0x14, Y11, Y11
+	VPXOR    Y13, Y11, Y11
+	VPADDD   Y11, Y7, Y7
+	VPXOR    Y7, Y3, Y3
+	VPSHUFB  ·rol8<>+0(SB), Y3, Y3
+	VPADDD   Y3, Y15, Y15
+	VPXOR    Y15, Y11, Y11
+	VPSLLD   $0x07, Y11, Y13
+	VPSRLD   $0x19, Y11, Y11
+	VPXOR    Y13, Y11, Y11
+	VMOVDQA  224(BP), Y13
+	VPALIGNR $0x0c, Y14, Y14, Y14
+	VPALIGNR $0x08, Y12, Y12, Y12
+	VPALIGNR $0x04, Y4, Y4, Y4
+	VPALIGNR $0x0c, Y9, Y9, Y9
+	VPALIGNR $0x08, Y13, Y13, Y13
+	VPALIGNR $0x04, Y1, Y1, Y1
+	VPALIGNR $0x0c, Y10, Y10, Y10
+	VPALIGNR $0x08, Y8, Y8, Y8
+	VPALIGNR $0x04, Y2, Y2, Y2
+	VPALIGNR $0x0c, Y11, Y11, Y11
+	VPALIGNR $0x08, Y15, Y15, Y15
+	VPALIGNR $0x04, Y3, Y3, Y3
+	VPADDD   Y14, Y0, Y0
+	VPADDD   Y9, Y5, Y5
+	VPADDD   Y10, Y6, Y6
+	VPADDD   Y11, Y7, Y7
+	VPXOR    Y0, Y4, Y4
+	VPXOR    Y5, Y1, Y1
+	VPXOR    Y6, Y2, Y2
+	VPXOR    Y7, Y3, Y3
+	VPSHUFB  ·rol16<>+0(SB), Y4, Y4
+	VPSHUFB  ·rol16<>+0(SB), Y1, Y1
+	VPSHUFB  ·rol16<>+0(SB), Y2, Y2
+	VPSHUFB  ·rol16<>+0(SB), Y3, Y3
+	VPADDD   Y4, Y12, Y12
+	VPADDD   Y1, Y13, Y13
+	VPADDD   Y2, Y8, Y8
+	VPADDD   Y3, Y15, Y15
+	VPXOR    Y12, Y14, Y14
+	VPXOR    Y13, Y9, Y9
+	VPXOR    Y8, Y10, Y10
+	VPXOR    Y15, Y11, Y11
+	VMOVDQA  Y15, 224(BP)
+	VPSLLD   $0x0c, Y14, Y15
+	VPSRLD   $0x14, Y14, Y14
+	VPXOR    Y15, Y14, Y14
+	VPSLLD   $0x0c, Y9, Y15
+	VPSRLD   $0x14, Y9, Y9
+	VPXOR    Y15, Y9, Y9
+	VPSLLD   $0x0c, Y10, Y15
+	VPSRLD   $0x14, Y10, Y10
+	VPXOR    Y15, Y10, Y10
+	VPSLLD   $0x0c, Y11, Y15
+	VPSRLD   $0x14, Y11, Y11
+	VPXOR    Y15, Y11, Y11
+	VMOVDQA  224(BP), Y15
+	SUBQ     $0x10, DI
+	MOVQ     $0x00000009, CX
+	JMP      sealAVX2InternalLoopStart
 
 sealAVX2MainLoop:
-	// Load state, increment counter blocks, store the incremented counters
-	VMOVDQU ·chacha20Constants<>(SB), AA0; VMOVDQA AA0, AA1; VMOVDQA AA0, AA2; VMOVDQA AA0, AA3
-	VMOVDQA state1StoreAVX2, BB0; VMOVDQA BB0, BB1; VMOVDQA BB0, BB2; VMOVDQA BB0, BB3
-	VMOVDQA state2StoreAVX2, CC0; VMOVDQA CC0, CC1; VMOVDQA CC0, CC2; VMOVDQA CC0, CC3
-	VMOVDQA ctr3StoreAVX2, DD0; VPADDD ·avx2IncMask<>(SB), DD0, DD0; VPADDD ·avx2IncMask<>(SB), DD0, DD1; VPADDD ·avx2IncMask<>(SB), DD1, DD2; VPADDD ·avx2IncMask<>(SB), DD2, DD3
-	VMOVDQA DD0, ctr0StoreAVX2; VMOVDQA DD1, ctr1StoreAVX2; VMOVDQA DD2, ctr2StoreAVX2; VMOVDQA DD3, ctr3StoreAVX2
-	MOVQ    $10, itr1
+	VMOVDQU ·chacha20Constants<>+0(SB), Y0
+	VMOVDQA Y0, Y5
+	VMOVDQA Y0, Y6
+	VMOVDQA Y0, Y7
+	VMOVDQA 32(BP), Y14
+	VMOVDQA Y14, Y9
+	VMOVDQA Y14, Y10
+	VMOVDQA Y14, Y11
+	VMOVDQA 64(BP), Y12
+	VMOVDQA Y12, Y13
+	VMOVDQA Y12, Y8
+	VMOVDQA Y12, Y15
+	VMOVDQA 192(BP), Y4
+	VPADDD  ·avx2IncMask<>+0(SB), Y4, Y4
+	VPADDD  ·avx2IncMask<>+0(SB), Y4, Y1
+	VPADDD  ·avx2IncMask<>+0(SB), Y1, Y2
+	VPADDD  ·avx2IncMask<>+0(SB), Y2, Y3
+	VMOVDQA Y4, 96(BP)
+	VMOVDQA Y1, 128(BP)
+	VMOVDQA Y2, 160(BP)
+	VMOVDQA Y3, 192(BP)
+	MOVQ    $0x0000000a, CX
 
 sealAVX2InternalLoop:
-	polyAdd(0*8(oup))
-	VPADDD  BB0, AA0, AA0; VPADDD BB1, AA1, AA1; VPADDD BB2, AA2, AA2; VPADDD BB3, AA3, AA3
-	polyMulStage1_AVX2
-	VPXOR   AA0, DD0, DD0; VPXOR AA1, DD1, DD1; VPXOR AA2, DD2, DD2; VPXOR AA3, DD3, DD3
-	VPSHUFB ·rol16<>(SB), DD0, DD0; VPSHUFB ·rol16<>(SB), DD1, DD1; VPSHUFB ·rol16<>(SB), DD2, DD2; VPSHUFB ·rol16<>(SB), DD3, DD3
-	polyMulStage2_AVX2
-	VPADDD  DD0, CC0, CC0; VPADDD DD1, CC1, CC1; VPADDD DD2, CC2, CC2; VPADDD DD3, CC3, CC3
-	VPXOR   CC0, BB0, BB0; VPXOR CC1, BB1, BB1; VPXOR CC2, BB2, BB2; VPXOR CC3, BB3, BB3
-	polyMulStage3_AVX2
-	VMOVDQA CC3, tmpStoreAVX2
-	VPSLLD  $12, BB0, CC3; VPSRLD $20, BB0, BB0; VPXOR CC3, BB0, BB0
-	VPSLLD  $12, BB1, CC3; VPSRLD $20, BB1, BB1; VPXOR CC3, BB1, BB1
-	VPSLLD  $12, BB2, CC3; VPSRLD $20, BB2, BB2; VPXOR CC3, BB2, BB2
-	VPSLLD  $12, BB3, CC3; VPSRLD $20, BB3, BB3; VPXOR CC3, BB3, BB3
-	VMOVDQA tmpStoreAVX2, CC3
-	polyMulReduceStage
+	ADDQ    (DI), R10
+	ADCQ    8(DI), R11
+	ADCQ    $0x01, R12
+	VPADDD  Y14, Y0, Y0
+	VPADDD  Y9, Y5, Y5
+	VPADDD  Y10, Y6, Y6
+	VPADDD  Y11, Y7, Y7
+	MOVQ    (BP), DX
+	MOVQ    DX, R15
+	MULXQ   R10, R13, R14
+	IMULQ   R12, R15
+	MULXQ   R11, AX, DX
+	ADDQ    AX, R14
+	ADCQ    DX, R15
+	VPXOR   Y0, Y4, Y4
+	VPXOR   Y5, Y1, Y1
+	VPXOR   Y6, Y2, Y2
+	VPXOR   Y7, Y3, Y3
+	VPSHUFB ·rol16<>+0(SB), Y4, Y4
+	VPSHUFB ·rol16<>+0(SB), Y1, Y1
+	VPSHUFB ·rol16<>+0(SB), Y2, Y2
+	VPSHUFB ·rol16<>+0(SB), Y3, Y3
+	MOVQ    8(BP), DX
+	MULXQ   R10, R10, AX
+	ADDQ    R10, R14
+	MULXQ   R11, R11, R8
+	ADCQ    R11, R15
+	ADCQ    $0x00, R8
+	VPADDD  Y4, Y12, Y12
+	VPADDD  Y1, Y13, Y13
+	VPADDD  Y2, Y8, Y8
+	VPADDD  Y3, Y15, Y15
+	VPXOR   Y12, Y14, Y14
+	VPXOR   Y13, Y9, Y9
+	VPXOR   Y8, Y10, Y10
+	VPXOR   Y15, Y11, Y11
+	IMULQ   R12, DX
+	ADDQ    AX, R15
+	ADCQ    DX, R8
+	VMOVDQA Y15, 224(BP)
+	VPSLLD  $0x0c, Y14, Y15
+	VPSRLD  $0x14, Y14, Y14
+	VPXOR   Y15, Y14, Y14
+	VPSLLD  $0x0c, Y9, Y15
+	VPSRLD  $0x14, Y9, Y9
+	VPXOR   Y15, Y9, Y9
+	VPSLLD  $0x0c, Y10, Y15
+	VPSRLD  $0x14, Y10, Y10
+	VPXOR   Y15, Y10, Y10
+	VPSLLD  $0x0c, Y11, Y15
+	VPSRLD  $0x14, Y11, Y11
+	VPXOR   Y15, Y11, Y11
+	VMOVDQA 224(BP), Y15
+	MOVQ    R13, R10
+	MOVQ    R14, R11
+	MOVQ    R15, R12
+	ANDQ    $0x03, R12
+	MOVQ    R15, R13
+	ANDQ    $-4, R13
+	MOVQ    R8, R14
+	SHRQ    $0x02, R8, R15
+	SHRQ    $0x02, R8
+	ADDQ    R13, R10
+	ADCQ    R14, R11
+	ADCQ    $0x00, R12
+	ADDQ    R15, R10
+	ADCQ    R8, R11
+	ADCQ    $0x00, R12
 
 sealAVX2InternalLoopStart:
-	VPADDD   BB0, AA0, AA0; VPADDD BB1, AA1, AA1; VPADDD BB2, AA2, AA2; VPADDD BB3, AA3, AA3
-	VPXOR    AA0, DD0, DD0; VPXOR AA1, DD1, DD1; VPXOR AA2, DD2, DD2; VPXOR AA3, DD3, DD3
-	VPSHUFB  ·rol8<>(SB), DD0, DD0; VPSHUFB ·rol8<>(SB), DD1, DD1; VPSHUFB ·rol8<>(SB), DD2, DD2; VPSHUFB ·rol8<>(SB), DD3, DD3
-	polyAdd(2*8(oup))
-	VPADDD   DD0, CC0, CC0; VPADDD DD1, CC1, CC1; VPADDD DD2, CC2, CC2; VPADDD DD3, CC3, CC3
-	polyMulStage1_AVX2
-	VPXOR    CC0, BB0, BB0; VPXOR CC1, BB1, BB1; VPXOR CC2, BB2, BB2; VPXOR CC3, BB3, BB3
-	VMOVDQA  CC3, tmpStoreAVX2
-	VPSLLD   $7, BB0, CC3; VPSRLD $25, BB0, BB0; VPXOR CC3, BB0, BB0
-	VPSLLD   $7, BB1, CC3; VPSRLD $25, BB1, BB1; VPXOR CC3, BB1, BB1
-	VPSLLD   $7, BB2, CC3; VPSRLD $25, BB2, BB2; VPXOR CC3, BB2, BB2
-	VPSLLD   $7, BB3, CC3; VPSRLD $25, BB3, BB3; VPXOR CC3, BB3, BB3
-	VMOVDQA  tmpStoreAVX2, CC3
-	polyMulStage2_AVX2
-	VPALIGNR $4, BB0, BB0, BB0; VPALIGNR $4, BB1, BB1, BB1; VPALIGNR $4, BB2, BB2, BB2; VPALIGNR $4, BB3, BB3, BB3
-	VPALIGNR $8, CC0, CC0, CC0; VPALIGNR $8, CC1, CC1, CC1; VPALIGNR $8, CC2, CC2, CC2; VPALIGNR $8, CC3, CC3, CC3
-	VPALIGNR $12, DD0, DD0, DD0; VPALIGNR $12, DD1, DD1, DD1; VPALIGNR $12, DD2, DD2, DD2; VPALIGNR $12, DD3, DD3, DD3
-	VPADDD   BB0, AA0, AA0; VPADDD BB1, AA1, AA1; VPADDD BB2, AA2, AA2; VPADDD BB3, AA3, AA3
-	polyMulStage3_AVX2
-	VPXOR    AA0, DD0, DD0; VPXOR AA1, DD1, DD1; VPXOR AA2, DD2, DD2; VPXOR AA3, DD3, DD3
-	VPSHUFB  ·rol16<>(SB), DD0, DD0; VPSHUFB ·rol16<>(SB), DD1, DD1; VPSHUFB ·rol16<>(SB), DD2, DD2; VPSHUFB ·rol16<>(SB), DD3, DD3
-	polyMulReduceStage
-	VPADDD   DD0, CC0, CC0; VPADDD DD1, CC1, CC1; VPADDD DD2, CC2, CC2; VPADDD DD3, CC3, CC3
-	VPXOR    CC0, BB0, BB0; VPXOR CC1, BB1, BB1; VPXOR CC2, BB2, BB2; VPXOR CC3, BB3, BB3
-	polyAdd(4*8(oup))
-	LEAQ     (6*8)(oup), oup
-	VMOVDQA  CC3, tmpStoreAVX2
-	VPSLLD   $12, BB0, CC3; VPSRLD $20, BB0, BB0; VPXOR CC3, BB0, BB0
-	VPSLLD   $12, BB1, CC3; VPSRLD $20, BB1, BB1; VPXOR CC3, BB1, BB1
-	VPSLLD   $12, BB2, CC3; VPSRLD $20, BB2, BB2; VPXOR CC3, BB2, BB2
-	VPSLLD   $12, BB3, CC3; VPSRLD $20, BB3, BB3; VPXOR CC3, BB3, BB3
-	VMOVDQA  tmpStoreAVX2, CC3
-	polyMulStage1_AVX2
-	VPADDD   BB0, AA0, AA0; VPADDD BB1, AA1, AA1; VPADDD BB2, AA2, AA2; VPADDD BB3, AA3, AA3
-	VPXOR    AA0, DD0, DD0; VPXOR AA1, DD1, DD1; VPXOR AA2, DD2, DD2; VPXOR AA3, DD3, DD3
-	polyMulStage2_AVX2
-	VPSHUFB  ·rol8<>(SB), DD0, DD0; VPSHUFB ·rol8<>(SB), DD1, DD1; VPSHUFB ·rol8<>(SB), DD2, DD2; VPSHUFB ·rol8<>(SB), DD3, DD3
-	VPADDD   DD0, CC0, CC0; VPADDD DD1, CC1, CC1; VPADDD DD2, CC2, CC2; VPADDD DD3, CC3, CC3
-	polyMulStage3_AVX2
-	VPXOR    CC0, BB0, BB0; VPXOR CC1, BB1, BB1; VPXOR CC2, BB2, BB2; VPXOR CC3, BB3, BB3
-	VMOVDQA  CC3, tmpStoreAVX2
-	VPSLLD   $7, BB0, CC3; VPSRLD $25, BB0, BB0; VPXOR CC3, BB0, BB0
-	VPSLLD   $7, BB1, CC3; VPSRLD $25, BB1, BB1; VPXOR CC3, BB1, BB1
-	VPSLLD   $7, BB2, CC3; VPSRLD $25, BB2, BB2; VPXOR CC3, BB2, BB2
-	VPSLLD   $7, BB3, CC3; VPSRLD $25, BB3, BB3; VPXOR CC3, BB3, BB3
-	VMOVDQA  tmpStoreAVX2, CC3
-	polyMulReduceStage
-	VPALIGNR $12, BB0, BB0, BB0; VPALIGNR $12, BB1, BB1, BB1; VPALIGNR $12, BB2, BB2, BB2; VPALIGNR $12, BB3, BB3, BB3
-	VPALIGNR $8, CC0, CC0, CC0; VPALIGNR $8, CC1, CC1, CC1; VPALIGNR $8, CC2, CC2, CC2; VPALIGNR $8, CC3, CC3, CC3
-	VPALIGNR $4, DD0, DD0, DD0; VPALIGNR $4, DD1, DD1, DD1; VPALIGNR $4, DD2, DD2, DD2; VPALIGNR $4, DD3, DD3, DD3
-	DECQ     itr1
+	VPADDD   Y14, Y0, Y0
+	VPADDD   Y9, Y5, Y5
+	VPADDD   Y10, Y6, Y6
+	VPADDD   Y11, Y7, Y7
+	VPXOR    Y0, Y4, Y4
+	VPXOR    Y5, Y1, Y1
+	VPXOR    Y6, Y2, Y2
+	VPXOR    Y7, Y3, Y3
+	VPSHUFB  ·rol8<>+0(SB), Y4, Y4
+	VPSHUFB  ·rol8<>+0(SB), Y1, Y1
+	VPSHUFB  ·rol8<>+0(SB), Y2, Y2
+	VPSHUFB  ·rol8<>+0(SB), Y3, Y3
+	ADDQ     16(DI), R10
+	ADCQ     24(DI), R11
+	ADCQ     $0x01, R12
+	VPADDD   Y4, Y12, Y12
+	VPADDD   Y1, Y13, Y13
+	VPADDD   Y2, Y8, Y8
+	VPADDD   Y3, Y15, Y15
+	MOVQ     (BP), DX
+	MOVQ     DX, R15
+	MULXQ    R10, R13, R14
+	IMULQ    R12, R15
+	MULXQ    R11, AX, DX
+	ADDQ     AX, R14
+	ADCQ     DX, R15
+	VPXOR    Y12, Y14, Y14
+	VPXOR    Y13, Y9, Y9
+	VPXOR    Y8, Y10, Y10
+	VPXOR    Y15, Y11, Y11
+	VMOVDQA  Y15, 224(BP)
+	VPSLLD   $0x07, Y14, Y15
+	VPSRLD   $0x19, Y14, Y14
+	VPXOR    Y15, Y14, Y14
+	VPSLLD   $0x07, Y9, Y15
+	VPSRLD   $0x19, Y9, Y9
+	VPXOR    Y15, Y9, Y9
+	VPSLLD   $0x07, Y10, Y15
+	VPSRLD   $0x19, Y10, Y10
+	VPXOR    Y15, Y10, Y10
+	VPSLLD   $0x07, Y11, Y15
+	VPSRLD   $0x19, Y11, Y11
+	VPXOR    Y15, Y11, Y11
+	VMOVDQA  224(BP), Y15
+	MOVQ     8(BP), DX
+	MULXQ    R10, R10, AX
+	ADDQ     R10, R14
+	MULXQ    R11, R11, R8
+	ADCQ     R11, R15
+	ADCQ     $0x00, R8
+	VPALIGNR $0x04, Y14, Y14, Y14
+	VPALIGNR $0x04, Y9, Y9, Y9
+	VPALIGNR $0x04, Y10, Y10, Y10
+	VPALIGNR $0x04, Y11, Y11, Y11
+	VPALIGNR $0x08, Y12, Y12, Y12
+	VPALIGNR $0x08, Y13, Y13, Y13
+	VPALIGNR $0x08, Y8, Y8, Y8
+	VPALIGNR $0x08, Y15, Y15, Y15
+	VPALIGNR $0x0c, Y4, Y4, Y4
+	VPALIGNR $0x0c, Y1, Y1, Y1
+	VPALIGNR $0x0c, Y2, Y2, Y2
+	VPALIGNR $0x0c, Y3, Y3, Y3
+	VPADDD   Y14, Y0, Y0
+	VPADDD   Y9, Y5, Y5
+	VPADDD   Y10, Y6, Y6
+	VPADDD   Y11, Y7, Y7
+	IMULQ    R12, DX
+	ADDQ     AX, R15
+	ADCQ     DX, R8
+	VPXOR    Y0, Y4, Y4
+	VPXOR    Y5, Y1, Y1
+	VPXOR    Y6, Y2, Y2
+	VPXOR    Y7, Y3, Y3
+	VPSHUFB  ·rol16<>+0(SB), Y4, Y4
+	VPSHUFB  ·rol16<>+0(SB), Y1, Y1
+	VPSHUFB  ·rol16<>+0(SB), Y2, Y2
+	VPSHUFB  ·rol16<>+0(SB), Y3, Y3
+	MOVQ     R13, R10
+	MOVQ     R14, R11
+	MOVQ     R15, R12
+	ANDQ     $0x03, R12
+	MOVQ     R15, R13
+	ANDQ     $-4, R13
+	MOVQ     R8, R14
+	SHRQ     $0x02, R8, R15
+	SHRQ     $0x02, R8
+	ADDQ     R13, R10
+	ADCQ     R14, R11
+	ADCQ     $0x00, R12
+	ADDQ     R15, R10
+	ADCQ     R8, R11
+	ADCQ     $0x00, R12
+	VPADDD   Y4, Y12, Y12
+	VPADDD   Y1, Y13, Y13
+	VPADDD   Y2, Y8, Y8
+	VPADDD   Y3, Y15, Y15
+	VPXOR    Y12, Y14, Y14
+	VPXOR    Y13, Y9, Y9
+	VPXOR    Y8, Y10, Y10
+	VPXOR    Y15, Y11, Y11
+	ADDQ     32(DI), R10
+	ADCQ     40(DI), R11
+	ADCQ     $0x01, R12
+	LEAQ     48(DI), DI
+	VMOVDQA  Y15, 224(BP)
+	VPSLLD   $0x0c, Y14, Y15
+	VPSRLD   $0x14, Y14, Y14
+	VPXOR    Y15, Y14, Y14
+	VPSLLD   $0x0c, Y9, Y15
+	VPSRLD   $0x14, Y9, Y9
+	VPXOR    Y15, Y9, Y9
+	VPSLLD   $0x0c, Y10, Y15
+	VPSRLD   $0x14, Y10, Y10
+	VPXOR    Y15, Y10, Y10
+	VPSLLD   $0x0c, Y11, Y15
+	VPSRLD   $0x14, Y11, Y11
+	VPXOR    Y15, Y11, Y11
+	VMOVDQA  224(BP), Y15
+	MOVQ     (BP), DX
+	MOVQ     DX, R15
+	MULXQ    R10, R13, R14
+	IMULQ    R12, R15
+	MULXQ    R11, AX, DX
+	ADDQ     AX, R14
+	ADCQ     DX, R15
+	VPADDD   Y14, Y0, Y0
+	VPADDD   Y9, Y5, Y5
+	VPADDD   Y10, Y6, Y6
+	VPADDD   Y11, Y7, Y7
+	VPXOR    Y0, Y4, Y4
+	VPXOR    Y5, Y1, Y1
+	VPXOR    Y6, Y2, Y2
+	VPXOR    Y7, Y3, Y3
+	MOVQ     8(BP), DX
+	MULXQ    R10, R10, AX
+	ADDQ     R10, R14
+	MULXQ    R11, R11, R8
+	ADCQ     R11, R15
+	ADCQ     $0x00, R8
+	VPSHUFB  ·rol8<>+0(SB), Y4, Y4
+	VPSHUFB  ·rol8<>+0(SB), Y1, Y1
+	VPSHUFB  ·rol8<>+0(SB), Y2, Y2
+	VPSHUFB  ·rol8<>+0(SB), Y3, Y3
+	VPADDD   Y4, Y12, Y12
+	VPADDD   Y1, Y13, Y13
+	VPADDD   Y2, Y8, Y8
+	VPADDD   Y3, Y15, Y15
+	IMULQ    R12, DX
+	ADDQ     AX, R15
+	ADCQ     DX, R8
+	VPXOR    Y12, Y14, Y14
+	VPXOR    Y13, Y9, Y9
+	VPXOR    Y8, Y10, Y10
+	VPXOR    Y15, Y11, Y11
+	VMOVDQA  Y15, 224(BP)
+	VPSLLD   $0x07, Y14, Y15
+	VPSRLD   $0x19, Y14, Y14
+	VPXOR    Y15, Y14, Y14
+	VPSLLD   $0x07, Y9, Y15
+	VPSRLD   $0x19, Y9, Y9
+	VPXOR    Y15, Y9, Y9
+	VPSLLD   $0x07, Y10, Y15
+	VPSRLD   $0x19, Y10, Y10
+	VPXOR    Y15, Y10, Y10
+	VPSLLD   $0x07, Y11, Y15
+	VPSRLD   $0x19, Y11, Y11
+	VPXOR    Y15, Y11, Y11
+	VMOVDQA  224(BP), Y15
+	MOVQ     R13, R10
+	MOVQ     R14, R11
+	MOVQ     R15, R12
+	ANDQ     $0x03, R12
+	MOVQ     R15, R13
+	ANDQ     $-4, R13
+	MOVQ     R8, R14
+	SHRQ     $0x02, R8, R15
+	SHRQ     $0x02, R8
+	ADDQ     R13, R10
+	ADCQ     R14, R11
+	ADCQ     $0x00, R12
+	ADDQ     R15, R10
+	ADCQ     R8, R11
+	ADCQ     $0x00, R12
+	VPALIGNR $0x0c, Y14, Y14, Y14
+	VPALIGNR $0x0c, Y9, Y9, Y9
+	VPALIGNR $0x0c, Y10, Y10, Y10
+	VPALIGNR $0x0c, Y11, Y11, Y11
+	VPALIGNR $0x08, Y12, Y12, Y12
+	VPALIGNR $0x08, Y13, Y13, Y13
+	VPALIGNR $0x08, Y8, Y8, Y8
+	VPALIGNR $0x08, Y15, Y15, Y15
+	VPALIGNR $0x04, Y4, Y4, Y4
+	VPALIGNR $0x04, Y1, Y1, Y1
+	VPALIGNR $0x04, Y2, Y2, Y2
+	VPALIGNR $0x04, Y3, Y3, Y3
+	DECQ     CX
 	JNE      sealAVX2InternalLoop
-
-	VPADDD  ·chacha20Constants<>(SB), AA0, AA0; VPADDD ·chacha20Constants<>(SB), AA1, AA1; VPADDD ·chacha20Constants<>(SB), AA2, AA2; VPADDD ·chacha20Constants<>(SB), AA3, AA3
-	VPADDD  state1StoreAVX2, BB0, BB0; VPADDD state1StoreAVX2, BB1, BB1; VPADDD state1StoreAVX2, BB2, BB2; VPADDD state1StoreAVX2, BB3, BB3
-	VPADDD  state2StoreAVX2, CC0, CC0; VPADDD state2StoreAVX2, CC1, CC1; VPADDD state2StoreAVX2, CC2, CC2; VPADDD state2StoreAVX2, CC3, CC3
-	VPADDD  ctr0StoreAVX2, DD0, DD0; VPADDD ctr1StoreAVX2, DD1, DD1; VPADDD ctr2StoreAVX2, DD2, DD2; VPADDD ctr3StoreAVX2, DD3, DD3
-	VMOVDQA CC3, tmpStoreAVX2
+	VPADDD   ·chacha20Constants<>+0(SB), Y0, Y0
+	VPADDD   ·chacha20Constants<>+0(SB), Y5, Y5
+	VPADDD   ·chacha20Constants<>+0(SB), Y6, Y6
+	VPADDD   ·chacha20Constants<>+0(SB), Y7, Y7
+	VPADDD   32(BP), Y14, Y14
+	VPADDD   32(BP), Y9, Y9
+	VPADDD   32(BP), Y10, Y10
+	VPADDD   32(BP), Y11, Y11
+	VPADDD   64(BP), Y12, Y12
+	VPADDD   64(BP), Y13, Y13
+	VPADDD   64(BP), Y8, Y8
+	VPADDD   64(BP), Y15, Y15
+	VPADDD   96(BP), Y4, Y4
+	VPADDD   128(BP), Y1, Y1
+	VPADDD   160(BP), Y2, Y2
+	VPADDD   192(BP), Y3, Y3
+	VMOVDQA  Y15, 224(BP)
 
 	// We only hashed 480 of the 512 bytes available - hash the remaining 32 here
-	polyAdd(0*8(oup))
-	polyMulAVX2
-	LEAQ       (4*8)(oup), oup
-	VPERM2I128 $0x02, AA0, BB0, CC3; VPERM2I128 $0x13, AA0, BB0, BB0; VPERM2I128 $0x02, CC0, DD0, AA0; VPERM2I128 $0x13, CC0, DD0, CC0
-	VPXOR      (0*32)(inp), CC3, CC3; VPXOR (1*32)(inp), AA0, AA0; VPXOR (2*32)(inp), BB0, BB0; VPXOR (3*32)(inp), CC0, CC0
-	VMOVDQU    CC3, (0*32)(oup); VMOVDQU AA0, (1*32)(oup); VMOVDQU BB0, (2*32)(oup); VMOVDQU CC0, (3*32)(oup)
-	VPERM2I128 $0x02, AA1, BB1, AA0; VPERM2I128 $0x02, CC1, DD1, BB0; VPERM2I128 $0x13, AA1, BB1, CC0; VPERM2I128 $0x13, CC1, DD1, DD0
-	VPXOR      (4*32)(inp), AA0, AA0; VPXOR (5*32)(inp), BB0, BB0; VPXOR (6*32)(inp), CC0, CC0; VPXOR (7*32)(inp), DD0, DD0
-	VMOVDQU    AA0, (4*32)(oup); VMOVDQU BB0, (5*32)(oup); VMOVDQU CC0, (6*32)(oup); VMOVDQU DD0, (7*32)(oup)
+	ADDQ       (DI), R10
+	ADCQ       8(DI), R11
+	ADCQ       $0x01, R12
+	MOVQ       (BP), DX
+	MOVQ       DX, R15
+	MULXQ      R10, R13, R14
+	IMULQ      R12, R15
+	MULXQ      R11, AX, DX
+	ADDQ       AX, R14
+	ADCQ       DX, R15
+	MOVQ       8(BP), DX
+	MULXQ      R10, R10, AX
+	ADDQ       R10, R14
+	MULXQ      R11, R11, R8
+	ADCQ       R11, R15
+	ADCQ       $0x00, R8
+	IMULQ      R12, DX
+	ADDQ       AX, R15
+	ADCQ       DX, R8
+	MOVQ       R13, R10
+	MOVQ       R14, R11
+	MOVQ       R15, R12
+	ANDQ       $0x03, R12
+	MOVQ       R15, R13
+	ANDQ       $-4, R13
+	MOVQ       R8, R14
+	SHRQ       $0x02, R8, R15
+	SHRQ       $0x02, R8
+	ADDQ       R13, R10
+	ADCQ       R14, R11
+	ADCQ       $0x00, R12
+	ADDQ       R15, R10
+	ADCQ       R8, R11
+	ADCQ       $0x00, R12
+	LEAQ       32(DI), DI
+	VPERM2I128 $0x02, Y0, Y14, Y15
+	VPERM2I128 $0x13, Y0, Y14, Y14
+	VPERM2I128 $0x02, Y12, Y4, Y0
+	VPERM2I128 $0x13, Y12, Y4, Y12
+	VPXOR      (SI), Y15, Y15
+	VPXOR      32(SI), Y0, Y0
+	VPXOR      64(SI), Y14, Y14
+	VPXOR      96(SI), Y12, Y12
+	VMOVDQU    Y15, (DI)
+	VMOVDQU    Y0, 32(DI)
+	VMOVDQU    Y14, 64(DI)
+	VMOVDQU    Y12, 96(DI)
+	VPERM2I128 $0x02, Y5, Y9, Y0
+	VPERM2I128 $0x02, Y13, Y1, Y14
+	VPERM2I128 $0x13, Y5, Y9, Y12
+	VPERM2I128 $0x13, Y13, Y1, Y4
+	VPXOR      128(SI), Y0, Y0
+	VPXOR      160(SI), Y14, Y14
+	VPXOR      192(SI), Y12, Y12
+	VPXOR      224(SI), Y4, Y4
+	VMOVDQU    Y0, 128(DI)
+	VMOVDQU    Y14, 160(DI)
+	VMOVDQU    Y12, 192(DI)
+	VMOVDQU    Y4, 224(DI)
 
 	// and here
-	polyAdd(-2*8(oup))
-	polyMulAVX2
-	VPERM2I128 $0x02, AA2, BB2, AA0; VPERM2I128 $0x02, CC2, DD2, BB0; VPERM2I128 $0x13, AA2, BB2, CC0; VPERM2I128 $0x13, CC2, DD2, DD0
-	VPXOR      (8*32)(inp), AA0, AA0; VPXOR (9*32)(inp), BB0, BB0; VPXOR (10*32)(inp), CC0, CC0; VPXOR (11*32)(inp), DD0, DD0
-	VMOVDQU    AA0, (8*32)(oup); VMOVDQU BB0, (9*32)(oup); VMOVDQU CC0, (10*32)(oup); VMOVDQU DD0, (11*32)(oup)
-	VPERM2I128 $0x02, AA3, BB3, AA0; VPERM2I128 $0x02, tmpStoreAVX2, DD3, BB0; VPERM2I128 $0x13, AA3, BB3, CC0; VPERM2I128 $0x13, tmpStoreAVX2, DD3, DD0
-	VPXOR      (12*32)(inp), AA0, AA0; VPXOR (13*32)(inp), BB0, BB0; VPXOR (14*32)(inp), CC0, CC0; VPXOR (15*32)(inp), DD0, DD0
-	VMOVDQU    AA0, (12*32)(oup); VMOVDQU BB0, (13*32)(oup); VMOVDQU CC0, (14*32)(oup); VMOVDQU DD0, (15*32)(oup)
-	LEAQ       (32*16)(inp), inp
-	SUBQ       $(32*16), inl
-	CMPQ       inl, $512
+	ADDQ       -16(DI), R10
+	ADCQ       -8(DI), R11
+	ADCQ       $0x01, R12
+	MOVQ       (BP), DX
+	MOVQ       DX, R15
+	MULXQ      R10, R13, R14
+	IMULQ      R12, R15
+	MULXQ      R11, AX, DX
+	ADDQ       AX, R14
+	ADCQ       DX, R15
+	MOVQ       8(BP), DX
+	MULXQ      R10, R10, AX
+	ADDQ       R10, R14
+	MULXQ      R11, R11, R8
+	ADCQ       R11, R15
+	ADCQ       $0x00, R8
+	IMULQ      R12, DX
+	ADDQ       AX, R15
+	ADCQ       DX, R8
+	MOVQ       R13, R10
+	MOVQ       R14, R11
+	MOVQ       R15, R12
+	ANDQ       $0x03, R12
+	MOVQ       R15, R13
+	ANDQ       $-4, R13
+	MOVQ       R8, R14
+	SHRQ       $0x02, R8, R15
+	SHRQ       $0x02, R8
+	ADDQ       R13, R10
+	ADCQ       R14, R11
+	ADCQ       $0x00, R12
+	ADDQ       R15, R10
+	ADCQ       R8, R11
+	ADCQ       $0x00, R12
+	VPERM2I128 $0x02, Y6, Y10, Y0
+	VPERM2I128 $0x02, Y8, Y2, Y14
+	VPERM2I128 $0x13, Y6, Y10, Y12
+	VPERM2I128 $0x13, Y8, Y2, Y4
+	VPXOR      256(SI), Y0, Y0
+	VPXOR      288(SI), Y14, Y14
+	VPXOR      320(SI), Y12, Y12
+	VPXOR      352(SI), Y4, Y4
+	VMOVDQU    Y0, 256(DI)
+	VMOVDQU    Y14, 288(DI)
+	VMOVDQU    Y12, 320(DI)
+	VMOVDQU    Y4, 352(DI)
+	VPERM2I128 $0x02, Y7, Y11, Y0
+	VPERM2I128 $0x02, 224(BP), Y3, Y14
+	VPERM2I128 $0x13, Y7, Y11, Y12
+	VPERM2I128 $0x13, 224(BP), Y3, Y4
+	VPXOR      384(SI), Y0, Y0
+	VPXOR      416(SI), Y14, Y14
+	VPXOR      448(SI), Y12, Y12
+	VPXOR      480(SI), Y4, Y4
+	VMOVDQU    Y0, 384(DI)
+	VMOVDQU    Y14, 416(DI)
+	VMOVDQU    Y12, 448(DI)
+	VMOVDQU    Y4, 480(DI)
+	LEAQ       512(SI), SI
+	SUBQ       $0x00000200, BX
+	CMPQ       BX, $0x00000200
 	JG         sealAVX2MainLoop
 
 	// Tail can only hash 480 bytes
-	polyAdd(0*8(oup))
-	polyMulAVX2
-	polyAdd(2*8(oup))
-	polyMulAVX2
-	LEAQ 32(oup), oup
-
-	MOVQ $10, itr1
-	MOVQ $0, itr2
-	CMPQ inl, $128
-	JBE  sealAVX2Tail128
-	CMPQ inl, $256
-	JBE  sealAVX2Tail256
-	CMPQ inl, $384
-	JBE  sealAVX2Tail384
-	JMP  sealAVX2Tail512
-
-// ----------------------------------------------------------------------------
-// Special optimization for buffers smaller than 193 bytes
+	ADDQ  (DI), R10
+	ADCQ  8(DI), R11
+	ADCQ  $0x01, R12
+	MOVQ  (BP), DX
+	MOVQ  DX, R15
+	MULXQ R10, R13, R14
+	IMULQ R12, R15
+	MULXQ R11, AX, DX
+	ADDQ  AX, R14
+	ADCQ  DX, R15
+	MOVQ  8(BP), DX
+	MULXQ R10, R10, AX
+	ADDQ  R10, R14
+	MULXQ R11, R11, R8
+	ADCQ  R11, R15
+	ADCQ  $0x00, R8
+	IMULQ R12, DX
+	ADDQ  AX, R15
+	ADCQ  DX, R8
+	MOVQ  R13, R10
+	MOVQ  R14, R11
+	MOVQ  R15, R12
+	ANDQ  $0x03, R12
+	MOVQ  R15, R13
+	ANDQ  $-4, R13
+	MOVQ  R8, R14
+	SHRQ  $0x02, R8, R15
+	SHRQ  $0x02, R8
+	ADDQ  R13, R10
+	ADCQ  R14, R11
+	ADCQ  $0x00, R12
+	ADDQ  R15, R10
+	ADCQ  R8, R11
+	ADCQ  $0x00, R12
+	ADDQ  16(DI), R10
+	ADCQ  24(DI), R11
+	ADCQ  $0x01, R12
+	MOVQ  (BP), DX
+	MOVQ  DX, R15
+	MULXQ R10, R13, R14
+	IMULQ R12, R15
+	MULXQ R11, AX, DX
+	ADDQ  AX, R14
+	ADCQ  DX, R15
+	MOVQ  8(BP), DX
+	MULXQ R10, R10, AX
+	ADDQ  R10, R14
+	MULXQ R11, R11, R8
+	ADCQ  R11, R15
+	ADCQ  $0x00, R8
+	IMULQ R12, DX
+	ADDQ  AX, R15
+	ADCQ  DX, R8
+	MOVQ  R13, R10
+	MOVQ  R14, R11
+	MOVQ  R15, R12
+	ANDQ  $0x03, R12
+	MOVQ  R15, R13
+	ANDQ  $-4, R13
+	MOVQ  R8, R14
+	SHRQ  $0x02, R8, R15
+	SHRQ  $0x02, R8
+	ADDQ  R13, R10
+	ADCQ  R14, R11
+	ADCQ  $0x00, R12
+	ADDQ  R15, R10
+	ADCQ  R8, R11
+	ADCQ  $0x00, R12
+	LEAQ  32(DI), DI
+	MOVQ  $0x0000000a, CX
+	MOVQ  $0x00000000, R9
+	CMPQ  BX, $0x80
+	JBE   sealAVX2Tail128
+	CMPQ  BX, $0x00000100
+	JBE   sealAVX2Tail256
+	CMPQ  BX, $0x00000180
+	JBE   sealAVX2Tail384
+	JMP   sealAVX2Tail512
+
 seal192AVX2:
-	// For up to 192 bytes of ciphertext and 64 bytes for the poly key, we process four blocks
-	VMOVDQA AA0, AA1
-	VMOVDQA BB0, BB1
-	VMOVDQA CC0, CC1
-	VPADDD  ·avx2IncMask<>(SB), DD0, DD1
-	VMOVDQA AA0, AA2
-	VMOVDQA BB0, BB2
-	VMOVDQA CC0, CC2
-	VMOVDQA DD0, DD2
-	VMOVDQA DD1, TT3
-	MOVQ    $10, itr2
+	VMOVDQA Y0, Y5
+	VMOVDQA Y14, Y9
+	VMOVDQA Y12, Y13
+	VPADDD  ·avx2IncMask<>+0(SB), Y4, Y1
+	VMOVDQA Y0, Y6
+	VMOVDQA Y14, Y10
+	VMOVDQA Y12, Y8
+	VMOVDQA Y4, Y2
+	VMOVDQA Y1, Y15
+	MOVQ    $0x0000000a, R9
 
 sealAVX2192InnerCipherLoop:
-	chachaQR_AVX2(AA0, BB0, CC0, DD0, TT0); chachaQR_AVX2(AA1, BB1, CC1, DD1, TT0)
-	VPALIGNR   $4, BB0, BB0, BB0; VPALIGNR $4, BB1, BB1, BB1
-	VPALIGNR   $8, CC0, CC0, CC0; VPALIGNR $8, CC1, CC1, CC1
-	VPALIGNR   $12, DD0, DD0, DD0; VPALIGNR $12, DD1, DD1, DD1
-	chachaQR_AVX2(AA0, BB0, CC0, DD0, TT0); chachaQR_AVX2(AA1, BB1, CC1, DD1, TT0)
-	VPALIGNR   $12, BB0, BB0, BB0; VPALIGNR $12, BB1, BB1, BB1
-	VPALIGNR   $8, CC0, CC0, CC0; VPALIGNR $8, CC1, CC1, CC1
-	VPALIGNR   $4, DD0, DD0, DD0; VPALIGNR $4, DD1, DD1, DD1
-	DECQ       itr2
+	VPADDD     Y14, Y0, Y0
+	VPXOR      Y0, Y4, Y4
+	VPSHUFB    ·rol16<>+0(SB), Y4, Y4
+	VPADDD     Y4, Y12, Y12
+	VPXOR      Y12, Y14, Y14
+	VPSLLD     $0x0c, Y14, Y3
+	VPSRLD     $0x14, Y14, Y14
+	VPXOR      Y3, Y14, Y14
+	VPADDD     Y14, Y0, Y0
+	VPXOR      Y0, Y4, Y4
+	VPSHUFB    ·rol8<>+0(SB), Y4, Y4
+	VPADDD     Y4, Y12, Y12
+	VPXOR      Y12, Y14, Y14
+	VPSLLD     $0x07, Y14, Y3
+	VPSRLD     $0x19, Y14, Y14
+	VPXOR      Y3, Y14, Y14
+	VPADDD     Y9, Y5, Y5
+	VPXOR      Y5, Y1, Y1
+	VPSHUFB    ·rol16<>+0(SB), Y1, Y1
+	VPADDD     Y1, Y13, Y13
+	VPXOR      Y13, Y9, Y9
+	VPSLLD     $0x0c, Y9, Y3
+	VPSRLD     $0x14, Y9, Y9
+	VPXOR      Y3, Y9, Y9
+	VPADDD     Y9, Y5, Y5
+	VPXOR      Y5, Y1, Y1
+	VPSHUFB    ·rol8<>+0(SB), Y1, Y1
+	VPADDD     Y1, Y13, Y13
+	VPXOR      Y13, Y9, Y9
+	VPSLLD     $0x07, Y9, Y3
+	VPSRLD     $0x19, Y9, Y9
+	VPXOR      Y3, Y9, Y9
+	VPALIGNR   $0x04, Y14, Y14, Y14
+	VPALIGNR   $0x04, Y9, Y9, Y9
+	VPALIGNR   $0x08, Y12, Y12, Y12
+	VPALIGNR   $0x08, Y13, Y13, Y13
+	VPALIGNR   $0x0c, Y4, Y4, Y4
+	VPALIGNR   $0x0c, Y1, Y1, Y1
+	VPADDD     Y14, Y0, Y0
+	VPXOR      Y0, Y4, Y4
+	VPSHUFB    ·rol16<>+0(SB), Y4, Y4
+	VPADDD     Y4, Y12, Y12
+	VPXOR      Y12, Y14, Y14
+	VPSLLD     $0x0c, Y14, Y3
+	VPSRLD     $0x14, Y14, Y14
+	VPXOR      Y3, Y14, Y14
+	VPADDD     Y14, Y0, Y0
+	VPXOR      Y0, Y4, Y4
+	VPSHUFB    ·rol8<>+0(SB), Y4, Y4
+	VPADDD     Y4, Y12, Y12
+	VPXOR      Y12, Y14, Y14
+	VPSLLD     $0x07, Y14, Y3
+	VPSRLD     $0x19, Y14, Y14
+	VPXOR      Y3, Y14, Y14
+	VPADDD     Y9, Y5, Y5
+	VPXOR      Y5, Y1, Y1
+	VPSHUFB    ·rol16<>+0(SB), Y1, Y1
+	VPADDD     Y1, Y13, Y13
+	VPXOR      Y13, Y9, Y9
+	VPSLLD     $0x0c, Y9, Y3
+	VPSRLD     $0x14, Y9, Y9
+	VPXOR      Y3, Y9, Y9
+	VPADDD     Y9, Y5, Y5
+	VPXOR      Y5, Y1, Y1
+	VPSHUFB    ·rol8<>+0(SB), Y1, Y1
+	VPADDD     Y1, Y13, Y13
+	VPXOR      Y13, Y9, Y9
+	VPSLLD     $0x07, Y9, Y3
+	VPSRLD     $0x19, Y9, Y9
+	VPXOR      Y3, Y9, Y9
+	VPALIGNR   $0x0c, Y14, Y14, Y14
+	VPALIGNR   $0x0c, Y9, Y9, Y9
+	VPALIGNR   $0x08, Y12, Y12, Y12
+	VPALIGNR   $0x08, Y13, Y13, Y13
+	VPALIGNR   $0x04, Y4, Y4, Y4
+	VPALIGNR   $0x04, Y1, Y1, Y1
+	DECQ       R9
 	JNE        sealAVX2192InnerCipherLoop
-	VPADDD     AA2, AA0, AA0; VPADDD AA2, AA1, AA1
-	VPADDD     BB2, BB0, BB0; VPADDD BB2, BB1, BB1
-	VPADDD     CC2, CC0, CC0; VPADDD CC2, CC1, CC1
-	VPADDD     DD2, DD0, DD0; VPADDD TT3, DD1, DD1
-	VPERM2I128 $0x02, AA0, BB0, TT0
+	VPADDD     Y6, Y0, Y0
+	VPADDD     Y6, Y5, Y5
+	VPADDD     Y10, Y14, Y14
+	VPADDD     Y10, Y9, Y9
+	VPADDD     Y8, Y12, Y12
+	VPADDD     Y8, Y13, Y13
+	VPADDD     Y2, Y4, Y4
+	VPADDD     Y15, Y1, Y1
+	VPERM2I128 $0x02, Y0, Y14, Y3
 
 	// Clamp and store poly key
-	VPAND   ·polyClampMask<>(SB), TT0, TT0
-	VMOVDQA TT0, rsStoreAVX2
+	VPAND   ·polyClampMask<>+0(SB), Y3, Y3
+	VMOVDQA Y3, (BP)
 
 	// Stream for up to 192 bytes
-	VPERM2I128 $0x13, AA0, BB0, AA0
-	VPERM2I128 $0x13, CC0, DD0, BB0
-	VPERM2I128 $0x02, AA1, BB1, CC0
-	VPERM2I128 $0x02, CC1, DD1, DD0
-	VPERM2I128 $0x13, AA1, BB1, AA1
-	VPERM2I128 $0x13, CC1, DD1, BB1
+	VPERM2I128 $0x13, Y0, Y14, Y0
+	VPERM2I128 $0x13, Y12, Y4, Y14
+	VPERM2I128 $0x02, Y5, Y9, Y12
+	VPERM2I128 $0x02, Y13, Y1, Y4
+	VPERM2I128 $0x13, Y5, Y9, Y5
+	VPERM2I128 $0x13, Y13, Y1, Y9
 
 sealAVX2ShortSeal:
 	// Hash aad
-	MOVQ ad_len+80(FP), itr2
+	MOVQ ad_len+80(FP), R9
 	CALL polyHashADInternal<>(SB)
-	XORQ itr1, itr1
+	XORQ CX, CX
 
 sealAVX2SealHash:
 	// itr1 holds the number of bytes encrypted but not yet hashed
-	CMPQ itr1, $16
-	JB   sealAVX2ShortSealLoop
-	polyAdd(0(oup))
-	polyMul
-	SUBQ $16, itr1
-	ADDQ $16, oup
-	JMP  sealAVX2SealHash
+	CMPQ  CX, $0x10
+	JB    sealAVX2ShortSealLoop
+	ADDQ  (DI), R10
+	ADCQ  8(DI), R11
+	ADCQ  $0x01, R12
+	MOVQ  (BP), AX
+	MOVQ  AX, R15
+	MULQ  R10
+	MOVQ  AX, R13
+	MOVQ  DX, R14
+	MOVQ  (BP), AX
+	MULQ  R11
+	IMULQ R12, R15
+	ADDQ  AX, R14
+	ADCQ  DX, R15
+	MOVQ  8(BP), AX
+	MOVQ  AX, R8
+	MULQ  R10
+	ADDQ  AX, R14
+	ADCQ  $0x00, DX
+	MOVQ  DX, R10
+	MOVQ  8(BP), AX
+	MULQ  R11
+	ADDQ  AX, R15
+	ADCQ  $0x00, DX
+	IMULQ R12, R8
+	ADDQ  R10, R15
+	ADCQ  DX, R8
+	MOVQ  R13, R10
+	MOVQ  R14, R11
+	MOVQ  R15, R12
+	ANDQ  $0x03, R12
+	MOVQ  R15, R13
+	ANDQ  $-4, R13
+	MOVQ  R8, R14
+	SHRQ  $0x02, R8, R15
+	SHRQ  $0x02, R8
+	ADDQ  R13, R10
+	ADCQ  R14, R11
+	ADCQ  $0x00, R12
+	ADDQ  R15, R10
+	ADCQ  R8, R11
+	ADCQ  $0x00, R12
+	SUBQ  $0x10, CX
+	ADDQ  $0x10, DI
+	JMP   sealAVX2SealHash
 
 sealAVX2ShortSealLoop:
-	CMPQ inl, $32
+	CMPQ BX, $0x20
 	JB   sealAVX2ShortTail32
-	SUBQ $32, inl
+	SUBQ $0x20, BX
 
 	// Load for encryption
-	VPXOR   (inp), AA0, AA0
-	VMOVDQU AA0, (oup)
-	LEAQ    (1*32)(inp), inp
+	VPXOR   (SI), Y0, Y0
+	VMOVDQU Y0, (DI)
+	LEAQ    32(SI), SI
 
 	// Now can hash
-	polyAdd(0*8(oup))
-	polyMulAVX2
-	polyAdd(2*8(oup))
-	polyMulAVX2
-	LEAQ (1*32)(oup), oup
+	ADDQ  (DI), R10
+	ADCQ  8(DI), R11
+	ADCQ  $0x01, R12
+	MOVQ  (BP), DX
+	MOVQ  DX, R15
+	MULXQ R10, R13, R14
+	IMULQ R12, R15
+	MULXQ R11, AX, DX
+	ADDQ  AX, R14
+	ADCQ  DX, R15
+	MOVQ  8(BP), DX
+	MULXQ R10, R10, AX
+	ADDQ  R10, R14
+	MULXQ R11, R11, R8
+	ADCQ  R11, R15
+	ADCQ  $0x00, R8
+	IMULQ R12, DX
+	ADDQ  AX, R15
+	ADCQ  DX, R8
+	MOVQ  R13, R10
+	MOVQ  R14, R11
+	MOVQ  R15, R12
+	ANDQ  $0x03, R12
+	MOVQ  R15, R13
+	ANDQ  $-4, R13
+	MOVQ  R8, R14
+	SHRQ  $0x02, R8, R15
+	SHRQ  $0x02, R8
+	ADDQ  R13, R10
+	ADCQ  R14, R11
+	ADCQ  $0x00, R12
+	ADDQ  R15, R10
+	ADCQ  R8, R11
+	ADCQ  $0x00, R12
+	ADDQ  16(DI), R10
+	ADCQ  24(DI), R11
+	ADCQ  $0x01, R12
+	MOVQ  (BP), DX
+	MOVQ  DX, R15
+	MULXQ R10, R13, R14
+	IMULQ R12, R15
+	MULXQ R11, AX, DX
+	ADDQ  AX, R14
+	ADCQ  DX, R15
+	MOVQ  8(BP), DX
+	MULXQ R10, R10, AX
+	ADDQ  R10, R14
+	MULXQ R11, R11, R8
+	ADCQ  R11, R15
+	ADCQ  $0x00, R8
+	IMULQ R12, DX
+	ADDQ  AX, R15
+	ADCQ  DX, R8
+	MOVQ  R13, R10
+	MOVQ  R14, R11
+	MOVQ  R15, R12
+	ANDQ  $0x03, R12
+	MOVQ  R15, R13
+	ANDQ  $-4, R13
+	MOVQ  R8, R14
+	SHRQ  $0x02, R8, R15
+	SHRQ  $0x02, R8
+	ADDQ  R13, R10
+	ADCQ  R14, R11
+	ADCQ  $0x00, R12
+	ADDQ  R15, R10
+	ADCQ  R8, R11
+	ADCQ  $0x00, R12
+	LEAQ  32(DI), DI
 
 	// Shift stream left
-	VMOVDQA BB0, AA0
-	VMOVDQA CC0, BB0
-	VMOVDQA DD0, CC0
-	VMOVDQA AA1, DD0
-	VMOVDQA BB1, AA1
-	VMOVDQA CC1, BB1
-	VMOVDQA DD1, CC1
-	VMOVDQA AA2, DD1
-	VMOVDQA BB2, AA2
+	VMOVDQA Y14, Y0
+	VMOVDQA Y12, Y14
+	VMOVDQA Y4, Y12
+	VMOVDQA Y5, Y4
+	VMOVDQA Y9, Y5
+	VMOVDQA Y13, Y9
+	VMOVDQA Y1, Y13
+	VMOVDQA Y6, Y1
+	VMOVDQA Y10, Y6
 	JMP     sealAVX2ShortSealLoop
 
 sealAVX2ShortTail32:
-	CMPQ    inl, $16
-	VMOVDQA A0, A1
+	CMPQ    BX, $0x10
+	VMOVDQA X0, X1
 	JB      sealAVX2ShortDone
-
-	SUBQ $16, inl
+	SUBQ    $0x10, BX
 
 	// Load for encryption
-	VPXOR   (inp), A0, T0
-	VMOVDQU T0, (oup)
-	LEAQ    (1*16)(inp), inp
+	VPXOR   (SI), X0, X12
+	VMOVDQU X12, (DI)
+	LEAQ    16(SI), SI
 
 	// Hash
-	polyAdd(0*8(oup))
-	polyMulAVX2
-	LEAQ       (1*16)(oup), oup
-	VPERM2I128 $0x11, AA0, AA0, AA0
-	VMOVDQA    A0, A1
+	ADDQ       (DI), R10
+	ADCQ       8(DI), R11
+	ADCQ       $0x01, R12
+	MOVQ       (BP), DX
+	MOVQ       DX, R15
+	MULXQ      R10, R13, R14
+	IMULQ      R12, R15
+	MULXQ      R11, AX, DX
+	ADDQ       AX, R14
+	ADCQ       DX, R15
+	MOVQ       8(BP), DX
+	MULXQ      R10, R10, AX
+	ADDQ       R10, R14
+	MULXQ      R11, R11, R8
+	ADCQ       R11, R15
+	ADCQ       $0x00, R8
+	IMULQ      R12, DX
+	ADDQ       AX, R15
+	ADCQ       DX, R8
+	MOVQ       R13, R10
+	MOVQ       R14, R11
+	MOVQ       R15, R12
+	ANDQ       $0x03, R12
+	MOVQ       R15, R13
+	ANDQ       $-4, R13
+	MOVQ       R8, R14
+	SHRQ       $0x02, R8, R15
+	SHRQ       $0x02, R8
+	ADDQ       R13, R10
+	ADCQ       R14, R11
+	ADCQ       $0x00, R12
+	ADDQ       R15, R10
+	ADCQ       R8, R11
+	ADCQ       $0x00, R12
+	LEAQ       16(DI), DI
+	VPERM2I128 $0x11, Y0, Y0, Y0
+	VMOVDQA    X0, X1
 
 sealAVX2ShortDone:
 	VZEROUPPER
 	JMP sealSSETail
 
-// ----------------------------------------------------------------------------
-// Special optimization for buffers smaller than 321 bytes
 seal320AVX2:
-	// For up to 320 bytes of ciphertext and 64 bytes for the poly key, we process six blocks
-	VMOVDQA AA0, AA1; VMOVDQA BB0, BB1; VMOVDQA CC0, CC1; VPADDD ·avx2IncMask<>(SB), DD0, DD1
-	VMOVDQA AA0, AA2; VMOVDQA BB0, BB2; VMOVDQA CC0, CC2; VPADDD ·avx2IncMask<>(SB), DD1, DD2
-	VMOVDQA BB0, TT1; VMOVDQA CC0, TT2; VMOVDQA DD0, TT3
-	MOVQ    $10, itr2
+	VMOVDQA Y0, Y5
+	VMOVDQA Y14, Y9
+	VMOVDQA Y12, Y13
+	VPADDD  ·avx2IncMask<>+0(SB), Y4, Y1
+	VMOVDQA Y0, Y6
+	VMOVDQA Y14, Y10
+	VMOVDQA Y12, Y8
+	VPADDD  ·avx2IncMask<>+0(SB), Y1, Y2
+	VMOVDQA Y14, Y7
+	VMOVDQA Y12, Y11
+	VMOVDQA Y4, Y15
+	MOVQ    $0x0000000a, R9
 
 sealAVX2320InnerCipherLoop:
-	chachaQR_AVX2(AA0, BB0, CC0, DD0, TT0); chachaQR_AVX2(AA1, BB1, CC1, DD1, TT0); chachaQR_AVX2(AA2, BB2, CC2, DD2, TT0)
-	VPALIGNR $4, BB0, BB0, BB0; VPALIGNR $4, BB1, BB1, BB1; VPALIGNR $4, BB2, BB2, BB2
-	VPALIGNR $8, CC0, CC0, CC0; VPALIGNR $8, CC1, CC1, CC1; VPALIGNR $8, CC2, CC2, CC2
-	VPALIGNR $12, DD0, DD0, DD0; VPALIGNR $12, DD1, DD1, DD1; VPALIGNR $12, DD2, DD2, DD2
-	chachaQR_AVX2(AA0, BB0, CC0, DD0, TT0); chachaQR_AVX2(AA1, BB1, CC1, DD1, TT0); chachaQR_AVX2(AA2, BB2, CC2, DD2, TT0)
-	VPALIGNR $12, BB0, BB0, BB0; VPALIGNR $12, BB1, BB1, BB1; VPALIGNR $12, BB2, BB2, BB2
-	VPALIGNR $8, CC0, CC0, CC0; VPALIGNR $8, CC1, CC1, CC1; VPALIGNR $8, CC2, CC2, CC2
-	VPALIGNR $4, DD0, DD0, DD0; VPALIGNR $4, DD1, DD1, DD1; VPALIGNR $4, DD2, DD2, DD2
-	DECQ     itr2
+	VPADDD   Y14, Y0, Y0
+	VPXOR    Y0, Y4, Y4
+	VPSHUFB  ·rol16<>+0(SB), Y4, Y4
+	VPADDD   Y4, Y12, Y12
+	VPXOR    Y12, Y14, Y14
+	VPSLLD   $0x0c, Y14, Y3
+	VPSRLD   $0x14, Y14, Y14
+	VPXOR    Y3, Y14, Y14
+	VPADDD   Y14, Y0, Y0
+	VPXOR    Y0, Y4, Y4
+	VPSHUFB  ·rol8<>+0(SB), Y4, Y4
+	VPADDD   Y4, Y12, Y12
+	VPXOR    Y12, Y14, Y14
+	VPSLLD   $0x07, Y14, Y3
+	VPSRLD   $0x19, Y14, Y14
+	VPXOR    Y3, Y14, Y14
+	VPADDD   Y9, Y5, Y5
+	VPXOR    Y5, Y1, Y1
+	VPSHUFB  ·rol16<>+0(SB), Y1, Y1
+	VPADDD   Y1, Y13, Y13
+	VPXOR    Y13, Y9, Y9
+	VPSLLD   $0x0c, Y9, Y3
+	VPSRLD   $0x14, Y9, Y9
+	VPXOR    Y3, Y9, Y9
+	VPADDD   Y9, Y5, Y5
+	VPXOR    Y5, Y1, Y1
+	VPSHUFB  ·rol8<>+0(SB), Y1, Y1
+	VPADDD   Y1, Y13, Y13
+	VPXOR    Y13, Y9, Y9
+	VPSLLD   $0x07, Y9, Y3
+	VPSRLD   $0x19, Y9, Y9
+	VPXOR    Y3, Y9, Y9
+	VPADDD   Y10, Y6, Y6
+	VPXOR    Y6, Y2, Y2
+	VPSHUFB  ·rol16<>+0(SB), Y2, Y2
+	VPADDD   Y2, Y8, Y8
+	VPXOR    Y8, Y10, Y10
+	VPSLLD   $0x0c, Y10, Y3
+	VPSRLD   $0x14, Y10, Y10
+	VPXOR    Y3, Y10, Y10
+	VPADDD   Y10, Y6, Y6
+	VPXOR    Y6, Y2, Y2
+	VPSHUFB  ·rol8<>+0(SB), Y2, Y2
+	VPADDD   Y2, Y8, Y8
+	VPXOR    Y8, Y10, Y10
+	VPSLLD   $0x07, Y10, Y3
+	VPSRLD   $0x19, Y10, Y10
+	VPXOR    Y3, Y10, Y10
+	VPALIGNR $0x04, Y14, Y14, Y14
+	VPALIGNR $0x04, Y9, Y9, Y9
+	VPALIGNR $0x04, Y10, Y10, Y10
+	VPALIGNR $0x08, Y12, Y12, Y12
+	VPALIGNR $0x08, Y13, Y13, Y13
+	VPALIGNR $0x08, Y8, Y8, Y8
+	VPALIGNR $0x0c, Y4, Y4, Y4
+	VPALIGNR $0x0c, Y1, Y1, Y1
+	VPALIGNR $0x0c, Y2, Y2, Y2
+	VPADDD   Y14, Y0, Y0
+	VPXOR    Y0, Y4, Y4
+	VPSHUFB  ·rol16<>+0(SB), Y4, Y4
+	VPADDD   Y4, Y12, Y12
+	VPXOR    Y12, Y14, Y14
+	VPSLLD   $0x0c, Y14, Y3
+	VPSRLD   $0x14, Y14, Y14
+	VPXOR    Y3, Y14, Y14
+	VPADDD   Y14, Y0, Y0
+	VPXOR    Y0, Y4, Y4
+	VPSHUFB  ·rol8<>+0(SB), Y4, Y4
+	VPADDD   Y4, Y12, Y12
+	VPXOR    Y12, Y14, Y14
+	VPSLLD   $0x07, Y14, Y3
+	VPSRLD   $0x19, Y14, Y14
+	VPXOR    Y3, Y14, Y14
+	VPADDD   Y9, Y5, Y5
+	VPXOR    Y5, Y1, Y1
+	VPSHUFB  ·rol16<>+0(SB), Y1, Y1
+	VPADDD   Y1, Y13, Y13
+	VPXOR    Y13, Y9, Y9
+	VPSLLD   $0x0c, Y9, Y3
+	VPSRLD   $0x14, Y9, Y9
+	VPXOR    Y3, Y9, Y9
+	VPADDD   Y9, Y5, Y5
+	VPXOR    Y5, Y1, Y1
+	VPSHUFB  ·rol8<>+0(SB), Y1, Y1
+	VPADDD   Y1, Y13, Y13
+	VPXOR    Y13, Y9, Y9
+	VPSLLD   $0x07, Y9, Y3
+	VPSRLD   $0x19, Y9, Y9
+	VPXOR    Y3, Y9, Y9
+	VPADDD   Y10, Y6, Y6
+	VPXOR    Y6, Y2, Y2
+	VPSHUFB  ·rol16<>+0(SB), Y2, Y2
+	VPADDD   Y2, Y8, Y8
+	VPXOR    Y8, Y10, Y10
+	VPSLLD   $0x0c, Y10, Y3
+	VPSRLD   $0x14, Y10, Y10
+	VPXOR    Y3, Y10, Y10
+	VPADDD   Y10, Y6, Y6
+	VPXOR    Y6, Y2, Y2
+	VPSHUFB  ·rol8<>+0(SB), Y2, Y2
+	VPADDD   Y2, Y8, Y8
+	VPXOR    Y8, Y10, Y10
+	VPSLLD   $0x07, Y10, Y3
+	VPSRLD   $0x19, Y10, Y10
+	VPXOR    Y3, Y10, Y10
+	VPALIGNR $0x0c, Y14, Y14, Y14
+	VPALIGNR $0x0c, Y9, Y9, Y9
+	VPALIGNR $0x0c, Y10, Y10, Y10
+	VPALIGNR $0x08, Y12, Y12, Y12
+	VPALIGNR $0x08, Y13, Y13, Y13
+	VPALIGNR $0x08, Y8, Y8, Y8
+	VPALIGNR $0x04, Y4, Y4, Y4
+	VPALIGNR $0x04, Y1, Y1, Y1
+	VPALIGNR $0x04, Y2, Y2, Y2
+	DECQ     R9
 	JNE      sealAVX2320InnerCipherLoop
-
-	VMOVDQA ·chacha20Constants<>(SB), TT0
-	VPADDD  TT0, AA0, AA0; VPADDD TT0, AA1, AA1; VPADDD TT0, AA2, AA2
-	VPADDD  TT1, BB0, BB0; VPADDD TT1, BB1, BB1; VPADDD TT1, BB2, BB2
-	VPADDD  TT2, CC0, CC0; VPADDD TT2, CC1, CC1; VPADDD TT2, CC2, CC2
-	VMOVDQA ·avx2IncMask<>(SB), TT0
-	VPADDD  TT3, DD0, DD0; VPADDD TT0, TT3, TT3
-	VPADDD  TT3, DD1, DD1; VPADDD TT0, TT3, TT3
-	VPADDD  TT3, DD2, DD2
+	VMOVDQA  ·chacha20Constants<>+0(SB), Y3
+	VPADDD   Y3, Y0, Y0
+	VPADDD   Y3, Y5, Y5
+	VPADDD   Y3, Y6, Y6
+	VPADDD   Y7, Y14, Y14
+	VPADDD   Y7, Y9, Y9
+	VPADDD   Y7, Y10, Y10
+	VPADDD   Y11, Y12, Y12
+	VPADDD   Y11, Y13, Y13
+	VPADDD   Y11, Y8, Y8
+	VMOVDQA  ·avx2IncMask<>+0(SB), Y3
+	VPADDD   Y15, Y4, Y4
+	VPADDD   Y3, Y15, Y15
+	VPADDD   Y15, Y1, Y1
+	VPADDD   Y3, Y15, Y15
+	VPADDD   Y15, Y2, Y2
 
 	// Clamp and store poly key
-	VPERM2I128 $0x02, AA0, BB0, TT0
-	VPAND      ·polyClampMask<>(SB), TT0, TT0
-	VMOVDQA    TT0, rsStoreAVX2
+	VPERM2I128 $0x02, Y0, Y14, Y3
+	VPAND      ·polyClampMask<>+0(SB), Y3, Y3
+	VMOVDQA    Y3, (BP)
 
 	// Stream for up to 320 bytes
-	VPERM2I128 $0x13, AA0, BB0, AA0
-	VPERM2I128 $0x13, CC0, DD0, BB0
-	VPERM2I128 $0x02, AA1, BB1, CC0
-	VPERM2I128 $0x02, CC1, DD1, DD0
-	VPERM2I128 $0x13, AA1, BB1, AA1
-	VPERM2I128 $0x13, CC1, DD1, BB1
-	VPERM2I128 $0x02, AA2, BB2, CC1
-	VPERM2I128 $0x02, CC2, DD2, DD1
-	VPERM2I128 $0x13, AA2, BB2, AA2
-	VPERM2I128 $0x13, CC2, DD2, BB2
+	VPERM2I128 $0x13, Y0, Y14, Y0
+	VPERM2I128 $0x13, Y12, Y4, Y14
+	VPERM2I128 $0x02, Y5, Y9, Y12
+	VPERM2I128 $0x02, Y13, Y1, Y4
+	VPERM2I128 $0x13, Y5, Y9, Y5
+	VPERM2I128 $0x13, Y13, Y1, Y9
+	VPERM2I128 $0x02, Y6, Y10, Y13
+	VPERM2I128 $0x02, Y8, Y2, Y1
+	VPERM2I128 $0x13, Y6, Y10, Y6
+	VPERM2I128 $0x13, Y8, Y2, Y10
 	JMP        sealAVX2ShortSeal
 
-// ----------------------------------------------------------------------------
-// Special optimization for the last 128 bytes of ciphertext
 sealAVX2Tail128:
-	// Need to decrypt up to 128 bytes - prepare two blocks
-	// If we got here after the main loop - there are 512 encrypted bytes waiting to be hashed
-	// If we got here before the main loop - there are 448 encrpyred bytes waiting to be hashed
-	VMOVDQA ·chacha20Constants<>(SB), AA0
-	VMOVDQA state1StoreAVX2, BB0
-	VMOVDQA state2StoreAVX2, CC0
-	VMOVDQA ctr3StoreAVX2, DD0
-	VPADDD  ·avx2IncMask<>(SB), DD0, DD0
-	VMOVDQA DD0, DD1
+	VMOVDQA ·chacha20Constants<>+0(SB), Y0
+	VMOVDQA 32(BP), Y14
+	VMOVDQA 64(BP), Y12
+	VMOVDQA 192(BP), Y4
+	VPADDD  ·avx2IncMask<>+0(SB), Y4, Y4
+	VMOVDQA Y4, Y1
 
 sealAVX2Tail128LoopA:
-	polyAdd(0(oup))
-	polyMul
-	LEAQ 16(oup), oup
+	ADDQ  (DI), R10
+	ADCQ  8(DI), R11
+	ADCQ  $0x01, R12
+	MOVQ  (BP), AX
+	MOVQ  AX, R15
+	MULQ  R10
+	MOVQ  AX, R13
+	MOVQ  DX, R14
+	MOVQ  (BP), AX
+	MULQ  R11
+	IMULQ R12, R15
+	ADDQ  AX, R14
+	ADCQ  DX, R15
+	MOVQ  8(BP), AX
+	MOVQ  AX, R8
+	MULQ  R10
+	ADDQ  AX, R14
+	ADCQ  $0x00, DX
+	MOVQ  DX, R10
+	MOVQ  8(BP), AX
+	MULQ  R11
+	ADDQ  AX, R15
+	ADCQ  $0x00, DX
+	IMULQ R12, R8
+	ADDQ  R10, R15
+	ADCQ  DX, R8
+	MOVQ  R13, R10
+	MOVQ  R14, R11
+	MOVQ  R15, R12
+	ANDQ  $0x03, R12
+	MOVQ  R15, R13
+	ANDQ  $-4, R13
+	MOVQ  R8, R14
+	SHRQ  $0x02, R8, R15
+	SHRQ  $0x02, R8
+	ADDQ  R13, R10
+	ADCQ  R14, R11
+	ADCQ  $0x00, R12
+	ADDQ  R15, R10
+	ADCQ  R8, R11
+	ADCQ  $0x00, R12
+	LEAQ  16(DI), DI
 
 sealAVX2Tail128LoopB:
-	chachaQR_AVX2(AA0, BB0, CC0, DD0, TT0)
-	polyAdd(0(oup))
-	polyMul
-	VPALIGNR $4, BB0, BB0, BB0
-	VPALIGNR $8, CC0, CC0, CC0
-	VPALIGNR $12, DD0, DD0, DD0
-	chachaQR_AVX2(AA0, BB0, CC0, DD0, TT0)
-	polyAdd(16(oup))
-	polyMul
-	LEAQ     32(oup), oup
-	VPALIGNR $12, BB0, BB0, BB0
-	VPALIGNR $8, CC0, CC0, CC0
-	VPALIGNR $4, DD0, DD0, DD0
-	DECQ     itr1
-	JG       sealAVX2Tail128LoopA
-	DECQ     itr2
-	JGE      sealAVX2Tail128LoopB
-
-	VPADDD ·chacha20Constants<>(SB), AA0, AA1
-	VPADDD state1StoreAVX2, BB0, BB1
-	VPADDD state2StoreAVX2, CC0, CC1
-	VPADDD DD1, DD0, DD1
-
-	VPERM2I128 $0x02, AA1, BB1, AA0
-	VPERM2I128 $0x02, CC1, DD1, BB0
-	VPERM2I128 $0x13, AA1, BB1, CC0
-	VPERM2I128 $0x13, CC1, DD1, DD0
+	VPADDD     Y14, Y0, Y0
+	VPXOR      Y0, Y4, Y4
+	VPSHUFB    ·rol16<>+0(SB), Y4, Y4
+	VPADDD     Y4, Y12, Y12
+	VPXOR      Y12, Y14, Y14
+	VPSLLD     $0x0c, Y14, Y3
+	VPSRLD     $0x14, Y14, Y14
+	VPXOR      Y3, Y14, Y14
+	VPADDD     Y14, Y0, Y0
+	VPXOR      Y0, Y4, Y4
+	VPSHUFB    ·rol8<>+0(SB), Y4, Y4
+	VPADDD     Y4, Y12, Y12
+	VPXOR      Y12, Y14, Y14
+	VPSLLD     $0x07, Y14, Y3
+	VPSRLD     $0x19, Y14, Y14
+	VPXOR      Y3, Y14, Y14
+	ADDQ       (DI), R10
+	ADCQ       8(DI), R11
+	ADCQ       $0x01, R12
+	MOVQ       (BP), AX
+	MOVQ       AX, R15
+	MULQ       R10
+	MOVQ       AX, R13
+	MOVQ       DX, R14
+	MOVQ       (BP), AX
+	MULQ       R11
+	IMULQ      R12, R15
+	ADDQ       AX, R14
+	ADCQ       DX, R15
+	MOVQ       8(BP), AX
+	MOVQ       AX, R8
+	MULQ       R10
+	ADDQ       AX, R14
+	ADCQ       $0x00, DX
+	MOVQ       DX, R10
+	MOVQ       8(BP), AX
+	MULQ       R11
+	ADDQ       AX, R15
+	ADCQ       $0x00, DX
+	IMULQ      R12, R8
+	ADDQ       R10, R15
+	ADCQ       DX, R8
+	MOVQ       R13, R10
+	MOVQ       R14, R11
+	MOVQ       R15, R12
+	ANDQ       $0x03, R12
+	MOVQ       R15, R13
+	ANDQ       $-4, R13
+	MOVQ       R8, R14
+	SHRQ       $0x02, R8, R15
+	SHRQ       $0x02, R8
+	ADDQ       R13, R10
+	ADCQ       R14, R11
+	ADCQ       $0x00, R12
+	ADDQ       R15, R10
+	ADCQ       R8, R11
+	ADCQ       $0x00, R12
+	VPALIGNR   $0x04, Y14, Y14, Y14
+	VPALIGNR   $0x08, Y12, Y12, Y12
+	VPALIGNR   $0x0c, Y4, Y4, Y4
+	VPADDD     Y14, Y0, Y0
+	VPXOR      Y0, Y4, Y4
+	VPSHUFB    ·rol16<>+0(SB), Y4, Y4
+	VPADDD     Y4, Y12, Y12
+	VPXOR      Y12, Y14, Y14
+	VPSLLD     $0x0c, Y14, Y3
+	VPSRLD     $0x14, Y14, Y14
+	VPXOR      Y3, Y14, Y14
+	VPADDD     Y14, Y0, Y0
+	VPXOR      Y0, Y4, Y4
+	VPSHUFB    ·rol8<>+0(SB), Y4, Y4
+	VPADDD     Y4, Y12, Y12
+	VPXOR      Y12, Y14, Y14
+	VPSLLD     $0x07, Y14, Y3
+	VPSRLD     $0x19, Y14, Y14
+	VPXOR      Y3, Y14, Y14
+	ADDQ       16(DI), R10
+	ADCQ       24(DI), R11
+	ADCQ       $0x01, R12
+	MOVQ       (BP), AX
+	MOVQ       AX, R15
+	MULQ       R10
+	MOVQ       AX, R13
+	MOVQ       DX, R14
+	MOVQ       (BP), AX
+	MULQ       R11
+	IMULQ      R12, R15
+	ADDQ       AX, R14
+	ADCQ       DX, R15
+	MOVQ       8(BP), AX
+	MOVQ       AX, R8
+	MULQ       R10
+	ADDQ       AX, R14
+	ADCQ       $0x00, DX
+	MOVQ       DX, R10
+	MOVQ       8(BP), AX
+	MULQ       R11
+	ADDQ       AX, R15
+	ADCQ       $0x00, DX
+	IMULQ      R12, R8
+	ADDQ       R10, R15
+	ADCQ       DX, R8
+	MOVQ       R13, R10
+	MOVQ       R14, R11
+	MOVQ       R15, R12
+	ANDQ       $0x03, R12
+	MOVQ       R15, R13
+	ANDQ       $-4, R13
+	MOVQ       R8, R14
+	SHRQ       $0x02, R8, R15
+	SHRQ       $0x02, R8
+	ADDQ       R13, R10
+	ADCQ       R14, R11
+	ADCQ       $0x00, R12
+	ADDQ       R15, R10
+	ADCQ       R8, R11
+	ADCQ       $0x00, R12
+	LEAQ       32(DI), DI
+	VPALIGNR   $0x0c, Y14, Y14, Y14
+	VPALIGNR   $0x08, Y12, Y12, Y12
+	VPALIGNR   $0x04, Y4, Y4, Y4
+	DECQ       CX
+	JG         sealAVX2Tail128LoopA
+	DECQ       R9
+	JGE        sealAVX2Tail128LoopB
+	VPADDD     ·chacha20Constants<>+0(SB), Y0, Y5
+	VPADDD     32(BP), Y14, Y9
+	VPADDD     64(BP), Y12, Y13
+	VPADDD     Y1, Y4, Y1
+	VPERM2I128 $0x02, Y5, Y9, Y0
+	VPERM2I128 $0x02, Y13, Y1, Y14
+	VPERM2I128 $0x13, Y5, Y9, Y12
+	VPERM2I128 $0x13, Y13, Y1, Y4
 	JMP        sealAVX2ShortSealLoop
 
-// ----------------------------------------------------------------------------
-// Special optimization for the last 256 bytes of ciphertext
 sealAVX2Tail256:
-	// Need to decrypt up to 256 bytes - prepare two blocks
-	// If we got here after the main loop - there are 512 encrypted bytes waiting to be hashed
-	// If we got here before the main loop - there are 448 encrpyred bytes waiting to be hashed
-	VMOVDQA ·chacha20Constants<>(SB), AA0; VMOVDQA ·chacha20Constants<>(SB), AA1
-	VMOVDQA state1StoreAVX2, BB0; VMOVDQA state1StoreAVX2, BB1
-	VMOVDQA state2StoreAVX2, CC0; VMOVDQA state2StoreAVX2, CC1
-	VMOVDQA ctr3StoreAVX2, DD0
-	VPADDD  ·avx2IncMask<>(SB), DD0, DD0
-	VPADDD  ·avx2IncMask<>(SB), DD0, DD1
-	VMOVDQA DD0, TT1
-	VMOVDQA DD1, TT2
+	VMOVDQA ·chacha20Constants<>+0(SB), Y0
+	VMOVDQA ·chacha20Constants<>+0(SB), Y5
+	VMOVDQA 32(BP), Y14
+	VMOVDQA 32(BP), Y9
+	VMOVDQA 64(BP), Y12
+	VMOVDQA 64(BP), Y13
+	VMOVDQA 192(BP), Y4
+	VPADDD  ·avx2IncMask<>+0(SB), Y4, Y4
+	VPADDD  ·avx2IncMask<>+0(SB), Y4, Y1
+	VMOVDQA Y4, Y7
+	VMOVDQA Y1, Y11
 
 sealAVX2Tail256LoopA:
-	polyAdd(0(oup))
-	polyMul
-	LEAQ 16(oup), oup
+	ADDQ  (DI), R10
+	ADCQ  8(DI), R11
+	ADCQ  $0x01, R12
+	MOVQ  (BP), AX
+	MOVQ  AX, R15
+	MULQ  R10
+	MOVQ  AX, R13
+	MOVQ  DX, R14
+	MOVQ  (BP), AX
+	MULQ  R11
+	IMULQ R12, R15
+	ADDQ  AX, R14
+	ADCQ  DX, R15
+	MOVQ  8(BP), AX
+	MOVQ  AX, R8
+	MULQ  R10
+	ADDQ  AX, R14
+	ADCQ  $0x00, DX
+	MOVQ  DX, R10
+	MOVQ  8(BP), AX
+	MULQ  R11
+	ADDQ  AX, R15
+	ADCQ  $0x00, DX
+	IMULQ R12, R8
+	ADDQ  R10, R15
+	ADCQ  DX, R8
+	MOVQ  R13, R10
+	MOVQ  R14, R11
+	MOVQ  R15, R12
+	ANDQ  $0x03, R12
+	MOVQ  R15, R13
+	ANDQ  $-4, R13
+	MOVQ  R8, R14
+	SHRQ  $0x02, R8, R15
+	SHRQ  $0x02, R8
+	ADDQ  R13, R10
+	ADCQ  R14, R11
+	ADCQ  $0x00, R12
+	ADDQ  R15, R10
+	ADCQ  R8, R11
+	ADCQ  $0x00, R12
+	LEAQ  16(DI), DI
 
 sealAVX2Tail256LoopB:
-	chachaQR_AVX2(AA0, BB0, CC0, DD0, TT0); chachaQR_AVX2(AA1, BB1, CC1, DD1, TT0)
-	polyAdd(0(oup))
-	polyMul
-	VPALIGNR $4, BB0, BB0, BB0; VPALIGNR $4, BB1, BB1, BB1
-	VPALIGNR $8, CC0, CC0, CC0; VPALIGNR $8, CC1, CC1, CC1
-	VPALIGNR $12, DD0, DD0, DD0; VPALIGNR $12, DD1, DD1, DD1
-	chachaQR_AVX2(AA0, BB0, CC0, DD0, TT0); chachaQR_AVX2(AA1, BB1, CC1, DD1, TT0)
-	polyAdd(16(oup))
-	polyMul
-	LEAQ     32(oup), oup
-	VPALIGNR $12, BB0, BB0, BB0; VPALIGNR $12, BB1, BB1, BB1
-	VPALIGNR $8, CC0, CC0, CC0; VPALIGNR $8, CC1, CC1, CC1
-	VPALIGNR $4, DD0, DD0, DD0; VPALIGNR $4, DD1, DD1, DD1
-	DECQ     itr1
-	JG       sealAVX2Tail256LoopA
-	DECQ     itr2
-	JGE      sealAVX2Tail256LoopB
-
-	VPADDD     ·chacha20Constants<>(SB), AA0, AA0; VPADDD ·chacha20Constants<>(SB), AA1, AA1
-	VPADDD     state1StoreAVX2, BB0, BB0; VPADDD state1StoreAVX2, BB1, BB1
-	VPADDD     state2StoreAVX2, CC0, CC0; VPADDD state2StoreAVX2, CC1, CC1
-	VPADDD     TT1, DD0, DD0; VPADDD TT2, DD1, DD1
-	VPERM2I128 $0x02, AA0, BB0, TT0
-	VPERM2I128 $0x02, CC0, DD0, TT1
-	VPERM2I128 $0x13, AA0, BB0, TT2
-	VPERM2I128 $0x13, CC0, DD0, TT3
-	VPXOR      (0*32)(inp), TT0, TT0; VPXOR (1*32)(inp), TT1, TT1; VPXOR (2*32)(inp), TT2, TT2; VPXOR (3*32)(inp), TT3, TT3
-	VMOVDQU    TT0, (0*32)(oup); VMOVDQU TT1, (1*32)(oup); VMOVDQU TT2, (2*32)(oup); VMOVDQU TT3, (3*32)(oup)
-	MOVQ       $128, itr1
-	LEAQ       128(inp), inp
-	SUBQ       $128, inl
-	VPERM2I128 $0x02, AA1, BB1, AA0
-	VPERM2I128 $0x02, CC1, DD1, BB0
-	VPERM2I128 $0x13, AA1, BB1, CC0
-	VPERM2I128 $0x13, CC1, DD1, DD0
-
-	JMP sealAVX2SealHash
-
-// ----------------------------------------------------------------------------
-// Special optimization for the last 384 bytes of ciphertext
+	VPADDD     Y14, Y0, Y0
+	VPXOR      Y0, Y4, Y4
+	VPSHUFB    ·rol16<>+0(SB), Y4, Y4
+	VPADDD     Y4, Y12, Y12
+	VPXOR      Y12, Y14, Y14
+	VPSLLD     $0x0c, Y14, Y3
+	VPSRLD     $0x14, Y14, Y14
+	VPXOR      Y3, Y14, Y14
+	VPADDD     Y14, Y0, Y0
+	VPXOR      Y0, Y4, Y4
+	VPSHUFB    ·rol8<>+0(SB), Y4, Y4
+	VPADDD     Y4, Y12, Y12
+	VPXOR      Y12, Y14, Y14
+	VPSLLD     $0x07, Y14, Y3
+	VPSRLD     $0x19, Y14, Y14
+	VPXOR      Y3, Y14, Y14
+	VPADDD     Y9, Y5, Y5
+	VPXOR      Y5, Y1, Y1
+	VPSHUFB    ·rol16<>+0(SB), Y1, Y1
+	VPADDD     Y1, Y13, Y13
+	VPXOR      Y13, Y9, Y9
+	VPSLLD     $0x0c, Y9, Y3
+	VPSRLD     $0x14, Y9, Y9
+	VPXOR      Y3, Y9, Y9
+	VPADDD     Y9, Y5, Y5
+	VPXOR      Y5, Y1, Y1
+	VPSHUFB    ·rol8<>+0(SB), Y1, Y1
+	VPADDD     Y1, Y13, Y13
+	VPXOR      Y13, Y9, Y9
+	VPSLLD     $0x07, Y9, Y3
+	VPSRLD     $0x19, Y9, Y9
+	VPXOR      Y3, Y9, Y9
+	ADDQ       (DI), R10
+	ADCQ       8(DI), R11
+	ADCQ       $0x01, R12
+	MOVQ       (BP), AX
+	MOVQ       AX, R15
+	MULQ       R10
+	MOVQ       AX, R13
+	MOVQ       DX, R14
+	MOVQ       (BP), AX
+	MULQ       R11
+	IMULQ      R12, R15
+	ADDQ       AX, R14
+	ADCQ       DX, R15
+	MOVQ       8(BP), AX
+	MOVQ       AX, R8
+	MULQ       R10
+	ADDQ       AX, R14
+	ADCQ       $0x00, DX
+	MOVQ       DX, R10
+	MOVQ       8(BP), AX
+	MULQ       R11
+	ADDQ       AX, R15
+	ADCQ       $0x00, DX
+	IMULQ      R12, R8
+	ADDQ       R10, R15
+	ADCQ       DX, R8
+	MOVQ       R13, R10
+	MOVQ       R14, R11
+	MOVQ       R15, R12
+	ANDQ       $0x03, R12
+	MOVQ       R15, R13
+	ANDQ       $-4, R13
+	MOVQ       R8, R14
+	SHRQ       $0x02, R8, R15
+	SHRQ       $0x02, R8
+	ADDQ       R13, R10
+	ADCQ       R14, R11
+	ADCQ       $0x00, R12
+	ADDQ       R15, R10
+	ADCQ       R8, R11
+	ADCQ       $0x00, R12
+	VPALIGNR   $0x04, Y14, Y14, Y14
+	VPALIGNR   $0x04, Y9, Y9, Y9
+	VPALIGNR   $0x08, Y12, Y12, Y12
+	VPALIGNR   $0x08, Y13, Y13, Y13
+	VPALIGNR   $0x0c, Y4, Y4, Y4
+	VPALIGNR   $0x0c, Y1, Y1, Y1
+	VPADDD     Y14, Y0, Y0
+	VPXOR      Y0, Y4, Y4
+	VPSHUFB    ·rol16<>+0(SB), Y4, Y4
+	VPADDD     Y4, Y12, Y12
+	VPXOR      Y12, Y14, Y14
+	VPSLLD     $0x0c, Y14, Y3
+	VPSRLD     $0x14, Y14, Y14
+	VPXOR      Y3, Y14, Y14
+	VPADDD     Y14, Y0, Y0
+	VPXOR      Y0, Y4, Y4
+	VPSHUFB    ·rol8<>+0(SB), Y4, Y4
+	VPADDD     Y4, Y12, Y12
+	VPXOR      Y12, Y14, Y14
+	VPSLLD     $0x07, Y14, Y3
+	VPSRLD     $0x19, Y14, Y14
+	VPXOR      Y3, Y14, Y14
+	VPADDD     Y9, Y5, Y5
+	VPXOR      Y5, Y1, Y1
+	VPSHUFB    ·rol16<>+0(SB), Y1, Y1
+	VPADDD     Y1, Y13, Y13
+	VPXOR      Y13, Y9, Y9
+	VPSLLD     $0x0c, Y9, Y3
+	VPSRLD     $0x14, Y9, Y9
+	VPXOR      Y3, Y9, Y9
+	VPADDD     Y9, Y5, Y5
+	VPXOR      Y5, Y1, Y1
+	VPSHUFB    ·rol8<>+0(SB), Y1, Y1
+	VPADDD     Y1, Y13, Y13
+	VPXOR      Y13, Y9, Y9
+	VPSLLD     $0x07, Y9, Y3
+	VPSRLD     $0x19, Y9, Y9
+	VPXOR      Y3, Y9, Y9
+	ADDQ       16(DI), R10
+	ADCQ       24(DI), R11
+	ADCQ       $0x01, R12
+	MOVQ       (BP), AX
+	MOVQ       AX, R15
+	MULQ       R10
+	MOVQ       AX, R13
+	MOVQ       DX, R14
+	MOVQ       (BP), AX
+	MULQ       R11
+	IMULQ      R12, R15
+	ADDQ       AX, R14
+	ADCQ       DX, R15
+	MOVQ       8(BP), AX
+	MOVQ       AX, R8
+	MULQ       R10
+	ADDQ       AX, R14
+	ADCQ       $0x00, DX
+	MOVQ       DX, R10
+	MOVQ       8(BP), AX
+	MULQ       R11
+	ADDQ       AX, R15
+	ADCQ       $0x00, DX
+	IMULQ      R12, R8
+	ADDQ       R10, R15
+	ADCQ       DX, R8
+	MOVQ       R13, R10
+	MOVQ       R14, R11
+	MOVQ       R15, R12
+	ANDQ       $0x03, R12
+	MOVQ       R15, R13
+	ANDQ       $-4, R13
+	MOVQ       R8, R14
+	SHRQ       $0x02, R8, R15
+	SHRQ       $0x02, R8
+	ADDQ       R13, R10
+	ADCQ       R14, R11
+	ADCQ       $0x00, R12
+	ADDQ       R15, R10
+	ADCQ       R8, R11
+	ADCQ       $0x00, R12
+	LEAQ       32(DI), DI
+	VPALIGNR   $0x0c, Y14, Y14, Y14
+	VPALIGNR   $0x0c, Y9, Y9, Y9
+	VPALIGNR   $0x08, Y12, Y12, Y12
+	VPALIGNR   $0x08, Y13, Y13, Y13
+	VPALIGNR   $0x04, Y4, Y4, Y4
+	VPALIGNR   $0x04, Y1, Y1, Y1
+	DECQ       CX
+	JG         sealAVX2Tail256LoopA
+	DECQ       R9
+	JGE        sealAVX2Tail256LoopB
+	VPADDD     ·chacha20Constants<>+0(SB), Y0, Y0
+	VPADDD     ·chacha20Constants<>+0(SB), Y5, Y5
+	VPADDD     32(BP), Y14, Y14
+	VPADDD     32(BP), Y9, Y9
+	VPADDD     64(BP), Y12, Y12
+	VPADDD     64(BP), Y13, Y13
+	VPADDD     Y7, Y4, Y4
+	VPADDD     Y11, Y1, Y1
+	VPERM2I128 $0x02, Y0, Y14, Y3
+	VPERM2I128 $0x02, Y12, Y4, Y7
+	VPERM2I128 $0x13, Y0, Y14, Y11
+	VPERM2I128 $0x13, Y12, Y4, Y15
+	VPXOR      (SI), Y3, Y3
+	VPXOR      32(SI), Y7, Y7
+	VPXOR      64(SI), Y11, Y11
+	VPXOR      96(SI), Y15, Y15
+	VMOVDQU    Y3, (DI)
+	VMOVDQU    Y7, 32(DI)
+	VMOVDQU    Y11, 64(DI)
+	VMOVDQU    Y15, 96(DI)
+	MOVQ       $0x00000080, CX
+	LEAQ       128(SI), SI
+	SUBQ       $0x80, BX
+	VPERM2I128 $0x02, Y5, Y9, Y0
+	VPERM2I128 $0x02, Y13, Y1, Y14
+	VPERM2I128 $0x13, Y5, Y9, Y12
+	VPERM2I128 $0x13, Y13, Y1, Y4
+	JMP        sealAVX2SealHash
+
 sealAVX2Tail384:
-	// Need to decrypt up to 384 bytes - prepare two blocks
-	// If we got here after the main loop - there are 512 encrypted bytes waiting to be hashed
-	// If we got here before the main loop - there are 448 encrpyred bytes waiting to be hashed
-	VMOVDQA ·chacha20Constants<>(SB), AA0; VMOVDQA AA0, AA1; VMOVDQA AA0, AA2
-	VMOVDQA state1StoreAVX2, BB0; VMOVDQA BB0, BB1; VMOVDQA BB0, BB2
-	VMOVDQA state2StoreAVX2, CC0; VMOVDQA CC0, CC1; VMOVDQA CC0, CC2
-	VMOVDQA ctr3StoreAVX2, DD0
-	VPADDD  ·avx2IncMask<>(SB), DD0, DD0; VPADDD ·avx2IncMask<>(SB), DD0, DD1; VPADDD ·avx2IncMask<>(SB), DD1, DD2
-	VMOVDQA DD0, TT1; VMOVDQA DD1, TT2; VMOVDQA DD2, TT3
+	VMOVDQA ·chacha20Constants<>+0(SB), Y0
+	VMOVDQA Y0, Y5
+	VMOVDQA Y0, Y6
+	VMOVDQA 32(BP), Y14
+	VMOVDQA Y14, Y9
+	VMOVDQA Y14, Y10
+	VMOVDQA 64(BP), Y12
+	VMOVDQA Y12, Y13
+	VMOVDQA Y12, Y8
+	VMOVDQA 192(BP), Y4
+	VPADDD  ·avx2IncMask<>+0(SB), Y4, Y4
+	VPADDD  ·avx2IncMask<>+0(SB), Y4, Y1
+	VPADDD  ·avx2IncMask<>+0(SB), Y1, Y2
+	VMOVDQA Y4, Y7
+	VMOVDQA Y1, Y11
+	VMOVDQA Y2, Y15
 
 sealAVX2Tail384LoopA:
-	polyAdd(0(oup))
-	polyMul
-	LEAQ 16(oup), oup
+	ADDQ  (DI), R10
+	ADCQ  8(DI), R11
+	ADCQ  $0x01, R12
+	MOVQ  (BP), AX
+	MOVQ  AX, R15
+	MULQ  R10
+	MOVQ  AX, R13
+	MOVQ  DX, R14
+	MOVQ  (BP), AX
+	MULQ  R11
+	IMULQ R12, R15
+	ADDQ  AX, R14
+	ADCQ  DX, R15
+	MOVQ  8(BP), AX
+	MOVQ  AX, R8
+	MULQ  R10
+	ADDQ  AX, R14
+	ADCQ  $0x00, DX
+	MOVQ  DX, R10
+	MOVQ  8(BP), AX
+	MULQ  R11
+	ADDQ  AX, R15
+	ADCQ  $0x00, DX
+	IMULQ R12, R8
+	ADDQ  R10, R15
+	ADCQ  DX, R8
+	MOVQ  R13, R10
+	MOVQ  R14, R11
+	MOVQ  R15, R12
+	ANDQ  $0x03, R12
+	MOVQ  R15, R13
+	ANDQ  $-4, R13
+	MOVQ  R8, R14
+	SHRQ  $0x02, R8, R15
+	SHRQ  $0x02, R8
+	ADDQ  R13, R10
+	ADCQ  R14, R11
+	ADCQ  $0x00, R12
+	ADDQ  R15, R10
+	ADCQ  R8, R11
+	ADCQ  $0x00, R12
+	LEAQ  16(DI), DI
 
 sealAVX2Tail384LoopB:
-	chachaQR_AVX2(AA0, BB0, CC0, DD0, TT0); chachaQR_AVX2(AA1, BB1, CC1, DD1, TT0); chachaQR_AVX2(AA2, BB2, CC2, DD2, TT0)
-	polyAdd(0(oup))
-	polyMul
-	VPALIGNR $4, BB0, BB0, BB0; VPALIGNR $4, BB1, BB1, BB1; VPALIGNR $4, BB2, BB2, BB2
-	VPALIGNR $8, CC0, CC0, CC0; VPALIGNR $8, CC1, CC1, CC1; VPALIGNR $8, CC2, CC2, CC2
-	VPALIGNR $12, DD0, DD0, DD0; VPALIGNR $12, DD1, DD1, DD1; VPALIGNR $12, DD2, DD2, DD2
-	chachaQR_AVX2(AA0, BB0, CC0, DD0, TT0); chachaQR_AVX2(AA1, BB1, CC1, DD1, TT0); chachaQR_AVX2(AA2, BB2, CC2, DD2, TT0)
-	polyAdd(16(oup))
-	polyMul
-	LEAQ     32(oup), oup
-	VPALIGNR $12, BB0, BB0, BB0; VPALIGNR $12, BB1, BB1, BB1; VPALIGNR $12, BB2, BB2, BB2
-	VPALIGNR $8, CC0, CC0, CC0; VPALIGNR $8, CC1, CC1, CC1; VPALIGNR $8, CC2, CC2, CC2
-	VPALIGNR $4, DD0, DD0, DD0; VPALIGNR $4, DD1, DD1, DD1; VPALIGNR $4, DD2, DD2, DD2
-	DECQ     itr1
-	JG       sealAVX2Tail384LoopA
-	DECQ     itr2
-	JGE      sealAVX2Tail384LoopB
-
-	VPADDD     ·chacha20Constants<>(SB), AA0, AA0; VPADDD ·chacha20Constants<>(SB), AA1, AA1; VPADDD ·chacha20Constants<>(SB), AA2, AA2
-	VPADDD     state1StoreAVX2, BB0, BB0; VPADDD state1StoreAVX2, BB1, BB1; VPADDD state1StoreAVX2, BB2, BB2
-	VPADDD     state2StoreAVX2, CC0, CC0; VPADDD state2StoreAVX2, CC1, CC1; VPADDD state2StoreAVX2, CC2, CC2
-	VPADDD     TT1, DD0, DD0; VPADDD TT2, DD1, DD1; VPADDD TT3, DD2, DD2
-	VPERM2I128 $0x02, AA0, BB0, TT0
-	VPERM2I128 $0x02, CC0, DD0, TT1
-	VPERM2I128 $0x13, AA0, BB0, TT2
-	VPERM2I128 $0x13, CC0, DD0, TT3
-	VPXOR      (0*32)(inp), TT0, TT0; VPXOR (1*32)(inp), TT1, TT1; VPXOR (2*32)(inp), TT2, TT2; VPXOR (3*32)(inp), TT3, TT3
-	VMOVDQU    TT0, (0*32)(oup); VMOVDQU TT1, (1*32)(oup); VMOVDQU TT2, (2*32)(oup); VMOVDQU TT3, (3*32)(oup)
-	VPERM2I128 $0x02, AA1, BB1, TT0
-	VPERM2I128 $0x02, CC1, DD1, TT1
-	VPERM2I128 $0x13, AA1, BB1, TT2
-	VPERM2I128 $0x13, CC1, DD1, TT3
-	VPXOR      (4*32)(inp), TT0, TT0; VPXOR (5*32)(inp), TT1, TT1; VPXOR (6*32)(inp), TT2, TT2; VPXOR (7*32)(inp), TT3, TT3
-	VMOVDQU    TT0, (4*32)(oup); VMOVDQU TT1, (5*32)(oup); VMOVDQU TT2, (6*32)(oup); VMOVDQU TT3, (7*32)(oup)
-	MOVQ       $256, itr1
-	LEAQ       256(inp), inp
-	SUBQ       $256, inl
-	VPERM2I128 $0x02, AA2, BB2, AA0
-	VPERM2I128 $0x02, CC2, DD2, BB0
-	VPERM2I128 $0x13, AA2, BB2, CC0
-	VPERM2I128 $0x13, CC2, DD2, DD0
-
-	JMP sealAVX2SealHash
-
-// ----------------------------------------------------------------------------
-// Special optimization for the last 512 bytes of ciphertext
+	VPADDD     Y14, Y0, Y0
+	VPXOR      Y0, Y4, Y4
+	VPSHUFB    ·rol16<>+0(SB), Y4, Y4
+	VPADDD     Y4, Y12, Y12
+	VPXOR      Y12, Y14, Y14
+	VPSLLD     $0x0c, Y14, Y3
+	VPSRLD     $0x14, Y14, Y14
+	VPXOR      Y3, Y14, Y14
+	VPADDD     Y14, Y0, Y0
+	VPXOR      Y0, Y4, Y4
+	VPSHUFB    ·rol8<>+0(SB), Y4, Y4
+	VPADDD     Y4, Y12, Y12
+	VPXOR      Y12, Y14, Y14
+	VPSLLD     $0x07, Y14, Y3
+	VPSRLD     $0x19, Y14, Y14
+	VPXOR      Y3, Y14, Y14
+	VPADDD     Y9, Y5, Y5
+	VPXOR      Y5, Y1, Y1
+	VPSHUFB    ·rol16<>+0(SB), Y1, Y1
+	VPADDD     Y1, Y13, Y13
+	VPXOR      Y13, Y9, Y9
+	VPSLLD     $0x0c, Y9, Y3
+	VPSRLD     $0x14, Y9, Y9
+	VPXOR      Y3, Y9, Y9
+	VPADDD     Y9, Y5, Y5
+	VPXOR      Y5, Y1, Y1
+	VPSHUFB    ·rol8<>+0(SB), Y1, Y1
+	VPADDD     Y1, Y13, Y13
+	VPXOR      Y13, Y9, Y9
+	VPSLLD     $0x07, Y9, Y3
+	VPSRLD     $0x19, Y9, Y9
+	VPXOR      Y3, Y9, Y9
+	VPADDD     Y10, Y6, Y6
+	VPXOR      Y6, Y2, Y2
+	VPSHUFB    ·rol16<>+0(SB), Y2, Y2
+	VPADDD     Y2, Y8, Y8
+	VPXOR      Y8, Y10, Y10
+	VPSLLD     $0x0c, Y10, Y3
+	VPSRLD     $0x14, Y10, Y10
+	VPXOR      Y3, Y10, Y10
+	VPADDD     Y10, Y6, Y6
+	VPXOR      Y6, Y2, Y2
+	VPSHUFB    ·rol8<>+0(SB), Y2, Y2
+	VPADDD     Y2, Y8, Y8
+	VPXOR      Y8, Y10, Y10
+	VPSLLD     $0x07, Y10, Y3
+	VPSRLD     $0x19, Y10, Y10
+	VPXOR      Y3, Y10, Y10
+	ADDQ       (DI), R10
+	ADCQ       8(DI), R11
+	ADCQ       $0x01, R12
+	MOVQ       (BP), AX
+	MOVQ       AX, R15
+	MULQ       R10
+	MOVQ       AX, R13
+	MOVQ       DX, R14
+	MOVQ       (BP), AX
+	MULQ       R11
+	IMULQ      R12, R15
+	ADDQ       AX, R14
+	ADCQ       DX, R15
+	MOVQ       8(BP), AX
+	MOVQ       AX, R8
+	MULQ       R10
+	ADDQ       AX, R14
+	ADCQ       $0x00, DX
+	MOVQ       DX, R10
+	MOVQ       8(BP), AX
+	MULQ       R11
+	ADDQ       AX, R15
+	ADCQ       $0x00, DX
+	IMULQ      R12, R8
+	ADDQ       R10, R15
+	ADCQ       DX, R8
+	MOVQ       R13, R10
+	MOVQ       R14, R11
+	MOVQ       R15, R12
+	ANDQ       $0x03, R12
+	MOVQ       R15, R13
+	ANDQ       $-4, R13
+	MOVQ       R8, R14
+	SHRQ       $0x02, R8, R15
+	SHRQ       $0x02, R8
+	ADDQ       R13, R10
+	ADCQ       R14, R11
+	ADCQ       $0x00, R12
+	ADDQ       R15, R10
+	ADCQ       R8, R11
+	ADCQ       $0x00, R12
+	VPALIGNR   $0x04, Y14, Y14, Y14
+	VPALIGNR   $0x04, Y9, Y9, Y9
+	VPALIGNR   $0x04, Y10, Y10, Y10
+	VPALIGNR   $0x08, Y12, Y12, Y12
+	VPALIGNR   $0x08, Y13, Y13, Y13
+	VPALIGNR   $0x08, Y8, Y8, Y8
+	VPALIGNR   $0x0c, Y4, Y4, Y4
+	VPALIGNR   $0x0c, Y1, Y1, Y1
+	VPALIGNR   $0x0c, Y2, Y2, Y2
+	VPADDD     Y14, Y0, Y0
+	VPXOR      Y0, Y4, Y4
+	VPSHUFB    ·rol16<>+0(SB), Y4, Y4
+	VPADDD     Y4, Y12, Y12
+	VPXOR      Y12, Y14, Y14
+	VPSLLD     $0x0c, Y14, Y3
+	VPSRLD     $0x14, Y14, Y14
+	VPXOR      Y3, Y14, Y14
+	VPADDD     Y14, Y0, Y0
+	VPXOR      Y0, Y4, Y4
+	VPSHUFB    ·rol8<>+0(SB), Y4, Y4
+	VPADDD     Y4, Y12, Y12
+	VPXOR      Y12, Y14, Y14
+	VPSLLD     $0x07, Y14, Y3
+	VPSRLD     $0x19, Y14, Y14
+	VPXOR      Y3, Y14, Y14
+	VPADDD     Y9, Y5, Y5
+	VPXOR      Y5, Y1, Y1
+	VPSHUFB    ·rol16<>+0(SB), Y1, Y1
+	VPADDD     Y1, Y13, Y13
+	VPXOR      Y13, Y9, Y9
+	VPSLLD     $0x0c, Y9, Y3
+	VPSRLD     $0x14, Y9, Y9
+	VPXOR      Y3, Y9, Y9
+	VPADDD     Y9, Y5, Y5
+	VPXOR      Y5, Y1, Y1
+	VPSHUFB    ·rol8<>+0(SB), Y1, Y1
+	VPADDD     Y1, Y13, Y13
+	VPXOR      Y13, Y9, Y9
+	VPSLLD     $0x07, Y9, Y3
+	VPSRLD     $0x19, Y9, Y9
+	VPXOR      Y3, Y9, Y9
+	VPADDD     Y10, Y6, Y6
+	VPXOR      Y6, Y2, Y2
+	VPSHUFB    ·rol16<>+0(SB), Y2, Y2
+	VPADDD     Y2, Y8, Y8
+	VPXOR      Y8, Y10, Y10
+	VPSLLD     $0x0c, Y10, Y3
+	VPSRLD     $0x14, Y10, Y10
+	VPXOR      Y3, Y10, Y10
+	VPADDD     Y10, Y6, Y6
+	VPXOR      Y6, Y2, Y2
+	VPSHUFB    ·rol8<>+0(SB), Y2, Y2
+	VPADDD     Y2, Y8, Y8
+	VPXOR      Y8, Y10, Y10
+	VPSLLD     $0x07, Y10, Y3
+	VPSRLD     $0x19, Y10, Y10
+	VPXOR      Y3, Y10, Y10
+	ADDQ       16(DI), R10
+	ADCQ       24(DI), R11
+	ADCQ       $0x01, R12
+	MOVQ       (BP), AX
+	MOVQ       AX, R15
+	MULQ       R10
+	MOVQ       AX, R13
+	MOVQ       DX, R14
+	MOVQ       (BP), AX
+	MULQ       R11
+	IMULQ      R12, R15
+	ADDQ       AX, R14
+	ADCQ       DX, R15
+	MOVQ       8(BP), AX
+	MOVQ       AX, R8
+	MULQ       R10
+	ADDQ       AX, R14
+	ADCQ       $0x00, DX
+	MOVQ       DX, R10
+	MOVQ       8(BP), AX
+	MULQ       R11
+	ADDQ       AX, R15
+	ADCQ       $0x00, DX
+	IMULQ      R12, R8
+	ADDQ       R10, R15
+	ADCQ       DX, R8
+	MOVQ       R13, R10
+	MOVQ       R14, R11
+	MOVQ       R15, R12
+	ANDQ       $0x03, R12
+	MOVQ       R15, R13
+	ANDQ       $-4, R13
+	MOVQ       R8, R14
+	SHRQ       $0x02, R8, R15
+	SHRQ       $0x02, R8
+	ADDQ       R13, R10
+	ADCQ       R14, R11
+	ADCQ       $0x00, R12
+	ADDQ       R15, R10
+	ADCQ       R8, R11
+	ADCQ       $0x00, R12
+	LEAQ       32(DI), DI
+	VPALIGNR   $0x0c, Y14, Y14, Y14
+	VPALIGNR   $0x0c, Y9, Y9, Y9
+	VPALIGNR   $0x0c, Y10, Y10, Y10
+	VPALIGNR   $0x08, Y12, Y12, Y12
+	VPALIGNR   $0x08, Y13, Y13, Y13
+	VPALIGNR   $0x08, Y8, Y8, Y8
+	VPALIGNR   $0x04, Y4, Y4, Y4
+	VPALIGNR   $0x04, Y1, Y1, Y1
+	VPALIGNR   $0x04, Y2, Y2, Y2
+	DECQ       CX
+	JG         sealAVX2Tail384LoopA
+	DECQ       R9
+	JGE        sealAVX2Tail384LoopB
+	VPADDD     ·chacha20Constants<>+0(SB), Y0, Y0
+	VPADDD     ·chacha20Constants<>+0(SB), Y5, Y5
+	VPADDD     ·chacha20Constants<>+0(SB), Y6, Y6
+	VPADDD     32(BP), Y14, Y14
+	VPADDD     32(BP), Y9, Y9
+	VPADDD     32(BP), Y10, Y10
+	VPADDD     64(BP), Y12, Y12
+	VPADDD     64(BP), Y13, Y13
+	VPADDD     64(BP), Y8, Y8
+	VPADDD     Y7, Y4, Y4
+	VPADDD     Y11, Y1, Y1
+	VPADDD     Y15, Y2, Y2
+	VPERM2I128 $0x02, Y0, Y14, Y3
+	VPERM2I128 $0x02, Y12, Y4, Y7
+	VPERM2I128 $0x13, Y0, Y14, Y11
+	VPERM2I128 $0x13, Y12, Y4, Y15
+	VPXOR      (SI), Y3, Y3
+	VPXOR      32(SI), Y7, Y7
+	VPXOR      64(SI), Y11, Y11
+	VPXOR      96(SI), Y15, Y15
+	VMOVDQU    Y3, (DI)
+	VMOVDQU    Y7, 32(DI)
+	VMOVDQU    Y11, 64(DI)
+	VMOVDQU    Y15, 96(DI)
+	VPERM2I128 $0x02, Y5, Y9, Y3
+	VPERM2I128 $0x02, Y13, Y1, Y7
+	VPERM2I128 $0x13, Y5, Y9, Y11
+	VPERM2I128 $0x13, Y13, Y1, Y15
+	VPXOR      128(SI), Y3, Y3
+	VPXOR      160(SI), Y7, Y7
+	VPXOR      192(SI), Y11, Y11
+	VPXOR      224(SI), Y15, Y15
+	VMOVDQU    Y3, 128(DI)
+	VMOVDQU    Y7, 160(DI)
+	VMOVDQU    Y11, 192(DI)
+	VMOVDQU    Y15, 224(DI)
+	MOVQ       $0x00000100, CX
+	LEAQ       256(SI), SI
+	SUBQ       $0x00000100, BX
+	VPERM2I128 $0x02, Y6, Y10, Y0
+	VPERM2I128 $0x02, Y8, Y2, Y14
+	VPERM2I128 $0x13, Y6, Y10, Y12
+	VPERM2I128 $0x13, Y8, Y2, Y4
+	JMP        sealAVX2SealHash
+
 sealAVX2Tail512:
-	// Need to decrypt up to 512 bytes - prepare two blocks
-	// If we got here after the main loop - there are 512 encrypted bytes waiting to be hashed
-	// If we got here before the main loop - there are 448 encrpyred bytes waiting to be hashed
-	VMOVDQA ·chacha20Constants<>(SB), AA0; VMOVDQA AA0, AA1; VMOVDQA AA0, AA2; VMOVDQA AA0, AA3
-	VMOVDQA state1StoreAVX2, BB0; VMOVDQA BB0, BB1; VMOVDQA BB0, BB2; VMOVDQA BB0, BB3
-	VMOVDQA state2StoreAVX2, CC0; VMOVDQA CC0, CC1; VMOVDQA CC0, CC2; VMOVDQA CC0, CC3
-	VMOVDQA ctr3StoreAVX2, DD0
-	VPADDD  ·avx2IncMask<>(SB), DD0, DD0; VPADDD ·avx2IncMask<>(SB), DD0, DD1; VPADDD ·avx2IncMask<>(SB), DD1, DD2; VPADDD ·avx2IncMask<>(SB), DD2, DD3
-	VMOVDQA DD0, ctr0StoreAVX2; VMOVDQA DD1, ctr1StoreAVX2; VMOVDQA DD2, ctr2StoreAVX2; VMOVDQA DD3, ctr3StoreAVX2
+	VMOVDQA ·chacha20Constants<>+0(SB), Y0
+	VMOVDQA Y0, Y5
+	VMOVDQA Y0, Y6
+	VMOVDQA Y0, Y7
+	VMOVDQA 32(BP), Y14
+	VMOVDQA Y14, Y9
+	VMOVDQA Y14, Y10
+	VMOVDQA Y14, Y11
+	VMOVDQA 64(BP), Y12
+	VMOVDQA Y12, Y13
+	VMOVDQA Y12, Y8
+	VMOVDQA Y12, Y15
+	VMOVDQA 192(BP), Y4
+	VPADDD  ·avx2IncMask<>+0(SB), Y4, Y4
+	VPADDD  ·avx2IncMask<>+0(SB), Y4, Y1
+	VPADDD  ·avx2IncMask<>+0(SB), Y1, Y2
+	VPADDD  ·avx2IncMask<>+0(SB), Y2, Y3
+	VMOVDQA Y4, 96(BP)
+	VMOVDQA Y1, 128(BP)
+	VMOVDQA Y2, 160(BP)
+	VMOVDQA Y3, 192(BP)
 
 sealAVX2Tail512LoopA:
-	polyAdd(0(oup))
-	polyMul
-	LEAQ 16(oup), oup
+	ADDQ  (DI), R10
+	ADCQ  8(DI), R11
+	ADCQ  $0x01, R12
+	MOVQ  (BP), AX
+	MOVQ  AX, R15
+	MULQ  R10
+	MOVQ  AX, R13
+	MOVQ  DX, R14
+	MOVQ  (BP), AX
+	MULQ  R11
+	IMULQ R12, R15
+	ADDQ  AX, R14
+	ADCQ  DX, R15
+	MOVQ  8(BP), AX
+	MOVQ  AX, R8
+	MULQ  R10
+	ADDQ  AX, R14
+	ADCQ  $0x00, DX
+	MOVQ  DX, R10
+	MOVQ  8(BP), AX
+	MULQ  R11
+	ADDQ  AX, R15
+	ADCQ  $0x00, DX
+	IMULQ R12, R8
+	ADDQ  R10, R15
+	ADCQ  DX, R8
+	MOVQ  R13, R10
+	MOVQ  R14, R11
+	MOVQ  R15, R12
+	ANDQ  $0x03, R12
+	MOVQ  R15, R13
+	ANDQ  $-4, R13
+	MOVQ  R8, R14
+	SHRQ  $0x02, R8, R15
+	SHRQ  $0x02, R8
+	ADDQ  R13, R10
+	ADCQ  R14, R11
+	ADCQ  $0x00, R12
+	ADDQ  R15, R10
+	ADCQ  R8, R11
+	ADCQ  $0x00, R12
+	LEAQ  16(DI), DI
 
 sealAVX2Tail512LoopB:
-	VPADDD   BB0, AA0, AA0; VPADDD BB1, AA1, AA1; VPADDD BB2, AA2, AA2; VPADDD BB3, AA3, AA3
-	VPXOR    AA0, DD0, DD0; VPXOR AA1, DD1, DD1; VPXOR AA2, DD2, DD2; VPXOR AA3, DD3, DD3
-	VPSHUFB  ·rol16<>(SB), DD0, DD0; VPSHUFB ·rol16<>(SB), DD1, DD1; VPSHUFB ·rol16<>(SB), DD2, DD2; VPSHUFB ·rol16<>(SB), DD3, DD3
-	VPADDD   DD0, CC0, CC0; VPADDD DD1, CC1, CC1; VPADDD DD2, CC2, CC2; VPADDD DD3, CC3, CC3
-	VPXOR    CC0, BB0, BB0; VPXOR CC1, BB1, BB1; VPXOR CC2, BB2, BB2; VPXOR CC3, BB3, BB3
-	VMOVDQA  CC3, tmpStoreAVX2
-	VPSLLD   $12, BB0, CC3; VPSRLD $20, BB0, BB0; VPXOR CC3, BB0, BB0
-	VPSLLD   $12, BB1, CC3; VPSRLD $20, BB1, BB1; VPXOR CC3, BB1, BB1
-	VPSLLD   $12, BB2, CC3; VPSRLD $20, BB2, BB2; VPXOR CC3, BB2, BB2
-	VPSLLD   $12, BB3, CC3; VPSRLD $20, BB3, BB3; VPXOR CC3, BB3, BB3
-	VMOVDQA  tmpStoreAVX2, CC3
-	polyAdd(0*8(oup))
-	polyMulAVX2
-	VPADDD   BB0, AA0, AA0; VPADDD BB1, AA1, AA1; VPADDD BB2, AA2, AA2; VPADDD BB3, AA3, AA3
-	VPXOR    AA0, DD0, DD0; VPXOR AA1, DD1, DD1; VPXOR AA2, DD2, DD2; VPXOR AA3, DD3, DD3
-	VPSHUFB  ·rol8<>(SB), DD0, DD0; VPSHUFB ·rol8<>(SB), DD1, DD1; VPSHUFB ·rol8<>(SB), DD2, DD2; VPSHUFB ·rol8<>(SB), DD3, DD3
-	VPADDD   DD0, CC0, CC0; VPADDD DD1, CC1, CC1; VPADDD DD2, CC2, CC2; VPADDD DD3, CC3, CC3
-	VPXOR    CC0, BB0, BB0; VPXOR CC1, BB1, BB1; VPXOR CC2, BB2, BB2; VPXOR CC3, BB3, BB3
-	VMOVDQA  CC3, tmpStoreAVX2
-	VPSLLD   $7, BB0, CC3; VPSRLD $25, BB0, BB0; VPXOR CC3, BB0, BB0
-	VPSLLD   $7, BB1, CC3; VPSRLD $25, BB1, BB1; VPXOR CC3, BB1, BB1
-	VPSLLD   $7, BB2, CC3; VPSRLD $25, BB2, BB2; VPXOR CC3, BB2, BB2
-	VPSLLD   $7, BB3, CC3; VPSRLD $25, BB3, BB3; VPXOR CC3, BB3, BB3
-	VMOVDQA  tmpStoreAVX2, CC3
-	VPALIGNR $4, BB0, BB0, BB0; VPALIGNR $4, BB1, BB1, BB1; VPALIGNR $4, BB2, BB2, BB2; VPALIGNR $4, BB3, BB3, BB3
-	VPALIGNR $8, CC0, CC0, CC0; VPALIGNR $8, CC1, CC1, CC1; VPALIGNR $8, CC2, CC2, CC2; VPALIGNR $8, CC3, CC3, CC3
-	VPALIGNR $12, DD0, DD0, DD0; VPALIGNR $12, DD1, DD1, DD1; VPALIGNR $12, DD2, DD2, DD2; VPALIGNR $12, DD3, DD3, DD3
-	VPADDD   BB0, AA0, AA0; VPADDD BB1, AA1, AA1; VPADDD BB2, AA2, AA2; VPADDD BB3, AA3, AA3
-	VPXOR    AA0, DD0, DD0; VPXOR AA1, DD1, DD1; VPXOR AA2, DD2, DD2; VPXOR AA3, DD3, DD3
-	VPSHUFB  ·rol16<>(SB), DD0, DD0; VPSHUFB ·rol16<>(SB), DD1, DD1; VPSHUFB ·rol16<>(SB), DD2, DD2; VPSHUFB ·rol16<>(SB), DD3, DD3
-	VPADDD   DD0, CC0, CC0; VPADDD DD1, CC1, CC1; VPADDD DD2, CC2, CC2; VPADDD DD3, CC3, CC3
-	VPXOR    CC0, BB0, BB0; VPXOR CC1, BB1, BB1; VPXOR CC2, BB2, BB2; VPXOR CC3, BB3, BB3
-	polyAdd(2*8(oup))
-	polyMulAVX2
-	LEAQ     (4*8)(oup), oup
-	VMOVDQA  CC3, tmpStoreAVX2
-	VPSLLD   $12, BB0, CC3; VPSRLD $20, BB0, BB0; VPXOR CC3, BB0, BB0
-	VPSLLD   $12, BB1, CC3; VPSRLD $20, BB1, BB1; VPXOR CC3, BB1, BB1
-	VPSLLD   $12, BB2, CC3; VPSRLD $20, BB2, BB2; VPXOR CC3, BB2, BB2
-	VPSLLD   $12, BB3, CC3; VPSRLD $20, BB3, BB3; VPXOR CC3, BB3, BB3
-	VMOVDQA  tmpStoreAVX2, CC3
-	VPADDD   BB0, AA0, AA0; VPADDD BB1, AA1, AA1; VPADDD BB2, AA2, AA2; VPADDD BB3, AA3, AA3
-	VPXOR    AA0, DD0, DD0; VPXOR AA1, DD1, DD1; VPXOR AA2, DD2, DD2; VPXOR AA3, DD3, DD3
-	VPSHUFB  ·rol8<>(SB), DD0, DD0; VPSHUFB ·rol8<>(SB), DD1, DD1; VPSHUFB ·rol8<>(SB), DD2, DD2; VPSHUFB ·rol8<>(SB), DD3, DD3
-	VPADDD   DD0, CC0, CC0; VPADDD DD1, CC1, CC1; VPADDD DD2, CC2, CC2; VPADDD DD3, CC3, CC3
-	VPXOR    CC0, BB0, BB0; VPXOR CC1, BB1, BB1; VPXOR CC2, BB2, BB2; VPXOR CC3, BB3, BB3
-	VMOVDQA  CC3, tmpStoreAVX2
-	VPSLLD   $7, BB0, CC3; VPSRLD $25, BB0, BB0; VPXOR CC3, BB0, BB0
-	VPSLLD   $7, BB1, CC3; VPSRLD $25, BB1, BB1; VPXOR CC3, BB1, BB1
-	VPSLLD   $7, BB2, CC3; VPSRLD $25, BB2, BB2; VPXOR CC3, BB2, BB2
-	VPSLLD   $7, BB3, CC3; VPSRLD $25, BB3, BB3; VPXOR CC3, BB3, BB3
-	VMOVDQA  tmpStoreAVX2, CC3
-	VPALIGNR $12, BB0, BB0, BB0; VPALIGNR $12, BB1, BB1, BB1; VPALIGNR $12, BB2, BB2, BB2; VPALIGNR $12, BB3, BB3, BB3
-	VPALIGNR $8, CC0, CC0, CC0; VPALIGNR $8, CC1, CC1, CC1; VPALIGNR $8, CC2, CC2, CC2; VPALIGNR $8, CC3, CC3, CC3
-	VPALIGNR $4, DD0, DD0, DD0; VPALIGNR $4, DD1, DD1, DD1; VPALIGNR $4, DD2, DD2, DD2; VPALIGNR $4, DD3, DD3, DD3
-
-	DECQ itr1
-	JG   sealAVX2Tail512LoopA
-	DECQ itr2
-	JGE  sealAVX2Tail512LoopB
-
-	VPADDD     ·chacha20Constants<>(SB), AA0, AA0; VPADDD ·chacha20Constants<>(SB), AA1, AA1; VPADDD ·chacha20Constants<>(SB), AA2, AA2; VPADDD ·chacha20Constants<>(SB), AA3, AA3
-	VPADDD     state1StoreAVX2, BB0, BB0; VPADDD state1StoreAVX2, BB1, BB1; VPADDD state1StoreAVX2, BB2, BB2; VPADDD state1StoreAVX2, BB3, BB3
-	VPADDD     state2StoreAVX2, CC0, CC0; VPADDD state2StoreAVX2, CC1, CC1; VPADDD state2StoreAVX2, CC2, CC2; VPADDD state2StoreAVX2, CC3, CC3
-	VPADDD     ctr0StoreAVX2, DD0, DD0; VPADDD ctr1StoreAVX2, DD1, DD1; VPADDD ctr2StoreAVX2, DD2, DD2; VPADDD ctr3StoreAVX2, DD3, DD3
-	VMOVDQA    CC3, tmpStoreAVX2
-	VPERM2I128 $0x02, AA0, BB0, CC3
-	VPXOR      (0*32)(inp), CC3, CC3
-	VMOVDQU    CC3, (0*32)(oup)
-	VPERM2I128 $0x02, CC0, DD0, CC3
-	VPXOR      (1*32)(inp), CC3, CC3
-	VMOVDQU    CC3, (1*32)(oup)
-	VPERM2I128 $0x13, AA0, BB0, CC3
-	VPXOR      (2*32)(inp), CC3, CC3
-	VMOVDQU    CC3, (2*32)(oup)
-	VPERM2I128 $0x13, CC0, DD0, CC3
-	VPXOR      (3*32)(inp), CC3, CC3
-	VMOVDQU    CC3, (3*32)(oup)
-
-	VPERM2I128 $0x02, AA1, BB1, AA0
-	VPERM2I128 $0x02, CC1, DD1, BB0
-	VPERM2I128 $0x13, AA1, BB1, CC0
-	VPERM2I128 $0x13, CC1, DD1, DD0
-	VPXOR      (4*32)(inp), AA0, AA0; VPXOR (5*32)(inp), BB0, BB0; VPXOR (6*32)(inp), CC0, CC0; VPXOR (7*32)(inp), DD0, DD0
-	VMOVDQU    AA0, (4*32)(oup); VMOVDQU BB0, (5*32)(oup); VMOVDQU CC0, (6*32)(oup); VMOVDQU DD0, (7*32)(oup)
-
-	VPERM2I128 $0x02, AA2, BB2, AA0
-	VPERM2I128 $0x02, CC2, DD2, BB0
-	VPERM2I128 $0x13, AA2, BB2, CC0
-	VPERM2I128 $0x13, CC2, DD2, DD0
-	VPXOR      (8*32)(inp), AA0, AA0; VPXOR (9*32)(inp), BB0, BB0; VPXOR (10*32)(inp), CC0, CC0; VPXOR (11*32)(inp), DD0, DD0
-	VMOVDQU    AA0, (8*32)(oup); VMOVDQU BB0, (9*32)(oup); VMOVDQU CC0, (10*32)(oup); VMOVDQU DD0, (11*32)(oup)
-
-	MOVQ       $384, itr1
-	LEAQ       384(inp), inp
-	SUBQ       $384, inl
-	VPERM2I128 $0x02, AA3, BB3, AA0
-	VPERM2I128 $0x02, tmpStoreAVX2, DD3, BB0
-	VPERM2I128 $0x13, AA3, BB3, CC0
-	VPERM2I128 $0x13, tmpStoreAVX2, DD3, DD0
-
-	JMP sealAVX2SealHash
+	VPADDD     Y14, Y0, Y0
+	VPADDD     Y9, Y5, Y5
+	VPADDD     Y10, Y6, Y6
+	VPADDD     Y11, Y7, Y7
+	VPXOR      Y0, Y4, Y4
+	VPXOR      Y5, Y1, Y1
+	VPXOR      Y6, Y2, Y2
+	VPXOR      Y7, Y3, Y3
+	VPSHUFB    ·rol16<>+0(SB), Y4, Y4
+	VPSHUFB    ·rol16<>+0(SB), Y1, Y1
+	VPSHUFB    ·rol16<>+0(SB), Y2, Y2
+	VPSHUFB    ·rol16<>+0(SB), Y3, Y3
+	VPADDD     Y4, Y12, Y12
+	VPADDD     Y1, Y13, Y13
+	VPADDD     Y2, Y8, Y8
+	VPADDD     Y3, Y15, Y15
+	VPXOR      Y12, Y14, Y14
+	VPXOR      Y13, Y9, Y9
+	VPXOR      Y8, Y10, Y10
+	VPXOR      Y15, Y11, Y11
+	VMOVDQA    Y15, 224(BP)
+	VPSLLD     $0x0c, Y14, Y15
+	VPSRLD     $0x14, Y14, Y14
+	VPXOR      Y15, Y14, Y14
+	VPSLLD     $0x0c, Y9, Y15
+	VPSRLD     $0x14, Y9, Y9
+	VPXOR      Y15, Y9, Y9
+	VPSLLD     $0x0c, Y10, Y15
+	VPSRLD     $0x14, Y10, Y10
+	VPXOR      Y15, Y10, Y10
+	VPSLLD     $0x0c, Y11, Y15
+	VPSRLD     $0x14, Y11, Y11
+	VPXOR      Y15, Y11, Y11
+	VMOVDQA    224(BP), Y15
+	ADDQ       (DI), R10
+	ADCQ       8(DI), R11
+	ADCQ       $0x01, R12
+	MOVQ       (BP), DX
+	MOVQ       DX, R15
+	MULXQ      R10, R13, R14
+	IMULQ      R12, R15
+	MULXQ      R11, AX, DX
+	ADDQ       AX, R14
+	ADCQ       DX, R15
+	MOVQ       8(BP), DX
+	MULXQ      R10, R10, AX
+	ADDQ       R10, R14
+	MULXQ      R11, R11, R8
+	ADCQ       R11, R15
+	ADCQ       $0x00, R8
+	IMULQ      R12, DX
+	ADDQ       AX, R15
+	ADCQ       DX, R8
+	MOVQ       R13, R10
+	MOVQ       R14, R11
+	MOVQ       R15, R12
+	ANDQ       $0x03, R12
+	MOVQ       R15, R13
+	ANDQ       $-4, R13
+	MOVQ       R8, R14
+	SHRQ       $0x02, R8, R15
+	SHRQ       $0x02, R8
+	ADDQ       R13, R10
+	ADCQ       R14, R11
+	ADCQ       $0x00, R12
+	ADDQ       R15, R10
+	ADCQ       R8, R11
+	ADCQ       $0x00, R12
+	VPADDD     Y14, Y0, Y0
+	VPADDD     Y9, Y5, Y5
+	VPADDD     Y10, Y6, Y6
+	VPADDD     Y11, Y7, Y7
+	VPXOR      Y0, Y4, Y4
+	VPXOR      Y5, Y1, Y1
+	VPXOR      Y6, Y2, Y2
+	VPXOR      Y7, Y3, Y3
+	VPSHUFB    ·rol8<>+0(SB), Y4, Y4
+	VPSHUFB    ·rol8<>+0(SB), Y1, Y1
+	VPSHUFB    ·rol8<>+0(SB), Y2, Y2
+	VPSHUFB    ·rol8<>+0(SB), Y3, Y3
+	VPADDD     Y4, Y12, Y12
+	VPADDD     Y1, Y13, Y13
+	VPADDD     Y2, Y8, Y8
+	VPADDD     Y3, Y15, Y15
+	VPXOR      Y12, Y14, Y14
+	VPXOR      Y13, Y9, Y9
+	VPXOR      Y8, Y10, Y10
+	VPXOR      Y15, Y11, Y11
+	VMOVDQA    Y15, 224(BP)
+	VPSLLD     $0x07, Y14, Y15
+	VPSRLD     $0x19, Y14, Y14
+	VPXOR      Y15, Y14, Y14
+	VPSLLD     $0x07, Y9, Y15
+	VPSRLD     $0x19, Y9, Y9
+	VPXOR      Y15, Y9, Y9
+	VPSLLD     $0x07, Y10, Y15
+	VPSRLD     $0x19, Y10, Y10
+	VPXOR      Y15, Y10, Y10
+	VPSLLD     $0x07, Y11, Y15
+	VPSRLD     $0x19, Y11, Y11
+	VPXOR      Y15, Y11, Y11
+	VMOVDQA    224(BP), Y15
+	VPALIGNR   $0x04, Y14, Y14, Y14
+	VPALIGNR   $0x04, Y9, Y9, Y9
+	VPALIGNR   $0x04, Y10, Y10, Y10
+	VPALIGNR   $0x04, Y11, Y11, Y11
+	VPALIGNR   $0x08, Y12, Y12, Y12
+	VPALIGNR   $0x08, Y13, Y13, Y13
+	VPALIGNR   $0x08, Y8, Y8, Y8
+	VPALIGNR   $0x08, Y15, Y15, Y15
+	VPALIGNR   $0x0c, Y4, Y4, Y4
+	VPALIGNR   $0x0c, Y1, Y1, Y1
+	VPALIGNR   $0x0c, Y2, Y2, Y2
+	VPALIGNR   $0x0c, Y3, Y3, Y3
+	VPADDD     Y14, Y0, Y0
+	VPADDD     Y9, Y5, Y5
+	VPADDD     Y10, Y6, Y6
+	VPADDD     Y11, Y7, Y7
+	VPXOR      Y0, Y4, Y4
+	VPXOR      Y5, Y1, Y1
+	VPXOR      Y6, Y2, Y2
+	VPXOR      Y7, Y3, Y3
+	VPSHUFB    ·rol16<>+0(SB), Y4, Y4
+	VPSHUFB    ·rol16<>+0(SB), Y1, Y1
+	VPSHUFB    ·rol16<>+0(SB), Y2, Y2
+	VPSHUFB    ·rol16<>+0(SB), Y3, Y3
+	VPADDD     Y4, Y12, Y12
+	VPADDD     Y1, Y13, Y13
+	VPADDD     Y2, Y8, Y8
+	VPADDD     Y3, Y15, Y15
+	VPXOR      Y12, Y14, Y14
+	VPXOR      Y13, Y9, Y9
+	VPXOR      Y8, Y10, Y10
+	VPXOR      Y15, Y11, Y11
+	ADDQ       16(DI), R10
+	ADCQ       24(DI), R11
+	ADCQ       $0x01, R12
+	MOVQ       (BP), DX
+	MOVQ       DX, R15
+	MULXQ      R10, R13, R14
+	IMULQ      R12, R15
+	MULXQ      R11, AX, DX
+	ADDQ       AX, R14
+	ADCQ       DX, R15
+	MOVQ       8(BP), DX
+	MULXQ      R10, R10, AX
+	ADDQ       R10, R14
+	MULXQ      R11, R11, R8
+	ADCQ       R11, R15
+	ADCQ       $0x00, R8
+	IMULQ      R12, DX
+	ADDQ       AX, R15
+	ADCQ       DX, R8
+	MOVQ       R13, R10
+	MOVQ       R14, R11
+	MOVQ       R15, R12
+	ANDQ       $0x03, R12
+	MOVQ       R15, R13
+	ANDQ       $-4, R13
+	MOVQ       R8, R14
+	SHRQ       $0x02, R8, R15
+	SHRQ       $0x02, R8
+	ADDQ       R13, R10
+	ADCQ       R14, R11
+	ADCQ       $0x00, R12
+	ADDQ       R15, R10
+	ADCQ       R8, R11
+	ADCQ       $0x00, R12
+	LEAQ       32(DI), DI
+	VMOVDQA    Y15, 224(BP)
+	VPSLLD     $0x0c, Y14, Y15
+	VPSRLD     $0x14, Y14, Y14
+	VPXOR      Y15, Y14, Y14
+	VPSLLD     $0x0c, Y9, Y15
+	VPSRLD     $0x14, Y9, Y9
+	VPXOR      Y15, Y9, Y9
+	VPSLLD     $0x0c, Y10, Y15
+	VPSRLD     $0x14, Y10, Y10
+	VPXOR      Y15, Y10, Y10
+	VPSLLD     $0x0c, Y11, Y15
+	VPSRLD     $0x14, Y11, Y11
+	VPXOR      Y15, Y11, Y11
+	VMOVDQA    224(BP), Y15
+	VPADDD     Y14, Y0, Y0
+	VPADDD     Y9, Y5, Y5
+	VPADDD     Y10, Y6, Y6
+	VPADDD     Y11, Y7, Y7
+	VPXOR      Y0, Y4, Y4
+	VPXOR      Y5, Y1, Y1
+	VPXOR      Y6, Y2, Y2
+	VPXOR      Y7, Y3, Y3
+	VPSHUFB    ·rol8<>+0(SB), Y4, Y4
+	VPSHUFB    ·rol8<>+0(SB), Y1, Y1
+	VPSHUFB    ·rol8<>+0(SB), Y2, Y2
+	VPSHUFB    ·rol8<>+0(SB), Y3, Y3
+	VPADDD     Y4, Y12, Y12
+	VPADDD     Y1, Y13, Y13
+	VPADDD     Y2, Y8, Y8
+	VPADDD     Y3, Y15, Y15
+	VPXOR      Y12, Y14, Y14
+	VPXOR      Y13, Y9, Y9
+	VPXOR      Y8, Y10, Y10
+	VPXOR      Y15, Y11, Y11
+	VMOVDQA    Y15, 224(BP)
+	VPSLLD     $0x07, Y14, Y15
+	VPSRLD     $0x19, Y14, Y14
+	VPXOR      Y15, Y14, Y14
+	VPSLLD     $0x07, Y9, Y15
+	VPSRLD     $0x19, Y9, Y9
+	VPXOR      Y15, Y9, Y9
+	VPSLLD     $0x07, Y10, Y15
+	VPSRLD     $0x19, Y10, Y10
+	VPXOR      Y15, Y10, Y10
+	VPSLLD     $0x07, Y11, Y15
+	VPSRLD     $0x19, Y11, Y11
+	VPXOR      Y15, Y11, Y11
+	VMOVDQA    224(BP), Y15
+	VPALIGNR   $0x0c, Y14, Y14, Y14
+	VPALIGNR   $0x0c, Y9, Y9, Y9
+	VPALIGNR   $0x0c, Y10, Y10, Y10
+	VPALIGNR   $0x0c, Y11, Y11, Y11
+	VPALIGNR   $0x08, Y12, Y12, Y12
+	VPALIGNR   $0x08, Y13, Y13, Y13
+	VPALIGNR   $0x08, Y8, Y8, Y8
+	VPALIGNR   $0x08, Y15, Y15, Y15
+	VPALIGNR   $0x04, Y4, Y4, Y4
+	VPALIGNR   $0x04, Y1, Y1, Y1
+	VPALIGNR   $0x04, Y2, Y2, Y2
+	VPALIGNR   $0x04, Y3, Y3, Y3
+	DECQ       CX
+	JG         sealAVX2Tail512LoopA
+	DECQ       R9
+	JGE        sealAVX2Tail512LoopB
+	VPADDD     ·chacha20Constants<>+0(SB), Y0, Y0
+	VPADDD     ·chacha20Constants<>+0(SB), Y5, Y5
+	VPADDD     ·chacha20Constants<>+0(SB), Y6, Y6
+	VPADDD     ·chacha20Constants<>+0(SB), Y7, Y7
+	VPADDD     32(BP), Y14, Y14
+	VPADDD     32(BP), Y9, Y9
+	VPADDD     32(BP), Y10, Y10
+	VPADDD     32(BP), Y11, Y11
+	VPADDD     64(BP), Y12, Y12
+	VPADDD     64(BP), Y13, Y13
+	VPADDD     64(BP), Y8, Y8
+	VPADDD     64(BP), Y15, Y15
+	VPADDD     96(BP), Y4, Y4
+	VPADDD     128(BP), Y1, Y1
+	VPADDD     160(BP), Y2, Y2
+	VPADDD     192(BP), Y3, Y3
+	VMOVDQA    Y15, 224(BP)
+	VPERM2I128 $0x02, Y0, Y14, Y15
+	VPXOR      (SI), Y15, Y15
+	VMOVDQU    Y15, (DI)
+	VPERM2I128 $0x02, Y12, Y4, Y15
+	VPXOR      32(SI), Y15, Y15
+	VMOVDQU    Y15, 32(DI)
+	VPERM2I128 $0x13, Y0, Y14, Y15
+	VPXOR      64(SI), Y15, Y15
+	VMOVDQU    Y15, 64(DI)
+	VPERM2I128 $0x13, Y12, Y4, Y15
+	VPXOR      96(SI), Y15, Y15
+	VMOVDQU    Y15, 96(DI)
+	VPERM2I128 $0x02, Y5, Y9, Y0
+	VPERM2I128 $0x02, Y13, Y1, Y14
+	VPERM2I128 $0x13, Y5, Y9, Y12
+	VPERM2I128 $0x13, Y13, Y1, Y4
+	VPXOR      128(SI), Y0, Y0
+	VPXOR      160(SI), Y14, Y14
+	VPXOR      192(SI), Y12, Y12
+	VPXOR      224(SI), Y4, Y4
+	VMOVDQU    Y0, 128(DI)
+	VMOVDQU    Y14, 160(DI)
+	VMOVDQU    Y12, 192(DI)
+	VMOVDQU    Y4, 224(DI)
+	VPERM2I128 $0x02, Y6, Y10, Y0
+	VPERM2I128 $0x02, Y8, Y2, Y14
+	VPERM2I128 $0x13, Y6, Y10, Y12
+	VPERM2I128 $0x13, Y8, Y2, Y4
+	VPXOR      256(SI), Y0, Y0
+	VPXOR      288(SI), Y14, Y14
+	VPXOR      320(SI), Y12, Y12
+	VPXOR      352(SI), Y4, Y4
+	VMOVDQU    Y0, 256(DI)
+	VMOVDQU    Y14, 288(DI)
+	VMOVDQU    Y12, 320(DI)
+	VMOVDQU    Y4, 352(DI)
+	MOVQ       $0x00000180, CX
+	LEAQ       384(SI), SI
+	SUBQ       $0x00000180, BX
+	VPERM2I128 $0x02, Y7, Y11, Y0
+	VPERM2I128 $0x02, 224(BP), Y3, Y14
+	VPERM2I128 $0x13, Y7, Y11, Y12
+	VPERM2I128 $0x13, 224(BP), Y3, Y4
+	JMP        sealAVX2SealHash
diff --git a/vendor/golang.org/x/crypto/internal/poly1305/mac_noasm.go b/vendor/golang.org/x/crypto/internal/poly1305/mac_noasm.go
index 333da285b..bd896bdc7 100644
--- a/vendor/golang.org/x/crypto/internal/poly1305/mac_noasm.go
+++ b/vendor/golang.org/x/crypto/internal/poly1305/mac_noasm.go
@@ -2,7 +2,7 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-//go:build (!amd64 && !ppc64le && !s390x) || !gc || purego
+//go:build (!amd64 && !ppc64le && !ppc64 && !s390x) || !gc || purego
 
 package poly1305
 
diff --git a/vendor/golang.org/x/crypto/internal/poly1305/sum_amd64.s b/vendor/golang.org/x/crypto/internal/poly1305/sum_amd64.s
index e0d3c6475..133757384 100644
--- a/vendor/golang.org/x/crypto/internal/poly1305/sum_amd64.s
+++ b/vendor/golang.org/x/crypto/internal/poly1305/sum_amd64.s
@@ -1,108 +1,93 @@
-// Copyright 2012 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
+// Code generated by command: go run sum_amd64_asm.go -out ../sum_amd64.s -pkg poly1305. DO NOT EDIT.
 
 //go:build gc && !purego
 
-#include "textflag.h"
-
-#define POLY1305_ADD(msg, h0, h1, h2) \
-	ADDQ 0(msg), h0;  \
-	ADCQ 8(msg), h1;  \
-	ADCQ $1, h2;      \
-	LEAQ 16(msg), msg
-
-#define POLY1305_MUL(h0, h1, h2, r0, r1, t0, t1, t2, t3) \
-	MOVQ  r0, AX;                  \
-	MULQ  h0;                      \
-	MOVQ  AX, t0;                  \
-	MOVQ  DX, t1;                  \
-	MOVQ  r0, AX;                  \
-	MULQ  h1;                      \
-	ADDQ  AX, t1;                  \
-	ADCQ  $0, DX;                  \
-	MOVQ  r0, t2;                  \
-	IMULQ h2, t2;                  \
-	ADDQ  DX, t2;                  \
-	                               \
-	MOVQ  r1, AX;                  \
-	MULQ  h0;                      \
-	ADDQ  AX, t1;                  \
-	ADCQ  $0, DX;                  \
-	MOVQ  DX, h0;                  \
-	MOVQ  r1, t3;                  \
-	IMULQ h2, t3;                  \
-	MOVQ  r1, AX;                  \
-	MULQ  h1;                      \
-	ADDQ  AX, t2;                  \
-	ADCQ  DX, t3;                  \
-	ADDQ  h0, t2;                  \
-	ADCQ  $0, t3;                  \
-	                               \
-	MOVQ  t0, h0;                  \
-	MOVQ  t1, h1;                  \
-	MOVQ  t2, h2;                  \
-	ANDQ  $3, h2;                  \
-	MOVQ  t2, t0;                  \
-	ANDQ  $0xFFFFFFFFFFFFFFFC, t0; \
-	ADDQ  t0, h0;                  \
-	ADCQ  t3, h1;                  \
-	ADCQ  $0, h2;                  \
-	SHRQ  $2, t3, t2;              \
-	SHRQ  $2, t3;                  \
-	ADDQ  t2, h0;                  \
-	ADCQ  t3, h1;                  \
-	ADCQ  $0, h2
-
-// func update(state *[7]uint64, msg []byte)
+// func update(state *macState, msg []byte)
 TEXT ·update(SB), $0-32
 	MOVQ state+0(FP), DI
 	MOVQ msg_base+8(FP), SI
 	MOVQ msg_len+16(FP), R15
-
-	MOVQ 0(DI), R8   // h0
-	MOVQ 8(DI), R9   // h1
-	MOVQ 16(DI), R10 // h2
-	MOVQ 24(DI), R11 // r0
-	MOVQ 32(DI), R12 // r1
-
-	CMPQ R15, $16
+	MOVQ (DI), R8
+	MOVQ 8(DI), R9
+	MOVQ 16(DI), R10
+	MOVQ 24(DI), R11
+	MOVQ 32(DI), R12
+	CMPQ R15, $0x10
 	JB   bytes_between_0_and_15
 
 loop:
-	POLY1305_ADD(SI, R8, R9, R10)
+	ADDQ (SI), R8
+	ADCQ 8(SI), R9
+	ADCQ $0x01, R10
+	LEAQ 16(SI), SI
 
 multiply:
-	POLY1305_MUL(R8, R9, R10, R11, R12, BX, CX, R13, R14)
-	SUBQ $16, R15
-	CMPQ R15, $16
-	JAE  loop
+	MOVQ  R11, AX
+	MULQ  R8
+	MOVQ  AX, BX
+	MOVQ  DX, CX
+	MOVQ  R11, AX
+	MULQ  R9
+	ADDQ  AX, CX
+	ADCQ  $0x00, DX
+	MOVQ  R11, R13
+	IMULQ R10, R13
+	ADDQ  DX, R13
+	MOVQ  R12, AX
+	MULQ  R8
+	ADDQ  AX, CX
+	ADCQ  $0x00, DX
+	MOVQ  DX, R8
+	MOVQ  R12, R14
+	IMULQ R10, R14
+	MOVQ  R12, AX
+	MULQ  R9
+	ADDQ  AX, R13
+	ADCQ  DX, R14
+	ADDQ  R8, R13
+	ADCQ  $0x00, R14
+	MOVQ  BX, R8
+	MOVQ  CX, R9
+	MOVQ  R13, R10
+	ANDQ  $0x03, R10
+	MOVQ  R13, BX
+	ANDQ  $-4, BX
+	ADDQ  BX, R8
+	ADCQ  R14, R9
+	ADCQ  $0x00, R10
+	SHRQ  $0x02, R14, R13
+	SHRQ  $0x02, R14
+	ADDQ  R13, R8
+	ADCQ  R14, R9
+	ADCQ  $0x00, R10
+	SUBQ  $0x10, R15
+	CMPQ  R15, $0x10
+	JAE   loop
 
 bytes_between_0_and_15:
 	TESTQ R15, R15
 	JZ    done
-	MOVQ  $1, BX
+	MOVQ  $0x00000001, BX
 	XORQ  CX, CX
 	XORQ  R13, R13
 	ADDQ  R15, SI
 
 flush_buffer:
-	SHLQ $8, BX, CX
-	SHLQ $8, BX
+	SHLQ $0x08, BX, CX
+	SHLQ $0x08, BX
 	MOVB -1(SI), R13
 	XORQ R13, BX
 	DECQ SI
 	DECQ R15
 	JNZ  flush_buffer
-
 	ADDQ BX, R8
 	ADCQ CX, R9
-	ADCQ $0, R10
-	MOVQ $16, R15
+	ADCQ $0x00, R10
+	MOVQ $0x00000010, R15
 	JMP  multiply
 
 done:
-	MOVQ R8, 0(DI)
+	MOVQ R8, (DI)
 	MOVQ R9, 8(DI)
 	MOVQ R10, 16(DI)
 	RET
diff --git a/vendor/golang.org/x/crypto/internal/poly1305/sum_ppc64le.go b/vendor/golang.org/x/crypto/internal/poly1305/sum_ppc64x.go
similarity index 95%
rename from vendor/golang.org/x/crypto/internal/poly1305/sum_ppc64le.go
rename to vendor/golang.org/x/crypto/internal/poly1305/sum_ppc64x.go
index 4aec4874b..1a1679aaa 100644
--- a/vendor/golang.org/x/crypto/internal/poly1305/sum_ppc64le.go
+++ b/vendor/golang.org/x/crypto/internal/poly1305/sum_ppc64x.go
@@ -2,7 +2,7 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-//go:build gc && !purego
+//go:build gc && !purego && (ppc64 || ppc64le)
 
 package poly1305
 
diff --git a/vendor/golang.org/x/crypto/internal/poly1305/sum_ppc64le.s b/vendor/golang.org/x/crypto/internal/poly1305/sum_ppc64x.s
similarity index 89%
rename from vendor/golang.org/x/crypto/internal/poly1305/sum_ppc64le.s
rename to vendor/golang.org/x/crypto/internal/poly1305/sum_ppc64x.s
index b3c1699bf..6899a1dab 100644
--- a/vendor/golang.org/x/crypto/internal/poly1305/sum_ppc64le.s
+++ b/vendor/golang.org/x/crypto/internal/poly1305/sum_ppc64x.s
@@ -2,15 +2,25 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-//go:build gc && !purego
+//go:build gc && !purego && (ppc64 || ppc64le)
 
 #include "textflag.h"
 
 // This was ported from the amd64 implementation.
 
+#ifdef GOARCH_ppc64le
+#define LE_MOVD MOVD
+#define LE_MOVWZ MOVWZ
+#define LE_MOVHZ MOVHZ
+#else
+#define LE_MOVD MOVDBR
+#define LE_MOVWZ MOVWBR
+#define LE_MOVHZ MOVHBR
+#endif
+
 #define POLY1305_ADD(msg, h0, h1, h2, t0, t1, t2) \
-	MOVD (msg), t0;  \
-	MOVD 8(msg), t1; \
+	LE_MOVD (msg)( R0), t0; \
+	LE_MOVD (msg)(R24), t1; \
 	MOVD $1, t2;     \
 	ADDC t0, h0, h0; \
 	ADDE t1, h1, h1; \
@@ -50,10 +60,6 @@
 	ADDE   t3, h1, h1;  \
 	ADDZE  h2
 
-DATA ·poly1305Mask<>+0x00(SB)/8, $0x0FFFFFFC0FFFFFFF
-DATA ·poly1305Mask<>+0x08(SB)/8, $0x0FFFFFFC0FFFFFFC
-GLOBL ·poly1305Mask<>(SB), RODATA, $16
-
 // func update(state *[7]uint64, msg []byte)
 TEXT ·update(SB), $0-32
 	MOVD state+0(FP), R3
@@ -66,6 +72,8 @@ TEXT ·update(SB), $0-32
 	MOVD 24(R3), R11 // r0
 	MOVD 32(R3), R12 // r1
 
+	MOVD $8, R24
+
 	CMP R5, $16
 	BLT bytes_between_0_and_15
 
@@ -94,7 +102,7 @@ flush_buffer:
 
 	// Greater than 8 -- load the rightmost remaining bytes in msg
 	// and put into R17 (h1)
-	MOVD (R4)(R21), R17
+	LE_MOVD (R4)(R21), R17
 	MOVD $16, R22
 
 	// Find the offset to those bytes
@@ -118,7 +126,7 @@ just1:
 	BLT less8
 
 	// Exactly 8
-	MOVD (R4), R16
+	LE_MOVD (R4), R16
 
 	CMP R17, $0
 
@@ -133,7 +141,7 @@ less8:
 	MOVD  $0, R22   // shift count
 	CMP   R5, $4
 	BLT   less4
-	MOVWZ (R4), R16
+	LE_MOVWZ (R4), R16
 	ADD   $4, R4
 	ADD   $-4, R5
 	MOVD  $32, R22
@@ -141,7 +149,7 @@ less8:
 less4:
 	CMP   R5, $2
 	BLT   less2
-	MOVHZ (R4), R21
+	LE_MOVHZ (R4), R21
 	SLD   R22, R21, R21
 	OR    R16, R21, R16
 	ADD   $16, R22
diff --git a/vendor/golang.org/x/crypto/salsa20/salsa/salsa20_amd64.s b/vendor/golang.org/x/crypto/salsa20/salsa/salsa20_amd64.s
index fcce0234b..3883e0ec2 100644
--- a/vendor/golang.org/x/crypto/salsa20/salsa/salsa20_amd64.s
+++ b/vendor/golang.org/x/crypto/salsa20/salsa/salsa20_amd64.s
@@ -1,880 +1,880 @@
-// Copyright 2012 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
+// Code generated by command: go run salsa20_amd64_asm.go -out ../salsa20_amd64.s -pkg salsa. DO NOT EDIT.
 
 //go:build amd64 && !purego && gc
 
-// This code was translated into a form compatible with 6a from the public
-// domain sources in SUPERCOP: https://bench.cr.yp.to/supercop.html
+// func salsa2020XORKeyStream(out *byte, in *byte, n uint64, nonce *byte, key *byte)
+// Requires: SSE2
+TEXT ·salsa2020XORKeyStream(SB), $456-40
+	// This needs up to 64 bytes at 360(R12); hence the non-obvious frame size.
+	MOVQ   out+0(FP), DI
+	MOVQ   in+8(FP), SI
+	MOVQ   n+16(FP), DX
+	MOVQ   nonce+24(FP), CX
+	MOVQ   key+32(FP), R8
+	MOVQ   SP, R12
+	ADDQ   $0x1f, R12
+	ANDQ   $-32, R12
+	MOVQ   DX, R9
+	MOVQ   CX, DX
+	MOVQ   R8, R10
+	CMPQ   R9, $0x00
+	JBE    DONE
+	MOVL   20(R10), CX
+	MOVL   (R10), R8
+	MOVL   (DX), AX
+	MOVL   16(R10), R11
+	MOVL   CX, (R12)
+	MOVL   R8, 4(R12)
+	MOVL   AX, 8(R12)
+	MOVL   R11, 12(R12)
+	MOVL   8(DX), CX
+	MOVL   24(R10), R8
+	MOVL   4(R10), AX
+	MOVL   4(DX), R11
+	MOVL   CX, 16(R12)
+	MOVL   R8, 20(R12)
+	MOVL   AX, 24(R12)
+	MOVL   R11, 28(R12)
+	MOVL   12(DX), CX
+	MOVL   12(R10), DX
+	MOVL   28(R10), R8
+	MOVL   8(R10), AX
+	MOVL   DX, 32(R12)
+	MOVL   CX, 36(R12)
+	MOVL   R8, 40(R12)
+	MOVL   AX, 44(R12)
+	MOVQ   $0x61707865, DX
+	MOVQ   $0x3320646e, CX
+	MOVQ   $0x79622d32, R8
+	MOVQ   $0x6b206574, AX
+	MOVL   DX, 48(R12)
+	MOVL   CX, 52(R12)
+	MOVL   R8, 56(R12)
+	MOVL   AX, 60(R12)
+	CMPQ   R9, $0x00000100
+	JB     BYTESBETWEEN1AND255
+	MOVOA  48(R12), X0
+	PSHUFL $0x55, X0, X1
+	PSHUFL $0xaa, X0, X2
+	PSHUFL $0xff, X0, X3
+	PSHUFL $0x00, X0, X0
+	MOVOA  X1, 64(R12)
+	MOVOA  X2, 80(R12)
+	MOVOA  X3, 96(R12)
+	MOVOA  X0, 112(R12)
+	MOVOA  (R12), X0
+	PSHUFL $0xaa, X0, X1
+	PSHUFL $0xff, X0, X2
+	PSHUFL $0x00, X0, X3
+	PSHUFL $0x55, X0, X0
+	MOVOA  X1, 128(R12)
+	MOVOA  X2, 144(R12)
+	MOVOA  X3, 160(R12)
+	MOVOA  X0, 176(R12)
+	MOVOA  16(R12), X0
+	PSHUFL $0xff, X0, X1
+	PSHUFL $0x55, X0, X2
+	PSHUFL $0xaa, X0, X0
+	MOVOA  X1, 192(R12)
+	MOVOA  X2, 208(R12)
+	MOVOA  X0, 224(R12)
+	MOVOA  32(R12), X0
+	PSHUFL $0x00, X0, X1
+	PSHUFL $0xaa, X0, X2
+	PSHUFL $0xff, X0, X0
+	MOVOA  X1, 240(R12)
+	MOVOA  X2, 256(R12)
+	MOVOA  X0, 272(R12)
 
-// func salsa2020XORKeyStream(out, in *byte, n uint64, nonce, key *byte)
-// This needs up to 64 bytes at 360(R12); hence the non-obvious frame size.
-TEXT ·salsa2020XORKeyStream(SB),0,$456-40 // frame = 424 + 32 byte alignment
-	MOVQ out+0(FP),DI
-	MOVQ in+8(FP),SI
-	MOVQ n+16(FP),DX
-	MOVQ nonce+24(FP),CX
-	MOVQ key+32(FP),R8
+BYTESATLEAST256:
+	MOVL  16(R12), DX
+	MOVL  36(R12), CX
+	MOVL  DX, 288(R12)
+	MOVL  CX, 304(R12)
+	SHLQ  $0x20, CX
+	ADDQ  CX, DX
+	ADDQ  $0x01, DX
+	MOVQ  DX, CX
+	SHRQ  $0x20, CX
+	MOVL  DX, 292(R12)
+	MOVL  CX, 308(R12)
+	ADDQ  $0x01, DX
+	MOVQ  DX, CX
+	SHRQ  $0x20, CX
+	MOVL  DX, 296(R12)
+	MOVL  CX, 312(R12)
+	ADDQ  $0x01, DX
+	MOVQ  DX, CX
+	SHRQ  $0x20, CX
+	MOVL  DX, 300(R12)
+	MOVL  CX, 316(R12)
+	ADDQ  $0x01, DX
+	MOVQ  DX, CX
+	SHRQ  $0x20, CX
+	MOVL  DX, 16(R12)
+	MOVL  CX, 36(R12)
+	MOVQ  R9, 352(R12)
+	MOVQ  $0x00000014, DX
+	MOVOA 64(R12), X0
+	MOVOA 80(R12), X1
+	MOVOA 96(R12), X2
+	MOVOA 256(R12), X3
+	MOVOA 272(R12), X4
+	MOVOA 128(R12), X5
+	MOVOA 144(R12), X6
+	MOVOA 176(R12), X7
+	MOVOA 192(R12), X8
+	MOVOA 208(R12), X9
+	MOVOA 224(R12), X10
+	MOVOA 304(R12), X11
+	MOVOA 112(R12), X12
+	MOVOA 160(R12), X13
+	MOVOA 240(R12), X14
+	MOVOA 288(R12), X15
 
-	MOVQ SP,R12
-	ADDQ $31, R12
-	ANDQ $~31, R12
+MAINLOOP1:
+	MOVOA  X1, 320(R12)
+	MOVOA  X2, 336(R12)
+	MOVOA  X13, X1
+	PADDL  X12, X1
+	MOVOA  X1, X2
+	PSLLL  $0x07, X1
+	PXOR   X1, X14
+	PSRLL  $0x19, X2
+	PXOR   X2, X14
+	MOVOA  X7, X1
+	PADDL  X0, X1
+	MOVOA  X1, X2
+	PSLLL  $0x07, X1
+	PXOR   X1, X11
+	PSRLL  $0x19, X2
+	PXOR   X2, X11
+	MOVOA  X12, X1
+	PADDL  X14, X1
+	MOVOA  X1, X2
+	PSLLL  $0x09, X1
+	PXOR   X1, X15
+	PSRLL  $0x17, X2
+	PXOR   X2, X15
+	MOVOA  X0, X1
+	PADDL  X11, X1
+	MOVOA  X1, X2
+	PSLLL  $0x09, X1
+	PXOR   X1, X9
+	PSRLL  $0x17, X2
+	PXOR   X2, X9
+	MOVOA  X14, X1
+	PADDL  X15, X1
+	MOVOA  X1, X2
+	PSLLL  $0x0d, X1
+	PXOR   X1, X13
+	PSRLL  $0x13, X2
+	PXOR   X2, X13
+	MOVOA  X11, X1
+	PADDL  X9, X1
+	MOVOA  X1, X2
+	PSLLL  $0x0d, X1
+	PXOR   X1, X7
+	PSRLL  $0x13, X2
+	PXOR   X2, X7
+	MOVOA  X15, X1
+	PADDL  X13, X1
+	MOVOA  X1, X2
+	PSLLL  $0x12, X1
+	PXOR   X1, X12
+	PSRLL  $0x0e, X2
+	PXOR   X2, X12
+	MOVOA  320(R12), X1
+	MOVOA  X12, 320(R12)
+	MOVOA  X9, X2
+	PADDL  X7, X2
+	MOVOA  X2, X12
+	PSLLL  $0x12, X2
+	PXOR   X2, X0
+	PSRLL  $0x0e, X12
+	PXOR   X12, X0
+	MOVOA  X5, X2
+	PADDL  X1, X2
+	MOVOA  X2, X12
+	PSLLL  $0x07, X2
+	PXOR   X2, X3
+	PSRLL  $0x19, X12
+	PXOR   X12, X3
+	MOVOA  336(R12), X2
+	MOVOA  X0, 336(R12)
+	MOVOA  X6, X0
+	PADDL  X2, X0
+	MOVOA  X0, X12
+	PSLLL  $0x07, X0
+	PXOR   X0, X4
+	PSRLL  $0x19, X12
+	PXOR   X12, X4
+	MOVOA  X1, X0
+	PADDL  X3, X0
+	MOVOA  X0, X12
+	PSLLL  $0x09, X0
+	PXOR   X0, X10
+	PSRLL  $0x17, X12
+	PXOR   X12, X10
+	MOVOA  X2, X0
+	PADDL  X4, X0
+	MOVOA  X0, X12
+	PSLLL  $0x09, X0
+	PXOR   X0, X8
+	PSRLL  $0x17, X12
+	PXOR   X12, X8
+	MOVOA  X3, X0
+	PADDL  X10, X0
+	MOVOA  X0, X12
+	PSLLL  $0x0d, X0
+	PXOR   X0, X5
+	PSRLL  $0x13, X12
+	PXOR   X12, X5
+	MOVOA  X4, X0
+	PADDL  X8, X0
+	MOVOA  X0, X12
+	PSLLL  $0x0d, X0
+	PXOR   X0, X6
+	PSRLL  $0x13, X12
+	PXOR   X12, X6
+	MOVOA  X10, X0
+	PADDL  X5, X0
+	MOVOA  X0, X12
+	PSLLL  $0x12, X0
+	PXOR   X0, X1
+	PSRLL  $0x0e, X12
+	PXOR   X12, X1
+	MOVOA  320(R12), X0
+	MOVOA  X1, 320(R12)
+	MOVOA  X4, X1
+	PADDL  X0, X1
+	MOVOA  X1, X12
+	PSLLL  $0x07, X1
+	PXOR   X1, X7
+	PSRLL  $0x19, X12
+	PXOR   X12, X7
+	MOVOA  X8, X1
+	PADDL  X6, X1
+	MOVOA  X1, X12
+	PSLLL  $0x12, X1
+	PXOR   X1, X2
+	PSRLL  $0x0e, X12
+	PXOR   X12, X2
+	MOVOA  336(R12), X12
+	MOVOA  X2, 336(R12)
+	MOVOA  X14, X1
+	PADDL  X12, X1
+	MOVOA  X1, X2
+	PSLLL  $0x07, X1
+	PXOR   X1, X5
+	PSRLL  $0x19, X2
+	PXOR   X2, X5
+	MOVOA  X0, X1
+	PADDL  X7, X1
+	MOVOA  X1, X2
+	PSLLL  $0x09, X1
+	PXOR   X1, X10
+	PSRLL  $0x17, X2
+	PXOR   X2, X10
+	MOVOA  X12, X1
+	PADDL  X5, X1
+	MOVOA  X1, X2
+	PSLLL  $0x09, X1
+	PXOR   X1, X8
+	PSRLL  $0x17, X2
+	PXOR   X2, X8
+	MOVOA  X7, X1
+	PADDL  X10, X1
+	MOVOA  X1, X2
+	PSLLL  $0x0d, X1
+	PXOR   X1, X4
+	PSRLL  $0x13, X2
+	PXOR   X2, X4
+	MOVOA  X5, X1
+	PADDL  X8, X1
+	MOVOA  X1, X2
+	PSLLL  $0x0d, X1
+	PXOR   X1, X14
+	PSRLL  $0x13, X2
+	PXOR   X2, X14
+	MOVOA  X10, X1
+	PADDL  X4, X1
+	MOVOA  X1, X2
+	PSLLL  $0x12, X1
+	PXOR   X1, X0
+	PSRLL  $0x0e, X2
+	PXOR   X2, X0
+	MOVOA  320(R12), X1
+	MOVOA  X0, 320(R12)
+	MOVOA  X8, X0
+	PADDL  X14, X0
+	MOVOA  X0, X2
+	PSLLL  $0x12, X0
+	PXOR   X0, X12
+	PSRLL  $0x0e, X2
+	PXOR   X2, X12
+	MOVOA  X11, X0
+	PADDL  X1, X0
+	MOVOA  X0, X2
+	PSLLL  $0x07, X0
+	PXOR   X0, X6
+	PSRLL  $0x19, X2
+	PXOR   X2, X6
+	MOVOA  336(R12), X2
+	MOVOA  X12, 336(R12)
+	MOVOA  X3, X0
+	PADDL  X2, X0
+	MOVOA  X0, X12
+	PSLLL  $0x07, X0
+	PXOR   X0, X13
+	PSRLL  $0x19, X12
+	PXOR   X12, X13
+	MOVOA  X1, X0
+	PADDL  X6, X0
+	MOVOA  X0, X12
+	PSLLL  $0x09, X0
+	PXOR   X0, X15
+	PSRLL  $0x17, X12
+	PXOR   X12, X15
+	MOVOA  X2, X0
+	PADDL  X13, X0
+	MOVOA  X0, X12
+	PSLLL  $0x09, X0
+	PXOR   X0, X9
+	PSRLL  $0x17, X12
+	PXOR   X12, X9
+	MOVOA  X6, X0
+	PADDL  X15, X0
+	MOVOA  X0, X12
+	PSLLL  $0x0d, X0
+	PXOR   X0, X11
+	PSRLL  $0x13, X12
+	PXOR   X12, X11
+	MOVOA  X13, X0
+	PADDL  X9, X0
+	MOVOA  X0, X12
+	PSLLL  $0x0d, X0
+	PXOR   X0, X3
+	PSRLL  $0x13, X12
+	PXOR   X12, X3
+	MOVOA  X15, X0
+	PADDL  X11, X0
+	MOVOA  X0, X12
+	PSLLL  $0x12, X0
+	PXOR   X0, X1
+	PSRLL  $0x0e, X12
+	PXOR   X12, X1
+	MOVOA  X9, X0
+	PADDL  X3, X0
+	MOVOA  X0, X12
+	PSLLL  $0x12, X0
+	PXOR   X0, X2
+	PSRLL  $0x0e, X12
+	PXOR   X12, X2
+	MOVOA  320(R12), X12
+	MOVOA  336(R12), X0
+	SUBQ   $0x02, DX
+	JA     MAINLOOP1
+	PADDL  112(R12), X12
+	PADDL  176(R12), X7
+	PADDL  224(R12), X10
+	PADDL  272(R12), X4
+	MOVD   X12, DX
+	MOVD   X7, CX
+	MOVD   X10, R8
+	MOVD   X4, R9
+	PSHUFL $0x39, X12, X12
+	PSHUFL $0x39, X7, X7
+	PSHUFL $0x39, X10, X10
+	PSHUFL $0x39, X4, X4
+	XORL   (SI), DX
+	XORL   4(SI), CX
+	XORL   8(SI), R8
+	XORL   12(SI), R9
+	MOVL   DX, (DI)
+	MOVL   CX, 4(DI)
+	MOVL   R8, 8(DI)
+	MOVL   R9, 12(DI)
+	MOVD   X12, DX
+	MOVD   X7, CX
+	MOVD   X10, R8
+	MOVD   X4, R9
+	PSHUFL $0x39, X12, X12
+	PSHUFL $0x39, X7, X7
+	PSHUFL $0x39, X10, X10
+	PSHUFL $0x39, X4, X4
+	XORL   64(SI), DX
+	XORL   68(SI), CX
+	XORL   72(SI), R8
+	XORL   76(SI), R9
+	MOVL   DX, 64(DI)
+	MOVL   CX, 68(DI)
+	MOVL   R8, 72(DI)
+	MOVL   R9, 76(DI)
+	MOVD   X12, DX
+	MOVD   X7, CX
+	MOVD   X10, R8
+	MOVD   X4, R9
+	PSHUFL $0x39, X12, X12
+	PSHUFL $0x39, X7, X7
+	PSHUFL $0x39, X10, X10
+	PSHUFL $0x39, X4, X4
+	XORL   128(SI), DX
+	XORL   132(SI), CX
+	XORL   136(SI), R8
+	XORL   140(SI), R9
+	MOVL   DX, 128(DI)
+	MOVL   CX, 132(DI)
+	MOVL   R8, 136(DI)
+	MOVL   R9, 140(DI)
+	MOVD   X12, DX
+	MOVD   X7, CX
+	MOVD   X10, R8
+	MOVD   X4, R9
+	XORL   192(SI), DX
+	XORL   196(SI), CX
+	XORL   200(SI), R8
+	XORL   204(SI), R9
+	MOVL   DX, 192(DI)
+	MOVL   CX, 196(DI)
+	MOVL   R8, 200(DI)
+	MOVL   R9, 204(DI)
+	PADDL  240(R12), X14
+	PADDL  64(R12), X0
+	PADDL  128(R12), X5
+	PADDL  192(R12), X8
+	MOVD   X14, DX
+	MOVD   X0, CX
+	MOVD   X5, R8
+	MOVD   X8, R9
+	PSHUFL $0x39, X14, X14
+	PSHUFL $0x39, X0, X0
+	PSHUFL $0x39, X5, X5
+	PSHUFL $0x39, X8, X8
+	XORL   16(SI), DX
+	XORL   20(SI), CX
+	XORL   24(SI), R8
+	XORL   28(SI), R9
+	MOVL   DX, 16(DI)
+	MOVL   CX, 20(DI)
+	MOVL   R8, 24(DI)
+	MOVL   R9, 28(DI)
+	MOVD   X14, DX
+	MOVD   X0, CX
+	MOVD   X5, R8
+	MOVD   X8, R9
+	PSHUFL $0x39, X14, X14
+	PSHUFL $0x39, X0, X0
+	PSHUFL $0x39, X5, X5
+	PSHUFL $0x39, X8, X8
+	XORL   80(SI), DX
+	XORL   84(SI), CX
+	XORL   88(SI), R8
+	XORL   92(SI), R9
+	MOVL   DX, 80(DI)
+	MOVL   CX, 84(DI)
+	MOVL   R8, 88(DI)
+	MOVL   R9, 92(DI)
+	MOVD   X14, DX
+	MOVD   X0, CX
+	MOVD   X5, R8
+	MOVD   X8, R9
+	PSHUFL $0x39, X14, X14
+	PSHUFL $0x39, X0, X0
+	PSHUFL $0x39, X5, X5
+	PSHUFL $0x39, X8, X8
+	XORL   144(SI), DX
+	XORL   148(SI), CX
+	XORL   152(SI), R8
+	XORL   156(SI), R9
+	MOVL   DX, 144(DI)
+	MOVL   CX, 148(DI)
+	MOVL   R8, 152(DI)
+	MOVL   R9, 156(DI)
+	MOVD   X14, DX
+	MOVD   X0, CX
+	MOVD   X5, R8
+	MOVD   X8, R9
+	XORL   208(SI), DX
+	XORL   212(SI), CX
+	XORL   216(SI), R8
+	XORL   220(SI), R9
+	MOVL   DX, 208(DI)
+	MOVL   CX, 212(DI)
+	MOVL   R8, 216(DI)
+	MOVL   R9, 220(DI)
+	PADDL  288(R12), X15
+	PADDL  304(R12), X11
+	PADDL  80(R12), X1
+	PADDL  144(R12), X6
+	MOVD   X15, DX
+	MOVD   X11, CX
+	MOVD   X1, R8
+	MOVD   X6, R9
+	PSHUFL $0x39, X15, X15
+	PSHUFL $0x39, X11, X11
+	PSHUFL $0x39, X1, X1
+	PSHUFL $0x39, X6, X6
+	XORL   32(SI), DX
+	XORL   36(SI), CX
+	XORL   40(SI), R8
+	XORL   44(SI), R9
+	MOVL   DX, 32(DI)
+	MOVL   CX, 36(DI)
+	MOVL   R8, 40(DI)
+	MOVL   R9, 44(DI)
+	MOVD   X15, DX
+	MOVD   X11, CX
+	MOVD   X1, R8
+	MOVD   X6, R9
+	PSHUFL $0x39, X15, X15
+	PSHUFL $0x39, X11, X11
+	PSHUFL $0x39, X1, X1
+	PSHUFL $0x39, X6, X6
+	XORL   96(SI), DX
+	XORL   100(SI), CX
+	XORL   104(SI), R8
+	XORL   108(SI), R9
+	MOVL   DX, 96(DI)
+	MOVL   CX, 100(DI)
+	MOVL   R8, 104(DI)
+	MOVL   R9, 108(DI)
+	MOVD   X15, DX
+	MOVD   X11, CX
+	MOVD   X1, R8
+	MOVD   X6, R9
+	PSHUFL $0x39, X15, X15
+	PSHUFL $0x39, X11, X11
+	PSHUFL $0x39, X1, X1
+	PSHUFL $0x39, X6, X6
+	XORL   160(SI), DX
+	XORL   164(SI), CX
+	XORL   168(SI), R8
+	XORL   172(SI), R9
+	MOVL   DX, 160(DI)
+	MOVL   CX, 164(DI)
+	MOVL   R8, 168(DI)
+	MOVL   R9, 172(DI)
+	MOVD   X15, DX
+	MOVD   X11, CX
+	MOVD   X1, R8
+	MOVD   X6, R9
+	XORL   224(SI), DX
+	XORL   228(SI), CX
+	XORL   232(SI), R8
+	XORL   236(SI), R9
+	MOVL   DX, 224(DI)
+	MOVL   CX, 228(DI)
+	MOVL   R8, 232(DI)
+	MOVL   R9, 236(DI)
+	PADDL  160(R12), X13
+	PADDL  208(R12), X9
+	PADDL  256(R12), X3
+	PADDL  96(R12), X2
+	MOVD   X13, DX
+	MOVD   X9, CX
+	MOVD   X3, R8
+	MOVD   X2, R9
+	PSHUFL $0x39, X13, X13
+	PSHUFL $0x39, X9, X9
+	PSHUFL $0x39, X3, X3
+	PSHUFL $0x39, X2, X2
+	XORL   48(SI), DX
+	XORL   52(SI), CX
+	XORL   56(SI), R8
+	XORL   60(SI), R9
+	MOVL   DX, 48(DI)
+	MOVL   CX, 52(DI)
+	MOVL   R8, 56(DI)
+	MOVL   R9, 60(DI)
+	MOVD   X13, DX
+	MOVD   X9, CX
+	MOVD   X3, R8
+	MOVD   X2, R9
+	PSHUFL $0x39, X13, X13
+	PSHUFL $0x39, X9, X9
+	PSHUFL $0x39, X3, X3
+	PSHUFL $0x39, X2, X2
+	XORL   112(SI), DX
+	XORL   116(SI), CX
+	XORL   120(SI), R8
+	XORL   124(SI), R9
+	MOVL   DX, 112(DI)
+	MOVL   CX, 116(DI)
+	MOVL   R8, 120(DI)
+	MOVL   R9, 124(DI)
+	MOVD   X13, DX
+	MOVD   X9, CX
+	MOVD   X3, R8
+	MOVD   X2, R9
+	PSHUFL $0x39, X13, X13
+	PSHUFL $0x39, X9, X9
+	PSHUFL $0x39, X3, X3
+	PSHUFL $0x39, X2, X2
+	XORL   176(SI), DX
+	XORL   180(SI), CX
+	XORL   184(SI), R8
+	XORL   188(SI), R9
+	MOVL   DX, 176(DI)
+	MOVL   CX, 180(DI)
+	MOVL   R8, 184(DI)
+	MOVL   R9, 188(DI)
+	MOVD   X13, DX
+	MOVD   X9, CX
+	MOVD   X3, R8
+	MOVD   X2, R9
+	XORL   240(SI), DX
+	XORL   244(SI), CX
+	XORL   248(SI), R8
+	XORL   252(SI), R9
+	MOVL   DX, 240(DI)
+	MOVL   CX, 244(DI)
+	MOVL   R8, 248(DI)
+	MOVL   R9, 252(DI)
+	MOVQ   352(R12), R9
+	SUBQ   $0x00000100, R9
+	ADDQ   $0x00000100, SI
+	ADDQ   $0x00000100, DI
+	CMPQ   R9, $0x00000100
+	JAE    BYTESATLEAST256
+	CMPQ   R9, $0x00
+	JBE    DONE
 
-	MOVQ DX,R9
-	MOVQ CX,DX
-	MOVQ R8,R10
-	CMPQ R9,$0
-	JBE DONE
-	START:
-	MOVL 20(R10),CX
-	MOVL 0(R10),R8
-	MOVL 0(DX),AX
-	MOVL 16(R10),R11
-	MOVL CX,0(R12)
-	MOVL R8, 4 (R12)
-	MOVL AX, 8 (R12)
-	MOVL R11, 12 (R12)
-	MOVL 8(DX),CX
-	MOVL 24(R10),R8
-	MOVL 4(R10),AX
-	MOVL 4(DX),R11
-	MOVL CX,16(R12)
-	MOVL R8, 20 (R12)
-	MOVL AX, 24 (R12)
-	MOVL R11, 28 (R12)
-	MOVL 12(DX),CX
-	MOVL 12(R10),DX
-	MOVL 28(R10),R8
-	MOVL 8(R10),AX
-	MOVL DX,32(R12)
-	MOVL CX, 36 (R12)
-	MOVL R8, 40 (R12)
-	MOVL AX, 44 (R12)
-	MOVQ $1634760805,DX
-	MOVQ $857760878,CX
-	MOVQ $2036477234,R8
-	MOVQ $1797285236,AX
-	MOVL DX,48(R12)
-	MOVL CX, 52 (R12)
-	MOVL R8, 56 (R12)
-	MOVL AX, 60 (R12)
-	CMPQ R9,$256
-	JB BYTESBETWEEN1AND255
-	MOVOA 48(R12),X0
-	PSHUFL $0X55,X0,X1
-	PSHUFL $0XAA,X0,X2
-	PSHUFL $0XFF,X0,X3
-	PSHUFL $0X00,X0,X0
-	MOVOA X1,64(R12)
-	MOVOA X2,80(R12)
-	MOVOA X3,96(R12)
-	MOVOA X0,112(R12)
-	MOVOA 0(R12),X0
-	PSHUFL $0XAA,X0,X1
-	PSHUFL $0XFF,X0,X2
-	PSHUFL $0X00,X0,X3
-	PSHUFL $0X55,X0,X0
-	MOVOA X1,128(R12)
-	MOVOA X2,144(R12)
-	MOVOA X3,160(R12)
-	MOVOA X0,176(R12)
-	MOVOA 16(R12),X0
-	PSHUFL $0XFF,X0,X1
-	PSHUFL $0X55,X0,X2
-	PSHUFL $0XAA,X0,X0
-	MOVOA X1,192(R12)
-	MOVOA X2,208(R12)
-	MOVOA X0,224(R12)
-	MOVOA 32(R12),X0
-	PSHUFL $0X00,X0,X1
-	PSHUFL $0XAA,X0,X2
-	PSHUFL $0XFF,X0,X0
-	MOVOA X1,240(R12)
-	MOVOA X2,256(R12)
-	MOVOA X0,272(R12)
-	BYTESATLEAST256:
-	MOVL 16(R12),DX
-	MOVL  36 (R12),CX
-	MOVL DX,288(R12)
-	MOVL CX,304(R12)
-	SHLQ $32,CX
-	ADDQ CX,DX
-	ADDQ $1,DX
-	MOVQ DX,CX
-	SHRQ $32,CX
-	MOVL DX, 292 (R12)
-	MOVL CX, 308 (R12)
-	ADDQ $1,DX
-	MOVQ DX,CX
-	SHRQ $32,CX
-	MOVL DX, 296 (R12)
-	MOVL CX, 312 (R12)
-	ADDQ $1,DX
-	MOVQ DX,CX
-	SHRQ $32,CX
-	MOVL DX, 300 (R12)
-	MOVL CX, 316 (R12)
-	ADDQ $1,DX
-	MOVQ DX,CX
-	SHRQ $32,CX
-	MOVL DX,16(R12)
-	MOVL CX, 36 (R12)
-	MOVQ R9,352(R12)
-	MOVQ $20,DX
-	MOVOA 64(R12),X0
-	MOVOA 80(R12),X1
-	MOVOA 96(R12),X2
-	MOVOA 256(R12),X3
-	MOVOA 272(R12),X4
-	MOVOA 128(R12),X5
-	MOVOA 144(R12),X6
-	MOVOA 176(R12),X7
-	MOVOA 192(R12),X8
-	MOVOA 208(R12),X9
-	MOVOA 224(R12),X10
-	MOVOA 304(R12),X11
-	MOVOA 112(R12),X12
-	MOVOA 160(R12),X13
-	MOVOA 240(R12),X14
-	MOVOA 288(R12),X15
-	MAINLOOP1:
-	MOVOA X1,320(R12)
-	MOVOA X2,336(R12)
-	MOVOA X13,X1
-	PADDL X12,X1
-	MOVOA X1,X2
-	PSLLL $7,X1
-	PXOR X1,X14
-	PSRLL $25,X2
-	PXOR X2,X14
-	MOVOA X7,X1
-	PADDL X0,X1
-	MOVOA X1,X2
-	PSLLL $7,X1
-	PXOR X1,X11
-	PSRLL $25,X2
-	PXOR X2,X11
-	MOVOA X12,X1
-	PADDL X14,X1
-	MOVOA X1,X2
-	PSLLL $9,X1
-	PXOR X1,X15
-	PSRLL $23,X2
-	PXOR X2,X15
-	MOVOA X0,X1
-	PADDL X11,X1
-	MOVOA X1,X2
-	PSLLL $9,X1
-	PXOR X1,X9
-	PSRLL $23,X2
-	PXOR X2,X9
-	MOVOA X14,X1
-	PADDL X15,X1
-	MOVOA X1,X2
-	PSLLL $13,X1
-	PXOR X1,X13
-	PSRLL $19,X2
-	PXOR X2,X13
-	MOVOA X11,X1
-	PADDL X9,X1
-	MOVOA X1,X2
-	PSLLL $13,X1
-	PXOR X1,X7
-	PSRLL $19,X2
-	PXOR X2,X7
-	MOVOA X15,X1
-	PADDL X13,X1
-	MOVOA X1,X2
-	PSLLL $18,X1
-	PXOR X1,X12
-	PSRLL $14,X2
-	PXOR X2,X12
-	MOVOA 320(R12),X1
-	MOVOA X12,320(R12)
-	MOVOA X9,X2
-	PADDL X7,X2
-	MOVOA X2,X12
-	PSLLL $18,X2
-	PXOR X2,X0
-	PSRLL $14,X12
-	PXOR X12,X0
-	MOVOA X5,X2
-	PADDL X1,X2
-	MOVOA X2,X12
-	PSLLL $7,X2
-	PXOR X2,X3
-	PSRLL $25,X12
-	PXOR X12,X3
-	MOVOA 336(R12),X2
-	MOVOA X0,336(R12)
-	MOVOA X6,X0
-	PADDL X2,X0
-	MOVOA X0,X12
-	PSLLL $7,X0
-	PXOR X0,X4
-	PSRLL $25,X12
-	PXOR X12,X4
-	MOVOA X1,X0
-	PADDL X3,X0
-	MOVOA X0,X12
-	PSLLL $9,X0
-	PXOR X0,X10
-	PSRLL $23,X12
-	PXOR X12,X10
-	MOVOA X2,X0
-	PADDL X4,X0
-	MOVOA X0,X12
-	PSLLL $9,X0
-	PXOR X0,X8
-	PSRLL $23,X12
-	PXOR X12,X8
-	MOVOA X3,X0
-	PADDL X10,X0
-	MOVOA X0,X12
-	PSLLL $13,X0
-	PXOR X0,X5
-	PSRLL $19,X12
-	PXOR X12,X5
-	MOVOA X4,X0
-	PADDL X8,X0
-	MOVOA X0,X12
-	PSLLL $13,X0
-	PXOR X0,X6
-	PSRLL $19,X12
-	PXOR X12,X6
-	MOVOA X10,X0
-	PADDL X5,X0
-	MOVOA X0,X12
-	PSLLL $18,X0
-	PXOR X0,X1
-	PSRLL $14,X12
-	PXOR X12,X1
-	MOVOA 320(R12),X0
-	MOVOA X1,320(R12)
-	MOVOA X4,X1
-	PADDL X0,X1
-	MOVOA X1,X12
-	PSLLL $7,X1
-	PXOR X1,X7
-	PSRLL $25,X12
-	PXOR X12,X7
-	MOVOA X8,X1
-	PADDL X6,X1
-	MOVOA X1,X12
-	PSLLL $18,X1
-	PXOR X1,X2
-	PSRLL $14,X12
-	PXOR X12,X2
-	MOVOA 336(R12),X12
-	MOVOA X2,336(R12)
-	MOVOA X14,X1
-	PADDL X12,X1
-	MOVOA X1,X2
-	PSLLL $7,X1
-	PXOR X1,X5
-	PSRLL $25,X2
-	PXOR X2,X5
-	MOVOA X0,X1
-	PADDL X7,X1
-	MOVOA X1,X2
-	PSLLL $9,X1
-	PXOR X1,X10
-	PSRLL $23,X2
-	PXOR X2,X10
-	MOVOA X12,X1
-	PADDL X5,X1
-	MOVOA X1,X2
-	PSLLL $9,X1
-	PXOR X1,X8
-	PSRLL $23,X2
-	PXOR X2,X8
-	MOVOA X7,X1
-	PADDL X10,X1
-	MOVOA X1,X2
-	PSLLL $13,X1
-	PXOR X1,X4
-	PSRLL $19,X2
-	PXOR X2,X4
-	MOVOA X5,X1
-	PADDL X8,X1
-	MOVOA X1,X2
-	PSLLL $13,X1
-	PXOR X1,X14
-	PSRLL $19,X2
-	PXOR X2,X14
-	MOVOA X10,X1
-	PADDL X4,X1
-	MOVOA X1,X2
-	PSLLL $18,X1
-	PXOR X1,X0
-	PSRLL $14,X2
-	PXOR X2,X0
-	MOVOA 320(R12),X1
-	MOVOA X0,320(R12)
-	MOVOA X8,X0
-	PADDL X14,X0
-	MOVOA X0,X2
-	PSLLL $18,X0
-	PXOR X0,X12
-	PSRLL $14,X2
-	PXOR X2,X12
-	MOVOA X11,X0
-	PADDL X1,X0
-	MOVOA X0,X2
-	PSLLL $7,X0
-	PXOR X0,X6
-	PSRLL $25,X2
-	PXOR X2,X6
-	MOVOA 336(R12),X2
-	MOVOA X12,336(R12)
-	MOVOA X3,X0
-	PADDL X2,X0
-	MOVOA X0,X12
-	PSLLL $7,X0
-	PXOR X0,X13
-	PSRLL $25,X12
-	PXOR X12,X13
-	MOVOA X1,X0
-	PADDL X6,X0
-	MOVOA X0,X12
-	PSLLL $9,X0
-	PXOR X0,X15
-	PSRLL $23,X12
-	PXOR X12,X15
-	MOVOA X2,X0
-	PADDL X13,X0
-	MOVOA X0,X12
-	PSLLL $9,X0
-	PXOR X0,X9
-	PSRLL $23,X12
-	PXOR X12,X9
-	MOVOA X6,X0
-	PADDL X15,X0
-	MOVOA X0,X12
-	PSLLL $13,X0
-	PXOR X0,X11
-	PSRLL $19,X12
-	PXOR X12,X11
-	MOVOA X13,X0
-	PADDL X9,X0
-	MOVOA X0,X12
-	PSLLL $13,X0
-	PXOR X0,X3
-	PSRLL $19,X12
-	PXOR X12,X3
-	MOVOA X15,X0
-	PADDL X11,X0
-	MOVOA X0,X12
-	PSLLL $18,X0
-	PXOR X0,X1
-	PSRLL $14,X12
-	PXOR X12,X1
-	MOVOA X9,X0
-	PADDL X3,X0
-	MOVOA X0,X12
-	PSLLL $18,X0
-	PXOR X0,X2
-	PSRLL $14,X12
-	PXOR X12,X2
-	MOVOA 320(R12),X12
-	MOVOA 336(R12),X0
-	SUBQ $2,DX
-	JA MAINLOOP1
-	PADDL 112(R12),X12
-	PADDL 176(R12),X7
-	PADDL 224(R12),X10
-	PADDL 272(R12),X4
-	MOVD X12,DX
-	MOVD X7,CX
-	MOVD X10,R8
-	MOVD X4,R9
-	PSHUFL $0X39,X12,X12
-	PSHUFL $0X39,X7,X7
-	PSHUFL $0X39,X10,X10
-	PSHUFL $0X39,X4,X4
-	XORL 0(SI),DX
-	XORL 4(SI),CX
-	XORL 8(SI),R8
-	XORL 12(SI),R9
-	MOVL DX,0(DI)
-	MOVL CX,4(DI)
-	MOVL R8,8(DI)
-	MOVL R9,12(DI)
-	MOVD X12,DX
-	MOVD X7,CX
-	MOVD X10,R8
-	MOVD X4,R9
-	PSHUFL $0X39,X12,X12
-	PSHUFL $0X39,X7,X7
-	PSHUFL $0X39,X10,X10
-	PSHUFL $0X39,X4,X4
-	XORL 64(SI),DX
-	XORL 68(SI),CX
-	XORL 72(SI),R8
-	XORL 76(SI),R9
-	MOVL DX,64(DI)
-	MOVL CX,68(DI)
-	MOVL R8,72(DI)
-	MOVL R9,76(DI)
-	MOVD X12,DX
-	MOVD X7,CX
-	MOVD X10,R8
-	MOVD X4,R9
-	PSHUFL $0X39,X12,X12
-	PSHUFL $0X39,X7,X7
-	PSHUFL $0X39,X10,X10
-	PSHUFL $0X39,X4,X4
-	XORL 128(SI),DX
-	XORL 132(SI),CX
-	XORL 136(SI),R8
-	XORL 140(SI),R9
-	MOVL DX,128(DI)
-	MOVL CX,132(DI)
-	MOVL R8,136(DI)
-	MOVL R9,140(DI)
-	MOVD X12,DX
-	MOVD X7,CX
-	MOVD X10,R8
-	MOVD X4,R9
-	XORL 192(SI),DX
-	XORL 196(SI),CX
-	XORL 200(SI),R8
-	XORL 204(SI),R9
-	MOVL DX,192(DI)
-	MOVL CX,196(DI)
-	MOVL R8,200(DI)
-	MOVL R9,204(DI)
-	PADDL 240(R12),X14
-	PADDL 64(R12),X0
-	PADDL 128(R12),X5
-	PADDL 192(R12),X8
-	MOVD X14,DX
-	MOVD X0,CX
-	MOVD X5,R8
-	MOVD X8,R9
-	PSHUFL $0X39,X14,X14
-	PSHUFL $0X39,X0,X0
-	PSHUFL $0X39,X5,X5
-	PSHUFL $0X39,X8,X8
-	XORL 16(SI),DX
-	XORL 20(SI),CX
-	XORL 24(SI),R8
-	XORL 28(SI),R9
-	MOVL DX,16(DI)
-	MOVL CX,20(DI)
-	MOVL R8,24(DI)
-	MOVL R9,28(DI)
-	MOVD X14,DX
-	MOVD X0,CX
-	MOVD X5,R8
-	MOVD X8,R9
-	PSHUFL $0X39,X14,X14
-	PSHUFL $0X39,X0,X0
-	PSHUFL $0X39,X5,X5
-	PSHUFL $0X39,X8,X8
-	XORL 80(SI),DX
-	XORL 84(SI),CX
-	XORL 88(SI),R8
-	XORL 92(SI),R9
-	MOVL DX,80(DI)
-	MOVL CX,84(DI)
-	MOVL R8,88(DI)
-	MOVL R9,92(DI)
-	MOVD X14,DX
-	MOVD X0,CX
-	MOVD X5,R8
-	MOVD X8,R9
-	PSHUFL $0X39,X14,X14
-	PSHUFL $0X39,X0,X0
-	PSHUFL $0X39,X5,X5
-	PSHUFL $0X39,X8,X8
-	XORL 144(SI),DX
-	XORL 148(SI),CX
-	XORL 152(SI),R8
-	XORL 156(SI),R9
-	MOVL DX,144(DI)
-	MOVL CX,148(DI)
-	MOVL R8,152(DI)
-	MOVL R9,156(DI)
-	MOVD X14,DX
-	MOVD X0,CX
-	MOVD X5,R8
-	MOVD X8,R9
-	XORL 208(SI),DX
-	XORL 212(SI),CX
-	XORL 216(SI),R8
-	XORL 220(SI),R9
-	MOVL DX,208(DI)
-	MOVL CX,212(DI)
-	MOVL R8,216(DI)
-	MOVL R9,220(DI)
-	PADDL 288(R12),X15
-	PADDL 304(R12),X11
-	PADDL 80(R12),X1
-	PADDL 144(R12),X6
-	MOVD X15,DX
-	MOVD X11,CX
-	MOVD X1,R8
-	MOVD X6,R9
-	PSHUFL $0X39,X15,X15
-	PSHUFL $0X39,X11,X11
-	PSHUFL $0X39,X1,X1
-	PSHUFL $0X39,X6,X6
-	XORL 32(SI),DX
-	XORL 36(SI),CX
-	XORL 40(SI),R8
-	XORL 44(SI),R9
-	MOVL DX,32(DI)
-	MOVL CX,36(DI)
-	MOVL R8,40(DI)
-	MOVL R9,44(DI)
-	MOVD X15,DX
-	MOVD X11,CX
-	MOVD X1,R8
-	MOVD X6,R9
-	PSHUFL $0X39,X15,X15
-	PSHUFL $0X39,X11,X11
-	PSHUFL $0X39,X1,X1
-	PSHUFL $0X39,X6,X6
-	XORL 96(SI),DX
-	XORL 100(SI),CX
-	XORL 104(SI),R8
-	XORL 108(SI),R9
-	MOVL DX,96(DI)
-	MOVL CX,100(DI)
-	MOVL R8,104(DI)
-	MOVL R9,108(DI)
-	MOVD X15,DX
-	MOVD X11,CX
-	MOVD X1,R8
-	MOVD X6,R9
-	PSHUFL $0X39,X15,X15
-	PSHUFL $0X39,X11,X11
-	PSHUFL $0X39,X1,X1
-	PSHUFL $0X39,X6,X6
-	XORL 160(SI),DX
-	XORL 164(SI),CX
-	XORL 168(SI),R8
-	XORL 172(SI),R9
-	MOVL DX,160(DI)
-	MOVL CX,164(DI)
-	MOVL R8,168(DI)
-	MOVL R9,172(DI)
-	MOVD X15,DX
-	MOVD X11,CX
-	MOVD X1,R8
-	MOVD X6,R9
-	XORL 224(SI),DX
-	XORL 228(SI),CX
-	XORL 232(SI),R8
-	XORL 236(SI),R9
-	MOVL DX,224(DI)
-	MOVL CX,228(DI)
-	MOVL R8,232(DI)
-	MOVL R9,236(DI)
-	PADDL 160(R12),X13
-	PADDL 208(R12),X9
-	PADDL 256(R12),X3
-	PADDL 96(R12),X2
-	MOVD X13,DX
-	MOVD X9,CX
-	MOVD X3,R8
-	MOVD X2,R9
-	PSHUFL $0X39,X13,X13
-	PSHUFL $0X39,X9,X9
-	PSHUFL $0X39,X3,X3
-	PSHUFL $0X39,X2,X2
-	XORL 48(SI),DX
-	XORL 52(SI),CX
-	XORL 56(SI),R8
-	XORL 60(SI),R9
-	MOVL DX,48(DI)
-	MOVL CX,52(DI)
-	MOVL R8,56(DI)
-	MOVL R9,60(DI)
-	MOVD X13,DX
-	MOVD X9,CX
-	MOVD X3,R8
-	MOVD X2,R9
-	PSHUFL $0X39,X13,X13
-	PSHUFL $0X39,X9,X9
-	PSHUFL $0X39,X3,X3
-	PSHUFL $0X39,X2,X2
-	XORL 112(SI),DX
-	XORL 116(SI),CX
-	XORL 120(SI),R8
-	XORL 124(SI),R9
-	MOVL DX,112(DI)
-	MOVL CX,116(DI)
-	MOVL R8,120(DI)
-	MOVL R9,124(DI)
-	MOVD X13,DX
-	MOVD X9,CX
-	MOVD X3,R8
-	MOVD X2,R9
-	PSHUFL $0X39,X13,X13
-	PSHUFL $0X39,X9,X9
-	PSHUFL $0X39,X3,X3
-	PSHUFL $0X39,X2,X2
-	XORL 176(SI),DX
-	XORL 180(SI),CX
-	XORL 184(SI),R8
-	XORL 188(SI),R9
-	MOVL DX,176(DI)
-	MOVL CX,180(DI)
-	MOVL R8,184(DI)
-	MOVL R9,188(DI)
-	MOVD X13,DX
-	MOVD X9,CX
-	MOVD X3,R8
-	MOVD X2,R9
-	XORL 240(SI),DX
-	XORL 244(SI),CX
-	XORL 248(SI),R8
-	XORL 252(SI),R9
-	MOVL DX,240(DI)
-	MOVL CX,244(DI)
-	MOVL R8,248(DI)
-	MOVL R9,252(DI)
-	MOVQ 352(R12),R9
-	SUBQ $256,R9
-	ADDQ $256,SI
-	ADDQ $256,DI
-	CMPQ R9,$256
-	JAE BYTESATLEAST256
-	CMPQ R9,$0
-	JBE DONE
-	BYTESBETWEEN1AND255:
-	CMPQ R9,$64
-	JAE NOCOPY
-	MOVQ DI,DX
-	LEAQ 360(R12),DI
-	MOVQ R9,CX
+BYTESBETWEEN1AND255:
+	CMPQ R9, $0x40
+	JAE  NOCOPY
+	MOVQ DI, DX
+	LEAQ 360(R12), DI
+	MOVQ R9, CX
 	REP; MOVSB
-	LEAQ 360(R12),DI
-	LEAQ 360(R12),SI
-	NOCOPY:
-	MOVQ R9,352(R12)
-	MOVOA 48(R12),X0
-	MOVOA 0(R12),X1
-	MOVOA 16(R12),X2
-	MOVOA 32(R12),X3
-	MOVOA X1,X4
-	MOVQ $20,CX
-	MAINLOOP2:
-	PADDL X0,X4
-	MOVOA X0,X5
-	MOVOA X4,X6
-	PSLLL $7,X4
-	PSRLL $25,X6
-	PXOR X4,X3
-	PXOR X6,X3
-	PADDL X3,X5
-	MOVOA X3,X4
-	MOVOA X5,X6
-	PSLLL $9,X5
-	PSRLL $23,X6
-	PXOR X5,X2
-	PSHUFL $0X93,X3,X3
-	PXOR X6,X2
-	PADDL X2,X4
-	MOVOA X2,X5
-	MOVOA X4,X6
-	PSLLL $13,X4
-	PSRLL $19,X6
-	PXOR X4,X1
-	PSHUFL $0X4E,X2,X2
-	PXOR X6,X1
-	PADDL X1,X5
-	MOVOA X3,X4
-	MOVOA X5,X6
-	PSLLL $18,X5
-	PSRLL $14,X6
-	PXOR X5,X0
-	PSHUFL $0X39,X1,X1
-	PXOR X6,X0
-	PADDL X0,X4
-	MOVOA X0,X5
-	MOVOA X4,X6
-	PSLLL $7,X4
-	PSRLL $25,X6
-	PXOR X4,X1
-	PXOR X6,X1
-	PADDL X1,X5
-	MOVOA X1,X4
-	MOVOA X5,X6
-	PSLLL $9,X5
-	PSRLL $23,X6
-	PXOR X5,X2
-	PSHUFL $0X93,X1,X1
-	PXOR X6,X2
-	PADDL X2,X4
-	MOVOA X2,X5
-	MOVOA X4,X6
-	PSLLL $13,X4
-	PSRLL $19,X6
-	PXOR X4,X3
-	PSHUFL $0X4E,X2,X2
-	PXOR X6,X3
-	PADDL X3,X5
-	MOVOA X1,X4
-	MOVOA X5,X6
-	PSLLL $18,X5
-	PSRLL $14,X6
-	PXOR X5,X0
-	PSHUFL $0X39,X3,X3
-	PXOR X6,X0
-	PADDL X0,X4
-	MOVOA X0,X5
-	MOVOA X4,X6
-	PSLLL $7,X4
-	PSRLL $25,X6
-	PXOR X4,X3
-	PXOR X6,X3
-	PADDL X3,X5
-	MOVOA X3,X4
-	MOVOA X5,X6
-	PSLLL $9,X5
-	PSRLL $23,X6
-	PXOR X5,X2
-	PSHUFL $0X93,X3,X3
-	PXOR X6,X2
-	PADDL X2,X4
-	MOVOA X2,X5
-	MOVOA X4,X6
-	PSLLL $13,X4
-	PSRLL $19,X6
-	PXOR X4,X1
-	PSHUFL $0X4E,X2,X2
-	PXOR X6,X1
-	PADDL X1,X5
-	MOVOA X3,X4
-	MOVOA X5,X6
-	PSLLL $18,X5
-	PSRLL $14,X6
-	PXOR X5,X0
-	PSHUFL $0X39,X1,X1
-	PXOR X6,X0
-	PADDL X0,X4
-	MOVOA X0,X5
-	MOVOA X4,X6
-	PSLLL $7,X4
-	PSRLL $25,X6
-	PXOR X4,X1
-	PXOR X6,X1
-	PADDL X1,X5
-	MOVOA X1,X4
-	MOVOA X5,X6
-	PSLLL $9,X5
-	PSRLL $23,X6
-	PXOR X5,X2
-	PSHUFL $0X93,X1,X1
-	PXOR X6,X2
-	PADDL X2,X4
-	MOVOA X2,X5
-	MOVOA X4,X6
-	PSLLL $13,X4
-	PSRLL $19,X6
-	PXOR X4,X3
-	PSHUFL $0X4E,X2,X2
-	PXOR X6,X3
-	SUBQ $4,CX
-	PADDL X3,X5
-	MOVOA X1,X4
-	MOVOA X5,X6
-	PSLLL $18,X5
-	PXOR X7,X7
-	PSRLL $14,X6
-	PXOR X5,X0
-	PSHUFL $0X39,X3,X3
-	PXOR X6,X0
-	JA MAINLOOP2
-	PADDL 48(R12),X0
-	PADDL 0(R12),X1
-	PADDL 16(R12),X2
-	PADDL 32(R12),X3
-	MOVD X0,CX
-	MOVD X1,R8
-	MOVD X2,R9
-	MOVD X3,AX
-	PSHUFL $0X39,X0,X0
-	PSHUFL $0X39,X1,X1
-	PSHUFL $0X39,X2,X2
-	PSHUFL $0X39,X3,X3
-	XORL 0(SI),CX
-	XORL 48(SI),R8
-	XORL 32(SI),R9
-	XORL 16(SI),AX
-	MOVL CX,0(DI)
-	MOVL R8,48(DI)
-	MOVL R9,32(DI)
-	MOVL AX,16(DI)
-	MOVD X0,CX
-	MOVD X1,R8
-	MOVD X2,R9
-	MOVD X3,AX
-	PSHUFL $0X39,X0,X0
-	PSHUFL $0X39,X1,X1
-	PSHUFL $0X39,X2,X2
-	PSHUFL $0X39,X3,X3
-	XORL 20(SI),CX
-	XORL 4(SI),R8
-	XORL 52(SI),R9
-	XORL 36(SI),AX
-	MOVL CX,20(DI)
-	MOVL R8,4(DI)
-	MOVL R9,52(DI)
-	MOVL AX,36(DI)
-	MOVD X0,CX
-	MOVD X1,R8
-	MOVD X2,R9
-	MOVD X3,AX
-	PSHUFL $0X39,X0,X0
-	PSHUFL $0X39,X1,X1
-	PSHUFL $0X39,X2,X2
-	PSHUFL $0X39,X3,X3
-	XORL 40(SI),CX
-	XORL 24(SI),R8
-	XORL 8(SI),R9
-	XORL 56(SI),AX
-	MOVL CX,40(DI)
-	MOVL R8,24(DI)
-	MOVL R9,8(DI)
-	MOVL AX,56(DI)
-	MOVD X0,CX
-	MOVD X1,R8
-	MOVD X2,R9
-	MOVD X3,AX
-	XORL 60(SI),CX
-	XORL 44(SI),R8
-	XORL 28(SI),R9
-	XORL 12(SI),AX
-	MOVL CX,60(DI)
-	MOVL R8,44(DI)
-	MOVL R9,28(DI)
-	MOVL AX,12(DI)
-	MOVQ 352(R12),R9
-	MOVL 16(R12),CX
-	MOVL  36 (R12),R8
-	ADDQ $1,CX
-	SHLQ $32,R8
-	ADDQ R8,CX
-	MOVQ CX,R8
-	SHRQ $32,R8
-	MOVL CX,16(R12)
-	MOVL R8, 36 (R12)
-	CMPQ R9,$64
-	JA BYTESATLEAST65
-	JAE BYTESATLEAST64
-	MOVQ DI,SI
-	MOVQ DX,DI
-	MOVQ R9,CX
+	LEAQ 360(R12), DI
+	LEAQ 360(R12), SI
+
+NOCOPY:
+	MOVQ  R9, 352(R12)
+	MOVOA 48(R12), X0
+	MOVOA (R12), X1
+	MOVOA 16(R12), X2
+	MOVOA 32(R12), X3
+	MOVOA X1, X4
+	MOVQ  $0x00000014, CX
+
+MAINLOOP2:
+	PADDL  X0, X4
+	MOVOA  X0, X5
+	MOVOA  X4, X6
+	PSLLL  $0x07, X4
+	PSRLL  $0x19, X6
+	PXOR   X4, X3
+	PXOR   X6, X3
+	PADDL  X3, X5
+	MOVOA  X3, X4
+	MOVOA  X5, X6
+	PSLLL  $0x09, X5
+	PSRLL  $0x17, X6
+	PXOR   X5, X2
+	PSHUFL $0x93, X3, X3
+	PXOR   X6, X2
+	PADDL  X2, X4
+	MOVOA  X2, X5
+	MOVOA  X4, X6
+	PSLLL  $0x0d, X4
+	PSRLL  $0x13, X6
+	PXOR   X4, X1
+	PSHUFL $0x4e, X2, X2
+	PXOR   X6, X1
+	PADDL  X1, X5
+	MOVOA  X3, X4
+	MOVOA  X5, X6
+	PSLLL  $0x12, X5
+	PSRLL  $0x0e, X6
+	PXOR   X5, X0
+	PSHUFL $0x39, X1, X1
+	PXOR   X6, X0
+	PADDL  X0, X4
+	MOVOA  X0, X5
+	MOVOA  X4, X6
+	PSLLL  $0x07, X4
+	PSRLL  $0x19, X6
+	PXOR   X4, X1
+	PXOR   X6, X1
+	PADDL  X1, X5
+	MOVOA  X1, X4
+	MOVOA  X5, X6
+	PSLLL  $0x09, X5
+	PSRLL  $0x17, X6
+	PXOR   X5, X2
+	PSHUFL $0x93, X1, X1
+	PXOR   X6, X2
+	PADDL  X2, X4
+	MOVOA  X2, X5
+	MOVOA  X4, X6
+	PSLLL  $0x0d, X4
+	PSRLL  $0x13, X6
+	PXOR   X4, X3
+	PSHUFL $0x4e, X2, X2
+	PXOR   X6, X3
+	PADDL  X3, X5
+	MOVOA  X1, X4
+	MOVOA  X5, X6
+	PSLLL  $0x12, X5
+	PSRLL  $0x0e, X6
+	PXOR   X5, X0
+	PSHUFL $0x39, X3, X3
+	PXOR   X6, X0
+	PADDL  X0, X4
+	MOVOA  X0, X5
+	MOVOA  X4, X6
+	PSLLL  $0x07, X4
+	PSRLL  $0x19, X6
+	PXOR   X4, X3
+	PXOR   X6, X3
+	PADDL  X3, X5
+	MOVOA  X3, X4
+	MOVOA  X5, X6
+	PSLLL  $0x09, X5
+	PSRLL  $0x17, X6
+	PXOR   X5, X2
+	PSHUFL $0x93, X3, X3
+	PXOR   X6, X2
+	PADDL  X2, X4
+	MOVOA  X2, X5
+	MOVOA  X4, X6
+	PSLLL  $0x0d, X4
+	PSRLL  $0x13, X6
+	PXOR   X4, X1
+	PSHUFL $0x4e, X2, X2
+	PXOR   X6, X1
+	PADDL  X1, X5
+	MOVOA  X3, X4
+	MOVOA  X5, X6
+	PSLLL  $0x12, X5
+	PSRLL  $0x0e, X6
+	PXOR   X5, X0
+	PSHUFL $0x39, X1, X1
+	PXOR   X6, X0
+	PADDL  X0, X4
+	MOVOA  X0, X5
+	MOVOA  X4, X6
+	PSLLL  $0x07, X4
+	PSRLL  $0x19, X6
+	PXOR   X4, X1
+	PXOR   X6, X1
+	PADDL  X1, X5
+	MOVOA  X1, X4
+	MOVOA  X5, X6
+	PSLLL  $0x09, X5
+	PSRLL  $0x17, X6
+	PXOR   X5, X2
+	PSHUFL $0x93, X1, X1
+	PXOR   X6, X2
+	PADDL  X2, X4
+	MOVOA  X2, X5
+	MOVOA  X4, X6
+	PSLLL  $0x0d, X4
+	PSRLL  $0x13, X6
+	PXOR   X4, X3
+	PSHUFL $0x4e, X2, X2
+	PXOR   X6, X3
+	SUBQ   $0x04, CX
+	PADDL  X3, X5
+	MOVOA  X1, X4
+	MOVOA  X5, X6
+	PSLLL  $0x12, X5
+	PXOR   X7, X7
+	PSRLL  $0x0e, X6
+	PXOR   X5, X0
+	PSHUFL $0x39, X3, X3
+	PXOR   X6, X0
+	JA     MAINLOOP2
+	PADDL  48(R12), X0
+	PADDL  (R12), X1
+	PADDL  16(R12), X2
+	PADDL  32(R12), X3
+	MOVD   X0, CX
+	MOVD   X1, R8
+	MOVD   X2, R9
+	MOVD   X3, AX
+	PSHUFL $0x39, X0, X0
+	PSHUFL $0x39, X1, X1
+	PSHUFL $0x39, X2, X2
+	PSHUFL $0x39, X3, X3
+	XORL   (SI), CX
+	XORL   48(SI), R8
+	XORL   32(SI), R9
+	XORL   16(SI), AX
+	MOVL   CX, (DI)
+	MOVL   R8, 48(DI)
+	MOVL   R9, 32(DI)
+	MOVL   AX, 16(DI)
+	MOVD   X0, CX
+	MOVD   X1, R8
+	MOVD   X2, R9
+	MOVD   X3, AX
+	PSHUFL $0x39, X0, X0
+	PSHUFL $0x39, X1, X1
+	PSHUFL $0x39, X2, X2
+	PSHUFL $0x39, X3, X3
+	XORL   20(SI), CX
+	XORL   4(SI), R8
+	XORL   52(SI), R9
+	XORL   36(SI), AX
+	MOVL   CX, 20(DI)
+	MOVL   R8, 4(DI)
+	MOVL   R9, 52(DI)
+	MOVL   AX, 36(DI)
+	MOVD   X0, CX
+	MOVD   X1, R8
+	MOVD   X2, R9
+	MOVD   X3, AX
+	PSHUFL $0x39, X0, X0
+	PSHUFL $0x39, X1, X1
+	PSHUFL $0x39, X2, X2
+	PSHUFL $0x39, X3, X3
+	XORL   40(SI), CX
+	XORL   24(SI), R8
+	XORL   8(SI), R9
+	XORL   56(SI), AX
+	MOVL   CX, 40(DI)
+	MOVL   R8, 24(DI)
+	MOVL   R9, 8(DI)
+	MOVL   AX, 56(DI)
+	MOVD   X0, CX
+	MOVD   X1, R8
+	MOVD   X2, R9
+	MOVD   X3, AX
+	XORL   60(SI), CX
+	XORL   44(SI), R8
+	XORL   28(SI), R9
+	XORL   12(SI), AX
+	MOVL   CX, 60(DI)
+	MOVL   R8, 44(DI)
+	MOVL   R9, 28(DI)
+	MOVL   AX, 12(DI)
+	MOVQ   352(R12), R9
+	MOVL   16(R12), CX
+	MOVL   36(R12), R8
+	ADDQ   $0x01, CX
+	SHLQ   $0x20, R8
+	ADDQ   R8, CX
+	MOVQ   CX, R8
+	SHRQ   $0x20, R8
+	MOVL   CX, 16(R12)
+	MOVL   R8, 36(R12)
+	CMPQ   R9, $0x40
+	JA     BYTESATLEAST65
+	JAE    BYTESATLEAST64
+	MOVQ   DI, SI
+	MOVQ   DX, DI
+	MOVQ   R9, CX
 	REP; MOVSB
-	BYTESATLEAST64:
-	DONE:
+
+BYTESATLEAST64:
+DONE:
 	RET
-	BYTESATLEAST65:
-	SUBQ $64,R9
-	ADDQ $64,DI
-	ADDQ $64,SI
-	JMP BYTESBETWEEN1AND255
+
+BYTESATLEAST65:
+	SUBQ $0x40, R9
+	ADDQ $0x40, DI
+	ADDQ $0x40, SI
+	JMP  BYTESBETWEEN1AND255
diff --git a/vendor/golang.org/x/crypto/ssh/client_auth.go b/vendor/golang.org/x/crypto/ssh/client_auth.go
index b93961010..b86dde151 100644
--- a/vendor/golang.org/x/crypto/ssh/client_auth.go
+++ b/vendor/golang.org/x/crypto/ssh/client_auth.go
@@ -555,6 +555,7 @@ func (cb KeyboardInteractiveChallenge) auth(session []byte, user string, c packe
 	}
 
 	gotMsgExtInfo := false
+	gotUserAuthInfoRequest := false
 	for {
 		packet, err := c.readPacket()
 		if err != nil {
@@ -585,6 +586,9 @@ func (cb KeyboardInteractiveChallenge) auth(session []byte, user string, c packe
 			if msg.PartialSuccess {
 				return authPartialSuccess, msg.Methods, nil
 			}
+			if !gotUserAuthInfoRequest {
+				return authFailure, msg.Methods, unexpectedMessageError(msgUserAuthInfoRequest, packet[0])
+			}
 			return authFailure, msg.Methods, nil
 		case msgUserAuthSuccess:
 			return authSuccess, nil, nil
@@ -596,6 +600,7 @@ func (cb KeyboardInteractiveChallenge) auth(session []byte, user string, c packe
 		if err := Unmarshal(packet, &msg); err != nil {
 			return authFailure, nil, err
 		}
+		gotUserAuthInfoRequest = true
 
 		// Manually unpack the prompt/echo pairs.
 		rest := msg.Prompts
diff --git a/vendor/golang.org/x/crypto/ssh/keys.go b/vendor/golang.org/x/crypto/ssh/keys.go
index 7967665f1..98e6706d5 100644
--- a/vendor/golang.org/x/crypto/ssh/keys.go
+++ b/vendor/golang.org/x/crypto/ssh/keys.go
@@ -488,7 +488,49 @@ func (r *rsaPublicKey) Verify(data []byte, sig *Signature) error {
 	h := hash.New()
 	h.Write(data)
 	digest := h.Sum(nil)
-	return rsa.VerifyPKCS1v15((*rsa.PublicKey)(r), hash, digest, sig.Blob)
+
+	// Signatures in PKCS1v15 must match the key's modulus in
+	// length. However with SSH, some signers provide RSA
+	// signatures which are missing the MSB 0's of the bignum
+	// represented. With ssh-rsa signatures, this is encouraged by
+	// the spec (even though e.g. OpenSSH will give the full
+	// length unconditionally). With rsa-sha2-* signatures, the
+	// verifier is allowed to support these, even though they are
+	// out of spec. See RFC 4253 Section 6.6 for ssh-rsa and RFC
+	// 8332 Section 3 for rsa-sha2-* details.
+	//
+	// In practice:
+	// * OpenSSH always allows "short" signatures:
+	//   https://github.com/openssh/openssh-portable/blob/V_9_8_P1/ssh-rsa.c#L526
+	//   but always generates padded signatures:
+	//   https://github.com/openssh/openssh-portable/blob/V_9_8_P1/ssh-rsa.c#L439
+	//
+	// * PuTTY versions 0.81 and earlier will generate short
+	//   signatures for all RSA signature variants. Note that
+	//   PuTTY is embedded in other software, such as WinSCP and
+	//   FileZilla. At the time of writing, a patch has been
+	//   applied to PuTTY to generate padded signatures for
+	//   rsa-sha2-*, but not yet released:
+	//   https://git.tartarus.org/?p=simon/putty.git;a=commitdiff;h=a5bcf3d384e1bf15a51a6923c3724cbbee022d8e
+	//
+	// * SSH.NET versions 2024.0.0 and earlier will generate short
+	//   signatures for all RSA signature variants, fixed in 2024.1.0:
+	//   https://github.com/sshnet/SSH.NET/releases/tag/2024.1.0
+	//
+	// As a result, we pad these up to the key size by inserting
+	// leading 0's.
+	//
+	// Note that support for short signatures with rsa-sha2-* may
+	// be removed in the future due to such signatures not being
+	// allowed by the spec.
+	blob := sig.Blob
+	keySize := (*rsa.PublicKey)(r).Size()
+	if len(blob) < keySize {
+		padded := make([]byte, keySize)
+		copy(padded[keySize-len(blob):], blob)
+		blob = padded
+	}
+	return rsa.VerifyPKCS1v15((*rsa.PublicKey)(r), hash, digest, blob)
 }
 
 func (r *rsaPublicKey) CryptoPublicKey() crypto.PublicKey {
diff --git a/vendor/golang.org/x/crypto/ssh/server.go b/vendor/golang.org/x/crypto/ssh/server.go
index 3ca9e89e2..5b5ccd96f 100644
--- a/vendor/golang.org/x/crypto/ssh/server.go
+++ b/vendor/golang.org/x/crypto/ssh/server.go
@@ -149,7 +149,7 @@ func (s *ServerConfig) AddHostKey(key Signer) {
 }
 
 // cachedPubKey contains the results of querying whether a public key is
-// acceptable for a user.
+// acceptable for a user. This is a FIFO cache.
 type cachedPubKey struct {
 	user       string
 	pubKeyData []byte
@@ -157,7 +157,13 @@ type cachedPubKey struct {
 	perms      *Permissions
 }
 
-const maxCachedPubKeys = 16
+// maxCachedPubKeys is the number of cache entries we store.
+//
+// Due to consistent misuse of the PublicKeyCallback API, we have reduced this
+// to 1, such that the only key in the cache is the most recently seen one. This
+// forces the behavior that the last call to PublicKeyCallback will always be
+// with the key that is used for authentication.
+const maxCachedPubKeys = 1
 
 // pubKeyCache caches tests for public keys.  Since SSH clients
 // will query whether a public key is acceptable before attempting to
@@ -179,9 +185,10 @@ func (c *pubKeyCache) get(user string, pubKeyData []byte) (cachedPubKey, bool) {
 
 // add adds the given tuple to the cache.
 func (c *pubKeyCache) add(candidate cachedPubKey) {
-	if len(c.keys) < maxCachedPubKeys {
-		c.keys = append(c.keys, candidate)
+	if len(c.keys) >= maxCachedPubKeys {
+		c.keys = c.keys[1:]
 	}
+	c.keys = append(c.keys, candidate)
 }
 
 // ServerConn is an authenticated SSH connection, as seen from the
@@ -510,8 +517,8 @@ userAuthLoop:
 			if err := s.transport.writePacket(Marshal(discMsg)); err != nil {
 				return nil, err
 			}
-
-			return nil, discMsg
+			authErrs = append(authErrs, discMsg)
+			return nil, &ServerAuthError{Errors: authErrs}
 		}
 
 		var userAuthReq userAuthRequestMsg
diff --git a/vendor/golang.org/x/sync/LICENSE b/vendor/golang.org/x/sync/LICENSE
index 6a66aea5e..2a7cf70da 100644
--- a/vendor/golang.org/x/sync/LICENSE
+++ b/vendor/golang.org/x/sync/LICENSE
@@ -1,4 +1,4 @@
-Copyright (c) 2009 The Go Authors. All rights reserved.
+Copyright 2009 The Go Authors.
 
 Redistribution and use in source and binary forms, with or without
 modification, are permitted provided that the following conditions are
@@ -10,7 +10,7 @@ notice, this list of conditions and the following disclaimer.
 copyright notice, this list of conditions and the following disclaimer
 in the documentation and/or other materials provided with the
 distribution.
-   * Neither the name of Google Inc. nor the names of its
+   * Neither the name of Google LLC nor the names of its
 contributors may be used to endorse or promote products derived from
 this software without specific prior written permission.
 
diff --git a/vendor/golang.org/x/sys/LICENSE b/vendor/golang.org/x/sys/LICENSE
index 6a66aea5e..2a7cf70da 100644
--- a/vendor/golang.org/x/sys/LICENSE
+++ b/vendor/golang.org/x/sys/LICENSE
@@ -1,4 +1,4 @@
-Copyright (c) 2009 The Go Authors. All rights reserved.
+Copyright 2009 The Go Authors.
 
 Redistribution and use in source and binary forms, with or without
 modification, are permitted provided that the following conditions are
@@ -10,7 +10,7 @@ notice, this list of conditions and the following disclaimer.
 copyright notice, this list of conditions and the following disclaimer
 in the documentation and/or other materials provided with the
 distribution.
-   * Neither the name of Google Inc. nor the names of its
+   * Neither the name of Google LLC nor the names of its
 contributors may be used to endorse or promote products derived from
 this software without specific prior written permission.
 
diff --git a/vendor/golang.org/x/sys/cpu/asm_darwin_x86_gc.s b/vendor/golang.org/x/sys/cpu/asm_darwin_x86_gc.s
new file mode 100644
index 000000000..ec2acfe54
--- /dev/null
+++ b/vendor/golang.org/x/sys/cpu/asm_darwin_x86_gc.s
@@ -0,0 +1,17 @@
+// Copyright 2024 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+//go:build darwin && amd64 && gc
+
+#include "textflag.h"
+
+TEXT libc_sysctl_trampoline<>(SB),NOSPLIT,$0-0
+	JMP	libc_sysctl(SB)
+GLOBL	·libc_sysctl_trampoline_addr(SB), RODATA, $8
+DATA	·libc_sysctl_trampoline_addr(SB)/8, $libc_sysctl_trampoline<>(SB)
+
+TEXT libc_sysctlbyname_trampoline<>(SB),NOSPLIT,$0-0
+	JMP	libc_sysctlbyname(SB)
+GLOBL	·libc_sysctlbyname_trampoline_addr(SB), RODATA, $8
+DATA	·libc_sysctlbyname_trampoline_addr(SB)/8, $libc_sysctlbyname_trampoline<>(SB)
diff --git a/vendor/golang.org/x/sys/cpu/cpu.go b/vendor/golang.org/x/sys/cpu/cpu.go
index 8fa707aa4..02609d5b2 100644
--- a/vendor/golang.org/x/sys/cpu/cpu.go
+++ b/vendor/golang.org/x/sys/cpu/cpu.go
@@ -105,6 +105,8 @@ var ARM64 struct {
 	HasSVE      bool // Scalable Vector Extensions
 	HasSVE2     bool // Scalable Vector Extensions 2
 	HasASIMDFHM bool // Advanced SIMD multiplication FP16 to FP32
+	HasDIT      bool // Data Independent Timing support
+	HasI8MM     bool // Advanced SIMD Int8 matrix multiplication instructions
 	_           CacheLinePad
 }
 
@@ -199,6 +201,25 @@ var S390X struct {
 	_         CacheLinePad
 }
 
+// RISCV64 contains the supported CPU features and performance characteristics for riscv64
+// platforms. The booleans in RISCV64, with the exception of HasFastMisaligned, indicate
+// the presence of RISC-V extensions.
+//
+// It is safe to assume that all the RV64G extensions are supported and so they are omitted from
+// this structure. As riscv64 Go programs require at least RV64G, the code that populates
+// this structure cannot run successfully if some of the RV64G extensions are missing.
+// The struct is padded to avoid false sharing.
+var RISCV64 struct {
+	_                 CacheLinePad
+	HasFastMisaligned bool // Fast misaligned accesses
+	HasC              bool // Compressed instruction-set extension
+	HasV              bool // Vector extension compatible with RVV 1.0
+	HasZba            bool // Address generation instructions extension
+	HasZbb            bool // Basic bit-manipulation extension
+	HasZbs            bool // Single-bit instructions extension
+	_                 CacheLinePad
+}
+
 func init() {
 	archInit()
 	initOptions()
diff --git a/vendor/golang.org/x/sys/cpu/cpu_arm64.go b/vendor/golang.org/x/sys/cpu/cpu_arm64.go
index 0e27a21e1..af2aa99f9 100644
--- a/vendor/golang.org/x/sys/cpu/cpu_arm64.go
+++ b/vendor/golang.org/x/sys/cpu/cpu_arm64.go
@@ -38,6 +38,8 @@ func initOptions() {
 		{Name: "dcpop", Feature: &ARM64.HasDCPOP},
 		{Name: "asimddp", Feature: &ARM64.HasASIMDDP},
 		{Name: "asimdfhm", Feature: &ARM64.HasASIMDFHM},
+		{Name: "dit", Feature: &ARM64.HasDIT},
+		{Name: "i8mm", Feature: &ARM64.HasI8MM},
 	}
 }
 
@@ -145,6 +147,11 @@ func parseARM64SystemRegisters(isar0, isar1, pfr0 uint64) {
 		ARM64.HasLRCPC = true
 	}
 
+	switch extractBits(isar1, 52, 55) {
+	case 1:
+		ARM64.HasI8MM = true
+	}
+
 	// ID_AA64PFR0_EL1
 	switch extractBits(pfr0, 16, 19) {
 	case 0:
@@ -168,6 +175,11 @@ func parseARM64SystemRegisters(isar0, isar1, pfr0 uint64) {
 
 		parseARM64SVERegister(getzfr0())
 	}
+
+	switch extractBits(pfr0, 48, 51) {
+	case 1:
+		ARM64.HasDIT = true
+	}
 }
 
 func parseARM64SVERegister(zfr0 uint64) {
diff --git a/vendor/golang.org/x/sys/cpu/cpu_darwin_x86.go b/vendor/golang.org/x/sys/cpu/cpu_darwin_x86.go
new file mode 100644
index 000000000..b838cb9e9
--- /dev/null
+++ b/vendor/golang.org/x/sys/cpu/cpu_darwin_x86.go
@@ -0,0 +1,61 @@
+// Copyright 2024 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+//go:build darwin && amd64 && gc
+
+package cpu
+
+// darwinSupportsAVX512 checks Darwin kernel for AVX512 support via sysctl
+// call (see issue 43089). It also restricts AVX512 support for Darwin to
+// kernel version 21.3.0 (MacOS 12.2.0) or later (see issue 49233).
+//
+// Background:
+// Darwin implements a special mechanism to economize on thread state when
+// AVX512 specific registers are not in use. This scheme minimizes state when
+// preempting threads that haven't yet used any AVX512 instructions, but adds
+// special requirements to check for AVX512 hardware support at runtime (e.g.
+// via sysctl call or commpage inspection). See issue 43089 and link below for
+// full background:
+// https://github.com/apple-oss-distributions/xnu/blob/xnu-11215.1.10/osfmk/i386/fpu.c#L214-L240
+//
+// Additionally, all versions of the Darwin kernel from 19.6.0 through 21.2.0
+// (corresponding to MacOS 10.15.6 - 12.1) have a bug that can cause corruption
+// of the AVX512 mask registers (K0-K7) upon signal return. For this reason
+// AVX512 is considered unsafe to use on Darwin for kernel versions prior to
+// 21.3.0, where a fix has been confirmed. See issue 49233 for full background.
+func darwinSupportsAVX512() bool {
+	return darwinSysctlEnabled([]byte("hw.optional.avx512f\x00")) && darwinKernelVersionCheck(21, 3, 0)
+}
+
+// Ensure Darwin kernel version is at least major.minor.patch, avoiding dependencies
+func darwinKernelVersionCheck(major, minor, patch int) bool {
+	var release [256]byte
+	err := darwinOSRelease(&release)
+	if err != nil {
+		return false
+	}
+
+	var mmp [3]int
+	c := 0
+Loop:
+	for _, b := range release[:] {
+		switch {
+		case b >= '0' && b <= '9':
+			mmp[c] = 10*mmp[c] + int(b-'0')
+		case b == '.':
+			c++
+			if c > 2 {
+				return false
+			}
+		case b == 0:
+			break Loop
+		default:
+			return false
+		}
+	}
+	if c != 2 {
+		return false
+	}
+	return mmp[0] > major || mmp[0] == major && (mmp[1] > minor || mmp[1] == minor && mmp[2] >= patch)
+}
diff --git a/vendor/golang.org/x/sys/cpu/cpu_gc_x86.go b/vendor/golang.org/x/sys/cpu/cpu_gc_x86.go
index 910728fb1..32a44514e 100644
--- a/vendor/golang.org/x/sys/cpu/cpu_gc_x86.go
+++ b/vendor/golang.org/x/sys/cpu/cpu_gc_x86.go
@@ -6,10 +6,10 @@
 
 package cpu
 
-// cpuid is implemented in cpu_x86.s for gc compiler
+// cpuid is implemented in cpu_gc_x86.s for gc compiler
 // and in cpu_gccgo.c for gccgo.
 func cpuid(eaxArg, ecxArg uint32) (eax, ebx, ecx, edx uint32)
 
-// xgetbv with ecx = 0 is implemented in cpu_x86.s for gc compiler
+// xgetbv with ecx = 0 is implemented in cpu_gc_x86.s for gc compiler
 // and in cpu_gccgo.c for gccgo.
 func xgetbv() (eax, edx uint32)
diff --git a/vendor/golang.org/x/sys/cpu/cpu_x86.s b/vendor/golang.org/x/sys/cpu/cpu_gc_x86.s
similarity index 94%
rename from vendor/golang.org/x/sys/cpu/cpu_x86.s
rename to vendor/golang.org/x/sys/cpu/cpu_gc_x86.s
index 7d7ba33ef..ce208ce6d 100644
--- a/vendor/golang.org/x/sys/cpu/cpu_x86.s
+++ b/vendor/golang.org/x/sys/cpu/cpu_gc_x86.s
@@ -18,7 +18,7 @@ TEXT ·cpuid(SB), NOSPLIT, $0-24
 	RET
 
 // func xgetbv() (eax, edx uint32)
-TEXT ·xgetbv(SB),NOSPLIT,$0-8
+TEXT ·xgetbv(SB), NOSPLIT, $0-8
 	MOVL $0, CX
 	XGETBV
 	MOVL AX, eax+0(FP)
diff --git a/vendor/golang.org/x/sys/cpu/cpu_gccgo_x86.go b/vendor/golang.org/x/sys/cpu/cpu_gccgo_x86.go
index 99c60fe9f..170d21ddf 100644
--- a/vendor/golang.org/x/sys/cpu/cpu_gccgo_x86.go
+++ b/vendor/golang.org/x/sys/cpu/cpu_gccgo_x86.go
@@ -23,9 +23,3 @@ func xgetbv() (eax, edx uint32) {
 	gccgoXgetbv(&a, &d)
 	return a, d
 }
-
-// gccgo doesn't build on Darwin, per:
-// https://github.com/Homebrew/homebrew-core/blob/HEAD/Formula/gcc.rb#L76
-func darwinSupportsAVX512() bool {
-	return false
-}
diff --git a/vendor/golang.org/x/sys/cpu/cpu_linux_arm64.go b/vendor/golang.org/x/sys/cpu/cpu_linux_arm64.go
index 3d386d0fc..f1caf0f78 100644
--- a/vendor/golang.org/x/sys/cpu/cpu_linux_arm64.go
+++ b/vendor/golang.org/x/sys/cpu/cpu_linux_arm64.go
@@ -35,8 +35,10 @@ const (
 	hwcap_SHA512   = 1 << 21
 	hwcap_SVE      = 1 << 22
 	hwcap_ASIMDFHM = 1 << 23
+	hwcap_DIT      = 1 << 24
 
 	hwcap2_SVE2 = 1 << 1
+	hwcap2_I8MM = 1 << 13
 )
 
 // linuxKernelCanEmulateCPUID reports whether we're running
@@ -106,9 +108,11 @@ func doinit() {
 	ARM64.HasSHA512 = isSet(hwCap, hwcap_SHA512)
 	ARM64.HasSVE = isSet(hwCap, hwcap_SVE)
 	ARM64.HasASIMDFHM = isSet(hwCap, hwcap_ASIMDFHM)
+	ARM64.HasDIT = isSet(hwCap, hwcap_DIT)
 
 	// HWCAP2 feature bits
 	ARM64.HasSVE2 = isSet(hwCap2, hwcap2_SVE2)
+	ARM64.HasI8MM = isSet(hwCap2, hwcap2_I8MM)
 }
 
 func isSet(hwc uint, value uint) bool {
diff --git a/vendor/golang.org/x/sys/cpu/cpu_linux_noinit.go b/vendor/golang.org/x/sys/cpu/cpu_linux_noinit.go
index cd63e7335..7d902b684 100644
--- a/vendor/golang.org/x/sys/cpu/cpu_linux_noinit.go
+++ b/vendor/golang.org/x/sys/cpu/cpu_linux_noinit.go
@@ -2,7 +2,7 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-//go:build linux && !arm && !arm64 && !mips64 && !mips64le && !ppc64 && !ppc64le && !s390x
+//go:build linux && !arm && !arm64 && !mips64 && !mips64le && !ppc64 && !ppc64le && !s390x && !riscv64
 
 package cpu
 
diff --git a/vendor/golang.org/x/sys/cpu/cpu_linux_riscv64.go b/vendor/golang.org/x/sys/cpu/cpu_linux_riscv64.go
new file mode 100644
index 000000000..cb4a0c572
--- /dev/null
+++ b/vendor/golang.org/x/sys/cpu/cpu_linux_riscv64.go
@@ -0,0 +1,137 @@
+// Copyright 2024 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package cpu
+
+import (
+	"syscall"
+	"unsafe"
+)
+
+// RISC-V extension discovery code for Linux. The approach here is to first try the riscv_hwprobe
+// syscall falling back to HWCAP to check for the C extension if riscv_hwprobe is not available.
+//
+// A note on detection of the Vector extension using HWCAP.
+//
+// Support for the Vector extension version 1.0 was added to the Linux kernel in release 6.5.
+// Support for the riscv_hwprobe syscall was added in 6.4. It follows that if the riscv_hwprobe
+// syscall is not available then neither is the Vector extension (which needs kernel support).
+// The riscv_hwprobe syscall should then be all we need to detect the Vector extension.
+// However, some RISC-V board manufacturers ship boards with an older kernel on top of which
+// they have back-ported various versions of the Vector extension patches but not the riscv_hwprobe
+// patches. These kernels advertise support for the Vector extension using HWCAP. Falling
+// back to HWCAP to detect the Vector extension, if riscv_hwprobe is not available, or simply not
+// bothering with riscv_hwprobe at all and just using HWCAP may then seem like an attractive option.
+//
+// Unfortunately, simply checking the 'V' bit in AT_HWCAP will not work as this bit is used by
+// RISC-V board and cloud instance providers to mean different things. The Lichee Pi 4A board
+// and the Scaleway RV1 cloud instances use the 'V' bit to advertise their support for the unratified
+// 0.7.1 version of the Vector Specification. The Banana Pi BPI-F3 and the CanMV-K230 board use
+// it to advertise support for 1.0 of the Vector extension. Versions 0.7.1 and 1.0 of the Vector
+// extension are binary incompatible. HWCAP can then not be used in isolation to populate the
+// HasV field as this field indicates that the underlying CPU is compatible with RVV 1.0.
+//
+// There is a way at runtime to distinguish between versions 0.7.1 and 1.0 of the Vector
+// specification by issuing a RVV 1.0 vsetvli instruction and checking the vill bit of the vtype
+// register. This check would allow us to safely detect version 1.0 of the Vector extension
+// with HWCAP, if riscv_hwprobe were not available. However, the check cannot
+// be added until the assembler supports the Vector instructions.
+//
+// Note the riscv_hwprobe syscall does not suffer from these ambiguities by design as all of the
+// extensions it advertises support for are explicitly versioned. It's also worth noting that
+// the riscv_hwprobe syscall is the only way to detect multi-letter RISC-V extensions, e.g., Zba.
+// These cannot be detected using HWCAP and so riscv_hwprobe must be used to detect the majority
+// of RISC-V extensions.
+//
+// Please see https://docs.kernel.org/arch/riscv/hwprobe.html for more information.
+
+// golang.org/x/sys/cpu is not allowed to depend on golang.org/x/sys/unix so we must
+// reproduce the constants, types and functions needed to make the riscv_hwprobe syscall
+// here.
+
+const (
+	// Copied from golang.org/x/sys/unix/ztypes_linux_riscv64.go.
+	riscv_HWPROBE_KEY_IMA_EXT_0   = 0x4
+	riscv_HWPROBE_IMA_C           = 0x2
+	riscv_HWPROBE_IMA_V           = 0x4
+	riscv_HWPROBE_EXT_ZBA         = 0x8
+	riscv_HWPROBE_EXT_ZBB         = 0x10
+	riscv_HWPROBE_EXT_ZBS         = 0x20
+	riscv_HWPROBE_KEY_CPUPERF_0   = 0x5
+	riscv_HWPROBE_MISALIGNED_FAST = 0x3
+	riscv_HWPROBE_MISALIGNED_MASK = 0x7
+)
+
+const (
+	// sys_RISCV_HWPROBE is copied from golang.org/x/sys/unix/zsysnum_linux_riscv64.go.
+	sys_RISCV_HWPROBE = 258
+)
+
+// riscvHWProbePairs is copied from golang.org/x/sys/unix/ztypes_linux_riscv64.go.
+type riscvHWProbePairs struct {
+	key   int64
+	value uint64
+}
+
+const (
+	// CPU features
+	hwcap_RISCV_ISA_C = 1 << ('C' - 'A')
+)
+
+func doinit() {
+	// A slice of key/value pair structures is passed to the RISCVHWProbe syscall. The key
+	// field should be initialised with one of the key constants defined above, e.g.,
+	// RISCV_HWPROBE_KEY_IMA_EXT_0. The syscall will set the value field to the appropriate value.
+	// If the kernel does not recognise a key it will set the key field to -1 and the value field to 0.
+
+	pairs := []riscvHWProbePairs{
+		{riscv_HWPROBE_KEY_IMA_EXT_0, 0},
+		{riscv_HWPROBE_KEY_CPUPERF_0, 0},
+	}
+
+	// This call only indicates that extensions are supported if they are implemented on all cores.
+	if riscvHWProbe(pairs, 0) {
+		if pairs[0].key != -1 {
+			v := uint(pairs[0].value)
+			RISCV64.HasC = isSet(v, riscv_HWPROBE_IMA_C)
+			RISCV64.HasV = isSet(v, riscv_HWPROBE_IMA_V)
+			RISCV64.HasZba = isSet(v, riscv_HWPROBE_EXT_ZBA)
+			RISCV64.HasZbb = isSet(v, riscv_HWPROBE_EXT_ZBB)
+			RISCV64.HasZbs = isSet(v, riscv_HWPROBE_EXT_ZBS)
+		}
+		if pairs[1].key != -1 {
+			v := pairs[1].value & riscv_HWPROBE_MISALIGNED_MASK
+			RISCV64.HasFastMisaligned = v == riscv_HWPROBE_MISALIGNED_FAST
+		}
+	}
+
+	// Let's double check with HWCAP if the C extension does not appear to be supported.
+	// This may happen if we're running on a kernel older than 6.4.
+
+	if !RISCV64.HasC {
+		RISCV64.HasC = isSet(hwCap, hwcap_RISCV_ISA_C)
+	}
+}
+
+func isSet(hwc uint, value uint) bool {
+	return hwc&value != 0
+}
+
+// riscvHWProbe is a simplified version of the generated wrapper function found in
+// golang.org/x/sys/unix/zsyscall_linux_riscv64.go. We simplify it by removing the
+// cpuCount and cpus parameters which we do not need. We always want to pass 0 for
+// these parameters here so the kernel only reports the extensions that are present
+// on all cores.
+func riscvHWProbe(pairs []riscvHWProbePairs, flags uint) bool {
+	var _zero uintptr
+	var p0 unsafe.Pointer
+	if len(pairs) > 0 {
+		p0 = unsafe.Pointer(&pairs[0])
+	} else {
+		p0 = unsafe.Pointer(&_zero)
+	}
+
+	_, _, e1 := syscall.Syscall6(sys_RISCV_HWPROBE, uintptr(p0), uintptr(len(pairs)), uintptr(0), uintptr(0), uintptr(flags), 0)
+	return e1 == 0
+}
diff --git a/vendor/golang.org/x/sys/cpu/cpu_other_x86.go b/vendor/golang.org/x/sys/cpu/cpu_other_x86.go
new file mode 100644
index 000000000..a0fd7e2f7
--- /dev/null
+++ b/vendor/golang.org/x/sys/cpu/cpu_other_x86.go
@@ -0,0 +1,11 @@
+// Copyright 2024 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+//go:build 386 || amd64p32 || (amd64 && (!darwin || !gc))
+
+package cpu
+
+func darwinSupportsAVX512() bool {
+	panic("only implemented for gc && amd64 && darwin")
+}
diff --git a/vendor/golang.org/x/sys/cpu/cpu_riscv64.go b/vendor/golang.org/x/sys/cpu/cpu_riscv64.go
index 7f0c79c00..aca3199c9 100644
--- a/vendor/golang.org/x/sys/cpu/cpu_riscv64.go
+++ b/vendor/golang.org/x/sys/cpu/cpu_riscv64.go
@@ -8,4 +8,13 @@ package cpu
 
 const cacheLineSize = 64
 
-func initOptions() {}
+func initOptions() {
+	options = []option{
+		{Name: "fastmisaligned", Feature: &RISCV64.HasFastMisaligned},
+		{Name: "c", Feature: &RISCV64.HasC},
+		{Name: "v", Feature: &RISCV64.HasV},
+		{Name: "zba", Feature: &RISCV64.HasZba},
+		{Name: "zbb", Feature: &RISCV64.HasZbb},
+		{Name: "zbs", Feature: &RISCV64.HasZbs},
+	}
+}
diff --git a/vendor/golang.org/x/sys/cpu/cpu_x86.go b/vendor/golang.org/x/sys/cpu/cpu_x86.go
index c29f5e4c5..600a68078 100644
--- a/vendor/golang.org/x/sys/cpu/cpu_x86.go
+++ b/vendor/golang.org/x/sys/cpu/cpu_x86.go
@@ -92,10 +92,8 @@ func archInit() {
 		osSupportsAVX = isSet(1, eax) && isSet(2, eax)
 
 		if runtime.GOOS == "darwin" {
-			// Darwin doesn't save/restore AVX-512 mask registers correctly across signal handlers.
-			// Since users can't rely on mask register contents, let's not advertise AVX-512 support.
-			// See issue 49233.
-			osSupportsAVX512 = false
+			// Darwin requires special AVX512 checks, see cpu_darwin_x86.go
+			osSupportsAVX512 = osSupportsAVX && darwinSupportsAVX512()
 		} else {
 			// Check if OPMASK and ZMM registers have OS support.
 			osSupportsAVX512 = osSupportsAVX && isSet(5, eax) && isSet(6, eax) && isSet(7, eax)
diff --git a/vendor/golang.org/x/sys/cpu/syscall_darwin_x86_gc.go b/vendor/golang.org/x/sys/cpu/syscall_darwin_x86_gc.go
new file mode 100644
index 000000000..4d0888b0c
--- /dev/null
+++ b/vendor/golang.org/x/sys/cpu/syscall_darwin_x86_gc.go
@@ -0,0 +1,98 @@
+// Copyright 2024 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Minimal copy of x/sys/unix so the cpu package can make a
+// system call on Darwin without depending on x/sys/unix.
+
+//go:build darwin && amd64 && gc
+
+package cpu
+
+import (
+	"syscall"
+	"unsafe"
+)
+
+type _C_int int32
+
+// adapted from unix.Uname() at x/sys/unix/syscall_darwin.go L419
+func darwinOSRelease(release *[256]byte) error {
+	// from x/sys/unix/zerrors_openbsd_amd64.go
+	const (
+		CTL_KERN       = 0x1
+		KERN_OSRELEASE = 0x2
+	)
+
+	mib := []_C_int{CTL_KERN, KERN_OSRELEASE}
+	n := unsafe.Sizeof(*release)
+
+	return sysctl(mib, &release[0], &n, nil, 0)
+}
+
+type Errno = syscall.Errno
+
+var _zero uintptr // Single-word zero for use when we need a valid pointer to 0 bytes.
+
+// from x/sys/unix/zsyscall_darwin_amd64.go L791-807
+func sysctl(mib []_C_int, old *byte, oldlen *uintptr, new *byte, newlen uintptr) error {
+	var _p0 unsafe.Pointer
+	if len(mib) > 0 {
+		_p0 = unsafe.Pointer(&mib[0])
+	} else {
+		_p0 = unsafe.Pointer(&_zero)
+	}
+	if _, _, err := syscall_syscall6(
+		libc_sysctl_trampoline_addr,
+		uintptr(_p0),
+		uintptr(len(mib)),
+		uintptr(unsafe.Pointer(old)),
+		uintptr(unsafe.Pointer(oldlen)),
+		uintptr(unsafe.Pointer(new)),
+		uintptr(newlen),
+	); err != 0 {
+		return err
+	}
+
+	return nil
+}
+
+var libc_sysctl_trampoline_addr uintptr
+
+// adapted from internal/cpu/cpu_arm64_darwin.go
+func darwinSysctlEnabled(name []byte) bool {
+	out := int32(0)
+	nout := unsafe.Sizeof(out)
+	if ret := sysctlbyname(&name[0], (*byte)(unsafe.Pointer(&out)), &nout, nil, 0); ret != nil {
+		return false
+	}
+	return out > 0
+}
+
+//go:cgo_import_dynamic libc_sysctl sysctl "/usr/lib/libSystem.B.dylib"
+
+var libc_sysctlbyname_trampoline_addr uintptr
+
+// adapted from runtime/sys_darwin.go in the pattern of sysctl() above, as defined in x/sys/unix
+func sysctlbyname(name *byte, old *byte, oldlen *uintptr, new *byte, newlen uintptr) error {
+	if _, _, err := syscall_syscall6(
+		libc_sysctlbyname_trampoline_addr,
+		uintptr(unsafe.Pointer(name)),
+		uintptr(unsafe.Pointer(old)),
+		uintptr(unsafe.Pointer(oldlen)),
+		uintptr(unsafe.Pointer(new)),
+		uintptr(newlen),
+		0,
+	); err != 0 {
+		return err
+	}
+
+	return nil
+}
+
+//go:cgo_import_dynamic libc_sysctlbyname sysctlbyname "/usr/lib/libSystem.B.dylib"
+
+// Implemented in the runtime package (runtime/sys_darwin.go)
+func syscall_syscall6(fn, a1, a2, a3, a4, a5, a6 uintptr) (r1, r2 uintptr, err Errno)
+
+//go:linkname syscall_syscall6 syscall.syscall6
diff --git a/vendor/golang.org/x/sys/unix/README.md b/vendor/golang.org/x/sys/unix/README.md
index 7d3c060e1..6e08a76a7 100644
--- a/vendor/golang.org/x/sys/unix/README.md
+++ b/vendor/golang.org/x/sys/unix/README.md
@@ -156,7 +156,7 @@ from the generated architecture-specific files listed below, and merge these
 into a common file for each OS.
 
 The merge is performed in the following steps:
-1. Construct the set of common code that is idential in all architecture-specific files.
+1. Construct the set of common code that is identical in all architecture-specific files.
 2. Write this common code to the merged file.
 3. Remove the common code from all architecture-specific files.
 
diff --git a/vendor/golang.org/x/sys/unix/ioctl_linux.go b/vendor/golang.org/x/sys/unix/ioctl_linux.go
index dbe680eab..7ca4fa12a 100644
--- a/vendor/golang.org/x/sys/unix/ioctl_linux.go
+++ b/vendor/golang.org/x/sys/unix/ioctl_linux.go
@@ -58,6 +58,102 @@ func IoctlGetEthtoolDrvinfo(fd int, ifname string) (*EthtoolDrvinfo, error) {
 	return &value, err
 }
 
+// IoctlGetEthtoolTsInfo fetches ethtool timestamping and PHC
+// association for the network device specified by ifname.
+func IoctlGetEthtoolTsInfo(fd int, ifname string) (*EthtoolTsInfo, error) {
+	ifr, err := NewIfreq(ifname)
+	if err != nil {
+		return nil, err
+	}
+
+	value := EthtoolTsInfo{Cmd: ETHTOOL_GET_TS_INFO}
+	ifrd := ifr.withData(unsafe.Pointer(&value))
+
+	err = ioctlIfreqData(fd, SIOCETHTOOL, &ifrd)
+	return &value, err
+}
+
+// IoctlGetHwTstamp retrieves the hardware timestamping configuration
+// for the network device specified by ifname.
+func IoctlGetHwTstamp(fd int, ifname string) (*HwTstampConfig, error) {
+	ifr, err := NewIfreq(ifname)
+	if err != nil {
+		return nil, err
+	}
+
+	value := HwTstampConfig{}
+	ifrd := ifr.withData(unsafe.Pointer(&value))
+
+	err = ioctlIfreqData(fd, SIOCGHWTSTAMP, &ifrd)
+	return &value, err
+}
+
+// IoctlSetHwTstamp updates the hardware timestamping configuration for
+// the network device specified by ifname.
+func IoctlSetHwTstamp(fd int, ifname string, cfg *HwTstampConfig) error {
+	ifr, err := NewIfreq(ifname)
+	if err != nil {
+		return err
+	}
+	ifrd := ifr.withData(unsafe.Pointer(cfg))
+	return ioctlIfreqData(fd, SIOCSHWTSTAMP, &ifrd)
+}
+
+// FdToClockID derives the clock ID from the file descriptor number
+// - see clock_gettime(3), FD_TO_CLOCKID macros. The resulting ID is
+// suitable for system calls like ClockGettime.
+func FdToClockID(fd int) int32 { return int32((int(^fd) << 3) | 3) }
+
+// IoctlPtpClockGetcaps returns the description of a given PTP device.
+func IoctlPtpClockGetcaps(fd int) (*PtpClockCaps, error) {
+	var value PtpClockCaps
+	err := ioctlPtr(fd, PTP_CLOCK_GETCAPS2, unsafe.Pointer(&value))
+	return &value, err
+}
+
+// IoctlPtpSysOffsetPrecise returns a description of the clock
+// offset compared to the system clock.
+func IoctlPtpSysOffsetPrecise(fd int) (*PtpSysOffsetPrecise, error) {
+	var value PtpSysOffsetPrecise
+	err := ioctlPtr(fd, PTP_SYS_OFFSET_PRECISE2, unsafe.Pointer(&value))
+	return &value, err
+}
+
+// IoctlPtpSysOffsetExtended returns an extended description of the
+// clock offset compared to the system clock. The samples parameter
+// specifies the desired number of measurements.
+func IoctlPtpSysOffsetExtended(fd int, samples uint) (*PtpSysOffsetExtended, error) {
+	value := PtpSysOffsetExtended{Samples: uint32(samples)}
+	err := ioctlPtr(fd, PTP_SYS_OFFSET_EXTENDED2, unsafe.Pointer(&value))
+	return &value, err
+}
+
+// IoctlPtpPinGetfunc returns the configuration of the specified
+// I/O pin on given PTP device.
+func IoctlPtpPinGetfunc(fd int, index uint) (*PtpPinDesc, error) {
+	value := PtpPinDesc{Index: uint32(index)}
+	err := ioctlPtr(fd, PTP_PIN_GETFUNC2, unsafe.Pointer(&value))
+	return &value, err
+}
+
+// IoctlPtpPinSetfunc updates configuration of the specified PTP
+// I/O pin.
+func IoctlPtpPinSetfunc(fd int, pd *PtpPinDesc) error {
+	return ioctlPtr(fd, PTP_PIN_SETFUNC2, unsafe.Pointer(pd))
+}
+
+// IoctlPtpPeroutRequest configures the periodic output mode of the
+// PTP I/O pins.
+func IoctlPtpPeroutRequest(fd int, r *PtpPeroutRequest) error {
+	return ioctlPtr(fd, PTP_PEROUT_REQUEST2, unsafe.Pointer(r))
+}
+
+// IoctlPtpExttsRequest configures the external timestamping mode
+// of the PTP I/O pins.
+func IoctlPtpExttsRequest(fd int, r *PtpExttsRequest) error {
+	return ioctlPtr(fd, PTP_EXTTS_REQUEST2, unsafe.Pointer(r))
+}
+
 // IoctlGetWatchdogInfo fetches information about a watchdog device from the
 // Linux watchdog API. For more information, see:
 // https://www.kernel.org/doc/html/latest/watchdog/watchdog-api.html.
diff --git a/vendor/golang.org/x/sys/unix/mkerrors.sh b/vendor/golang.org/x/sys/unix/mkerrors.sh
index 4ed2e488b..6ab02b6c3 100644
--- a/vendor/golang.org/x/sys/unix/mkerrors.sh
+++ b/vendor/golang.org/x/sys/unix/mkerrors.sh
@@ -58,6 +58,7 @@ includes_Darwin='
 #define _DARWIN_USE_64_BIT_INODE
 #define __APPLE_USE_RFC_3542
 #include <stdint.h>
+#include <sys/stdio.h>
 #include <sys/attr.h>
 #include <sys/clonefile.h>
 #include <sys/kern_control.h>
@@ -157,6 +158,16 @@ includes_Linux='
 #endif
 #define _GNU_SOURCE
 
+// See the description in unix/linux/types.go
+#if defined(__ARM_EABI__) || \
+	(defined(__mips__) && (_MIPS_SIM == _ABIO32)) || \
+	(defined(__powerpc__) && (!defined(__powerpc64__)))
+# ifdef   _TIME_BITS
+#  undef  _TIME_BITS
+# endif
+# define  _TIME_BITS 32
+#endif
+
 // <sys/ioctl.h> is broken on powerpc64, as it fails to include definitions of
 // these structures. We just include them copied from <bits/termios.h>.
 #if defined(__powerpc__)
@@ -255,6 +266,7 @@ struct ltchars {
 #include <linux/nsfs.h>
 #include <linux/perf_event.h>
 #include <linux/pps.h>
+#include <linux/ptp_clock.h>
 #include <linux/ptrace.h>
 #include <linux/random.h>
 #include <linux/reboot.h>
@@ -526,6 +538,7 @@ ccflags="$@"
 		$2 ~ /^(AF|SOCK|SO|SOL|IPPROTO|IP|IPV6|TCP|MCAST|EVFILT|NOTE|SHUT|PROT|MAP|MREMAP|MFD|T?PACKET|MSG|SCM|MCL|DT|MADV|PR|LOCAL|TCPOPT|UDP)_/ ||
 		$2 ~ /^NFC_(GENL|PROTO|COMM|RF|SE|DIRECTION|LLCP|SOCKPROTO)_/ ||
 		$2 ~ /^NFC_.*_(MAX)?SIZE$/ ||
+		$2 ~ /^PTP_/ ||
 		$2 ~ /^RAW_PAYLOAD_/ ||
 		$2 ~ /^[US]F_/ ||
 		$2 ~ /^TP_STATUS_/ ||
@@ -551,6 +564,7 @@ ccflags="$@"
 		$2 !~ /^RTC_VL_(ACCURACY|BACKUP|DATA)/ &&
 		$2 ~ /^(NETLINK|NLM|NLMSG|NLA|IFA|IFAN|RT|RTC|RTCF|RTN|RTPROT|RTNH|ARPHRD|ETH_P|NETNSA)_/ ||
 		$2 ~ /^SOCK_|SK_DIAG_|SKNLGRP_$/ ||
+		$2 ~ /^(CONNECT|SAE)_/ ||
 		$2 ~ /^FIORDCHK$/ ||
 		$2 ~ /^SIOC/ ||
 		$2 ~ /^TIOC/ ||
@@ -654,7 +668,7 @@ errors=$(
 signals=$(
 	echo '#include <signal.h>' | $CC -x c - -E -dM $ccflags |
 	awk '$1=="#define" && $2 ~ /^SIG[A-Z0-9]+$/ { print $2 }' |
-	grep -v 'SIGSTKSIZE\|SIGSTKSZ\|SIGRT\|SIGMAX64' |
+	grep -E -v '(SIGSTKSIZE|SIGSTKSZ|SIGRT|SIGMAX64)' |
 	sort
 )
 
@@ -664,7 +678,7 @@ echo '#include <errno.h>' | $CC -x c - -E -dM $ccflags |
 	sort >_error.grep
 echo '#include <signal.h>' | $CC -x c - -E -dM $ccflags |
 	awk '$1=="#define" && $2 ~ /^SIG[A-Z0-9]+$/ { print "^\t" $2 "[ \t]*=" }' |
-	grep -v 'SIGSTKSIZE\|SIGSTKSZ\|SIGRT\|SIGMAX64' |
+	grep -E -v '(SIGSTKSIZE|SIGSTKSZ|SIGRT|SIGMAX64)' |
 	sort >_signal.grep
 
 echo '// mkerrors.sh' "$@"
diff --git a/vendor/golang.org/x/sys/unix/syscall_aix.go b/vendor/golang.org/x/sys/unix/syscall_aix.go
index 67ce6cef2..6f15ba1ea 100644
--- a/vendor/golang.org/x/sys/unix/syscall_aix.go
+++ b/vendor/golang.org/x/sys/unix/syscall_aix.go
@@ -360,7 +360,7 @@ func Wait4(pid int, wstatus *WaitStatus, options int, rusage *Rusage) (wpid int,
 	var status _C_int
 	var r Pid_t
 	err = ERESTART
-	// AIX wait4 may return with ERESTART errno, while the processus is still
+	// AIX wait4 may return with ERESTART errno, while the process is still
 	// active.
 	for err == ERESTART {
 		r, err = wait4(Pid_t(pid), &status, options, rusage)
diff --git a/vendor/golang.org/x/sys/unix/syscall_darwin.go b/vendor/golang.org/x/sys/unix/syscall_darwin.go
index 4cc7b0059..099867dee 100644
--- a/vendor/golang.org/x/sys/unix/syscall_darwin.go
+++ b/vendor/golang.org/x/sys/unix/syscall_darwin.go
@@ -402,6 +402,18 @@ func IoctlSetIfreqMTU(fd int, ifreq *IfreqMTU) error {
 	return ioctlPtr(fd, SIOCSIFMTU, unsafe.Pointer(ifreq))
 }
 
+//sys	renamexNp(from string, to string, flag uint32) (err error)
+
+func RenamexNp(from string, to string, flag uint32) (err error) {
+	return renamexNp(from, to, flag)
+}
+
+//sys	renameatxNp(fromfd int, from string, tofd int, to string, flag uint32) (err error)
+
+func RenameatxNp(fromfd int, from string, tofd int, to string, flag uint32) (err error) {
+	return renameatxNp(fromfd, from, tofd, to, flag)
+}
+
 //sys	sysctl(mib []_C_int, old *byte, oldlen *uintptr, new *byte, newlen uintptr) (err error) = SYS_SYSCTL
 
 func Uname(uname *Utsname) error {
@@ -554,6 +566,43 @@ func PthreadFchdir(fd int) (err error) {
 	return pthread_fchdir_np(fd)
 }
 
+// Connectx calls connectx(2) to initiate a connection on a socket.
+//
+// srcIf, srcAddr, and dstAddr are filled into a [SaEndpoints] struct and passed as the endpoints argument.
+//
+//   - srcIf is the optional source interface index. 0 means unspecified.
+//   - srcAddr is the optional source address. nil means unspecified.
+//   - dstAddr is the destination address.
+//
+// On success, Connectx returns the number of bytes enqueued for transmission.
+func Connectx(fd int, srcIf uint32, srcAddr, dstAddr Sockaddr, associd SaeAssocID, flags uint32, iov []Iovec, connid *SaeConnID) (n uintptr, err error) {
+	endpoints := SaEndpoints{
+		Srcif: srcIf,
+	}
+
+	if srcAddr != nil {
+		addrp, addrlen, err := srcAddr.sockaddr()
+		if err != nil {
+			return 0, err
+		}
+		endpoints.Srcaddr = (*RawSockaddr)(addrp)
+		endpoints.Srcaddrlen = uint32(addrlen)
+	}
+
+	if dstAddr != nil {
+		addrp, addrlen, err := dstAddr.sockaddr()
+		if err != nil {
+			return 0, err
+		}
+		endpoints.Dstaddr = (*RawSockaddr)(addrp)
+		endpoints.Dstaddrlen = uint32(addrlen)
+	}
+
+	err = connectx(fd, &endpoints, associd, flags, iov, &n, connid)
+	return
+}
+
+//sys	connectx(fd int, endpoints *SaEndpoints, associd SaeAssocID, flags uint32, iov []Iovec, n *uintptr, connid *SaeConnID) (err error)
 //sys	sendfile(infd int, outfd int, offset int64, len *int64, hdtr unsafe.Pointer, flags int) (err error)
 
 //sys	shmat(id int, addr uintptr, flag int) (ret uintptr, err error)
diff --git a/vendor/golang.org/x/sys/unix/syscall_hurd.go b/vendor/golang.org/x/sys/unix/syscall_hurd.go
index ba46651f8..a6a2d2fc2 100644
--- a/vendor/golang.org/x/sys/unix/syscall_hurd.go
+++ b/vendor/golang.org/x/sys/unix/syscall_hurd.go
@@ -11,6 +11,7 @@ package unix
 int ioctl(int, unsigned long int, uintptr_t);
 */
 import "C"
+import "unsafe"
 
 func ioctl(fd int, req uint, arg uintptr) (err error) {
 	r0, er := C.ioctl(C.int(fd), C.ulong(req), C.uintptr_t(arg))
diff --git a/vendor/golang.org/x/sys/unix/syscall_linux.go b/vendor/golang.org/x/sys/unix/syscall_linux.go
index 5682e2628..230a94549 100644
--- a/vendor/golang.org/x/sys/unix/syscall_linux.go
+++ b/vendor/golang.org/x/sys/unix/syscall_linux.go
@@ -1295,6 +1295,48 @@ func GetsockoptTCPInfo(fd, level, opt int) (*TCPInfo, error) {
 	return &value, err
 }
 
+// GetsockoptTCPCCVegasInfo returns algorithm specific congestion control information for a socket using the "vegas"
+// algorithm.
+//
+// The socket's congestion control algorighm can be retrieved via [GetsockoptString] with the [TCP_CONGESTION] option:
+//
+//	algo, err := unix.GetsockoptString(fd, unix.IPPROTO_TCP, unix.TCP_CONGESTION)
+func GetsockoptTCPCCVegasInfo(fd, level, opt int) (*TCPVegasInfo, error) {
+	var value [SizeofTCPCCInfo / 4]uint32 // ensure proper alignment
+	vallen := _Socklen(SizeofTCPCCInfo)
+	err := getsockopt(fd, level, opt, unsafe.Pointer(&value[0]), &vallen)
+	out := (*TCPVegasInfo)(unsafe.Pointer(&value[0]))
+	return out, err
+}
+
+// GetsockoptTCPCCDCTCPInfo returns algorithm specific congestion control information for a socket using the "dctp"
+// algorithm.
+//
+// The socket's congestion control algorighm can be retrieved via [GetsockoptString] with the [TCP_CONGESTION] option:
+//
+//	algo, err := unix.GetsockoptString(fd, unix.IPPROTO_TCP, unix.TCP_CONGESTION)
+func GetsockoptTCPCCDCTCPInfo(fd, level, opt int) (*TCPDCTCPInfo, error) {
+	var value [SizeofTCPCCInfo / 4]uint32 // ensure proper alignment
+	vallen := _Socklen(SizeofTCPCCInfo)
+	err := getsockopt(fd, level, opt, unsafe.Pointer(&value[0]), &vallen)
+	out := (*TCPDCTCPInfo)(unsafe.Pointer(&value[0]))
+	return out, err
+}
+
+// GetsockoptTCPCCBBRInfo returns algorithm specific congestion control information for a socket using the "bbr"
+// algorithm.
+//
+// The socket's congestion control algorighm can be retrieved via [GetsockoptString] with the [TCP_CONGESTION] option:
+//
+//	algo, err := unix.GetsockoptString(fd, unix.IPPROTO_TCP, unix.TCP_CONGESTION)
+func GetsockoptTCPCCBBRInfo(fd, level, opt int) (*TCPBBRInfo, error) {
+	var value [SizeofTCPCCInfo / 4]uint32 // ensure proper alignment
+	vallen := _Socklen(SizeofTCPCCInfo)
+	err := getsockopt(fd, level, opt, unsafe.Pointer(&value[0]), &vallen)
+	out := (*TCPBBRInfo)(unsafe.Pointer(&value[0]))
+	return out, err
+}
+
 // GetsockoptString returns the string value of the socket option opt for the
 // socket associated with fd at the given socket level.
 func GetsockoptString(fd, level, opt int) (string, error) {
@@ -1818,6 +1860,7 @@ func Sendfile(outfd int, infd int, offset *int64, count int) (written int, err e
 //sys	ClockAdjtime(clockid int32, buf *Timex) (state int, err error)
 //sys	ClockGetres(clockid int32, res *Timespec) (err error)
 //sys	ClockGettime(clockid int32, time *Timespec) (err error)
+//sys	ClockSettime(clockid int32, time *Timespec) (err error)
 //sys	ClockNanosleep(clockid int32, flags int, request *Timespec, remain *Timespec) (err error)
 //sys	Close(fd int) (err error)
 //sys	CloseRange(first uint, last uint, flags uint) (err error)
@@ -1959,7 +2002,26 @@ func Getpgrp() (pid int) {
 //sysnb	Getpid() (pid int)
 //sysnb	Getppid() (ppid int)
 //sys	Getpriority(which int, who int) (prio int, err error)
-//sys	Getrandom(buf []byte, flags int) (n int, err error)
+
+func Getrandom(buf []byte, flags int) (n int, err error) {
+	vdsoRet, supported := vgetrandom(buf, uint32(flags))
+	if supported {
+		if vdsoRet < 0 {
+			return 0, errnoErr(syscall.Errno(-vdsoRet))
+		}
+		return vdsoRet, nil
+	}
+	var p *byte
+	if len(buf) > 0 {
+		p = &buf[0]
+	}
+	r, _, e := Syscall(SYS_GETRANDOM, uintptr(unsafe.Pointer(p)), uintptr(len(buf)), uintptr(flags))
+	if e != 0 {
+		return 0, errnoErr(e)
+	}
+	return int(r), nil
+}
+
 //sysnb	Getrusage(who int, rusage *Rusage) (err error)
 //sysnb	Getsid(pid int) (sid int, err error)
 //sysnb	Gettid() (tid int)
@@ -2592,3 +2654,4 @@ func SchedGetAttr(pid int, flags uint) (*SchedAttr, error) {
 }
 
 //sys	Cachestat(fd uint, crange *CachestatRange, cstat *Cachestat_t, flags uint) (err error)
+//sys	Mseal(b []byte, flags uint) (err error)
diff --git a/vendor/golang.org/x/sys/unix/syscall_linux_arm64.go b/vendor/golang.org/x/sys/unix/syscall_linux_arm64.go
index cf2ee6c75..745e5c7e6 100644
--- a/vendor/golang.org/x/sys/unix/syscall_linux_arm64.go
+++ b/vendor/golang.org/x/sys/unix/syscall_linux_arm64.go
@@ -182,3 +182,5 @@ func KexecFileLoad(kernelFd int, initrdFd int, cmdline string, flags int) error
 	}
 	return kexecFileLoad(kernelFd, initrdFd, cmdlineLen, cmdline, flags)
 }
+
+const SYS_FSTATAT = SYS_NEWFSTATAT
diff --git a/vendor/golang.org/x/sys/unix/syscall_linux_loong64.go b/vendor/golang.org/x/sys/unix/syscall_linux_loong64.go
index 3d0e98451..dd2262a40 100644
--- a/vendor/golang.org/x/sys/unix/syscall_linux_loong64.go
+++ b/vendor/golang.org/x/sys/unix/syscall_linux_loong64.go
@@ -214,3 +214,5 @@ func KexecFileLoad(kernelFd int, initrdFd int, cmdline string, flags int) error
 	}
 	return kexecFileLoad(kernelFd, initrdFd, cmdlineLen, cmdline, flags)
 }
+
+const SYS_FSTATAT = SYS_NEWFSTATAT
diff --git a/vendor/golang.org/x/sys/unix/syscall_linux_riscv64.go b/vendor/golang.org/x/sys/unix/syscall_linux_riscv64.go
index 6f5a28894..8cf3670bd 100644
--- a/vendor/golang.org/x/sys/unix/syscall_linux_riscv64.go
+++ b/vendor/golang.org/x/sys/unix/syscall_linux_riscv64.go
@@ -187,3 +187,5 @@ func RISCVHWProbe(pairs []RISCVHWProbePairs, set *CPUSet, flags uint) (err error
 	}
 	return riscvHWProbe(pairs, setSize, set, flags)
 }
+
+const SYS_FSTATAT = SYS_NEWFSTATAT
diff --git a/vendor/golang.org/x/sys/unix/syscall_openbsd.go b/vendor/golang.org/x/sys/unix/syscall_openbsd.go
index b25343c71..b86ded549 100644
--- a/vendor/golang.org/x/sys/unix/syscall_openbsd.go
+++ b/vendor/golang.org/x/sys/unix/syscall_openbsd.go
@@ -293,6 +293,7 @@ func Uname(uname *Utsname) error {
 //sys	Mkfifoat(dirfd int, path string, mode uint32) (err error)
 //sys	Mknod(path string, mode uint32, dev int) (err error)
 //sys	Mknodat(dirfd int, path string, mode uint32, dev int) (err error)
+//sys	Mount(fsType string, dir string, flags int, data unsafe.Pointer) (err error)
 //sys	Nanosleep(time *Timespec, leftover *Timespec) (err error)
 //sys	Open(path string, mode int, perm uint32) (fd int, err error)
 //sys	Openat(dirfd int, path string, mode int, perm uint32) (fd int, err error)
diff --git a/vendor/golang.org/x/sys/unix/syscall_zos_s390x.go b/vendor/golang.org/x/sys/unix/syscall_zos_s390x.go
index 312ae6ac1..7bf5c04bb 100644
--- a/vendor/golang.org/x/sys/unix/syscall_zos_s390x.go
+++ b/vendor/golang.org/x/sys/unix/syscall_zos_s390x.go
@@ -768,6 +768,15 @@ func Munmap(b []byte) (err error) {
 	return mapper.Munmap(b)
 }
 
+func MmapPtr(fd int, offset int64, addr unsafe.Pointer, length uintptr, prot int, flags int) (ret unsafe.Pointer, err error) {
+	xaddr, err := mapper.mmap(uintptr(addr), length, prot, flags, fd, offset)
+	return unsafe.Pointer(xaddr), err
+}
+
+func MunmapPtr(addr unsafe.Pointer, length uintptr) (err error) {
+	return mapper.munmap(uintptr(addr), length)
+}
+
 //sys   Gethostname(buf []byte) (err error) = SYS___GETHOSTNAME_A
 //sysnb	Getgid() (gid int)
 //sysnb	Getpid() (pid int)
@@ -816,10 +825,10 @@ func Lstat(path string, stat *Stat_t) (err error) {
 // for checking symlinks begins with $VERSION/ $SYSNAME/ $SYSSYMR/ $SYSSYMA/
 func isSpecialPath(path []byte) (v bool) {
 	var special = [4][8]byte{
-		[8]byte{'V', 'E', 'R', 'S', 'I', 'O', 'N', '/'},
-		[8]byte{'S', 'Y', 'S', 'N', 'A', 'M', 'E', '/'},
-		[8]byte{'S', 'Y', 'S', 'S', 'Y', 'M', 'R', '/'},
-		[8]byte{'S', 'Y', 'S', 'S', 'Y', 'M', 'A', '/'}}
+		{'V', 'E', 'R', 'S', 'I', 'O', 'N', '/'},
+		{'S', 'Y', 'S', 'N', 'A', 'M', 'E', '/'},
+		{'S', 'Y', 'S', 'S', 'Y', 'M', 'R', '/'},
+		{'S', 'Y', 'S', 'S', 'Y', 'M', 'A', '/'}}
 
 	var i, j int
 	for i = 0; i < len(special); i++ {
@@ -3115,3 +3124,90 @@ func legacy_Mkfifoat(dirfd int, path string, mode uint32) (err error) {
 //sys	Posix_openpt(oflag int) (fd int, err error) = SYS_POSIX_OPENPT
 //sys	Grantpt(fildes int) (rc int, err error) = SYS_GRANTPT
 //sys	Unlockpt(fildes int) (rc int, err error) = SYS_UNLOCKPT
+
+func fcntlAsIs(fd uintptr, cmd int, arg uintptr) (val int, err error) {
+	runtime.EnterSyscall()
+	r0, e2, e1 := CallLeFuncWithErr(GetZosLibVec()+SYS_FCNTL<<4, uintptr(fd), uintptr(cmd), arg)
+	runtime.ExitSyscall()
+	val = int(r0)
+	if int64(r0) == -1 {
+		err = errnoErr2(e1, e2)
+	}
+	return
+}
+
+func Fcntl(fd uintptr, cmd int, op interface{}) (ret int, err error) {
+	switch op.(type) {
+	case *Flock_t:
+		err = FcntlFlock(fd, cmd, op.(*Flock_t))
+		if err != nil {
+			ret = -1
+		}
+		return
+	case int:
+		return FcntlInt(fd, cmd, op.(int))
+	case *F_cnvrt:
+		return fcntlAsIs(fd, cmd, uintptr(unsafe.Pointer(op.(*F_cnvrt))))
+	case unsafe.Pointer:
+		return fcntlAsIs(fd, cmd, uintptr(op.(unsafe.Pointer)))
+	default:
+		return -1, EINVAL
+	}
+	return
+}
+
+func Sendfile(outfd int, infd int, offset *int64, count int) (written int, err error) {
+	if raceenabled {
+		raceReleaseMerge(unsafe.Pointer(&ioSync))
+	}
+	return sendfile(outfd, infd, offset, count)
+}
+
+func sendfile(outfd int, infd int, offset *int64, count int) (written int, err error) {
+	// TODO: use LE call instead if the call is implemented
+	originalOffset, err := Seek(infd, 0, SEEK_CUR)
+	if err != nil {
+		return -1, err
+	}
+	//start reading data from in_fd
+	if offset != nil {
+		_, err := Seek(infd, *offset, SEEK_SET)
+		if err != nil {
+			return -1, err
+		}
+	}
+
+	buf := make([]byte, count)
+	readBuf := make([]byte, 0)
+	var n int = 0
+	for i := 0; i < count; i += n {
+		n, err := Read(infd, buf)
+		if n == 0 {
+			if err != nil {
+				return -1, err
+			} else { // EOF
+				break
+			}
+		}
+		readBuf = append(readBuf, buf...)
+		buf = buf[0:0]
+	}
+
+	n2, err := Write(outfd, readBuf)
+	if err != nil {
+		return -1, err
+	}
+
+	//When sendfile() returns, this variable will be set to the
+	// offset of the byte following the last byte that was read.
+	if offset != nil {
+		*offset = *offset + int64(n)
+		// If offset is not NULL, then sendfile() does not modify the file
+		// offset of in_fd
+		_, err := Seek(infd, originalOffset, SEEK_SET)
+		if err != nil {
+			return -1, err
+		}
+	}
+	return n2, nil
+}
diff --git a/vendor/golang.org/x/sys/unix/vgetrandom_linux.go b/vendor/golang.org/x/sys/unix/vgetrandom_linux.go
new file mode 100644
index 000000000..07ac8e09d
--- /dev/null
+++ b/vendor/golang.org/x/sys/unix/vgetrandom_linux.go
@@ -0,0 +1,13 @@
+// Copyright 2024 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+//go:build linux && go1.24
+
+package unix
+
+import _ "unsafe"
+
+//go:linkname vgetrandom runtime.vgetrandom
+//go:noescape
+func vgetrandom(p []byte, flags uint32) (ret int, supported bool)
diff --git a/vendor/golang.org/x/sys/unix/vgetrandom_unsupported.go b/vendor/golang.org/x/sys/unix/vgetrandom_unsupported.go
new file mode 100644
index 000000000..297e97bce
--- /dev/null
+++ b/vendor/golang.org/x/sys/unix/vgetrandom_unsupported.go
@@ -0,0 +1,11 @@
+// Copyright 2024 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+//go:build !linux || !go1.24
+
+package unix
+
+func vgetrandom(p []byte, flags uint32) (ret int, supported bool) {
+	return -1, false
+}
diff --git a/vendor/golang.org/x/sys/unix/zerrors_darwin_amd64.go b/vendor/golang.org/x/sys/unix/zerrors_darwin_amd64.go
index e40fa8524..d73c4652e 100644
--- a/vendor/golang.org/x/sys/unix/zerrors_darwin_amd64.go
+++ b/vendor/golang.org/x/sys/unix/zerrors_darwin_amd64.go
@@ -237,6 +237,9 @@ const (
 	CLOCK_UPTIME_RAW_APPROX                 = 0x9
 	CLONE_NOFOLLOW                          = 0x1
 	CLONE_NOOWNERCOPY                       = 0x2
+	CONNECT_DATA_AUTHENTICATED              = 0x4
+	CONNECT_DATA_IDEMPOTENT                 = 0x2
+	CONNECT_RESUME_ON_READ_WRITE            = 0x1
 	CR0                                     = 0x0
 	CR1                                     = 0x1000
 	CR2                                     = 0x2000
@@ -1169,6 +1172,11 @@ const (
 	PT_WRITE_D                              = 0x5
 	PT_WRITE_I                              = 0x4
 	PT_WRITE_U                              = 0x6
+	RENAME_EXCL                             = 0x4
+	RENAME_NOFOLLOW_ANY                     = 0x10
+	RENAME_RESERVED1                        = 0x8
+	RENAME_SECLUDE                          = 0x1
+	RENAME_SWAP                             = 0x2
 	RLIMIT_AS                               = 0x5
 	RLIMIT_CORE                             = 0x4
 	RLIMIT_CPU                              = 0x0
@@ -1260,6 +1268,10 @@ const (
 	RTV_SSTHRESH                            = 0x20
 	RUSAGE_CHILDREN                         = -0x1
 	RUSAGE_SELF                             = 0x0
+	SAE_ASSOCID_ALL                         = 0xffffffff
+	SAE_ASSOCID_ANY                         = 0x0
+	SAE_CONNID_ALL                          = 0xffffffff
+	SAE_CONNID_ANY                          = 0x0
 	SCM_CREDS                               = 0x3
 	SCM_RIGHTS                              = 0x1
 	SCM_TIMESTAMP                           = 0x2
diff --git a/vendor/golang.org/x/sys/unix/zerrors_darwin_arm64.go b/vendor/golang.org/x/sys/unix/zerrors_darwin_arm64.go
index bb02aa6c0..4a55a4005 100644
--- a/vendor/golang.org/x/sys/unix/zerrors_darwin_arm64.go
+++ b/vendor/golang.org/x/sys/unix/zerrors_darwin_arm64.go
@@ -237,6 +237,9 @@ const (
 	CLOCK_UPTIME_RAW_APPROX                 = 0x9
 	CLONE_NOFOLLOW                          = 0x1
 	CLONE_NOOWNERCOPY                       = 0x2
+	CONNECT_DATA_AUTHENTICATED              = 0x4
+	CONNECT_DATA_IDEMPOTENT                 = 0x2
+	CONNECT_RESUME_ON_READ_WRITE            = 0x1
 	CR0                                     = 0x0
 	CR1                                     = 0x1000
 	CR2                                     = 0x2000
@@ -1169,6 +1172,11 @@ const (
 	PT_WRITE_D                              = 0x5
 	PT_WRITE_I                              = 0x4
 	PT_WRITE_U                              = 0x6
+	RENAME_EXCL                             = 0x4
+	RENAME_NOFOLLOW_ANY                     = 0x10
+	RENAME_RESERVED1                        = 0x8
+	RENAME_SECLUDE                          = 0x1
+	RENAME_SWAP                             = 0x2
 	RLIMIT_AS                               = 0x5
 	RLIMIT_CORE                             = 0x4
 	RLIMIT_CPU                              = 0x0
@@ -1260,6 +1268,10 @@ const (
 	RTV_SSTHRESH                            = 0x20
 	RUSAGE_CHILDREN                         = -0x1
 	RUSAGE_SELF                             = 0x0
+	SAE_ASSOCID_ALL                         = 0xffffffff
+	SAE_ASSOCID_ANY                         = 0x0
+	SAE_CONNID_ALL                          = 0xffffffff
+	SAE_CONNID_ANY                          = 0x0
 	SCM_CREDS                               = 0x3
 	SCM_RIGHTS                              = 0x1
 	SCM_TIMESTAMP                           = 0x2
diff --git a/vendor/golang.org/x/sys/unix/zerrors_linux.go b/vendor/golang.org/x/sys/unix/zerrors_linux.go
index 877a62b47..6ebc48b3f 100644
--- a/vendor/golang.org/x/sys/unix/zerrors_linux.go
+++ b/vendor/golang.org/x/sys/unix/zerrors_linux.go
@@ -321,6 +321,9 @@ const (
 	AUDIT_INTEGRITY_STATUS                      = 0x70a
 	AUDIT_IPC                                   = 0x517
 	AUDIT_IPC_SET_PERM                          = 0x51f
+	AUDIT_IPE_ACCESS                            = 0x58c
+	AUDIT_IPE_CONFIG_CHANGE                     = 0x58d
+	AUDIT_IPE_POLICY_LOAD                       = 0x58e
 	AUDIT_KERNEL                                = 0x7d0
 	AUDIT_KERNEL_OTHER                          = 0x524
 	AUDIT_KERN_MODULE                           = 0x532
@@ -457,6 +460,7 @@ const (
 	B600                                        = 0x8
 	B75                                         = 0x2
 	B9600                                       = 0xd
+	BCACHEFS_SUPER_MAGIC                        = 0xca451a4e
 	BDEVFS_MAGIC                                = 0x62646576
 	BINDERFS_SUPER_MAGIC                        = 0x6c6f6f70
 	BINFMTFS_MAGIC                              = 0x42494e4d
@@ -488,12 +492,14 @@ const (
 	BPF_F_ID                                    = 0x20
 	BPF_F_NETFILTER_IP_DEFRAG                   = 0x1
 	BPF_F_QUERY_EFFECTIVE                       = 0x1
+	BPF_F_REDIRECT_FLAGS                        = 0x19
 	BPF_F_REPLACE                               = 0x4
 	BPF_F_SLEEPABLE                             = 0x10
 	BPF_F_STRICT_ALIGNMENT                      = 0x1
 	BPF_F_TEST_REG_INVARIANTS                   = 0x80
 	BPF_F_TEST_RND_HI32                         = 0x4
 	BPF_F_TEST_RUN_ON_CPU                       = 0x1
+	BPF_F_TEST_SKB_CHECKSUM_COMPLETE            = 0x4
 	BPF_F_TEST_STATE_FREQ                       = 0x8
 	BPF_F_TEST_XDP_LIVE_FRAMES                  = 0x2
 	BPF_F_XDP_DEV_BOUND_ONLY                    = 0x40
@@ -928,6 +934,7 @@ const (
 	EPOLL_CTL_ADD                               = 0x1
 	EPOLL_CTL_DEL                               = 0x2
 	EPOLL_CTL_MOD                               = 0x3
+	EPOLL_IOC_TYPE                              = 0x8a
 	EROFS_SUPER_MAGIC_V1                        = 0xe0f5e1e2
 	ESP_V4_FLOW                                 = 0xa
 	ESP_V6_FLOW                                 = 0xc
@@ -941,9 +948,6 @@ const (
 	ETHTOOL_FEC_OFF                             = 0x4
 	ETHTOOL_FEC_RS                              = 0x8
 	ETHTOOL_FLAG_ALL                            = 0x7
-	ETHTOOL_FLAG_COMPACT_BITSETS                = 0x1
-	ETHTOOL_FLAG_OMIT_REPLY                     = 0x2
-	ETHTOOL_FLAG_STATS                          = 0x4
 	ETHTOOL_FLASHDEV                            = 0x33
 	ETHTOOL_FLASH_MAX_FILENAME                  = 0x80
 	ETHTOOL_FWVERS_LEN                          = 0x20
@@ -1166,6 +1170,7 @@ const (
 	EXTA                                        = 0xe
 	EXTB                                        = 0xf
 	F2FS_SUPER_MAGIC                            = 0xf2f52010
+	FALLOC_FL_ALLOCATE_RANGE                    = 0x0
 	FALLOC_FL_COLLAPSE_RANGE                    = 0x8
 	FALLOC_FL_INSERT_RANGE                      = 0x20
 	FALLOC_FL_KEEP_SIZE                         = 0x1
@@ -1705,6 +1710,7 @@ const (
 	KEXEC_ARCH_S390                             = 0x160000
 	KEXEC_ARCH_SH                               = 0x2a0000
 	KEXEC_ARCH_X86_64                           = 0x3e0000
+	KEXEC_CRASH_HOTPLUG_SUPPORT                 = 0x8
 	KEXEC_FILE_DEBUG                            = 0x8
 	KEXEC_FILE_NO_INITRAMFS                     = 0x4
 	KEXEC_FILE_ON_CRASH                         = 0x2
@@ -1780,6 +1786,7 @@ const (
 	KEY_SPEC_USER_KEYRING                       = -0x4
 	KEY_SPEC_USER_SESSION_KEYRING               = -0x5
 	LANDLOCK_ACCESS_FS_EXECUTE                  = 0x1
+	LANDLOCK_ACCESS_FS_IOCTL_DEV                = 0x8000
 	LANDLOCK_ACCESS_FS_MAKE_BLOCK               = 0x800
 	LANDLOCK_ACCESS_FS_MAKE_CHAR                = 0x40
 	LANDLOCK_ACCESS_FS_MAKE_DIR                 = 0x80
@@ -1797,6 +1804,8 @@ const (
 	LANDLOCK_ACCESS_NET_BIND_TCP                = 0x1
 	LANDLOCK_ACCESS_NET_CONNECT_TCP             = 0x2
 	LANDLOCK_CREATE_RULESET_VERSION             = 0x1
+	LANDLOCK_SCOPE_ABSTRACT_UNIX_SOCKET         = 0x1
+	LANDLOCK_SCOPE_SIGNAL                       = 0x2
 	LINUX_REBOOT_CMD_CAD_OFF                    = 0x0
 	LINUX_REBOOT_CMD_CAD_ON                     = 0x89abcdef
 	LINUX_REBOOT_CMD_HALT                       = 0xcdef0123
@@ -1861,6 +1870,19 @@ const (
 	MAP_FILE                                    = 0x0
 	MAP_FIXED                                   = 0x10
 	MAP_FIXED_NOREPLACE                         = 0x100000
+	MAP_HUGE_16GB                               = 0x88000000
+	MAP_HUGE_16KB                               = 0x38000000
+	MAP_HUGE_16MB                               = 0x60000000
+	MAP_HUGE_1GB                                = 0x78000000
+	MAP_HUGE_1MB                                = 0x50000000
+	MAP_HUGE_256MB                              = 0x70000000
+	MAP_HUGE_2GB                                = 0x7c000000
+	MAP_HUGE_2MB                                = 0x54000000
+	MAP_HUGE_32MB                               = 0x64000000
+	MAP_HUGE_512KB                              = 0x4c000000
+	MAP_HUGE_512MB                              = 0x74000000
+	MAP_HUGE_64KB                               = 0x40000000
+	MAP_HUGE_8MB                                = 0x5c000000
 	MAP_HUGE_MASK                               = 0x3f
 	MAP_HUGE_SHIFT                              = 0x1a
 	MAP_PRIVATE                                 = 0x2
@@ -1908,6 +1930,8 @@ const (
 	MNT_EXPIRE                                  = 0x4
 	MNT_FORCE                                   = 0x1
 	MNT_ID_REQ_SIZE_VER0                        = 0x18
+	MNT_ID_REQ_SIZE_VER1                        = 0x20
+	MNT_NS_INFO_SIZE_VER0                       = 0x10
 	MODULE_INIT_COMPRESSED_FILE                 = 0x4
 	MODULE_INIT_IGNORE_MODVERSIONS              = 0x1
 	MODULE_INIT_IGNORE_VERMAGIC                 = 0x2
@@ -2173,7 +2197,7 @@ const (
 	NFT_REG_SIZE                                = 0x10
 	NFT_REJECT_ICMPX_MAX                        = 0x3
 	NFT_RT_MAX                                  = 0x4
-	NFT_SECMARK_CTX_MAXLEN                      = 0x100
+	NFT_SECMARK_CTX_MAXLEN                      = 0x1000
 	NFT_SET_MAXNAMELEN                          = 0x100
 	NFT_SOCKET_MAX                              = 0x3
 	NFT_TABLE_F_MASK                            = 0x7
@@ -2342,9 +2366,11 @@ const (
 	PERF_MEM_LVLNUM_IO                          = 0xa
 	PERF_MEM_LVLNUM_L1                          = 0x1
 	PERF_MEM_LVLNUM_L2                          = 0x2
+	PERF_MEM_LVLNUM_L2_MHB                      = 0x5
 	PERF_MEM_LVLNUM_L3                          = 0x3
 	PERF_MEM_LVLNUM_L4                          = 0x4
 	PERF_MEM_LVLNUM_LFB                         = 0xc
+	PERF_MEM_LVLNUM_MSC                         = 0x6
 	PERF_MEM_LVLNUM_NA                          = 0xf
 	PERF_MEM_LVLNUM_PMEM                        = 0xe
 	PERF_MEM_LVLNUM_RAM                         = 0xd
@@ -2417,6 +2443,7 @@ const (
 	PRIO_PGRP                                   = 0x1
 	PRIO_PROCESS                                = 0x0
 	PRIO_USER                                   = 0x2
+	PROCFS_IOCTL_MAGIC                          = 'f'
 	PROC_SUPER_MAGIC                            = 0x9fa0
 	PROT_EXEC                                   = 0x4
 	PROT_GROWSDOWN                              = 0x1000000
@@ -2498,6 +2525,23 @@ const (
 	PR_PAC_GET_ENABLED_KEYS                     = 0x3d
 	PR_PAC_RESET_KEYS                           = 0x36
 	PR_PAC_SET_ENABLED_KEYS                     = 0x3c
+	PR_PPC_DEXCR_CTRL_CLEAR                     = 0x4
+	PR_PPC_DEXCR_CTRL_CLEAR_ONEXEC              = 0x10
+	PR_PPC_DEXCR_CTRL_EDITABLE                  = 0x1
+	PR_PPC_DEXCR_CTRL_MASK                      = 0x1f
+	PR_PPC_DEXCR_CTRL_SET                       = 0x2
+	PR_PPC_DEXCR_CTRL_SET_ONEXEC                = 0x8
+	PR_PPC_DEXCR_IBRTPD                         = 0x1
+	PR_PPC_DEXCR_NPHIE                          = 0x3
+	PR_PPC_DEXCR_SBHE                           = 0x0
+	PR_PPC_DEXCR_SRAPD                          = 0x2
+	PR_PPC_GET_DEXCR                            = 0x48
+	PR_PPC_SET_DEXCR                            = 0x49
+	PR_RISCV_CTX_SW_FENCEI_OFF                  = 0x1
+	PR_RISCV_CTX_SW_FENCEI_ON                   = 0x0
+	PR_RISCV_SCOPE_PER_PROCESS                  = 0x0
+	PR_RISCV_SCOPE_PER_THREAD                   = 0x1
+	PR_RISCV_SET_ICACHE_FLUSH_CTX               = 0x47
 	PR_RISCV_V_GET_CONTROL                      = 0x46
 	PR_RISCV_V_SET_CONTROL                      = 0x45
 	PR_RISCV_V_VSTATE_CTRL_CUR_MASK             = 0x3
@@ -2589,6 +2633,28 @@ const (
 	PR_UNALIGN_NOPRINT                          = 0x1
 	PR_UNALIGN_SIGBUS                           = 0x2
 	PSTOREFS_MAGIC                              = 0x6165676c
+	PTP_CLK_MAGIC                               = '='
+	PTP_ENABLE_FEATURE                          = 0x1
+	PTP_EXTTS_EDGES                             = 0x6
+	PTP_EXTTS_EVENT_VALID                       = 0x1
+	PTP_EXTTS_V1_VALID_FLAGS                    = 0x7
+	PTP_EXTTS_VALID_FLAGS                       = 0x1f
+	PTP_EXT_OFFSET                              = 0x10
+	PTP_FALLING_EDGE                            = 0x4
+	PTP_MAX_SAMPLES                             = 0x19
+	PTP_PEROUT_DUTY_CYCLE                       = 0x2
+	PTP_PEROUT_ONE_SHOT                         = 0x1
+	PTP_PEROUT_PHASE                            = 0x4
+	PTP_PEROUT_V1_VALID_FLAGS                   = 0x0
+	PTP_PEROUT_VALID_FLAGS                      = 0x7
+	PTP_PIN_GETFUNC                             = 0xc0603d06
+	PTP_PIN_GETFUNC2                            = 0xc0603d0f
+	PTP_RISING_EDGE                             = 0x2
+	PTP_STRICT_FLAGS                            = 0x8
+	PTP_SYS_OFFSET_EXTENDED                     = 0xc4c03d09
+	PTP_SYS_OFFSET_EXTENDED2                    = 0xc4c03d12
+	PTP_SYS_OFFSET_PRECISE                      = 0xc0403d08
+	PTP_SYS_OFFSET_PRECISE2                     = 0xc0403d11
 	PTRACE_ATTACH                               = 0x10
 	PTRACE_CONT                                 = 0x7
 	PTRACE_DETACH                               = 0x11
@@ -2902,15 +2968,17 @@ const (
 	RUSAGE_SELF                                 = 0x0
 	RUSAGE_THREAD                               = 0x1
 	RWF_APPEND                                  = 0x10
+	RWF_ATOMIC                                  = 0x40
 	RWF_DSYNC                                   = 0x2
 	RWF_HIPRI                                   = 0x1
 	RWF_NOAPPEND                                = 0x20
 	RWF_NOWAIT                                  = 0x8
-	RWF_SUPPORTED                               = 0x3f
+	RWF_SUPPORTED                               = 0x7f
 	RWF_SYNC                                    = 0x4
 	RWF_WRITE_LIFE_NOT_SET                      = 0x0
 	SCHED_BATCH                                 = 0x3
 	SCHED_DEADLINE                              = 0x6
+	SCHED_EXT                                   = 0x7
 	SCHED_FIFO                                  = 0x1
 	SCHED_FLAG_ALL                              = 0x7f
 	SCHED_FLAG_DL_OVERRUN                       = 0x4
@@ -3179,6 +3247,7 @@ const (
 	STATX_ATTR_MOUNT_ROOT                       = 0x2000
 	STATX_ATTR_NODUMP                           = 0x40
 	STATX_ATTR_VERITY                           = 0x100000
+	STATX_ATTR_WRITE_ATOMIC                     = 0x400000
 	STATX_BASIC_STATS                           = 0x7ff
 	STATX_BLOCKS                                = 0x400
 	STATX_BTIME                                 = 0x800
@@ -3192,8 +3261,10 @@ const (
 	STATX_MTIME                                 = 0x40
 	STATX_NLINK                                 = 0x4
 	STATX_SIZE                                  = 0x200
+	STATX_SUBVOL                                = 0x8000
 	STATX_TYPE                                  = 0x1
 	STATX_UID                                   = 0x8
+	STATX_WRITE_ATOMIC                          = 0x10000
 	STATX__RESERVED                             = 0x80000000
 	SYNC_FILE_RANGE_WAIT_AFTER                  = 0x4
 	SYNC_FILE_RANGE_WAIT_BEFORE                 = 0x1
@@ -3592,6 +3663,7 @@ const (
 	XDP_UMEM_PGOFF_COMPLETION_RING              = 0x180000000
 	XDP_UMEM_PGOFF_FILL_RING                    = 0x100000000
 	XDP_UMEM_REG                                = 0x4
+	XDP_UMEM_TX_METADATA_LEN                    = 0x4
 	XDP_UMEM_TX_SW_CSUM                         = 0x2
 	XDP_UMEM_UNALIGNED_CHUNK_FLAG               = 0x1
 	XDP_USE_NEED_WAKEUP                         = 0x8
diff --git a/vendor/golang.org/x/sys/unix/zerrors_linux_386.go b/vendor/golang.org/x/sys/unix/zerrors_linux_386.go
index e4bc0bd57..c0d45e320 100644
--- a/vendor/golang.org/x/sys/unix/zerrors_linux_386.go
+++ b/vendor/golang.org/x/sys/unix/zerrors_linux_386.go
@@ -78,6 +78,8 @@ const (
 	ECHOPRT                          = 0x400
 	EFD_CLOEXEC                      = 0x80000
 	EFD_NONBLOCK                     = 0x800
+	EPIOCGPARAMS                     = 0x80088a02
+	EPIOCSPARAMS                     = 0x40088a01
 	EPOLL_CLOEXEC                    = 0x80000
 	EXTPROC                          = 0x10000
 	FF1                              = 0x8000
@@ -107,6 +109,7 @@ const (
 	HIDIOCGRAWINFO                   = 0x80084803
 	HIDIOCGRDESC                     = 0x90044802
 	HIDIOCGRDESCSIZE                 = 0x80044801
+	HIDIOCREVOKE                     = 0x4004480d
 	HUPCL                            = 0x400
 	ICANON                           = 0x2
 	IEXTEN                           = 0x8000
@@ -151,9 +154,14 @@ const (
 	NFDBITS                          = 0x20
 	NLDLY                            = 0x100
 	NOFLSH                           = 0x80
+	NS_GET_MNTNS_ID                  = 0x8008b705
 	NS_GET_NSTYPE                    = 0xb703
 	NS_GET_OWNER_UID                 = 0xb704
 	NS_GET_PARENT                    = 0xb702
+	NS_GET_PID_FROM_PIDNS            = 0x8004b706
+	NS_GET_PID_IN_PIDNS              = 0x8004b708
+	NS_GET_TGID_FROM_PIDNS           = 0x8004b707
+	NS_GET_TGID_IN_PIDNS             = 0x8004b709
 	NS_GET_USERNS                    = 0xb701
 	OLCUC                            = 0x2
 	ONLCR                            = 0x4
@@ -230,6 +238,20 @@ const (
 	PPPIOCUNBRIDGECHAN               = 0x7434
 	PPPIOCXFERUNIT                   = 0x744e
 	PR_SET_PTRACER_ANY               = 0xffffffff
+	PTP_CLOCK_GETCAPS                = 0x80503d01
+	PTP_CLOCK_GETCAPS2               = 0x80503d0a
+	PTP_ENABLE_PPS                   = 0x40043d04
+	PTP_ENABLE_PPS2                  = 0x40043d0d
+	PTP_EXTTS_REQUEST                = 0x40103d02
+	PTP_EXTTS_REQUEST2               = 0x40103d0b
+	PTP_MASK_CLEAR_ALL               = 0x3d13
+	PTP_MASK_EN_SINGLE               = 0x40043d14
+	PTP_PEROUT_REQUEST               = 0x40383d03
+	PTP_PEROUT_REQUEST2              = 0x40383d0c
+	PTP_PIN_SETFUNC                  = 0x40603d07
+	PTP_PIN_SETFUNC2                 = 0x40603d10
+	PTP_SYS_OFFSET                   = 0x43403d05
+	PTP_SYS_OFFSET2                  = 0x43403d0e
 	PTRACE_GETFPREGS                 = 0xe
 	PTRACE_GETFPXREGS                = 0x12
 	PTRACE_GET_THREAD_AREA           = 0x19
@@ -276,6 +298,8 @@ const (
 	RTC_WIE_ON                       = 0x700f
 	RTC_WKALM_RD                     = 0x80287010
 	RTC_WKALM_SET                    = 0x4028700f
+	SCM_DEVMEM_DMABUF                = 0x4f
+	SCM_DEVMEM_LINEAR                = 0x4e
 	SCM_TIMESTAMPING                 = 0x25
 	SCM_TIMESTAMPING_OPT_STATS       = 0x36
 	SCM_TIMESTAMPING_PKTINFO         = 0x3a
@@ -314,6 +338,9 @@ const (
 	SO_CNX_ADVICE                    = 0x35
 	SO_COOKIE                        = 0x39
 	SO_DETACH_REUSEPORT_BPF          = 0x44
+	SO_DEVMEM_DMABUF                 = 0x4f
+	SO_DEVMEM_DONTNEED               = 0x50
+	SO_DEVMEM_LINEAR                 = 0x4e
 	SO_DOMAIN                        = 0x27
 	SO_DONTROUTE                     = 0x5
 	SO_ERROR                         = 0x4
diff --git a/vendor/golang.org/x/sys/unix/zerrors_linux_amd64.go b/vendor/golang.org/x/sys/unix/zerrors_linux_amd64.go
index 689317afd..c731d24f0 100644
--- a/vendor/golang.org/x/sys/unix/zerrors_linux_amd64.go
+++ b/vendor/golang.org/x/sys/unix/zerrors_linux_amd64.go
@@ -78,6 +78,8 @@ const (
 	ECHOPRT                          = 0x400
 	EFD_CLOEXEC                      = 0x80000
 	EFD_NONBLOCK                     = 0x800
+	EPIOCGPARAMS                     = 0x80088a02
+	EPIOCSPARAMS                     = 0x40088a01
 	EPOLL_CLOEXEC                    = 0x80000
 	EXTPROC                          = 0x10000
 	FF1                              = 0x8000
@@ -107,6 +109,7 @@ const (
 	HIDIOCGRAWINFO                   = 0x80084803
 	HIDIOCGRDESC                     = 0x90044802
 	HIDIOCGRDESCSIZE                 = 0x80044801
+	HIDIOCREVOKE                     = 0x4004480d
 	HUPCL                            = 0x400
 	ICANON                           = 0x2
 	IEXTEN                           = 0x8000
@@ -151,9 +154,14 @@ const (
 	NFDBITS                          = 0x40
 	NLDLY                            = 0x100
 	NOFLSH                           = 0x80
+	NS_GET_MNTNS_ID                  = 0x8008b705
 	NS_GET_NSTYPE                    = 0xb703
 	NS_GET_OWNER_UID                 = 0xb704
 	NS_GET_PARENT                    = 0xb702
+	NS_GET_PID_FROM_PIDNS            = 0x8004b706
+	NS_GET_PID_IN_PIDNS              = 0x8004b708
+	NS_GET_TGID_FROM_PIDNS           = 0x8004b707
+	NS_GET_TGID_IN_PIDNS             = 0x8004b709
 	NS_GET_USERNS                    = 0xb701
 	OLCUC                            = 0x2
 	ONLCR                            = 0x4
@@ -230,6 +238,20 @@ const (
 	PPPIOCUNBRIDGECHAN               = 0x7434
 	PPPIOCXFERUNIT                   = 0x744e
 	PR_SET_PTRACER_ANY               = 0xffffffffffffffff
+	PTP_CLOCK_GETCAPS                = 0x80503d01
+	PTP_CLOCK_GETCAPS2               = 0x80503d0a
+	PTP_ENABLE_PPS                   = 0x40043d04
+	PTP_ENABLE_PPS2                  = 0x40043d0d
+	PTP_EXTTS_REQUEST                = 0x40103d02
+	PTP_EXTTS_REQUEST2               = 0x40103d0b
+	PTP_MASK_CLEAR_ALL               = 0x3d13
+	PTP_MASK_EN_SINGLE               = 0x40043d14
+	PTP_PEROUT_REQUEST               = 0x40383d03
+	PTP_PEROUT_REQUEST2              = 0x40383d0c
+	PTP_PIN_SETFUNC                  = 0x40603d07
+	PTP_PIN_SETFUNC2                 = 0x40603d10
+	PTP_SYS_OFFSET                   = 0x43403d05
+	PTP_SYS_OFFSET2                  = 0x43403d0e
 	PTRACE_ARCH_PRCTL                = 0x1e
 	PTRACE_GETFPREGS                 = 0xe
 	PTRACE_GETFPXREGS                = 0x12
@@ -277,6 +299,8 @@ const (
 	RTC_WIE_ON                       = 0x700f
 	RTC_WKALM_RD                     = 0x80287010
 	RTC_WKALM_SET                    = 0x4028700f
+	SCM_DEVMEM_DMABUF                = 0x4f
+	SCM_DEVMEM_LINEAR                = 0x4e
 	SCM_TIMESTAMPING                 = 0x25
 	SCM_TIMESTAMPING_OPT_STATS       = 0x36
 	SCM_TIMESTAMPING_PKTINFO         = 0x3a
@@ -315,6 +339,9 @@ const (
 	SO_CNX_ADVICE                    = 0x35
 	SO_COOKIE                        = 0x39
 	SO_DETACH_REUSEPORT_BPF          = 0x44
+	SO_DEVMEM_DMABUF                 = 0x4f
+	SO_DEVMEM_DONTNEED               = 0x50
+	SO_DEVMEM_LINEAR                 = 0x4e
 	SO_DOMAIN                        = 0x27
 	SO_DONTROUTE                     = 0x5
 	SO_ERROR                         = 0x4
diff --git a/vendor/golang.org/x/sys/unix/zerrors_linux_arm.go b/vendor/golang.org/x/sys/unix/zerrors_linux_arm.go
index 5cca668ac..680018a4a 100644
--- a/vendor/golang.org/x/sys/unix/zerrors_linux_arm.go
+++ b/vendor/golang.org/x/sys/unix/zerrors_linux_arm.go
@@ -78,6 +78,8 @@ const (
 	ECHOPRT                          = 0x400
 	EFD_CLOEXEC                      = 0x80000
 	EFD_NONBLOCK                     = 0x800
+	EPIOCGPARAMS                     = 0x80088a02
+	EPIOCSPARAMS                     = 0x40088a01
 	EPOLL_CLOEXEC                    = 0x80000
 	EXTPROC                          = 0x10000
 	FF1                              = 0x8000
@@ -106,6 +108,7 @@ const (
 	HIDIOCGRAWINFO                   = 0x80084803
 	HIDIOCGRDESC                     = 0x90044802
 	HIDIOCGRDESCSIZE                 = 0x80044801
+	HIDIOCREVOKE                     = 0x4004480d
 	HUPCL                            = 0x400
 	ICANON                           = 0x2
 	IEXTEN                           = 0x8000
@@ -148,9 +151,14 @@ const (
 	NFDBITS                          = 0x20
 	NLDLY                            = 0x100
 	NOFLSH                           = 0x80
+	NS_GET_MNTNS_ID                  = 0x8008b705
 	NS_GET_NSTYPE                    = 0xb703
 	NS_GET_OWNER_UID                 = 0xb704
 	NS_GET_PARENT                    = 0xb702
+	NS_GET_PID_FROM_PIDNS            = 0x8004b706
+	NS_GET_PID_IN_PIDNS              = 0x8004b708
+	NS_GET_TGID_FROM_PIDNS           = 0x8004b707
+	NS_GET_TGID_IN_PIDNS             = 0x8004b709
 	NS_GET_USERNS                    = 0xb701
 	OLCUC                            = 0x2
 	ONLCR                            = 0x4
@@ -227,6 +235,20 @@ const (
 	PPPIOCUNBRIDGECHAN               = 0x7434
 	PPPIOCXFERUNIT                   = 0x744e
 	PR_SET_PTRACER_ANY               = 0xffffffff
+	PTP_CLOCK_GETCAPS                = 0x80503d01
+	PTP_CLOCK_GETCAPS2               = 0x80503d0a
+	PTP_ENABLE_PPS                   = 0x40043d04
+	PTP_ENABLE_PPS2                  = 0x40043d0d
+	PTP_EXTTS_REQUEST                = 0x40103d02
+	PTP_EXTTS_REQUEST2               = 0x40103d0b
+	PTP_MASK_CLEAR_ALL               = 0x3d13
+	PTP_MASK_EN_SINGLE               = 0x40043d14
+	PTP_PEROUT_REQUEST               = 0x40383d03
+	PTP_PEROUT_REQUEST2              = 0x40383d0c
+	PTP_PIN_SETFUNC                  = 0x40603d07
+	PTP_PIN_SETFUNC2                 = 0x40603d10
+	PTP_SYS_OFFSET                   = 0x43403d05
+	PTP_SYS_OFFSET2                  = 0x43403d0e
 	PTRACE_GETCRUNCHREGS             = 0x19
 	PTRACE_GETFDPIC                  = 0x1f
 	PTRACE_GETFDPIC_EXEC             = 0x0
@@ -282,6 +304,8 @@ const (
 	RTC_WIE_ON                       = 0x700f
 	RTC_WKALM_RD                     = 0x80287010
 	RTC_WKALM_SET                    = 0x4028700f
+	SCM_DEVMEM_DMABUF                = 0x4f
+	SCM_DEVMEM_LINEAR                = 0x4e
 	SCM_TIMESTAMPING                 = 0x25
 	SCM_TIMESTAMPING_OPT_STATS       = 0x36
 	SCM_TIMESTAMPING_PKTINFO         = 0x3a
@@ -320,6 +344,9 @@ const (
 	SO_CNX_ADVICE                    = 0x35
 	SO_COOKIE                        = 0x39
 	SO_DETACH_REUSEPORT_BPF          = 0x44
+	SO_DEVMEM_DMABUF                 = 0x4f
+	SO_DEVMEM_DONTNEED               = 0x50
+	SO_DEVMEM_LINEAR                 = 0x4e
 	SO_DOMAIN                        = 0x27
 	SO_DONTROUTE                     = 0x5
 	SO_ERROR                         = 0x4
diff --git a/vendor/golang.org/x/sys/unix/zerrors_linux_arm64.go b/vendor/golang.org/x/sys/unix/zerrors_linux_arm64.go
index 14270508b..a63909f30 100644
--- a/vendor/golang.org/x/sys/unix/zerrors_linux_arm64.go
+++ b/vendor/golang.org/x/sys/unix/zerrors_linux_arm64.go
@@ -78,6 +78,8 @@ const (
 	ECHOPRT                          = 0x400
 	EFD_CLOEXEC                      = 0x80000
 	EFD_NONBLOCK                     = 0x800
+	EPIOCGPARAMS                     = 0x80088a02
+	EPIOCSPARAMS                     = 0x40088a01
 	EPOLL_CLOEXEC                    = 0x80000
 	ESR_MAGIC                        = 0x45535201
 	EXTPROC                          = 0x10000
@@ -110,6 +112,7 @@ const (
 	HIDIOCGRAWINFO                   = 0x80084803
 	HIDIOCGRDESC                     = 0x90044802
 	HIDIOCGRDESCSIZE                 = 0x80044801
+	HIDIOCREVOKE                     = 0x4004480d
 	HUPCL                            = 0x400
 	ICANON                           = 0x2
 	IEXTEN                           = 0x8000
@@ -152,9 +155,14 @@ const (
 	NFDBITS                          = 0x40
 	NLDLY                            = 0x100
 	NOFLSH                           = 0x80
+	NS_GET_MNTNS_ID                  = 0x8008b705
 	NS_GET_NSTYPE                    = 0xb703
 	NS_GET_OWNER_UID                 = 0xb704
 	NS_GET_PARENT                    = 0xb702
+	NS_GET_PID_FROM_PIDNS            = 0x8004b706
+	NS_GET_PID_IN_PIDNS              = 0x8004b708
+	NS_GET_TGID_FROM_PIDNS           = 0x8004b707
+	NS_GET_TGID_IN_PIDNS             = 0x8004b709
 	NS_GET_USERNS                    = 0xb701
 	OLCUC                            = 0x2
 	ONLCR                            = 0x4
@@ -198,6 +206,7 @@ const (
 	PERF_EVENT_IOC_SET_BPF           = 0x40042408
 	PERF_EVENT_IOC_SET_FILTER        = 0x40082406
 	PERF_EVENT_IOC_SET_OUTPUT        = 0x2405
+	POE_MAGIC                        = 0x504f4530
 	PPPIOCATTACH                     = 0x4004743d
 	PPPIOCATTCHAN                    = 0x40047438
 	PPPIOCBRIDGECHAN                 = 0x40047435
@@ -233,6 +242,20 @@ const (
 	PROT_BTI                         = 0x10
 	PROT_MTE                         = 0x20
 	PR_SET_PTRACER_ANY               = 0xffffffffffffffff
+	PTP_CLOCK_GETCAPS                = 0x80503d01
+	PTP_CLOCK_GETCAPS2               = 0x80503d0a
+	PTP_ENABLE_PPS                   = 0x40043d04
+	PTP_ENABLE_PPS2                  = 0x40043d0d
+	PTP_EXTTS_REQUEST                = 0x40103d02
+	PTP_EXTTS_REQUEST2               = 0x40103d0b
+	PTP_MASK_CLEAR_ALL               = 0x3d13
+	PTP_MASK_EN_SINGLE               = 0x40043d14
+	PTP_PEROUT_REQUEST               = 0x40383d03
+	PTP_PEROUT_REQUEST2              = 0x40383d0c
+	PTP_PIN_SETFUNC                  = 0x40603d07
+	PTP_PIN_SETFUNC2                 = 0x40603d10
+	PTP_SYS_OFFSET                   = 0x43403d05
+	PTP_SYS_OFFSET2                  = 0x43403d0e
 	PTRACE_PEEKMTETAGS               = 0x21
 	PTRACE_POKEMTETAGS               = 0x22
 	PTRACE_SYSEMU                    = 0x1f
@@ -273,6 +296,8 @@ const (
 	RTC_WIE_ON                       = 0x700f
 	RTC_WKALM_RD                     = 0x80287010
 	RTC_WKALM_SET                    = 0x4028700f
+	SCM_DEVMEM_DMABUF                = 0x4f
+	SCM_DEVMEM_LINEAR                = 0x4e
 	SCM_TIMESTAMPING                 = 0x25
 	SCM_TIMESTAMPING_OPT_STATS       = 0x36
 	SCM_TIMESTAMPING_PKTINFO         = 0x3a
@@ -311,6 +336,9 @@ const (
 	SO_CNX_ADVICE                    = 0x35
 	SO_COOKIE                        = 0x39
 	SO_DETACH_REUSEPORT_BPF          = 0x44
+	SO_DEVMEM_DMABUF                 = 0x4f
+	SO_DEVMEM_DONTNEED               = 0x50
+	SO_DEVMEM_LINEAR                 = 0x4e
 	SO_DOMAIN                        = 0x27
 	SO_DONTROUTE                     = 0x5
 	SO_ERROR                         = 0x4
diff --git a/vendor/golang.org/x/sys/unix/zerrors_linux_loong64.go b/vendor/golang.org/x/sys/unix/zerrors_linux_loong64.go
index 28e39afdc..9b0a2573f 100644
--- a/vendor/golang.org/x/sys/unix/zerrors_linux_loong64.go
+++ b/vendor/golang.org/x/sys/unix/zerrors_linux_loong64.go
@@ -78,6 +78,8 @@ const (
 	ECHOPRT                          = 0x400
 	EFD_CLOEXEC                      = 0x80000
 	EFD_NONBLOCK                     = 0x800
+	EPIOCGPARAMS                     = 0x80088a02
+	EPIOCSPARAMS                     = 0x40088a01
 	EPOLL_CLOEXEC                    = 0x80000
 	EXTPROC                          = 0x10000
 	FF1                              = 0x8000
@@ -107,6 +109,7 @@ const (
 	HIDIOCGRAWINFO                   = 0x80084803
 	HIDIOCGRDESC                     = 0x90044802
 	HIDIOCGRDESCSIZE                 = 0x80044801
+	HIDIOCREVOKE                     = 0x4004480d
 	HUPCL                            = 0x400
 	ICANON                           = 0x2
 	IEXTEN                           = 0x8000
@@ -152,9 +155,14 @@ const (
 	NFDBITS                          = 0x40
 	NLDLY                            = 0x100
 	NOFLSH                           = 0x80
+	NS_GET_MNTNS_ID                  = 0x8008b705
 	NS_GET_NSTYPE                    = 0xb703
 	NS_GET_OWNER_UID                 = 0xb704
 	NS_GET_PARENT                    = 0xb702
+	NS_GET_PID_FROM_PIDNS            = 0x8004b706
+	NS_GET_PID_IN_PIDNS              = 0x8004b708
+	NS_GET_TGID_FROM_PIDNS           = 0x8004b707
+	NS_GET_TGID_IN_PIDNS             = 0x8004b709
 	NS_GET_USERNS                    = 0xb701
 	OLCUC                            = 0x2
 	ONLCR                            = 0x4
@@ -231,6 +239,20 @@ const (
 	PPPIOCUNBRIDGECHAN               = 0x7434
 	PPPIOCXFERUNIT                   = 0x744e
 	PR_SET_PTRACER_ANY               = 0xffffffffffffffff
+	PTP_CLOCK_GETCAPS                = 0x80503d01
+	PTP_CLOCK_GETCAPS2               = 0x80503d0a
+	PTP_ENABLE_PPS                   = 0x40043d04
+	PTP_ENABLE_PPS2                  = 0x40043d0d
+	PTP_EXTTS_REQUEST                = 0x40103d02
+	PTP_EXTTS_REQUEST2               = 0x40103d0b
+	PTP_MASK_CLEAR_ALL               = 0x3d13
+	PTP_MASK_EN_SINGLE               = 0x40043d14
+	PTP_PEROUT_REQUEST               = 0x40383d03
+	PTP_PEROUT_REQUEST2              = 0x40383d0c
+	PTP_PIN_SETFUNC                  = 0x40603d07
+	PTP_PIN_SETFUNC2                 = 0x40603d10
+	PTP_SYS_OFFSET                   = 0x43403d05
+	PTP_SYS_OFFSET2                  = 0x43403d0e
 	PTRACE_SYSEMU                    = 0x1f
 	PTRACE_SYSEMU_SINGLESTEP         = 0x20
 	RLIMIT_AS                        = 0x9
@@ -269,6 +291,8 @@ const (
 	RTC_WIE_ON                       = 0x700f
 	RTC_WKALM_RD                     = 0x80287010
 	RTC_WKALM_SET                    = 0x4028700f
+	SCM_DEVMEM_DMABUF                = 0x4f
+	SCM_DEVMEM_LINEAR                = 0x4e
 	SCM_TIMESTAMPING                 = 0x25
 	SCM_TIMESTAMPING_OPT_STATS       = 0x36
 	SCM_TIMESTAMPING_PKTINFO         = 0x3a
@@ -307,6 +331,9 @@ const (
 	SO_CNX_ADVICE                    = 0x35
 	SO_COOKIE                        = 0x39
 	SO_DETACH_REUSEPORT_BPF          = 0x44
+	SO_DEVMEM_DMABUF                 = 0x4f
+	SO_DEVMEM_DONTNEED               = 0x50
+	SO_DEVMEM_LINEAR                 = 0x4e
 	SO_DOMAIN                        = 0x27
 	SO_DONTROUTE                     = 0x5
 	SO_ERROR                         = 0x4
diff --git a/vendor/golang.org/x/sys/unix/zerrors_linux_mips.go b/vendor/golang.org/x/sys/unix/zerrors_linux_mips.go
index cd66e92cb..958e6e064 100644
--- a/vendor/golang.org/x/sys/unix/zerrors_linux_mips.go
+++ b/vendor/golang.org/x/sys/unix/zerrors_linux_mips.go
@@ -78,6 +78,8 @@ const (
 	ECHOPRT                          = 0x400
 	EFD_CLOEXEC                      = 0x80000
 	EFD_NONBLOCK                     = 0x80
+	EPIOCGPARAMS                     = 0x40088a02
+	EPIOCSPARAMS                     = 0x80088a01
 	EPOLL_CLOEXEC                    = 0x80000
 	EXTPROC                          = 0x10000
 	FF1                              = 0x8000
@@ -106,6 +108,7 @@ const (
 	HIDIOCGRAWINFO                   = 0x40084803
 	HIDIOCGRDESC                     = 0x50044802
 	HIDIOCGRDESCSIZE                 = 0x40044801
+	HIDIOCREVOKE                     = 0x8004480d
 	HUPCL                            = 0x400
 	ICANON                           = 0x2
 	IEXTEN                           = 0x100
@@ -148,9 +151,14 @@ const (
 	NFDBITS                          = 0x20
 	NLDLY                            = 0x100
 	NOFLSH                           = 0x80
+	NS_GET_MNTNS_ID                  = 0x4008b705
 	NS_GET_NSTYPE                    = 0x2000b703
 	NS_GET_OWNER_UID                 = 0x2000b704
 	NS_GET_PARENT                    = 0x2000b702
+	NS_GET_PID_FROM_PIDNS            = 0x4004b706
+	NS_GET_PID_IN_PIDNS              = 0x4004b708
+	NS_GET_TGID_FROM_PIDNS           = 0x4004b707
+	NS_GET_TGID_IN_PIDNS             = 0x4004b709
 	NS_GET_USERNS                    = 0x2000b701
 	OLCUC                            = 0x2
 	ONLCR                            = 0x4
@@ -227,6 +235,20 @@ const (
 	PPPIOCUNBRIDGECHAN               = 0x20007434
 	PPPIOCXFERUNIT                   = 0x2000744e
 	PR_SET_PTRACER_ANY               = 0xffffffff
+	PTP_CLOCK_GETCAPS                = 0x40503d01
+	PTP_CLOCK_GETCAPS2               = 0x40503d0a
+	PTP_ENABLE_PPS                   = 0x80043d04
+	PTP_ENABLE_PPS2                  = 0x80043d0d
+	PTP_EXTTS_REQUEST                = 0x80103d02
+	PTP_EXTTS_REQUEST2               = 0x80103d0b
+	PTP_MASK_CLEAR_ALL               = 0x20003d13
+	PTP_MASK_EN_SINGLE               = 0x80043d14
+	PTP_PEROUT_REQUEST               = 0x80383d03
+	PTP_PEROUT_REQUEST2              = 0x80383d0c
+	PTP_PIN_SETFUNC                  = 0x80603d07
+	PTP_PIN_SETFUNC2                 = 0x80603d10
+	PTP_SYS_OFFSET                   = 0x83403d05
+	PTP_SYS_OFFSET2                  = 0x83403d0e
 	PTRACE_GETFPREGS                 = 0xe
 	PTRACE_GET_THREAD_AREA           = 0x19
 	PTRACE_GET_THREAD_AREA_3264      = 0xc4
@@ -275,6 +297,8 @@ const (
 	RTC_WIE_ON                       = 0x2000700f
 	RTC_WKALM_RD                     = 0x40287010
 	RTC_WKALM_SET                    = 0x8028700f
+	SCM_DEVMEM_DMABUF                = 0x4f
+	SCM_DEVMEM_LINEAR                = 0x4e
 	SCM_TIMESTAMPING                 = 0x25
 	SCM_TIMESTAMPING_OPT_STATS       = 0x36
 	SCM_TIMESTAMPING_PKTINFO         = 0x3a
@@ -313,6 +337,9 @@ const (
 	SO_CNX_ADVICE                    = 0x35
 	SO_COOKIE                        = 0x39
 	SO_DETACH_REUSEPORT_BPF          = 0x44
+	SO_DEVMEM_DMABUF                 = 0x4f
+	SO_DEVMEM_DONTNEED               = 0x50
+	SO_DEVMEM_LINEAR                 = 0x4e
 	SO_DOMAIN                        = 0x1029
 	SO_DONTROUTE                     = 0x10
 	SO_ERROR                         = 0x1007
diff --git a/vendor/golang.org/x/sys/unix/zerrors_linux_mips64.go b/vendor/golang.org/x/sys/unix/zerrors_linux_mips64.go
index c1595eba7..50c7f25bd 100644
--- a/vendor/golang.org/x/sys/unix/zerrors_linux_mips64.go
+++ b/vendor/golang.org/x/sys/unix/zerrors_linux_mips64.go
@@ -78,6 +78,8 @@ const (
 	ECHOPRT                          = 0x400
 	EFD_CLOEXEC                      = 0x80000
 	EFD_NONBLOCK                     = 0x80
+	EPIOCGPARAMS                     = 0x40088a02
+	EPIOCSPARAMS                     = 0x80088a01
 	EPOLL_CLOEXEC                    = 0x80000
 	EXTPROC                          = 0x10000
 	FF1                              = 0x8000
@@ -106,6 +108,7 @@ const (
 	HIDIOCGRAWINFO                   = 0x40084803
 	HIDIOCGRDESC                     = 0x50044802
 	HIDIOCGRDESCSIZE                 = 0x40044801
+	HIDIOCREVOKE                     = 0x8004480d
 	HUPCL                            = 0x400
 	ICANON                           = 0x2
 	IEXTEN                           = 0x100
@@ -148,9 +151,14 @@ const (
 	NFDBITS                          = 0x40
 	NLDLY                            = 0x100
 	NOFLSH                           = 0x80
+	NS_GET_MNTNS_ID                  = 0x4008b705
 	NS_GET_NSTYPE                    = 0x2000b703
 	NS_GET_OWNER_UID                 = 0x2000b704
 	NS_GET_PARENT                    = 0x2000b702
+	NS_GET_PID_FROM_PIDNS            = 0x4004b706
+	NS_GET_PID_IN_PIDNS              = 0x4004b708
+	NS_GET_TGID_FROM_PIDNS           = 0x4004b707
+	NS_GET_TGID_IN_PIDNS             = 0x4004b709
 	NS_GET_USERNS                    = 0x2000b701
 	OLCUC                            = 0x2
 	ONLCR                            = 0x4
@@ -227,6 +235,20 @@ const (
 	PPPIOCUNBRIDGECHAN               = 0x20007434
 	PPPIOCXFERUNIT                   = 0x2000744e
 	PR_SET_PTRACER_ANY               = 0xffffffffffffffff
+	PTP_CLOCK_GETCAPS                = 0x40503d01
+	PTP_CLOCK_GETCAPS2               = 0x40503d0a
+	PTP_ENABLE_PPS                   = 0x80043d04
+	PTP_ENABLE_PPS2                  = 0x80043d0d
+	PTP_EXTTS_REQUEST                = 0x80103d02
+	PTP_EXTTS_REQUEST2               = 0x80103d0b
+	PTP_MASK_CLEAR_ALL               = 0x20003d13
+	PTP_MASK_EN_SINGLE               = 0x80043d14
+	PTP_PEROUT_REQUEST               = 0x80383d03
+	PTP_PEROUT_REQUEST2              = 0x80383d0c
+	PTP_PIN_SETFUNC                  = 0x80603d07
+	PTP_PIN_SETFUNC2                 = 0x80603d10
+	PTP_SYS_OFFSET                   = 0x83403d05
+	PTP_SYS_OFFSET2                  = 0x83403d0e
 	PTRACE_GETFPREGS                 = 0xe
 	PTRACE_GET_THREAD_AREA           = 0x19
 	PTRACE_GET_THREAD_AREA_3264      = 0xc4
@@ -275,6 +297,8 @@ const (
 	RTC_WIE_ON                       = 0x2000700f
 	RTC_WKALM_RD                     = 0x40287010
 	RTC_WKALM_SET                    = 0x8028700f
+	SCM_DEVMEM_DMABUF                = 0x4f
+	SCM_DEVMEM_LINEAR                = 0x4e
 	SCM_TIMESTAMPING                 = 0x25
 	SCM_TIMESTAMPING_OPT_STATS       = 0x36
 	SCM_TIMESTAMPING_PKTINFO         = 0x3a
@@ -313,6 +337,9 @@ const (
 	SO_CNX_ADVICE                    = 0x35
 	SO_COOKIE                        = 0x39
 	SO_DETACH_REUSEPORT_BPF          = 0x44
+	SO_DEVMEM_DMABUF                 = 0x4f
+	SO_DEVMEM_DONTNEED               = 0x50
+	SO_DEVMEM_LINEAR                 = 0x4e
 	SO_DOMAIN                        = 0x1029
 	SO_DONTROUTE                     = 0x10
 	SO_ERROR                         = 0x1007
diff --git a/vendor/golang.org/x/sys/unix/zerrors_linux_mips64le.go b/vendor/golang.org/x/sys/unix/zerrors_linux_mips64le.go
index ee9456b0d..ced21d66d 100644
--- a/vendor/golang.org/x/sys/unix/zerrors_linux_mips64le.go
+++ b/vendor/golang.org/x/sys/unix/zerrors_linux_mips64le.go
@@ -78,6 +78,8 @@ const (
 	ECHOPRT                          = 0x400
 	EFD_CLOEXEC                      = 0x80000
 	EFD_NONBLOCK                     = 0x80
+	EPIOCGPARAMS                     = 0x40088a02
+	EPIOCSPARAMS                     = 0x80088a01
 	EPOLL_CLOEXEC                    = 0x80000
 	EXTPROC                          = 0x10000
 	FF1                              = 0x8000
@@ -106,6 +108,7 @@ const (
 	HIDIOCGRAWINFO                   = 0x40084803
 	HIDIOCGRDESC                     = 0x50044802
 	HIDIOCGRDESCSIZE                 = 0x40044801
+	HIDIOCREVOKE                     = 0x8004480d
 	HUPCL                            = 0x400
 	ICANON                           = 0x2
 	IEXTEN                           = 0x100
@@ -148,9 +151,14 @@ const (
 	NFDBITS                          = 0x40
 	NLDLY                            = 0x100
 	NOFLSH                           = 0x80
+	NS_GET_MNTNS_ID                  = 0x4008b705
 	NS_GET_NSTYPE                    = 0x2000b703
 	NS_GET_OWNER_UID                 = 0x2000b704
 	NS_GET_PARENT                    = 0x2000b702
+	NS_GET_PID_FROM_PIDNS            = 0x4004b706
+	NS_GET_PID_IN_PIDNS              = 0x4004b708
+	NS_GET_TGID_FROM_PIDNS           = 0x4004b707
+	NS_GET_TGID_IN_PIDNS             = 0x4004b709
 	NS_GET_USERNS                    = 0x2000b701
 	OLCUC                            = 0x2
 	ONLCR                            = 0x4
@@ -227,6 +235,20 @@ const (
 	PPPIOCUNBRIDGECHAN               = 0x20007434
 	PPPIOCXFERUNIT                   = 0x2000744e
 	PR_SET_PTRACER_ANY               = 0xffffffffffffffff
+	PTP_CLOCK_GETCAPS                = 0x40503d01
+	PTP_CLOCK_GETCAPS2               = 0x40503d0a
+	PTP_ENABLE_PPS                   = 0x80043d04
+	PTP_ENABLE_PPS2                  = 0x80043d0d
+	PTP_EXTTS_REQUEST                = 0x80103d02
+	PTP_EXTTS_REQUEST2               = 0x80103d0b
+	PTP_MASK_CLEAR_ALL               = 0x20003d13
+	PTP_MASK_EN_SINGLE               = 0x80043d14
+	PTP_PEROUT_REQUEST               = 0x80383d03
+	PTP_PEROUT_REQUEST2              = 0x80383d0c
+	PTP_PIN_SETFUNC                  = 0x80603d07
+	PTP_PIN_SETFUNC2                 = 0x80603d10
+	PTP_SYS_OFFSET                   = 0x83403d05
+	PTP_SYS_OFFSET2                  = 0x83403d0e
 	PTRACE_GETFPREGS                 = 0xe
 	PTRACE_GET_THREAD_AREA           = 0x19
 	PTRACE_GET_THREAD_AREA_3264      = 0xc4
@@ -275,6 +297,8 @@ const (
 	RTC_WIE_ON                       = 0x2000700f
 	RTC_WKALM_RD                     = 0x40287010
 	RTC_WKALM_SET                    = 0x8028700f
+	SCM_DEVMEM_DMABUF                = 0x4f
+	SCM_DEVMEM_LINEAR                = 0x4e
 	SCM_TIMESTAMPING                 = 0x25
 	SCM_TIMESTAMPING_OPT_STATS       = 0x36
 	SCM_TIMESTAMPING_PKTINFO         = 0x3a
@@ -313,6 +337,9 @@ const (
 	SO_CNX_ADVICE                    = 0x35
 	SO_COOKIE                        = 0x39
 	SO_DETACH_REUSEPORT_BPF          = 0x44
+	SO_DEVMEM_DMABUF                 = 0x4f
+	SO_DEVMEM_DONTNEED               = 0x50
+	SO_DEVMEM_LINEAR                 = 0x4e
 	SO_DOMAIN                        = 0x1029
 	SO_DONTROUTE                     = 0x10
 	SO_ERROR                         = 0x1007
diff --git a/vendor/golang.org/x/sys/unix/zerrors_linux_mipsle.go b/vendor/golang.org/x/sys/unix/zerrors_linux_mipsle.go
index 8cfca81e1..226c04419 100644
--- a/vendor/golang.org/x/sys/unix/zerrors_linux_mipsle.go
+++ b/vendor/golang.org/x/sys/unix/zerrors_linux_mipsle.go
@@ -78,6 +78,8 @@ const (
 	ECHOPRT                          = 0x400
 	EFD_CLOEXEC                      = 0x80000
 	EFD_NONBLOCK                     = 0x80
+	EPIOCGPARAMS                     = 0x40088a02
+	EPIOCSPARAMS                     = 0x80088a01
 	EPOLL_CLOEXEC                    = 0x80000
 	EXTPROC                          = 0x10000
 	FF1                              = 0x8000
@@ -106,6 +108,7 @@ const (
 	HIDIOCGRAWINFO                   = 0x40084803
 	HIDIOCGRDESC                     = 0x50044802
 	HIDIOCGRDESCSIZE                 = 0x40044801
+	HIDIOCREVOKE                     = 0x8004480d
 	HUPCL                            = 0x400
 	ICANON                           = 0x2
 	IEXTEN                           = 0x100
@@ -148,9 +151,14 @@ const (
 	NFDBITS                          = 0x20
 	NLDLY                            = 0x100
 	NOFLSH                           = 0x80
+	NS_GET_MNTNS_ID                  = 0x4008b705
 	NS_GET_NSTYPE                    = 0x2000b703
 	NS_GET_OWNER_UID                 = 0x2000b704
 	NS_GET_PARENT                    = 0x2000b702
+	NS_GET_PID_FROM_PIDNS            = 0x4004b706
+	NS_GET_PID_IN_PIDNS              = 0x4004b708
+	NS_GET_TGID_FROM_PIDNS           = 0x4004b707
+	NS_GET_TGID_IN_PIDNS             = 0x4004b709
 	NS_GET_USERNS                    = 0x2000b701
 	OLCUC                            = 0x2
 	ONLCR                            = 0x4
@@ -227,6 +235,20 @@ const (
 	PPPIOCUNBRIDGECHAN               = 0x20007434
 	PPPIOCXFERUNIT                   = 0x2000744e
 	PR_SET_PTRACER_ANY               = 0xffffffff
+	PTP_CLOCK_GETCAPS                = 0x40503d01
+	PTP_CLOCK_GETCAPS2               = 0x40503d0a
+	PTP_ENABLE_PPS                   = 0x80043d04
+	PTP_ENABLE_PPS2                  = 0x80043d0d
+	PTP_EXTTS_REQUEST                = 0x80103d02
+	PTP_EXTTS_REQUEST2               = 0x80103d0b
+	PTP_MASK_CLEAR_ALL               = 0x20003d13
+	PTP_MASK_EN_SINGLE               = 0x80043d14
+	PTP_PEROUT_REQUEST               = 0x80383d03
+	PTP_PEROUT_REQUEST2              = 0x80383d0c
+	PTP_PIN_SETFUNC                  = 0x80603d07
+	PTP_PIN_SETFUNC2                 = 0x80603d10
+	PTP_SYS_OFFSET                   = 0x83403d05
+	PTP_SYS_OFFSET2                  = 0x83403d0e
 	PTRACE_GETFPREGS                 = 0xe
 	PTRACE_GET_THREAD_AREA           = 0x19
 	PTRACE_GET_THREAD_AREA_3264      = 0xc4
@@ -275,6 +297,8 @@ const (
 	RTC_WIE_ON                       = 0x2000700f
 	RTC_WKALM_RD                     = 0x40287010
 	RTC_WKALM_SET                    = 0x8028700f
+	SCM_DEVMEM_DMABUF                = 0x4f
+	SCM_DEVMEM_LINEAR                = 0x4e
 	SCM_TIMESTAMPING                 = 0x25
 	SCM_TIMESTAMPING_OPT_STATS       = 0x36
 	SCM_TIMESTAMPING_PKTINFO         = 0x3a
@@ -313,6 +337,9 @@ const (
 	SO_CNX_ADVICE                    = 0x35
 	SO_COOKIE                        = 0x39
 	SO_DETACH_REUSEPORT_BPF          = 0x44
+	SO_DEVMEM_DMABUF                 = 0x4f
+	SO_DEVMEM_DONTNEED               = 0x50
+	SO_DEVMEM_LINEAR                 = 0x4e
 	SO_DOMAIN                        = 0x1029
 	SO_DONTROUTE                     = 0x10
 	SO_ERROR                         = 0x1007
diff --git a/vendor/golang.org/x/sys/unix/zerrors_linux_ppc.go b/vendor/golang.org/x/sys/unix/zerrors_linux_ppc.go
index 60b0deb3a..3122737cd 100644
--- a/vendor/golang.org/x/sys/unix/zerrors_linux_ppc.go
+++ b/vendor/golang.org/x/sys/unix/zerrors_linux_ppc.go
@@ -78,6 +78,8 @@ const (
 	ECHOPRT                          = 0x20
 	EFD_CLOEXEC                      = 0x80000
 	EFD_NONBLOCK                     = 0x800
+	EPIOCGPARAMS                     = 0x40088a02
+	EPIOCSPARAMS                     = 0x80088a01
 	EPOLL_CLOEXEC                    = 0x80000
 	EXTPROC                          = 0x10000000
 	FF1                              = 0x4000
@@ -106,6 +108,7 @@ const (
 	HIDIOCGRAWINFO                   = 0x40084803
 	HIDIOCGRDESC                     = 0x50044802
 	HIDIOCGRDESCSIZE                 = 0x40044801
+	HIDIOCREVOKE                     = 0x8004480d
 	HUPCL                            = 0x4000
 	ICANON                           = 0x100
 	IEXTEN                           = 0x400
@@ -150,9 +153,14 @@ const (
 	NL3                              = 0x300
 	NLDLY                            = 0x300
 	NOFLSH                           = 0x80000000
+	NS_GET_MNTNS_ID                  = 0x4008b705
 	NS_GET_NSTYPE                    = 0x2000b703
 	NS_GET_OWNER_UID                 = 0x2000b704
 	NS_GET_PARENT                    = 0x2000b702
+	NS_GET_PID_FROM_PIDNS            = 0x4004b706
+	NS_GET_PID_IN_PIDNS              = 0x4004b708
+	NS_GET_TGID_FROM_PIDNS           = 0x4004b707
+	NS_GET_TGID_IN_PIDNS             = 0x4004b709
 	NS_GET_USERNS                    = 0x2000b701
 	OLCUC                            = 0x4
 	ONLCR                            = 0x2
@@ -230,6 +238,20 @@ const (
 	PPPIOCXFERUNIT                   = 0x2000744e
 	PROT_SAO                         = 0x10
 	PR_SET_PTRACER_ANY               = 0xffffffff
+	PTP_CLOCK_GETCAPS                = 0x40503d01
+	PTP_CLOCK_GETCAPS2               = 0x40503d0a
+	PTP_ENABLE_PPS                   = 0x80043d04
+	PTP_ENABLE_PPS2                  = 0x80043d0d
+	PTP_EXTTS_REQUEST                = 0x80103d02
+	PTP_EXTTS_REQUEST2               = 0x80103d0b
+	PTP_MASK_CLEAR_ALL               = 0x20003d13
+	PTP_MASK_EN_SINGLE               = 0x80043d14
+	PTP_PEROUT_REQUEST               = 0x80383d03
+	PTP_PEROUT_REQUEST2              = 0x80383d0c
+	PTP_PIN_SETFUNC                  = 0x80603d07
+	PTP_PIN_SETFUNC2                 = 0x80603d10
+	PTP_SYS_OFFSET                   = 0x83403d05
+	PTP_SYS_OFFSET2                  = 0x83403d0e
 	PTRACE_GETEVRREGS                = 0x14
 	PTRACE_GETFPREGS                 = 0xe
 	PTRACE_GETREGS64                 = 0x16
@@ -330,6 +352,8 @@ const (
 	RTC_WIE_ON                       = 0x2000700f
 	RTC_WKALM_RD                     = 0x40287010
 	RTC_WKALM_SET                    = 0x8028700f
+	SCM_DEVMEM_DMABUF                = 0x4f
+	SCM_DEVMEM_LINEAR                = 0x4e
 	SCM_TIMESTAMPING                 = 0x25
 	SCM_TIMESTAMPING_OPT_STATS       = 0x36
 	SCM_TIMESTAMPING_PKTINFO         = 0x3a
@@ -368,6 +392,9 @@ const (
 	SO_CNX_ADVICE                    = 0x35
 	SO_COOKIE                        = 0x39
 	SO_DETACH_REUSEPORT_BPF          = 0x44
+	SO_DEVMEM_DMABUF                 = 0x4f
+	SO_DEVMEM_DONTNEED               = 0x50
+	SO_DEVMEM_LINEAR                 = 0x4e
 	SO_DOMAIN                        = 0x27
 	SO_DONTROUTE                     = 0x5
 	SO_ERROR                         = 0x4
diff --git a/vendor/golang.org/x/sys/unix/zerrors_linux_ppc64.go b/vendor/golang.org/x/sys/unix/zerrors_linux_ppc64.go
index f90aa7281..eb5d3467e 100644
--- a/vendor/golang.org/x/sys/unix/zerrors_linux_ppc64.go
+++ b/vendor/golang.org/x/sys/unix/zerrors_linux_ppc64.go
@@ -78,6 +78,8 @@ const (
 	ECHOPRT                          = 0x20
 	EFD_CLOEXEC                      = 0x80000
 	EFD_NONBLOCK                     = 0x800
+	EPIOCGPARAMS                     = 0x40088a02
+	EPIOCSPARAMS                     = 0x80088a01
 	EPOLL_CLOEXEC                    = 0x80000
 	EXTPROC                          = 0x10000000
 	FF1                              = 0x4000
@@ -106,6 +108,7 @@ const (
 	HIDIOCGRAWINFO                   = 0x40084803
 	HIDIOCGRDESC                     = 0x50044802
 	HIDIOCGRDESCSIZE                 = 0x40044801
+	HIDIOCREVOKE                     = 0x8004480d
 	HUPCL                            = 0x4000
 	ICANON                           = 0x100
 	IEXTEN                           = 0x400
@@ -150,9 +153,14 @@ const (
 	NL3                              = 0x300
 	NLDLY                            = 0x300
 	NOFLSH                           = 0x80000000
+	NS_GET_MNTNS_ID                  = 0x4008b705
 	NS_GET_NSTYPE                    = 0x2000b703
 	NS_GET_OWNER_UID                 = 0x2000b704
 	NS_GET_PARENT                    = 0x2000b702
+	NS_GET_PID_FROM_PIDNS            = 0x4004b706
+	NS_GET_PID_IN_PIDNS              = 0x4004b708
+	NS_GET_TGID_FROM_PIDNS           = 0x4004b707
+	NS_GET_TGID_IN_PIDNS             = 0x4004b709
 	NS_GET_USERNS                    = 0x2000b701
 	OLCUC                            = 0x4
 	ONLCR                            = 0x2
@@ -230,6 +238,20 @@ const (
 	PPPIOCXFERUNIT                   = 0x2000744e
 	PROT_SAO                         = 0x10
 	PR_SET_PTRACER_ANY               = 0xffffffffffffffff
+	PTP_CLOCK_GETCAPS                = 0x40503d01
+	PTP_CLOCK_GETCAPS2               = 0x40503d0a
+	PTP_ENABLE_PPS                   = 0x80043d04
+	PTP_ENABLE_PPS2                  = 0x80043d0d
+	PTP_EXTTS_REQUEST                = 0x80103d02
+	PTP_EXTTS_REQUEST2               = 0x80103d0b
+	PTP_MASK_CLEAR_ALL               = 0x20003d13
+	PTP_MASK_EN_SINGLE               = 0x80043d14
+	PTP_PEROUT_REQUEST               = 0x80383d03
+	PTP_PEROUT_REQUEST2              = 0x80383d0c
+	PTP_PIN_SETFUNC                  = 0x80603d07
+	PTP_PIN_SETFUNC2                 = 0x80603d10
+	PTP_SYS_OFFSET                   = 0x83403d05
+	PTP_SYS_OFFSET2                  = 0x83403d0e
 	PTRACE_GETEVRREGS                = 0x14
 	PTRACE_GETFPREGS                 = 0xe
 	PTRACE_GETREGS64                 = 0x16
@@ -334,6 +356,8 @@ const (
 	RTC_WIE_ON                       = 0x2000700f
 	RTC_WKALM_RD                     = 0x40287010
 	RTC_WKALM_SET                    = 0x8028700f
+	SCM_DEVMEM_DMABUF                = 0x4f
+	SCM_DEVMEM_LINEAR                = 0x4e
 	SCM_TIMESTAMPING                 = 0x25
 	SCM_TIMESTAMPING_OPT_STATS       = 0x36
 	SCM_TIMESTAMPING_PKTINFO         = 0x3a
@@ -372,6 +396,9 @@ const (
 	SO_CNX_ADVICE                    = 0x35
 	SO_COOKIE                        = 0x39
 	SO_DETACH_REUSEPORT_BPF          = 0x44
+	SO_DEVMEM_DMABUF                 = 0x4f
+	SO_DEVMEM_DONTNEED               = 0x50
+	SO_DEVMEM_LINEAR                 = 0x4e
 	SO_DOMAIN                        = 0x27
 	SO_DONTROUTE                     = 0x5
 	SO_ERROR                         = 0x4
diff --git a/vendor/golang.org/x/sys/unix/zerrors_linux_ppc64le.go b/vendor/golang.org/x/sys/unix/zerrors_linux_ppc64le.go
index ba9e01503..e921ebc60 100644
--- a/vendor/golang.org/x/sys/unix/zerrors_linux_ppc64le.go
+++ b/vendor/golang.org/x/sys/unix/zerrors_linux_ppc64le.go
@@ -78,6 +78,8 @@ const (
 	ECHOPRT                          = 0x20
 	EFD_CLOEXEC                      = 0x80000
 	EFD_NONBLOCK                     = 0x800
+	EPIOCGPARAMS                     = 0x40088a02
+	EPIOCSPARAMS                     = 0x80088a01
 	EPOLL_CLOEXEC                    = 0x80000
 	EXTPROC                          = 0x10000000
 	FF1                              = 0x4000
@@ -106,6 +108,7 @@ const (
 	HIDIOCGRAWINFO                   = 0x40084803
 	HIDIOCGRDESC                     = 0x50044802
 	HIDIOCGRDESCSIZE                 = 0x40044801
+	HIDIOCREVOKE                     = 0x8004480d
 	HUPCL                            = 0x4000
 	ICANON                           = 0x100
 	IEXTEN                           = 0x400
@@ -150,9 +153,14 @@ const (
 	NL3                              = 0x300
 	NLDLY                            = 0x300
 	NOFLSH                           = 0x80000000
+	NS_GET_MNTNS_ID                  = 0x4008b705
 	NS_GET_NSTYPE                    = 0x2000b703
 	NS_GET_OWNER_UID                 = 0x2000b704
 	NS_GET_PARENT                    = 0x2000b702
+	NS_GET_PID_FROM_PIDNS            = 0x4004b706
+	NS_GET_PID_IN_PIDNS              = 0x4004b708
+	NS_GET_TGID_FROM_PIDNS           = 0x4004b707
+	NS_GET_TGID_IN_PIDNS             = 0x4004b709
 	NS_GET_USERNS                    = 0x2000b701
 	OLCUC                            = 0x4
 	ONLCR                            = 0x2
@@ -230,6 +238,20 @@ const (
 	PPPIOCXFERUNIT                   = 0x2000744e
 	PROT_SAO                         = 0x10
 	PR_SET_PTRACER_ANY               = 0xffffffffffffffff
+	PTP_CLOCK_GETCAPS                = 0x40503d01
+	PTP_CLOCK_GETCAPS2               = 0x40503d0a
+	PTP_ENABLE_PPS                   = 0x80043d04
+	PTP_ENABLE_PPS2                  = 0x80043d0d
+	PTP_EXTTS_REQUEST                = 0x80103d02
+	PTP_EXTTS_REQUEST2               = 0x80103d0b
+	PTP_MASK_CLEAR_ALL               = 0x20003d13
+	PTP_MASK_EN_SINGLE               = 0x80043d14
+	PTP_PEROUT_REQUEST               = 0x80383d03
+	PTP_PEROUT_REQUEST2              = 0x80383d0c
+	PTP_PIN_SETFUNC                  = 0x80603d07
+	PTP_PIN_SETFUNC2                 = 0x80603d10
+	PTP_SYS_OFFSET                   = 0x83403d05
+	PTP_SYS_OFFSET2                  = 0x83403d0e
 	PTRACE_GETEVRREGS                = 0x14
 	PTRACE_GETFPREGS                 = 0xe
 	PTRACE_GETREGS64                 = 0x16
@@ -334,6 +356,8 @@ const (
 	RTC_WIE_ON                       = 0x2000700f
 	RTC_WKALM_RD                     = 0x40287010
 	RTC_WKALM_SET                    = 0x8028700f
+	SCM_DEVMEM_DMABUF                = 0x4f
+	SCM_DEVMEM_LINEAR                = 0x4e
 	SCM_TIMESTAMPING                 = 0x25
 	SCM_TIMESTAMPING_OPT_STATS       = 0x36
 	SCM_TIMESTAMPING_PKTINFO         = 0x3a
@@ -372,6 +396,9 @@ const (
 	SO_CNX_ADVICE                    = 0x35
 	SO_COOKIE                        = 0x39
 	SO_DETACH_REUSEPORT_BPF          = 0x44
+	SO_DEVMEM_DMABUF                 = 0x4f
+	SO_DEVMEM_DONTNEED               = 0x50
+	SO_DEVMEM_LINEAR                 = 0x4e
 	SO_DOMAIN                        = 0x27
 	SO_DONTROUTE                     = 0x5
 	SO_ERROR                         = 0x4
diff --git a/vendor/golang.org/x/sys/unix/zerrors_linux_riscv64.go b/vendor/golang.org/x/sys/unix/zerrors_linux_riscv64.go
index 07cdfd6e9..38ba81c55 100644
--- a/vendor/golang.org/x/sys/unix/zerrors_linux_riscv64.go
+++ b/vendor/golang.org/x/sys/unix/zerrors_linux_riscv64.go
@@ -78,6 +78,8 @@ const (
 	ECHOPRT                          = 0x400
 	EFD_CLOEXEC                      = 0x80000
 	EFD_NONBLOCK                     = 0x800
+	EPIOCGPARAMS                     = 0x80088a02
+	EPIOCSPARAMS                     = 0x40088a01
 	EPOLL_CLOEXEC                    = 0x80000
 	EXTPROC                          = 0x10000
 	FF1                              = 0x8000
@@ -106,6 +108,7 @@ const (
 	HIDIOCGRAWINFO                   = 0x80084803
 	HIDIOCGRDESC                     = 0x90044802
 	HIDIOCGRDESCSIZE                 = 0x80044801
+	HIDIOCREVOKE                     = 0x4004480d
 	HUPCL                            = 0x400
 	ICANON                           = 0x2
 	IEXTEN                           = 0x8000
@@ -148,9 +151,14 @@ const (
 	NFDBITS                          = 0x40
 	NLDLY                            = 0x100
 	NOFLSH                           = 0x80
+	NS_GET_MNTNS_ID                  = 0x8008b705
 	NS_GET_NSTYPE                    = 0xb703
 	NS_GET_OWNER_UID                 = 0xb704
 	NS_GET_PARENT                    = 0xb702
+	NS_GET_PID_FROM_PIDNS            = 0x8004b706
+	NS_GET_PID_IN_PIDNS              = 0x8004b708
+	NS_GET_TGID_FROM_PIDNS           = 0x8004b707
+	NS_GET_TGID_IN_PIDNS             = 0x8004b709
 	NS_GET_USERNS                    = 0xb701
 	OLCUC                            = 0x2
 	ONLCR                            = 0x4
@@ -227,6 +235,20 @@ const (
 	PPPIOCUNBRIDGECHAN               = 0x7434
 	PPPIOCXFERUNIT                   = 0x744e
 	PR_SET_PTRACER_ANY               = 0xffffffffffffffff
+	PTP_CLOCK_GETCAPS                = 0x80503d01
+	PTP_CLOCK_GETCAPS2               = 0x80503d0a
+	PTP_ENABLE_PPS                   = 0x40043d04
+	PTP_ENABLE_PPS2                  = 0x40043d0d
+	PTP_EXTTS_REQUEST                = 0x40103d02
+	PTP_EXTTS_REQUEST2               = 0x40103d0b
+	PTP_MASK_CLEAR_ALL               = 0x3d13
+	PTP_MASK_EN_SINGLE               = 0x40043d14
+	PTP_PEROUT_REQUEST               = 0x40383d03
+	PTP_PEROUT_REQUEST2              = 0x40383d0c
+	PTP_PIN_SETFUNC                  = 0x40603d07
+	PTP_PIN_SETFUNC2                 = 0x40603d10
+	PTP_SYS_OFFSET                   = 0x43403d05
+	PTP_SYS_OFFSET2                  = 0x43403d0e
 	PTRACE_GETFDPIC                  = 0x21
 	PTRACE_GETFDPIC_EXEC             = 0x0
 	PTRACE_GETFDPIC_INTERP           = 0x1
@@ -266,6 +288,8 @@ const (
 	RTC_WIE_ON                       = 0x700f
 	RTC_WKALM_RD                     = 0x80287010
 	RTC_WKALM_SET                    = 0x4028700f
+	SCM_DEVMEM_DMABUF                = 0x4f
+	SCM_DEVMEM_LINEAR                = 0x4e
 	SCM_TIMESTAMPING                 = 0x25
 	SCM_TIMESTAMPING_OPT_STATS       = 0x36
 	SCM_TIMESTAMPING_PKTINFO         = 0x3a
@@ -304,6 +328,9 @@ const (
 	SO_CNX_ADVICE                    = 0x35
 	SO_COOKIE                        = 0x39
 	SO_DETACH_REUSEPORT_BPF          = 0x44
+	SO_DEVMEM_DMABUF                 = 0x4f
+	SO_DEVMEM_DONTNEED               = 0x50
+	SO_DEVMEM_LINEAR                 = 0x4e
 	SO_DOMAIN                        = 0x27
 	SO_DONTROUTE                     = 0x5
 	SO_ERROR                         = 0x4
diff --git a/vendor/golang.org/x/sys/unix/zerrors_linux_s390x.go b/vendor/golang.org/x/sys/unix/zerrors_linux_s390x.go
index 2f1dd214a..71f040097 100644
--- a/vendor/golang.org/x/sys/unix/zerrors_linux_s390x.go
+++ b/vendor/golang.org/x/sys/unix/zerrors_linux_s390x.go
@@ -78,6 +78,8 @@ const (
 	ECHOPRT                          = 0x400
 	EFD_CLOEXEC                      = 0x80000
 	EFD_NONBLOCK                     = 0x800
+	EPIOCGPARAMS                     = 0x80088a02
+	EPIOCSPARAMS                     = 0x40088a01
 	EPOLL_CLOEXEC                    = 0x80000
 	EXTPROC                          = 0x10000
 	FF1                              = 0x8000
@@ -106,6 +108,7 @@ const (
 	HIDIOCGRAWINFO                   = 0x80084803
 	HIDIOCGRDESC                     = 0x90044802
 	HIDIOCGRDESCSIZE                 = 0x80044801
+	HIDIOCREVOKE                     = 0x4004480d
 	HUPCL                            = 0x400
 	ICANON                           = 0x2
 	IEXTEN                           = 0x8000
@@ -148,9 +151,14 @@ const (
 	NFDBITS                          = 0x40
 	NLDLY                            = 0x100
 	NOFLSH                           = 0x80
+	NS_GET_MNTNS_ID                  = 0x8008b705
 	NS_GET_NSTYPE                    = 0xb703
 	NS_GET_OWNER_UID                 = 0xb704
 	NS_GET_PARENT                    = 0xb702
+	NS_GET_PID_FROM_PIDNS            = 0x8004b706
+	NS_GET_PID_IN_PIDNS              = 0x8004b708
+	NS_GET_TGID_FROM_PIDNS           = 0x8004b707
+	NS_GET_TGID_IN_PIDNS             = 0x8004b709
 	NS_GET_USERNS                    = 0xb701
 	OLCUC                            = 0x2
 	ONLCR                            = 0x4
@@ -227,6 +235,20 @@ const (
 	PPPIOCUNBRIDGECHAN               = 0x7434
 	PPPIOCXFERUNIT                   = 0x744e
 	PR_SET_PTRACER_ANY               = 0xffffffffffffffff
+	PTP_CLOCK_GETCAPS                = 0x80503d01
+	PTP_CLOCK_GETCAPS2               = 0x80503d0a
+	PTP_ENABLE_PPS                   = 0x40043d04
+	PTP_ENABLE_PPS2                  = 0x40043d0d
+	PTP_EXTTS_REQUEST                = 0x40103d02
+	PTP_EXTTS_REQUEST2               = 0x40103d0b
+	PTP_MASK_CLEAR_ALL               = 0x3d13
+	PTP_MASK_EN_SINGLE               = 0x40043d14
+	PTP_PEROUT_REQUEST               = 0x40383d03
+	PTP_PEROUT_REQUEST2              = 0x40383d0c
+	PTP_PIN_SETFUNC                  = 0x40603d07
+	PTP_PIN_SETFUNC2                 = 0x40603d10
+	PTP_SYS_OFFSET                   = 0x43403d05
+	PTP_SYS_OFFSET2                  = 0x43403d0e
 	PTRACE_DISABLE_TE                = 0x5010
 	PTRACE_ENABLE_TE                 = 0x5009
 	PTRACE_GET_LAST_BREAK            = 0x5006
@@ -338,6 +360,8 @@ const (
 	RTC_WIE_ON                       = 0x700f
 	RTC_WKALM_RD                     = 0x80287010
 	RTC_WKALM_SET                    = 0x4028700f
+	SCM_DEVMEM_DMABUF                = 0x4f
+	SCM_DEVMEM_LINEAR                = 0x4e
 	SCM_TIMESTAMPING                 = 0x25
 	SCM_TIMESTAMPING_OPT_STATS       = 0x36
 	SCM_TIMESTAMPING_PKTINFO         = 0x3a
@@ -376,6 +400,9 @@ const (
 	SO_CNX_ADVICE                    = 0x35
 	SO_COOKIE                        = 0x39
 	SO_DETACH_REUSEPORT_BPF          = 0x44
+	SO_DEVMEM_DMABUF                 = 0x4f
+	SO_DEVMEM_DONTNEED               = 0x50
+	SO_DEVMEM_LINEAR                 = 0x4e
 	SO_DOMAIN                        = 0x27
 	SO_DONTROUTE                     = 0x5
 	SO_ERROR                         = 0x4
diff --git a/vendor/golang.org/x/sys/unix/zerrors_linux_sparc64.go b/vendor/golang.org/x/sys/unix/zerrors_linux_sparc64.go
index f40519d90..c44a31332 100644
--- a/vendor/golang.org/x/sys/unix/zerrors_linux_sparc64.go
+++ b/vendor/golang.org/x/sys/unix/zerrors_linux_sparc64.go
@@ -82,6 +82,8 @@ const (
 	EFD_CLOEXEC                      = 0x400000
 	EFD_NONBLOCK                     = 0x4000
 	EMT_TAGOVF                       = 0x1
+	EPIOCGPARAMS                     = 0x40088a02
+	EPIOCSPARAMS                     = 0x80088a01
 	EPOLL_CLOEXEC                    = 0x400000
 	EXTPROC                          = 0x10000
 	FF1                              = 0x8000
@@ -110,6 +112,7 @@ const (
 	HIDIOCGRAWINFO                   = 0x40084803
 	HIDIOCGRDESC                     = 0x50044802
 	HIDIOCGRDESCSIZE                 = 0x40044801
+	HIDIOCREVOKE                     = 0x8004480d
 	HUPCL                            = 0x400
 	ICANON                           = 0x2
 	IEXTEN                           = 0x8000
@@ -153,9 +156,14 @@ const (
 	NFDBITS                          = 0x40
 	NLDLY                            = 0x100
 	NOFLSH                           = 0x80
+	NS_GET_MNTNS_ID                  = 0x4008b705
 	NS_GET_NSTYPE                    = 0x2000b703
 	NS_GET_OWNER_UID                 = 0x2000b704
 	NS_GET_PARENT                    = 0x2000b702
+	NS_GET_PID_FROM_PIDNS            = 0x4004b706
+	NS_GET_PID_IN_PIDNS              = 0x4004b708
+	NS_GET_TGID_FROM_PIDNS           = 0x4004b707
+	NS_GET_TGID_IN_PIDNS             = 0x4004b709
 	NS_GET_USERNS                    = 0x2000b701
 	OLCUC                            = 0x2
 	ONLCR                            = 0x4
@@ -232,6 +240,20 @@ const (
 	PPPIOCUNBRIDGECHAN               = 0x20007434
 	PPPIOCXFERUNIT                   = 0x2000744e
 	PR_SET_PTRACER_ANY               = 0xffffffffffffffff
+	PTP_CLOCK_GETCAPS                = 0x40503d01
+	PTP_CLOCK_GETCAPS2               = 0x40503d0a
+	PTP_ENABLE_PPS                   = 0x80043d04
+	PTP_ENABLE_PPS2                  = 0x80043d0d
+	PTP_EXTTS_REQUEST                = 0x80103d02
+	PTP_EXTTS_REQUEST2               = 0x80103d0b
+	PTP_MASK_CLEAR_ALL               = 0x20003d13
+	PTP_MASK_EN_SINGLE               = 0x80043d14
+	PTP_PEROUT_REQUEST               = 0x80383d03
+	PTP_PEROUT_REQUEST2              = 0x80383d0c
+	PTP_PIN_SETFUNC                  = 0x80603d07
+	PTP_PIN_SETFUNC2                 = 0x80603d10
+	PTP_SYS_OFFSET                   = 0x83403d05
+	PTP_SYS_OFFSET2                  = 0x83403d0e
 	PTRACE_GETFPAREGS                = 0x14
 	PTRACE_GETFPREGS                 = 0xe
 	PTRACE_GETFPREGS64               = 0x19
@@ -329,6 +351,8 @@ const (
 	RTC_WIE_ON                       = 0x2000700f
 	RTC_WKALM_RD                     = 0x40287010
 	RTC_WKALM_SET                    = 0x8028700f
+	SCM_DEVMEM_DMABUF                = 0x58
+	SCM_DEVMEM_LINEAR                = 0x57
 	SCM_TIMESTAMPING                 = 0x23
 	SCM_TIMESTAMPING_OPT_STATS       = 0x38
 	SCM_TIMESTAMPING_PKTINFO         = 0x3c
@@ -415,6 +439,9 @@ const (
 	SO_CNX_ADVICE                    = 0x37
 	SO_COOKIE                        = 0x3b
 	SO_DETACH_REUSEPORT_BPF          = 0x47
+	SO_DEVMEM_DMABUF                 = 0x58
+	SO_DEVMEM_DONTNEED               = 0x59
+	SO_DEVMEM_LINEAR                 = 0x57
 	SO_DOMAIN                        = 0x1029
 	SO_DONTROUTE                     = 0x10
 	SO_ERROR                         = 0x1007
diff --git a/vendor/golang.org/x/sys/unix/zerrors_zos_s390x.go b/vendor/golang.org/x/sys/unix/zerrors_zos_s390x.go
index da08b2ab3..1ec2b1407 100644
--- a/vendor/golang.org/x/sys/unix/zerrors_zos_s390x.go
+++ b/vendor/golang.org/x/sys/unix/zerrors_zos_s390x.go
@@ -581,6 +581,8 @@ const (
 	AT_EMPTY_PATH                   = 0x1000
 	AT_REMOVEDIR                    = 0x200
 	RENAME_NOREPLACE                = 1 << 0
+	ST_RDONLY                       = 1
+	ST_NOSUID                       = 2
 )
 
 const (
diff --git a/vendor/golang.org/x/sys/unix/zsyscall_darwin_amd64.go b/vendor/golang.org/x/sys/unix/zsyscall_darwin_amd64.go
index 07642c308..24b346e1a 100644
--- a/vendor/golang.org/x/sys/unix/zsyscall_darwin_amd64.go
+++ b/vendor/golang.org/x/sys/unix/zsyscall_darwin_amd64.go
@@ -740,6 +740,54 @@ func ioctlPtr(fd int, req uint, arg unsafe.Pointer) (err error) {
 
 // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT
 
+func renamexNp(from string, to string, flag uint32) (err error) {
+	var _p0 *byte
+	_p0, err = BytePtrFromString(from)
+	if err != nil {
+		return
+	}
+	var _p1 *byte
+	_p1, err = BytePtrFromString(to)
+	if err != nil {
+		return
+	}
+	_, _, e1 := syscall_syscall(libc_renamex_np_trampoline_addr, uintptr(unsafe.Pointer(_p0)), uintptr(unsafe.Pointer(_p1)), uintptr(flag))
+	if e1 != 0 {
+		err = errnoErr(e1)
+	}
+	return
+}
+
+var libc_renamex_np_trampoline_addr uintptr
+
+//go:cgo_import_dynamic libc_renamex_np renamex_np "/usr/lib/libSystem.B.dylib"
+
+// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT
+
+func renameatxNp(fromfd int, from string, tofd int, to string, flag uint32) (err error) {
+	var _p0 *byte
+	_p0, err = BytePtrFromString(from)
+	if err != nil {
+		return
+	}
+	var _p1 *byte
+	_p1, err = BytePtrFromString(to)
+	if err != nil {
+		return
+	}
+	_, _, e1 := syscall_syscall6(libc_renameatx_np_trampoline_addr, uintptr(fromfd), uintptr(unsafe.Pointer(_p0)), uintptr(tofd), uintptr(unsafe.Pointer(_p1)), uintptr(flag), 0)
+	if e1 != 0 {
+		err = errnoErr(e1)
+	}
+	return
+}
+
+var libc_renameatx_np_trampoline_addr uintptr
+
+//go:cgo_import_dynamic libc_renameatx_np renameatx_np "/usr/lib/libSystem.B.dylib"
+
+// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT
+
 func sysctl(mib []_C_int, old *byte, oldlen *uintptr, new *byte, newlen uintptr) (err error) {
 	var _p0 unsafe.Pointer
 	if len(mib) > 0 {
@@ -793,6 +841,26 @@ var libc_pthread_fchdir_np_trampoline_addr uintptr
 
 // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT
 
+func connectx(fd int, endpoints *SaEndpoints, associd SaeAssocID, flags uint32, iov []Iovec, n *uintptr, connid *SaeConnID) (err error) {
+	var _p0 unsafe.Pointer
+	if len(iov) > 0 {
+		_p0 = unsafe.Pointer(&iov[0])
+	} else {
+		_p0 = unsafe.Pointer(&_zero)
+	}
+	_, _, e1 := syscall_syscall9(libc_connectx_trampoline_addr, uintptr(fd), uintptr(unsafe.Pointer(endpoints)), uintptr(associd), uintptr(flags), uintptr(_p0), uintptr(len(iov)), uintptr(unsafe.Pointer(n)), uintptr(unsafe.Pointer(connid)), 0)
+	if e1 != 0 {
+		err = errnoErr(e1)
+	}
+	return
+}
+
+var libc_connectx_trampoline_addr uintptr
+
+//go:cgo_import_dynamic libc_connectx connectx "/usr/lib/libSystem.B.dylib"
+
+// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT
+
 func sendfile(infd int, outfd int, offset int64, len *int64, hdtr unsafe.Pointer, flags int) (err error) {
 	_, _, e1 := syscall_syscall6(libc_sendfile_trampoline_addr, uintptr(infd), uintptr(outfd), uintptr(offset), uintptr(unsafe.Pointer(len)), uintptr(hdtr), uintptr(flags))
 	if e1 != 0 {
diff --git a/vendor/golang.org/x/sys/unix/zsyscall_darwin_amd64.s b/vendor/golang.org/x/sys/unix/zsyscall_darwin_amd64.s
index 923e08cb7..ebd213100 100644
--- a/vendor/golang.org/x/sys/unix/zsyscall_darwin_amd64.s
+++ b/vendor/golang.org/x/sys/unix/zsyscall_darwin_amd64.s
@@ -223,6 +223,16 @@ TEXT libc_ioctl_trampoline<>(SB),NOSPLIT,$0-0
 GLOBL	·libc_ioctl_trampoline_addr(SB), RODATA, $8
 DATA	·libc_ioctl_trampoline_addr(SB)/8, $libc_ioctl_trampoline<>(SB)
 
+TEXT libc_renamex_np_trampoline<>(SB),NOSPLIT,$0-0
+	JMP	libc_renamex_np(SB)
+GLOBL	·libc_renamex_np_trampoline_addr(SB), RODATA, $8
+DATA	·libc_renamex_np_trampoline_addr(SB)/8, $libc_renamex_np_trampoline<>(SB)
+
+TEXT libc_renameatx_np_trampoline<>(SB),NOSPLIT,$0-0
+	JMP	libc_renameatx_np(SB)
+GLOBL	·libc_renameatx_np_trampoline_addr(SB), RODATA, $8
+DATA	·libc_renameatx_np_trampoline_addr(SB)/8, $libc_renameatx_np_trampoline<>(SB)
+
 TEXT libc_sysctl_trampoline<>(SB),NOSPLIT,$0-0
 	JMP	libc_sysctl(SB)
 GLOBL	·libc_sysctl_trampoline_addr(SB), RODATA, $8
@@ -238,6 +248,11 @@ TEXT libc_pthread_fchdir_np_trampoline<>(SB),NOSPLIT,$0-0
 GLOBL	·libc_pthread_fchdir_np_trampoline_addr(SB), RODATA, $8
 DATA	·libc_pthread_fchdir_np_trampoline_addr(SB)/8, $libc_pthread_fchdir_np_trampoline<>(SB)
 
+TEXT libc_connectx_trampoline<>(SB),NOSPLIT,$0-0
+	JMP	libc_connectx(SB)
+GLOBL	·libc_connectx_trampoline_addr(SB), RODATA, $8
+DATA	·libc_connectx_trampoline_addr(SB)/8, $libc_connectx_trampoline<>(SB)
+
 TEXT libc_sendfile_trampoline<>(SB),NOSPLIT,$0-0
 	JMP	libc_sendfile(SB)
 GLOBL	·libc_sendfile_trampoline_addr(SB), RODATA, $8
diff --git a/vendor/golang.org/x/sys/unix/zsyscall_darwin_arm64.go b/vendor/golang.org/x/sys/unix/zsyscall_darwin_arm64.go
index 7d73dda64..824b9c2d5 100644
--- a/vendor/golang.org/x/sys/unix/zsyscall_darwin_arm64.go
+++ b/vendor/golang.org/x/sys/unix/zsyscall_darwin_arm64.go
@@ -740,6 +740,54 @@ func ioctlPtr(fd int, req uint, arg unsafe.Pointer) (err error) {
 
 // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT
 
+func renamexNp(from string, to string, flag uint32) (err error) {
+	var _p0 *byte
+	_p0, err = BytePtrFromString(from)
+	if err != nil {
+		return
+	}
+	var _p1 *byte
+	_p1, err = BytePtrFromString(to)
+	if err != nil {
+		return
+	}
+	_, _, e1 := syscall_syscall(libc_renamex_np_trampoline_addr, uintptr(unsafe.Pointer(_p0)), uintptr(unsafe.Pointer(_p1)), uintptr(flag))
+	if e1 != 0 {
+		err = errnoErr(e1)
+	}
+	return
+}
+
+var libc_renamex_np_trampoline_addr uintptr
+
+//go:cgo_import_dynamic libc_renamex_np renamex_np "/usr/lib/libSystem.B.dylib"
+
+// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT
+
+func renameatxNp(fromfd int, from string, tofd int, to string, flag uint32) (err error) {
+	var _p0 *byte
+	_p0, err = BytePtrFromString(from)
+	if err != nil {
+		return
+	}
+	var _p1 *byte
+	_p1, err = BytePtrFromString(to)
+	if err != nil {
+		return
+	}
+	_, _, e1 := syscall_syscall6(libc_renameatx_np_trampoline_addr, uintptr(fromfd), uintptr(unsafe.Pointer(_p0)), uintptr(tofd), uintptr(unsafe.Pointer(_p1)), uintptr(flag), 0)
+	if e1 != 0 {
+		err = errnoErr(e1)
+	}
+	return
+}
+
+var libc_renameatx_np_trampoline_addr uintptr
+
+//go:cgo_import_dynamic libc_renameatx_np renameatx_np "/usr/lib/libSystem.B.dylib"
+
+// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT
+
 func sysctl(mib []_C_int, old *byte, oldlen *uintptr, new *byte, newlen uintptr) (err error) {
 	var _p0 unsafe.Pointer
 	if len(mib) > 0 {
@@ -793,6 +841,26 @@ var libc_pthread_fchdir_np_trampoline_addr uintptr
 
 // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT
 
+func connectx(fd int, endpoints *SaEndpoints, associd SaeAssocID, flags uint32, iov []Iovec, n *uintptr, connid *SaeConnID) (err error) {
+	var _p0 unsafe.Pointer
+	if len(iov) > 0 {
+		_p0 = unsafe.Pointer(&iov[0])
+	} else {
+		_p0 = unsafe.Pointer(&_zero)
+	}
+	_, _, e1 := syscall_syscall9(libc_connectx_trampoline_addr, uintptr(fd), uintptr(unsafe.Pointer(endpoints)), uintptr(associd), uintptr(flags), uintptr(_p0), uintptr(len(iov)), uintptr(unsafe.Pointer(n)), uintptr(unsafe.Pointer(connid)), 0)
+	if e1 != 0 {
+		err = errnoErr(e1)
+	}
+	return
+}
+
+var libc_connectx_trampoline_addr uintptr
+
+//go:cgo_import_dynamic libc_connectx connectx "/usr/lib/libSystem.B.dylib"
+
+// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT
+
 func sendfile(infd int, outfd int, offset int64, len *int64, hdtr unsafe.Pointer, flags int) (err error) {
 	_, _, e1 := syscall_syscall6(libc_sendfile_trampoline_addr, uintptr(infd), uintptr(outfd), uintptr(offset), uintptr(unsafe.Pointer(len)), uintptr(hdtr), uintptr(flags))
 	if e1 != 0 {
diff --git a/vendor/golang.org/x/sys/unix/zsyscall_darwin_arm64.s b/vendor/golang.org/x/sys/unix/zsyscall_darwin_arm64.s
index 057700111..4f178a229 100644
--- a/vendor/golang.org/x/sys/unix/zsyscall_darwin_arm64.s
+++ b/vendor/golang.org/x/sys/unix/zsyscall_darwin_arm64.s
@@ -223,6 +223,16 @@ TEXT libc_ioctl_trampoline<>(SB),NOSPLIT,$0-0
 GLOBL	·libc_ioctl_trampoline_addr(SB), RODATA, $8
 DATA	·libc_ioctl_trampoline_addr(SB)/8, $libc_ioctl_trampoline<>(SB)
 
+TEXT libc_renamex_np_trampoline<>(SB),NOSPLIT,$0-0
+	JMP	libc_renamex_np(SB)
+GLOBL	·libc_renamex_np_trampoline_addr(SB), RODATA, $8
+DATA	·libc_renamex_np_trampoline_addr(SB)/8, $libc_renamex_np_trampoline<>(SB)
+
+TEXT libc_renameatx_np_trampoline<>(SB),NOSPLIT,$0-0
+	JMP	libc_renameatx_np(SB)
+GLOBL	·libc_renameatx_np_trampoline_addr(SB), RODATA, $8
+DATA	·libc_renameatx_np_trampoline_addr(SB)/8, $libc_renameatx_np_trampoline<>(SB)
+
 TEXT libc_sysctl_trampoline<>(SB),NOSPLIT,$0-0
 	JMP	libc_sysctl(SB)
 GLOBL	·libc_sysctl_trampoline_addr(SB), RODATA, $8
@@ -238,6 +248,11 @@ TEXT libc_pthread_fchdir_np_trampoline<>(SB),NOSPLIT,$0-0
 GLOBL	·libc_pthread_fchdir_np_trampoline_addr(SB), RODATA, $8
 DATA	·libc_pthread_fchdir_np_trampoline_addr(SB)/8, $libc_pthread_fchdir_np_trampoline<>(SB)
 
+TEXT libc_connectx_trampoline<>(SB),NOSPLIT,$0-0
+	JMP	libc_connectx(SB)
+GLOBL	·libc_connectx_trampoline_addr(SB), RODATA, $8
+DATA	·libc_connectx_trampoline_addr(SB)/8, $libc_connectx_trampoline<>(SB)
+
 TEXT libc_sendfile_trampoline<>(SB),NOSPLIT,$0-0
 	JMP	libc_sendfile(SB)
 GLOBL	·libc_sendfile_trampoline_addr(SB), RODATA, $8
diff --git a/vendor/golang.org/x/sys/unix/zsyscall_linux.go b/vendor/golang.org/x/sys/unix/zsyscall_linux.go
index 87d8612a1..5cc1e8eb2 100644
--- a/vendor/golang.org/x/sys/unix/zsyscall_linux.go
+++ b/vendor/golang.org/x/sys/unix/zsyscall_linux.go
@@ -592,6 +592,16 @@ func ClockGettime(clockid int32, time *Timespec) (err error) {
 
 // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT
 
+func ClockSettime(clockid int32, time *Timespec) (err error) {
+	_, _, e1 := Syscall(SYS_CLOCK_SETTIME, uintptr(clockid), uintptr(unsafe.Pointer(time)), 0)
+	if e1 != 0 {
+		err = errnoErr(e1)
+	}
+	return
+}
+
+// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT
+
 func ClockNanosleep(clockid int32, flags int, request *Timespec, remain *Timespec) (err error) {
 	_, _, e1 := Syscall6(SYS_CLOCK_NANOSLEEP, uintptr(clockid), uintptr(flags), uintptr(unsafe.Pointer(request)), uintptr(unsafe.Pointer(remain)), 0, 0)
 	if e1 != 0 {
@@ -971,23 +981,6 @@ func Getpriority(which int, who int) (prio int, err error) {
 
 // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT
 
-func Getrandom(buf []byte, flags int) (n int, err error) {
-	var _p0 unsafe.Pointer
-	if len(buf) > 0 {
-		_p0 = unsafe.Pointer(&buf[0])
-	} else {
-		_p0 = unsafe.Pointer(&_zero)
-	}
-	r0, _, e1 := Syscall(SYS_GETRANDOM, uintptr(_p0), uintptr(len(buf)), uintptr(flags))
-	n = int(r0)
-	if e1 != 0 {
-		err = errnoErr(e1)
-	}
-	return
-}
-
-// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT
-
 func Getrusage(who int, rusage *Rusage) (err error) {
 	_, _, e1 := RawSyscall(SYS_GETRUSAGE, uintptr(who), uintptr(unsafe.Pointer(rusage)), 0)
 	if e1 != 0 {
@@ -2229,3 +2222,19 @@ func Cachestat(fd uint, crange *CachestatRange, cstat *Cachestat_t, flags uint)
 	}
 	return
 }
+
+// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT
+
+func Mseal(b []byte, flags uint) (err error) {
+	var _p0 unsafe.Pointer
+	if len(b) > 0 {
+		_p0 = unsafe.Pointer(&b[0])
+	} else {
+		_p0 = unsafe.Pointer(&_zero)
+	}
+	_, _, e1 := Syscall(SYS_MSEAL, uintptr(_p0), uintptr(len(b)), uintptr(flags))
+	if e1 != 0 {
+		err = errnoErr(e1)
+	}
+	return
+}
diff --git a/vendor/golang.org/x/sys/unix/zsyscall_openbsd_386.go b/vendor/golang.org/x/sys/unix/zsyscall_openbsd_386.go
index 9dc42410b..1851df14e 100644
--- a/vendor/golang.org/x/sys/unix/zsyscall_openbsd_386.go
+++ b/vendor/golang.org/x/sys/unix/zsyscall_openbsd_386.go
@@ -1493,6 +1493,30 @@ var libc_mknodat_trampoline_addr uintptr
 
 // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT
 
+func Mount(fsType string, dir string, flags int, data unsafe.Pointer) (err error) {
+	var _p0 *byte
+	_p0, err = BytePtrFromString(fsType)
+	if err != nil {
+		return
+	}
+	var _p1 *byte
+	_p1, err = BytePtrFromString(dir)
+	if err != nil {
+		return
+	}
+	_, _, e1 := syscall_syscall6(libc_mount_trampoline_addr, uintptr(unsafe.Pointer(_p0)), uintptr(unsafe.Pointer(_p1)), uintptr(flags), uintptr(data), 0, 0)
+	if e1 != 0 {
+		err = errnoErr(e1)
+	}
+	return
+}
+
+var libc_mount_trampoline_addr uintptr
+
+//go:cgo_import_dynamic libc_mount mount "libc.so"
+
+// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT
+
 func Nanosleep(time *Timespec, leftover *Timespec) (err error) {
 	_, _, e1 := syscall_syscall(libc_nanosleep_trampoline_addr, uintptr(unsafe.Pointer(time)), uintptr(unsafe.Pointer(leftover)), 0)
 	if e1 != 0 {
diff --git a/vendor/golang.org/x/sys/unix/zsyscall_openbsd_386.s b/vendor/golang.org/x/sys/unix/zsyscall_openbsd_386.s
index 41b561731..0b43c6936 100644
--- a/vendor/golang.org/x/sys/unix/zsyscall_openbsd_386.s
+++ b/vendor/golang.org/x/sys/unix/zsyscall_openbsd_386.s
@@ -463,6 +463,11 @@ TEXT libc_mknodat_trampoline<>(SB),NOSPLIT,$0-0
 GLOBL	·libc_mknodat_trampoline_addr(SB), RODATA, $4
 DATA	·libc_mknodat_trampoline_addr(SB)/4, $libc_mknodat_trampoline<>(SB)
 
+TEXT libc_mount_trampoline<>(SB),NOSPLIT,$0-0
+	JMP	libc_mount(SB)
+GLOBL	·libc_mount_trampoline_addr(SB), RODATA, $4
+DATA	·libc_mount_trampoline_addr(SB)/4, $libc_mount_trampoline<>(SB)
+
 TEXT libc_nanosleep_trampoline<>(SB),NOSPLIT,$0-0
 	JMP	libc_nanosleep(SB)
 GLOBL	·libc_nanosleep_trampoline_addr(SB), RODATA, $4
diff --git a/vendor/golang.org/x/sys/unix/zsyscall_openbsd_amd64.go b/vendor/golang.org/x/sys/unix/zsyscall_openbsd_amd64.go
index 0d3a0751c..e1ec0dbe4 100644
--- a/vendor/golang.org/x/sys/unix/zsyscall_openbsd_amd64.go
+++ b/vendor/golang.org/x/sys/unix/zsyscall_openbsd_amd64.go
@@ -1493,6 +1493,30 @@ var libc_mknodat_trampoline_addr uintptr
 
 // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT
 
+func Mount(fsType string, dir string, flags int, data unsafe.Pointer) (err error) {
+	var _p0 *byte
+	_p0, err = BytePtrFromString(fsType)
+	if err != nil {
+		return
+	}
+	var _p1 *byte
+	_p1, err = BytePtrFromString(dir)
+	if err != nil {
+		return
+	}
+	_, _, e1 := syscall_syscall6(libc_mount_trampoline_addr, uintptr(unsafe.Pointer(_p0)), uintptr(unsafe.Pointer(_p1)), uintptr(flags), uintptr(data), 0, 0)
+	if e1 != 0 {
+		err = errnoErr(e1)
+	}
+	return
+}
+
+var libc_mount_trampoline_addr uintptr
+
+//go:cgo_import_dynamic libc_mount mount "libc.so"
+
+// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT
+
 func Nanosleep(time *Timespec, leftover *Timespec) (err error) {
 	_, _, e1 := syscall_syscall(libc_nanosleep_trampoline_addr, uintptr(unsafe.Pointer(time)), uintptr(unsafe.Pointer(leftover)), 0)
 	if e1 != 0 {
diff --git a/vendor/golang.org/x/sys/unix/zsyscall_openbsd_amd64.s b/vendor/golang.org/x/sys/unix/zsyscall_openbsd_amd64.s
index 4019a656f..880c6d6e3 100644
--- a/vendor/golang.org/x/sys/unix/zsyscall_openbsd_amd64.s
+++ b/vendor/golang.org/x/sys/unix/zsyscall_openbsd_amd64.s
@@ -463,6 +463,11 @@ TEXT libc_mknodat_trampoline<>(SB),NOSPLIT,$0-0
 GLOBL	·libc_mknodat_trampoline_addr(SB), RODATA, $8
 DATA	·libc_mknodat_trampoline_addr(SB)/8, $libc_mknodat_trampoline<>(SB)
 
+TEXT libc_mount_trampoline<>(SB),NOSPLIT,$0-0
+	JMP	libc_mount(SB)
+GLOBL	·libc_mount_trampoline_addr(SB), RODATA, $8
+DATA	·libc_mount_trampoline_addr(SB)/8, $libc_mount_trampoline<>(SB)
+
 TEXT libc_nanosleep_trampoline<>(SB),NOSPLIT,$0-0
 	JMP	libc_nanosleep(SB)
 GLOBL	·libc_nanosleep_trampoline_addr(SB), RODATA, $8
diff --git a/vendor/golang.org/x/sys/unix/zsyscall_openbsd_arm.go b/vendor/golang.org/x/sys/unix/zsyscall_openbsd_arm.go
index c39f7776d..7c8452a63 100644
--- a/vendor/golang.org/x/sys/unix/zsyscall_openbsd_arm.go
+++ b/vendor/golang.org/x/sys/unix/zsyscall_openbsd_arm.go
@@ -1493,6 +1493,30 @@ var libc_mknodat_trampoline_addr uintptr
 
 // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT
 
+func Mount(fsType string, dir string, flags int, data unsafe.Pointer) (err error) {
+	var _p0 *byte
+	_p0, err = BytePtrFromString(fsType)
+	if err != nil {
+		return
+	}
+	var _p1 *byte
+	_p1, err = BytePtrFromString(dir)
+	if err != nil {
+		return
+	}
+	_, _, e1 := syscall_syscall6(libc_mount_trampoline_addr, uintptr(unsafe.Pointer(_p0)), uintptr(unsafe.Pointer(_p1)), uintptr(flags), uintptr(data), 0, 0)
+	if e1 != 0 {
+		err = errnoErr(e1)
+	}
+	return
+}
+
+var libc_mount_trampoline_addr uintptr
+
+//go:cgo_import_dynamic libc_mount mount "libc.so"
+
+// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT
+
 func Nanosleep(time *Timespec, leftover *Timespec) (err error) {
 	_, _, e1 := syscall_syscall(libc_nanosleep_trampoline_addr, uintptr(unsafe.Pointer(time)), uintptr(unsafe.Pointer(leftover)), 0)
 	if e1 != 0 {
diff --git a/vendor/golang.org/x/sys/unix/zsyscall_openbsd_arm.s b/vendor/golang.org/x/sys/unix/zsyscall_openbsd_arm.s
index ac4af24f9..b8ef95b0f 100644
--- a/vendor/golang.org/x/sys/unix/zsyscall_openbsd_arm.s
+++ b/vendor/golang.org/x/sys/unix/zsyscall_openbsd_arm.s
@@ -463,6 +463,11 @@ TEXT libc_mknodat_trampoline<>(SB),NOSPLIT,$0-0
 GLOBL	·libc_mknodat_trampoline_addr(SB), RODATA, $4
 DATA	·libc_mknodat_trampoline_addr(SB)/4, $libc_mknodat_trampoline<>(SB)
 
+TEXT libc_mount_trampoline<>(SB),NOSPLIT,$0-0
+	JMP	libc_mount(SB)
+GLOBL	·libc_mount_trampoline_addr(SB), RODATA, $4
+DATA	·libc_mount_trampoline_addr(SB)/4, $libc_mount_trampoline<>(SB)
+
 TEXT libc_nanosleep_trampoline<>(SB),NOSPLIT,$0-0
 	JMP	libc_nanosleep(SB)
 GLOBL	·libc_nanosleep_trampoline_addr(SB), RODATA, $4
diff --git a/vendor/golang.org/x/sys/unix/zsyscall_openbsd_arm64.go b/vendor/golang.org/x/sys/unix/zsyscall_openbsd_arm64.go
index 57571d072..2ffdf861f 100644
--- a/vendor/golang.org/x/sys/unix/zsyscall_openbsd_arm64.go
+++ b/vendor/golang.org/x/sys/unix/zsyscall_openbsd_arm64.go
@@ -1493,6 +1493,30 @@ var libc_mknodat_trampoline_addr uintptr
 
 // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT
 
+func Mount(fsType string, dir string, flags int, data unsafe.Pointer) (err error) {
+	var _p0 *byte
+	_p0, err = BytePtrFromString(fsType)
+	if err != nil {
+		return
+	}
+	var _p1 *byte
+	_p1, err = BytePtrFromString(dir)
+	if err != nil {
+		return
+	}
+	_, _, e1 := syscall_syscall6(libc_mount_trampoline_addr, uintptr(unsafe.Pointer(_p0)), uintptr(unsafe.Pointer(_p1)), uintptr(flags), uintptr(data), 0, 0)
+	if e1 != 0 {
+		err = errnoErr(e1)
+	}
+	return
+}
+
+var libc_mount_trampoline_addr uintptr
+
+//go:cgo_import_dynamic libc_mount mount "libc.so"
+
+// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT
+
 func Nanosleep(time *Timespec, leftover *Timespec) (err error) {
 	_, _, e1 := syscall_syscall(libc_nanosleep_trampoline_addr, uintptr(unsafe.Pointer(time)), uintptr(unsafe.Pointer(leftover)), 0)
 	if e1 != 0 {
diff --git a/vendor/golang.org/x/sys/unix/zsyscall_openbsd_arm64.s b/vendor/golang.org/x/sys/unix/zsyscall_openbsd_arm64.s
index f77d53212..2af3b5c76 100644
--- a/vendor/golang.org/x/sys/unix/zsyscall_openbsd_arm64.s
+++ b/vendor/golang.org/x/sys/unix/zsyscall_openbsd_arm64.s
@@ -463,6 +463,11 @@ TEXT libc_mknodat_trampoline<>(SB),NOSPLIT,$0-0
 GLOBL	·libc_mknodat_trampoline_addr(SB), RODATA, $8
 DATA	·libc_mknodat_trampoline_addr(SB)/8, $libc_mknodat_trampoline<>(SB)
 
+TEXT libc_mount_trampoline<>(SB),NOSPLIT,$0-0
+	JMP	libc_mount(SB)
+GLOBL	·libc_mount_trampoline_addr(SB), RODATA, $8
+DATA	·libc_mount_trampoline_addr(SB)/8, $libc_mount_trampoline<>(SB)
+
 TEXT libc_nanosleep_trampoline<>(SB),NOSPLIT,$0-0
 	JMP	libc_nanosleep(SB)
 GLOBL	·libc_nanosleep_trampoline_addr(SB), RODATA, $8
diff --git a/vendor/golang.org/x/sys/unix/zsyscall_openbsd_mips64.go b/vendor/golang.org/x/sys/unix/zsyscall_openbsd_mips64.go
index e62963e67..1da08d526 100644
--- a/vendor/golang.org/x/sys/unix/zsyscall_openbsd_mips64.go
+++ b/vendor/golang.org/x/sys/unix/zsyscall_openbsd_mips64.go
@@ -1493,6 +1493,30 @@ var libc_mknodat_trampoline_addr uintptr
 
 // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT
 
+func Mount(fsType string, dir string, flags int, data unsafe.Pointer) (err error) {
+	var _p0 *byte
+	_p0, err = BytePtrFromString(fsType)
+	if err != nil {
+		return
+	}
+	var _p1 *byte
+	_p1, err = BytePtrFromString(dir)
+	if err != nil {
+		return
+	}
+	_, _, e1 := syscall_syscall6(libc_mount_trampoline_addr, uintptr(unsafe.Pointer(_p0)), uintptr(unsafe.Pointer(_p1)), uintptr(flags), uintptr(data), 0, 0)
+	if e1 != 0 {
+		err = errnoErr(e1)
+	}
+	return
+}
+
+var libc_mount_trampoline_addr uintptr
+
+//go:cgo_import_dynamic libc_mount mount "libc.so"
+
+// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT
+
 func Nanosleep(time *Timespec, leftover *Timespec) (err error) {
 	_, _, e1 := syscall_syscall(libc_nanosleep_trampoline_addr, uintptr(unsafe.Pointer(time)), uintptr(unsafe.Pointer(leftover)), 0)
 	if e1 != 0 {
diff --git a/vendor/golang.org/x/sys/unix/zsyscall_openbsd_mips64.s b/vendor/golang.org/x/sys/unix/zsyscall_openbsd_mips64.s
index fae140b62..b7a251353 100644
--- a/vendor/golang.org/x/sys/unix/zsyscall_openbsd_mips64.s
+++ b/vendor/golang.org/x/sys/unix/zsyscall_openbsd_mips64.s
@@ -463,6 +463,11 @@ TEXT libc_mknodat_trampoline<>(SB),NOSPLIT,$0-0
 GLOBL	·libc_mknodat_trampoline_addr(SB), RODATA, $8
 DATA	·libc_mknodat_trampoline_addr(SB)/8, $libc_mknodat_trampoline<>(SB)
 
+TEXT libc_mount_trampoline<>(SB),NOSPLIT,$0-0
+	JMP	libc_mount(SB)
+GLOBL	·libc_mount_trampoline_addr(SB), RODATA, $8
+DATA	·libc_mount_trampoline_addr(SB)/8, $libc_mount_trampoline<>(SB)
+
 TEXT libc_nanosleep_trampoline<>(SB),NOSPLIT,$0-0
 	JMP	libc_nanosleep(SB)
 GLOBL	·libc_nanosleep_trampoline_addr(SB), RODATA, $8
diff --git a/vendor/golang.org/x/sys/unix/zsyscall_openbsd_ppc64.go b/vendor/golang.org/x/sys/unix/zsyscall_openbsd_ppc64.go
index 00831354c..6e85b0aac 100644
--- a/vendor/golang.org/x/sys/unix/zsyscall_openbsd_ppc64.go
+++ b/vendor/golang.org/x/sys/unix/zsyscall_openbsd_ppc64.go
@@ -1493,6 +1493,30 @@ var libc_mknodat_trampoline_addr uintptr
 
 // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT
 
+func Mount(fsType string, dir string, flags int, data unsafe.Pointer) (err error) {
+	var _p0 *byte
+	_p0, err = BytePtrFromString(fsType)
+	if err != nil {
+		return
+	}
+	var _p1 *byte
+	_p1, err = BytePtrFromString(dir)
+	if err != nil {
+		return
+	}
+	_, _, e1 := syscall_syscall6(libc_mount_trampoline_addr, uintptr(unsafe.Pointer(_p0)), uintptr(unsafe.Pointer(_p1)), uintptr(flags), uintptr(data), 0, 0)
+	if e1 != 0 {
+		err = errnoErr(e1)
+	}
+	return
+}
+
+var libc_mount_trampoline_addr uintptr
+
+//go:cgo_import_dynamic libc_mount mount "libc.so"
+
+// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT
+
 func Nanosleep(time *Timespec, leftover *Timespec) (err error) {
 	_, _, e1 := syscall_syscall(libc_nanosleep_trampoline_addr, uintptr(unsafe.Pointer(time)), uintptr(unsafe.Pointer(leftover)), 0)
 	if e1 != 0 {
diff --git a/vendor/golang.org/x/sys/unix/zsyscall_openbsd_ppc64.s b/vendor/golang.org/x/sys/unix/zsyscall_openbsd_ppc64.s
index 9d1e0ff06..f15dadf05 100644
--- a/vendor/golang.org/x/sys/unix/zsyscall_openbsd_ppc64.s
+++ b/vendor/golang.org/x/sys/unix/zsyscall_openbsd_ppc64.s
@@ -555,6 +555,12 @@ TEXT libc_mknodat_trampoline<>(SB),NOSPLIT,$0-0
 GLOBL	·libc_mknodat_trampoline_addr(SB), RODATA, $8
 DATA	·libc_mknodat_trampoline_addr(SB)/8, $libc_mknodat_trampoline<>(SB)
 
+TEXT libc_mount_trampoline<>(SB),NOSPLIT,$0-0
+	CALL	libc_mount(SB)
+	RET
+GLOBL	·libc_mount_trampoline_addr(SB), RODATA, $8
+DATA	·libc_mount_trampoline_addr(SB)/8, $libc_mount_trampoline<>(SB)
+
 TEXT libc_nanosleep_trampoline<>(SB),NOSPLIT,$0-0
 	CALL	libc_nanosleep(SB)
 	RET
diff --git a/vendor/golang.org/x/sys/unix/zsyscall_openbsd_riscv64.go b/vendor/golang.org/x/sys/unix/zsyscall_openbsd_riscv64.go
index 79029ed58..28b487df2 100644
--- a/vendor/golang.org/x/sys/unix/zsyscall_openbsd_riscv64.go
+++ b/vendor/golang.org/x/sys/unix/zsyscall_openbsd_riscv64.go
@@ -1493,6 +1493,30 @@ var libc_mknodat_trampoline_addr uintptr
 
 // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT
 
+func Mount(fsType string, dir string, flags int, data unsafe.Pointer) (err error) {
+	var _p0 *byte
+	_p0, err = BytePtrFromString(fsType)
+	if err != nil {
+		return
+	}
+	var _p1 *byte
+	_p1, err = BytePtrFromString(dir)
+	if err != nil {
+		return
+	}
+	_, _, e1 := syscall_syscall6(libc_mount_trampoline_addr, uintptr(unsafe.Pointer(_p0)), uintptr(unsafe.Pointer(_p1)), uintptr(flags), uintptr(data), 0, 0)
+	if e1 != 0 {
+		err = errnoErr(e1)
+	}
+	return
+}
+
+var libc_mount_trampoline_addr uintptr
+
+//go:cgo_import_dynamic libc_mount mount "libc.so"
+
+// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT
+
 func Nanosleep(time *Timespec, leftover *Timespec) (err error) {
 	_, _, e1 := syscall_syscall(libc_nanosleep_trampoline_addr, uintptr(unsafe.Pointer(time)), uintptr(unsafe.Pointer(leftover)), 0)
 	if e1 != 0 {
diff --git a/vendor/golang.org/x/sys/unix/zsyscall_openbsd_riscv64.s b/vendor/golang.org/x/sys/unix/zsyscall_openbsd_riscv64.s
index da115f9a4..1e7f321e4 100644
--- a/vendor/golang.org/x/sys/unix/zsyscall_openbsd_riscv64.s
+++ b/vendor/golang.org/x/sys/unix/zsyscall_openbsd_riscv64.s
@@ -463,6 +463,11 @@ TEXT libc_mknodat_trampoline<>(SB),NOSPLIT,$0-0
 GLOBL	·libc_mknodat_trampoline_addr(SB), RODATA, $8
 DATA	·libc_mknodat_trampoline_addr(SB)/8, $libc_mknodat_trampoline<>(SB)
 
+TEXT libc_mount_trampoline<>(SB),NOSPLIT,$0-0
+	JMP	libc_mount(SB)
+GLOBL	·libc_mount_trampoline_addr(SB), RODATA, $8
+DATA	·libc_mount_trampoline_addr(SB)/8, $libc_mount_trampoline<>(SB)
+
 TEXT libc_nanosleep_trampoline<>(SB),NOSPLIT,$0-0
 	JMP	libc_nanosleep(SB)
 GLOBL	·libc_nanosleep_trampoline_addr(SB), RODATA, $8
diff --git a/vendor/golang.org/x/sys/unix/zsysnum_linux_386.go b/vendor/golang.org/x/sys/unix/zsysnum_linux_386.go
index 53aef5dc5..524b0820c 100644
--- a/vendor/golang.org/x/sys/unix/zsysnum_linux_386.go
+++ b/vendor/golang.org/x/sys/unix/zsysnum_linux_386.go
@@ -457,4 +457,5 @@ const (
 	SYS_LSM_GET_SELF_ATTR            = 459
 	SYS_LSM_SET_SELF_ATTR            = 460
 	SYS_LSM_LIST_MODULES             = 461
+	SYS_MSEAL                        = 462
 )
diff --git a/vendor/golang.org/x/sys/unix/zsysnum_linux_amd64.go b/vendor/golang.org/x/sys/unix/zsysnum_linux_amd64.go
index 71d524763..f485dbf45 100644
--- a/vendor/golang.org/x/sys/unix/zsysnum_linux_amd64.go
+++ b/vendor/golang.org/x/sys/unix/zsysnum_linux_amd64.go
@@ -341,6 +341,7 @@ const (
 	SYS_STATX                   = 332
 	SYS_IO_PGETEVENTS           = 333
 	SYS_RSEQ                    = 334
+	SYS_URETPROBE               = 335
 	SYS_PIDFD_SEND_SIGNAL       = 424
 	SYS_IO_URING_SETUP          = 425
 	SYS_IO_URING_ENTER          = 426
@@ -379,4 +380,5 @@ const (
 	SYS_LSM_GET_SELF_ATTR       = 459
 	SYS_LSM_SET_SELF_ATTR       = 460
 	SYS_LSM_LIST_MODULES        = 461
+	SYS_MSEAL                   = 462
 )
diff --git a/vendor/golang.org/x/sys/unix/zsysnum_linux_arm.go b/vendor/golang.org/x/sys/unix/zsysnum_linux_arm.go
index c74770613..70b35bf3b 100644
--- a/vendor/golang.org/x/sys/unix/zsysnum_linux_arm.go
+++ b/vendor/golang.org/x/sys/unix/zsysnum_linux_arm.go
@@ -421,4 +421,5 @@ const (
 	SYS_LSM_GET_SELF_ATTR            = 459
 	SYS_LSM_SET_SELF_ATTR            = 460
 	SYS_LSM_LIST_MODULES             = 461
+	SYS_MSEAL                        = 462
 )
diff --git a/vendor/golang.org/x/sys/unix/zsysnum_linux_arm64.go b/vendor/golang.org/x/sys/unix/zsysnum_linux_arm64.go
index f96e214f6..1893e2fe8 100644
--- a/vendor/golang.org/x/sys/unix/zsysnum_linux_arm64.go
+++ b/vendor/golang.org/x/sys/unix/zsysnum_linux_arm64.go
@@ -85,7 +85,7 @@ const (
 	SYS_SPLICE                  = 76
 	SYS_TEE                     = 77
 	SYS_READLINKAT              = 78
-	SYS_FSTATAT                 = 79
+	SYS_NEWFSTATAT              = 79
 	SYS_FSTAT                   = 80
 	SYS_SYNC                    = 81
 	SYS_FSYNC                   = 82
@@ -324,4 +324,5 @@ const (
 	SYS_LSM_GET_SELF_ATTR       = 459
 	SYS_LSM_SET_SELF_ATTR       = 460
 	SYS_LSM_LIST_MODULES        = 461
+	SYS_MSEAL                   = 462
 )
diff --git a/vendor/golang.org/x/sys/unix/zsysnum_linux_loong64.go b/vendor/golang.org/x/sys/unix/zsysnum_linux_loong64.go
index 28425346c..16a4017da 100644
--- a/vendor/golang.org/x/sys/unix/zsysnum_linux_loong64.go
+++ b/vendor/golang.org/x/sys/unix/zsysnum_linux_loong64.go
@@ -84,6 +84,8 @@ const (
 	SYS_SPLICE                  = 76
 	SYS_TEE                     = 77
 	SYS_READLINKAT              = 78
+	SYS_NEWFSTATAT              = 79
+	SYS_FSTAT                   = 80
 	SYS_SYNC                    = 81
 	SYS_FSYNC                   = 82
 	SYS_FDATASYNC               = 83
@@ -318,4 +320,5 @@ const (
 	SYS_LSM_GET_SELF_ATTR       = 459
 	SYS_LSM_SET_SELF_ATTR       = 460
 	SYS_LSM_LIST_MODULES        = 461
+	SYS_MSEAL                   = 462
 )
diff --git a/vendor/golang.org/x/sys/unix/zsysnum_linux_mips.go b/vendor/golang.org/x/sys/unix/zsysnum_linux_mips.go
index d0953018d..7e567f1ef 100644
--- a/vendor/golang.org/x/sys/unix/zsysnum_linux_mips.go
+++ b/vendor/golang.org/x/sys/unix/zsysnum_linux_mips.go
@@ -441,4 +441,5 @@ const (
 	SYS_LSM_GET_SELF_ATTR            = 4459
 	SYS_LSM_SET_SELF_ATTR            = 4460
 	SYS_LSM_LIST_MODULES             = 4461
+	SYS_MSEAL                        = 4462
 )
diff --git a/vendor/golang.org/x/sys/unix/zsysnum_linux_mips64.go b/vendor/golang.org/x/sys/unix/zsysnum_linux_mips64.go
index 295c7f4b8..38ae55e5e 100644
--- a/vendor/golang.org/x/sys/unix/zsysnum_linux_mips64.go
+++ b/vendor/golang.org/x/sys/unix/zsysnum_linux_mips64.go
@@ -371,4 +371,5 @@ const (
 	SYS_LSM_GET_SELF_ATTR       = 5459
 	SYS_LSM_SET_SELF_ATTR       = 5460
 	SYS_LSM_LIST_MODULES        = 5461
+	SYS_MSEAL                   = 5462
 )
diff --git a/vendor/golang.org/x/sys/unix/zsysnum_linux_mips64le.go b/vendor/golang.org/x/sys/unix/zsysnum_linux_mips64le.go
index d1a9eaca7..55e92e60a 100644
--- a/vendor/golang.org/x/sys/unix/zsysnum_linux_mips64le.go
+++ b/vendor/golang.org/x/sys/unix/zsysnum_linux_mips64le.go
@@ -371,4 +371,5 @@ const (
 	SYS_LSM_GET_SELF_ATTR       = 5459
 	SYS_LSM_SET_SELF_ATTR       = 5460
 	SYS_LSM_LIST_MODULES        = 5461
+	SYS_MSEAL                   = 5462
 )
diff --git a/vendor/golang.org/x/sys/unix/zsysnum_linux_mipsle.go b/vendor/golang.org/x/sys/unix/zsysnum_linux_mipsle.go
index bec157c39..60658d6a0 100644
--- a/vendor/golang.org/x/sys/unix/zsysnum_linux_mipsle.go
+++ b/vendor/golang.org/x/sys/unix/zsysnum_linux_mipsle.go
@@ -441,4 +441,5 @@ const (
 	SYS_LSM_GET_SELF_ATTR            = 4459
 	SYS_LSM_SET_SELF_ATTR            = 4460
 	SYS_LSM_LIST_MODULES             = 4461
+	SYS_MSEAL                        = 4462
 )
diff --git a/vendor/golang.org/x/sys/unix/zsysnum_linux_ppc.go b/vendor/golang.org/x/sys/unix/zsysnum_linux_ppc.go
index 7ee7bdc43..e203e8a7e 100644
--- a/vendor/golang.org/x/sys/unix/zsysnum_linux_ppc.go
+++ b/vendor/golang.org/x/sys/unix/zsysnum_linux_ppc.go
@@ -448,4 +448,5 @@ const (
 	SYS_LSM_GET_SELF_ATTR            = 459
 	SYS_LSM_SET_SELF_ATTR            = 460
 	SYS_LSM_LIST_MODULES             = 461
+	SYS_MSEAL                        = 462
 )
diff --git a/vendor/golang.org/x/sys/unix/zsysnum_linux_ppc64.go b/vendor/golang.org/x/sys/unix/zsysnum_linux_ppc64.go
index fad1f25b4..5944b97d5 100644
--- a/vendor/golang.org/x/sys/unix/zsysnum_linux_ppc64.go
+++ b/vendor/golang.org/x/sys/unix/zsysnum_linux_ppc64.go
@@ -420,4 +420,5 @@ const (
 	SYS_LSM_GET_SELF_ATTR       = 459
 	SYS_LSM_SET_SELF_ATTR       = 460
 	SYS_LSM_LIST_MODULES        = 461
+	SYS_MSEAL                   = 462
 )
diff --git a/vendor/golang.org/x/sys/unix/zsysnum_linux_ppc64le.go b/vendor/golang.org/x/sys/unix/zsysnum_linux_ppc64le.go
index 7d3e16357..c66d416da 100644
--- a/vendor/golang.org/x/sys/unix/zsysnum_linux_ppc64le.go
+++ b/vendor/golang.org/x/sys/unix/zsysnum_linux_ppc64le.go
@@ -420,4 +420,5 @@ const (
 	SYS_LSM_GET_SELF_ATTR       = 459
 	SYS_LSM_SET_SELF_ATTR       = 460
 	SYS_LSM_LIST_MODULES        = 461
+	SYS_MSEAL                   = 462
 )
diff --git a/vendor/golang.org/x/sys/unix/zsysnum_linux_riscv64.go b/vendor/golang.org/x/sys/unix/zsysnum_linux_riscv64.go
index 0ed53ad9f..a5459e766 100644
--- a/vendor/golang.org/x/sys/unix/zsysnum_linux_riscv64.go
+++ b/vendor/golang.org/x/sys/unix/zsysnum_linux_riscv64.go
@@ -84,7 +84,7 @@ const (
 	SYS_SPLICE                  = 76
 	SYS_TEE                     = 77
 	SYS_READLINKAT              = 78
-	SYS_FSTATAT                 = 79
+	SYS_NEWFSTATAT              = 79
 	SYS_FSTAT                   = 80
 	SYS_SYNC                    = 81
 	SYS_FSYNC                   = 82
@@ -325,4 +325,5 @@ const (
 	SYS_LSM_GET_SELF_ATTR       = 459
 	SYS_LSM_SET_SELF_ATTR       = 460
 	SYS_LSM_LIST_MODULES        = 461
+	SYS_MSEAL                   = 462
 )
diff --git a/vendor/golang.org/x/sys/unix/zsysnum_linux_s390x.go b/vendor/golang.org/x/sys/unix/zsysnum_linux_s390x.go
index 2fba04ad5..01d86825b 100644
--- a/vendor/golang.org/x/sys/unix/zsysnum_linux_s390x.go
+++ b/vendor/golang.org/x/sys/unix/zsysnum_linux_s390x.go
@@ -386,4 +386,5 @@ const (
 	SYS_LSM_GET_SELF_ATTR       = 459
 	SYS_LSM_SET_SELF_ATTR       = 460
 	SYS_LSM_LIST_MODULES        = 461
+	SYS_MSEAL                   = 462
 )
diff --git a/vendor/golang.org/x/sys/unix/zsysnum_linux_sparc64.go b/vendor/golang.org/x/sys/unix/zsysnum_linux_sparc64.go
index 621d00d74..7b703e77c 100644
--- a/vendor/golang.org/x/sys/unix/zsysnum_linux_sparc64.go
+++ b/vendor/golang.org/x/sys/unix/zsysnum_linux_sparc64.go
@@ -399,4 +399,5 @@ const (
 	SYS_LSM_GET_SELF_ATTR       = 459
 	SYS_LSM_SET_SELF_ATTR       = 460
 	SYS_LSM_LIST_MODULES        = 461
+	SYS_MSEAL                   = 462
 )
diff --git a/vendor/golang.org/x/sys/unix/ztypes_darwin_amd64.go b/vendor/golang.org/x/sys/unix/ztypes_darwin_amd64.go
index 091d107f3..17c53bd9b 100644
--- a/vendor/golang.org/x/sys/unix/ztypes_darwin_amd64.go
+++ b/vendor/golang.org/x/sys/unix/ztypes_darwin_amd64.go
@@ -306,6 +306,19 @@ type XVSockPgen struct {
 
 type _Socklen uint32
 
+type SaeAssocID uint32
+
+type SaeConnID uint32
+
+type SaEndpoints struct {
+	Srcif      uint32
+	Srcaddr    *RawSockaddr
+	Srcaddrlen uint32
+	Dstaddr    *RawSockaddr
+	Dstaddrlen uint32
+	_          [4]byte
+}
+
 type Xucred struct {
 	Version uint32
 	Uid     uint32
@@ -449,11 +462,14 @@ type FdSet struct {
 
 const (
 	SizeofIfMsghdr    = 0x70
+	SizeofIfMsghdr2   = 0xa0
 	SizeofIfData      = 0x60
+	SizeofIfData64    = 0x80
 	SizeofIfaMsghdr   = 0x14
 	SizeofIfmaMsghdr  = 0x10
 	SizeofIfmaMsghdr2 = 0x14
 	SizeofRtMsghdr    = 0x5c
+	SizeofRtMsghdr2   = 0x5c
 	SizeofRtMetrics   = 0x38
 )
 
@@ -467,6 +483,20 @@ type IfMsghdr struct {
 	Data    IfData
 }
 
+type IfMsghdr2 struct {
+	Msglen     uint16
+	Version    uint8
+	Type       uint8
+	Addrs      int32
+	Flags      int32
+	Index      uint16
+	Snd_len    int32
+	Snd_maxlen int32
+	Snd_drops  int32
+	Timer      int32
+	Data       IfData64
+}
+
 type IfData struct {
 	Type       uint8
 	Typelen    uint8
@@ -499,6 +529,34 @@ type IfData struct {
 	Reserved2  uint32
 }
 
+type IfData64 struct {
+	Type       uint8
+	Typelen    uint8
+	Physical   uint8
+	Addrlen    uint8
+	Hdrlen     uint8
+	Recvquota  uint8
+	Xmitquota  uint8
+	Unused1    uint8
+	Mtu        uint32
+	Metric     uint32
+	Baudrate   uint64
+	Ipackets   uint64
+	Ierrors    uint64
+	Opackets   uint64
+	Oerrors    uint64
+	Collisions uint64
+	Ibytes     uint64
+	Obytes     uint64
+	Imcasts    uint64
+	Omcasts    uint64
+	Iqdrops    uint64
+	Noproto    uint64
+	Recvtiming uint32
+	Xmittiming uint32
+	Lastchange Timeval32
+}
+
 type IfaMsghdr struct {
 	Msglen  uint16
 	Version uint8
@@ -544,6 +602,21 @@ type RtMsghdr struct {
 	Rmx     RtMetrics
 }
 
+type RtMsghdr2 struct {
+	Msglen      uint16
+	Version     uint8
+	Type        uint8
+	Index       uint16
+	Flags       int32
+	Addrs       int32
+	Refcnt      int32
+	Parentflags int32
+	Reserved    int32
+	Use         int32
+	Inits       uint32
+	Rmx         RtMetrics
+}
+
 type RtMetrics struct {
 	Locks    uint32
 	Mtu      uint32
diff --git a/vendor/golang.org/x/sys/unix/ztypes_darwin_arm64.go b/vendor/golang.org/x/sys/unix/ztypes_darwin_arm64.go
index 28ff4ef74..2392226a7 100644
--- a/vendor/golang.org/x/sys/unix/ztypes_darwin_arm64.go
+++ b/vendor/golang.org/x/sys/unix/ztypes_darwin_arm64.go
@@ -306,6 +306,19 @@ type XVSockPgen struct {
 
 type _Socklen uint32
 
+type SaeAssocID uint32
+
+type SaeConnID uint32
+
+type SaEndpoints struct {
+	Srcif      uint32
+	Srcaddr    *RawSockaddr
+	Srcaddrlen uint32
+	Dstaddr    *RawSockaddr
+	Dstaddrlen uint32
+	_          [4]byte
+}
+
 type Xucred struct {
 	Version uint32
 	Uid     uint32
@@ -449,11 +462,14 @@ type FdSet struct {
 
 const (
 	SizeofIfMsghdr    = 0x70
+	SizeofIfMsghdr2   = 0xa0
 	SizeofIfData      = 0x60
+	SizeofIfData64    = 0x80
 	SizeofIfaMsghdr   = 0x14
 	SizeofIfmaMsghdr  = 0x10
 	SizeofIfmaMsghdr2 = 0x14
 	SizeofRtMsghdr    = 0x5c
+	SizeofRtMsghdr2   = 0x5c
 	SizeofRtMetrics   = 0x38
 )
 
@@ -467,6 +483,20 @@ type IfMsghdr struct {
 	Data    IfData
 }
 
+type IfMsghdr2 struct {
+	Msglen     uint16
+	Version    uint8
+	Type       uint8
+	Addrs      int32
+	Flags      int32
+	Index      uint16
+	Snd_len    int32
+	Snd_maxlen int32
+	Snd_drops  int32
+	Timer      int32
+	Data       IfData64
+}
+
 type IfData struct {
 	Type       uint8
 	Typelen    uint8
@@ -499,6 +529,34 @@ type IfData struct {
 	Reserved2  uint32
 }
 
+type IfData64 struct {
+	Type       uint8
+	Typelen    uint8
+	Physical   uint8
+	Addrlen    uint8
+	Hdrlen     uint8
+	Recvquota  uint8
+	Xmitquota  uint8
+	Unused1    uint8
+	Mtu        uint32
+	Metric     uint32
+	Baudrate   uint64
+	Ipackets   uint64
+	Ierrors    uint64
+	Opackets   uint64
+	Oerrors    uint64
+	Collisions uint64
+	Ibytes     uint64
+	Obytes     uint64
+	Imcasts    uint64
+	Omcasts    uint64
+	Iqdrops    uint64
+	Noproto    uint64
+	Recvtiming uint32
+	Xmittiming uint32
+	Lastchange Timeval32
+}
+
 type IfaMsghdr struct {
 	Msglen  uint16
 	Version uint8
@@ -544,6 +602,21 @@ type RtMsghdr struct {
 	Rmx     RtMetrics
 }
 
+type RtMsghdr2 struct {
+	Msglen      uint16
+	Version     uint8
+	Type        uint8
+	Index       uint16
+	Flags       int32
+	Addrs       int32
+	Refcnt      int32
+	Parentflags int32
+	Reserved    int32
+	Use         int32
+	Inits       uint32
+	Rmx         RtMetrics
+}
+
 type RtMetrics struct {
 	Locks    uint32
 	Mtu      uint32
diff --git a/vendor/golang.org/x/sys/unix/ztypes_freebsd_386.go b/vendor/golang.org/x/sys/unix/ztypes_freebsd_386.go
index 6cbd094a3..51e13eb05 100644
--- a/vendor/golang.org/x/sys/unix/ztypes_freebsd_386.go
+++ b/vendor/golang.org/x/sys/unix/ztypes_freebsd_386.go
@@ -625,6 +625,7 @@ const (
 	POLLRDNORM   = 0x40
 	POLLWRBAND   = 0x100
 	POLLWRNORM   = 0x4
+	POLLRDHUP    = 0x4000
 )
 
 type CapRights struct {
diff --git a/vendor/golang.org/x/sys/unix/ztypes_freebsd_amd64.go b/vendor/golang.org/x/sys/unix/ztypes_freebsd_amd64.go
index 7c03b6ee7..d002d8ef3 100644
--- a/vendor/golang.org/x/sys/unix/ztypes_freebsd_amd64.go
+++ b/vendor/golang.org/x/sys/unix/ztypes_freebsd_amd64.go
@@ -630,6 +630,7 @@ const (
 	POLLRDNORM   = 0x40
 	POLLWRBAND   = 0x100
 	POLLWRNORM   = 0x4
+	POLLRDHUP    = 0x4000
 )
 
 type CapRights struct {
diff --git a/vendor/golang.org/x/sys/unix/ztypes_freebsd_arm.go b/vendor/golang.org/x/sys/unix/ztypes_freebsd_arm.go
index 422107ee8..3f863d898 100644
--- a/vendor/golang.org/x/sys/unix/ztypes_freebsd_arm.go
+++ b/vendor/golang.org/x/sys/unix/ztypes_freebsd_arm.go
@@ -616,6 +616,7 @@ const (
 	POLLRDNORM   = 0x40
 	POLLWRBAND   = 0x100
 	POLLWRNORM   = 0x4
+	POLLRDHUP    = 0x4000
 )
 
 type CapRights struct {
diff --git a/vendor/golang.org/x/sys/unix/ztypes_freebsd_arm64.go b/vendor/golang.org/x/sys/unix/ztypes_freebsd_arm64.go
index 505a12acf..61c729310 100644
--- a/vendor/golang.org/x/sys/unix/ztypes_freebsd_arm64.go
+++ b/vendor/golang.org/x/sys/unix/ztypes_freebsd_arm64.go
@@ -610,6 +610,7 @@ const (
 	POLLRDNORM   = 0x40
 	POLLWRBAND   = 0x100
 	POLLWRNORM   = 0x4
+	POLLRDHUP    = 0x4000
 )
 
 type CapRights struct {
diff --git a/vendor/golang.org/x/sys/unix/ztypes_freebsd_riscv64.go b/vendor/golang.org/x/sys/unix/ztypes_freebsd_riscv64.go
index cc986c790..b5d17414f 100644
--- a/vendor/golang.org/x/sys/unix/ztypes_freebsd_riscv64.go
+++ b/vendor/golang.org/x/sys/unix/ztypes_freebsd_riscv64.go
@@ -612,6 +612,7 @@ const (
 	POLLRDNORM   = 0x40
 	POLLWRBAND   = 0x100
 	POLLWRNORM   = 0x4
+	POLLRDHUP    = 0x4000
 )
 
 type CapRights struct {
diff --git a/vendor/golang.org/x/sys/unix/ztypes_linux.go b/vendor/golang.org/x/sys/unix/ztypes_linux.go
index 4740b8348..5537148dc 100644
--- a/vendor/golang.org/x/sys/unix/ztypes_linux.go
+++ b/vendor/golang.org/x/sys/unix/ztypes_linux.go
@@ -87,30 +87,35 @@ type StatxTimestamp struct {
 }
 
 type Statx_t struct {
-	Mask             uint32
-	Blksize          uint32
-	Attributes       uint64
-	Nlink            uint32
-	Uid              uint32
-	Gid              uint32
-	Mode             uint16
-	_                [1]uint16
-	Ino              uint64
-	Size             uint64
-	Blocks           uint64
-	Attributes_mask  uint64
-	Atime            StatxTimestamp
-	Btime            StatxTimestamp
-	Ctime            StatxTimestamp
-	Mtime            StatxTimestamp
-	Rdev_major       uint32
-	Rdev_minor       uint32
-	Dev_major        uint32
-	Dev_minor        uint32
-	Mnt_id           uint64
-	Dio_mem_align    uint32
-	Dio_offset_align uint32
-	_                [12]uint64
+	Mask                      uint32
+	Blksize                   uint32
+	Attributes                uint64
+	Nlink                     uint32
+	Uid                       uint32
+	Gid                       uint32
+	Mode                      uint16
+	_                         [1]uint16
+	Ino                       uint64
+	Size                      uint64
+	Blocks                    uint64
+	Attributes_mask           uint64
+	Atime                     StatxTimestamp
+	Btime                     StatxTimestamp
+	Ctime                     StatxTimestamp
+	Mtime                     StatxTimestamp
+	Rdev_major                uint32
+	Rdev_minor                uint32
+	Dev_major                 uint32
+	Dev_minor                 uint32
+	Mnt_id                    uint64
+	Dio_mem_align             uint32
+	Dio_offset_align          uint32
+	Subvol                    uint64
+	Atomic_write_unit_min     uint32
+	Atomic_write_unit_max     uint32
+	Atomic_write_segments_max uint32
+	_                         [1]uint32
+	_                         [9]uint64
 }
 
 type Fsid struct {
@@ -515,6 +520,29 @@ type TCPInfo struct {
 	Total_rto_time       uint32
 }
 
+type TCPVegasInfo struct {
+	Enabled uint32
+	Rttcnt  uint32
+	Rtt     uint32
+	Minrtt  uint32
+}
+
+type TCPDCTCPInfo struct {
+	Enabled  uint16
+	Ce_state uint16
+	Alpha    uint32
+	Ab_ecn   uint32
+	Ab_tot   uint32
+}
+
+type TCPBBRInfo struct {
+	Bw_lo       uint32
+	Bw_hi       uint32
+	Min_rtt     uint32
+	Pacing_gain uint32
+	Cwnd_gain   uint32
+}
+
 type CanFilter struct {
 	Id   uint32
 	Mask uint32
@@ -556,6 +584,7 @@ const (
 	SizeofICMPv6Filter      = 0x20
 	SizeofUcred             = 0xc
 	SizeofTCPInfo           = 0xf8
+	SizeofTCPCCInfo         = 0x14
 	SizeofCanFilter         = 0x8
 	SizeofTCPRepairOpt      = 0x8
 )
@@ -1723,12 +1752,6 @@ const (
 	IFLA_IPVLAN_UNSPEC                         = 0x0
 	IFLA_IPVLAN_MODE                           = 0x1
 	IFLA_IPVLAN_FLAGS                          = 0x2
-	NETKIT_NEXT                                = -0x1
-	NETKIT_PASS                                = 0x0
-	NETKIT_DROP                                = 0x2
-	NETKIT_REDIRECT                            = 0x7
-	NETKIT_L2                                  = 0x0
-	NETKIT_L3                                  = 0x1
 	IFLA_NETKIT_UNSPEC                         = 0x0
 	IFLA_NETKIT_PEER_INFO                      = 0x1
 	IFLA_NETKIT_PRIMARY                        = 0x2
@@ -1767,6 +1790,7 @@ const (
 	IFLA_VXLAN_DF                              = 0x1d
 	IFLA_VXLAN_VNIFILTER                       = 0x1e
 	IFLA_VXLAN_LOCALBYPASS                     = 0x1f
+	IFLA_VXLAN_LABEL_POLICY                    = 0x20
 	IFLA_GENEVE_UNSPEC                         = 0x0
 	IFLA_GENEVE_ID                             = 0x1
 	IFLA_GENEVE_REMOTE                         = 0x2
@@ -1796,6 +1820,8 @@ const (
 	IFLA_GTP_ROLE                              = 0x4
 	IFLA_GTP_CREATE_SOCKETS                    = 0x5
 	IFLA_GTP_RESTART_COUNT                     = 0x6
+	IFLA_GTP_LOCAL                             = 0x7
+	IFLA_GTP_LOCAL6                            = 0x8
 	IFLA_BOND_UNSPEC                           = 0x0
 	IFLA_BOND_MODE                             = 0x1
 	IFLA_BOND_ACTIVE_SLAVE                     = 0x2
@@ -1828,6 +1854,7 @@ const (
 	IFLA_BOND_AD_LACP_ACTIVE                   = 0x1d
 	IFLA_BOND_MISSED_MAX                       = 0x1e
 	IFLA_BOND_NS_IP6_TARGET                    = 0x1f
+	IFLA_BOND_COUPLED_CONTROL                  = 0x20
 	IFLA_BOND_AD_INFO_UNSPEC                   = 0x0
 	IFLA_BOND_AD_INFO_AGGREGATOR               = 0x1
 	IFLA_BOND_AD_INFO_NUM_PORTS                = 0x2
@@ -1896,6 +1923,7 @@ const (
 	IFLA_HSR_SEQ_NR                            = 0x5
 	IFLA_HSR_VERSION                           = 0x6
 	IFLA_HSR_PROTOCOL                          = 0x7
+	IFLA_HSR_INTERLINK                         = 0x8
 	IFLA_STATS_UNSPEC                          = 0x0
 	IFLA_STATS_LINK_64                         = 0x1
 	IFLA_STATS_LINK_XSTATS                     = 0x2
@@ -1948,6 +1976,15 @@ const (
 	IFLA_DSA_MASTER                            = 0x1
 )
 
+const (
+	NETKIT_NEXT     = -0x1
+	NETKIT_PASS     = 0x0
+	NETKIT_DROP     = 0x2
+	NETKIT_REDIRECT = 0x7
+	NETKIT_L2       = 0x0
+	NETKIT_L3       = 0x1
+)
+
 const (
 	NF_INET_PRE_ROUTING  = 0x0
 	NF_INET_LOCAL_IN     = 0x1
@@ -2485,7 +2522,7 @@ type XDPMmapOffsets struct {
 type XDPUmemReg struct {
 	Addr            uint64
 	Len             uint64
-	Chunk_size      uint32
+	Size            uint32
 	Headroom        uint32
 	Flags           uint32
 	Tx_metadata_len uint32
@@ -2557,8 +2594,8 @@ const (
 	SOF_TIMESTAMPING_BIND_PHC     = 0x8000
 	SOF_TIMESTAMPING_OPT_ID_TCP   = 0x10000
 
-	SOF_TIMESTAMPING_LAST = 0x10000
-	SOF_TIMESTAMPING_MASK = 0x1ffff
+	SOF_TIMESTAMPING_LAST = 0x20000
+	SOF_TIMESTAMPING_MASK = 0x3ffff
 
 	SCM_TSTAMP_SND   = 0x0
 	SCM_TSTAMP_SCHED = 0x1
@@ -3473,7 +3510,7 @@ const (
 	DEVLINK_PORT_FN_ATTR_STATE                         = 0x2
 	DEVLINK_PORT_FN_ATTR_OPSTATE                       = 0x3
 	DEVLINK_PORT_FN_ATTR_CAPS                          = 0x4
-	DEVLINK_PORT_FUNCTION_ATTR_MAX                     = 0x5
+	DEVLINK_PORT_FUNCTION_ATTR_MAX                     = 0x6
 )
 
 type FsverityDigest struct {
@@ -3504,7 +3541,7 @@ type Nhmsg struct {
 type NexthopGrp struct {
 	Id     uint32
 	Weight uint8
-	Resvd1 uint8
+	High   uint8
 	Resvd2 uint16
 }
 
@@ -3765,7 +3802,7 @@ const (
 	ETHTOOL_MSG_PSE_GET                       = 0x24
 	ETHTOOL_MSG_PSE_SET                       = 0x25
 	ETHTOOL_MSG_RSS_GET                       = 0x26
-	ETHTOOL_MSG_USER_MAX                      = 0x2b
+	ETHTOOL_MSG_USER_MAX                      = 0x2d
 	ETHTOOL_MSG_KERNEL_NONE                   = 0x0
 	ETHTOOL_MSG_STRSET_GET_REPLY              = 0x1
 	ETHTOOL_MSG_LINKINFO_GET_REPLY            = 0x2
@@ -3805,12 +3842,15 @@ const (
 	ETHTOOL_MSG_MODULE_NTF                    = 0x24
 	ETHTOOL_MSG_PSE_GET_REPLY                 = 0x25
 	ETHTOOL_MSG_RSS_GET_REPLY                 = 0x26
-	ETHTOOL_MSG_KERNEL_MAX                    = 0x2b
+	ETHTOOL_MSG_KERNEL_MAX                    = 0x2e
+	ETHTOOL_FLAG_COMPACT_BITSETS              = 0x1
+	ETHTOOL_FLAG_OMIT_REPLY                   = 0x2
+	ETHTOOL_FLAG_STATS                        = 0x4
 	ETHTOOL_A_HEADER_UNSPEC                   = 0x0
 	ETHTOOL_A_HEADER_DEV_INDEX                = 0x1
 	ETHTOOL_A_HEADER_DEV_NAME                 = 0x2
 	ETHTOOL_A_HEADER_FLAGS                    = 0x3
-	ETHTOOL_A_HEADER_MAX                      = 0x3
+	ETHTOOL_A_HEADER_MAX                      = 0x4
 	ETHTOOL_A_BITSET_BIT_UNSPEC               = 0x0
 	ETHTOOL_A_BITSET_BIT_INDEX                = 0x1
 	ETHTOOL_A_BITSET_BIT_NAME                 = 0x2
@@ -3947,7 +3987,7 @@ const (
 	ETHTOOL_A_COALESCE_RATE_SAMPLE_INTERVAL   = 0x17
 	ETHTOOL_A_COALESCE_USE_CQE_MODE_TX        = 0x18
 	ETHTOOL_A_COALESCE_USE_CQE_MODE_RX        = 0x19
-	ETHTOOL_A_COALESCE_MAX                    = 0x1c
+	ETHTOOL_A_COALESCE_MAX                    = 0x1e
 	ETHTOOL_A_PAUSE_UNSPEC                    = 0x0
 	ETHTOOL_A_PAUSE_HEADER                    = 0x1
 	ETHTOOL_A_PAUSE_AUTONEG                   = 0x2
@@ -3975,7 +4015,7 @@ const (
 	ETHTOOL_A_TSINFO_TX_TYPES                 = 0x3
 	ETHTOOL_A_TSINFO_RX_FILTERS               = 0x4
 	ETHTOOL_A_TSINFO_PHC_INDEX                = 0x5
-	ETHTOOL_A_TSINFO_MAX                      = 0x5
+	ETHTOOL_A_TSINFO_MAX                      = 0x6
 	ETHTOOL_A_CABLE_TEST_UNSPEC               = 0x0
 	ETHTOOL_A_CABLE_TEST_HEADER               = 0x1
 	ETHTOOL_A_CABLE_TEST_MAX                  = 0x1
@@ -3991,11 +4031,11 @@ const (
 	ETHTOOL_A_CABLE_RESULT_UNSPEC             = 0x0
 	ETHTOOL_A_CABLE_RESULT_PAIR               = 0x1
 	ETHTOOL_A_CABLE_RESULT_CODE               = 0x2
-	ETHTOOL_A_CABLE_RESULT_MAX                = 0x2
+	ETHTOOL_A_CABLE_RESULT_MAX                = 0x3
 	ETHTOOL_A_CABLE_FAULT_LENGTH_UNSPEC       = 0x0
 	ETHTOOL_A_CABLE_FAULT_LENGTH_PAIR         = 0x1
 	ETHTOOL_A_CABLE_FAULT_LENGTH_CM           = 0x2
-	ETHTOOL_A_CABLE_FAULT_LENGTH_MAX          = 0x2
+	ETHTOOL_A_CABLE_FAULT_LENGTH_MAX          = 0x3
 	ETHTOOL_A_CABLE_TEST_NTF_STATUS_UNSPEC    = 0x0
 	ETHTOOL_A_CABLE_TEST_NTF_STATUS_STARTED   = 0x1
 	ETHTOOL_A_CABLE_TEST_NTF_STATUS_COMPLETED = 0x2
@@ -4078,6 +4118,107 @@ type EthtoolDrvinfo struct {
 	Regdump_len  uint32
 }
 
+type EthtoolTsInfo struct {
+	Cmd             uint32
+	So_timestamping uint32
+	Phc_index       int32
+	Tx_types        uint32
+	Tx_reserved     [3]uint32
+	Rx_filters      uint32
+	Rx_reserved     [3]uint32
+}
+
+type HwTstampConfig struct {
+	Flags     int32
+	Tx_type   int32
+	Rx_filter int32
+}
+
+const (
+	HWTSTAMP_FILTER_NONE            = 0x0
+	HWTSTAMP_FILTER_ALL             = 0x1
+	HWTSTAMP_FILTER_SOME            = 0x2
+	HWTSTAMP_FILTER_PTP_V1_L4_EVENT = 0x3
+	HWTSTAMP_FILTER_PTP_V2_L4_EVENT = 0x6
+	HWTSTAMP_FILTER_PTP_V2_L2_EVENT = 0x9
+	HWTSTAMP_FILTER_PTP_V2_EVENT    = 0xc
+)
+
+const (
+	HWTSTAMP_TX_OFF          = 0x0
+	HWTSTAMP_TX_ON           = 0x1
+	HWTSTAMP_TX_ONESTEP_SYNC = 0x2
+)
+
+type (
+	PtpClockCaps struct {
+		Max_adj            int32
+		N_alarm            int32
+		N_ext_ts           int32
+		N_per_out          int32
+		Pps                int32
+		N_pins             int32
+		Cross_timestamping int32
+		Adjust_phase       int32
+		Max_phase_adj      int32
+		Rsv                [11]int32
+	}
+	PtpClockTime struct {
+		Sec      int64
+		Nsec     uint32
+		Reserved uint32
+	}
+	PtpExttsEvent struct {
+		T     PtpClockTime
+		Index uint32
+		Flags uint32
+		Rsv   [2]uint32
+	}
+	PtpExttsRequest struct {
+		Index uint32
+		Flags uint32
+		Rsv   [2]uint32
+	}
+	PtpPeroutRequest struct {
+		StartOrPhase PtpClockTime
+		Period       PtpClockTime
+		Index        uint32
+		Flags        uint32
+		On           PtpClockTime
+	}
+	PtpPinDesc struct {
+		Name  [64]byte
+		Index uint32
+		Func  uint32
+		Chan  uint32
+		Rsv   [5]uint32
+	}
+	PtpSysOffset struct {
+		Samples uint32
+		Rsv     [3]uint32
+		Ts      [51]PtpClockTime
+	}
+	PtpSysOffsetExtended struct {
+		Samples uint32
+		Clockid int32
+		Rsv     [2]uint32
+		Ts      [25][3]PtpClockTime
+	}
+	PtpSysOffsetPrecise struct {
+		Device   PtpClockTime
+		Realtime PtpClockTime
+		Monoraw  PtpClockTime
+		Rsv      [4]uint32
+	}
+)
+
+const (
+	PTP_PF_NONE    = 0x0
+	PTP_PF_EXTTS   = 0x1
+	PTP_PF_PEROUT  = 0x2
+	PTP_PF_PHYSYNC = 0x3
+)
+
 type (
 	HIDRawReportDescriptor struct {
 		Size  uint32
@@ -4259,6 +4400,7 @@ const (
 type LandlockRulesetAttr struct {
 	Access_fs  uint64
 	Access_net uint64
+	Scoped     uint64
 }
 
 type LandlockPathBeneathAttr struct {
@@ -4605,7 +4747,7 @@ const (
 	NL80211_ATTR_MAC_HINT                                   = 0xc8
 	NL80211_ATTR_MAC_MASK                                   = 0xd7
 	NL80211_ATTR_MAX_AP_ASSOC_STA                           = 0xca
-	NL80211_ATTR_MAX                                        = 0x14a
+	NL80211_ATTR_MAX                                        = 0x14c
 	NL80211_ATTR_MAX_CRIT_PROT_DURATION                     = 0xb4
 	NL80211_ATTR_MAX_CSA_COUNTERS                           = 0xce
 	NL80211_ATTR_MAX_MATCH_SETS                             = 0x85
@@ -5209,7 +5351,7 @@ const (
 	NL80211_FREQUENCY_ATTR_GO_CONCURRENT                    = 0xf
 	NL80211_FREQUENCY_ATTR_INDOOR_ONLY                      = 0xe
 	NL80211_FREQUENCY_ATTR_IR_CONCURRENT                    = 0xf
-	NL80211_FREQUENCY_ATTR_MAX                              = 0x20
+	NL80211_FREQUENCY_ATTR_MAX                              = 0x21
 	NL80211_FREQUENCY_ATTR_MAX_TX_POWER                     = 0x6
 	NL80211_FREQUENCY_ATTR_NO_10MHZ                         = 0x11
 	NL80211_FREQUENCY_ATTR_NO_160MHZ                        = 0xc
diff --git a/vendor/golang.org/x/sys/unix/ztypes_linux_riscv64.go b/vendor/golang.org/x/sys/unix/ztypes_linux_riscv64.go
index 15adc0414..ad05b51a6 100644
--- a/vendor/golang.org/x/sys/unix/ztypes_linux_riscv64.go
+++ b/vendor/golang.org/x/sys/unix/ztypes_linux_riscv64.go
@@ -727,6 +727,37 @@ const (
 	RISCV_HWPROBE_EXT_ZBA                = 0x8
 	RISCV_HWPROBE_EXT_ZBB                = 0x10
 	RISCV_HWPROBE_EXT_ZBS                = 0x20
+	RISCV_HWPROBE_EXT_ZICBOZ             = 0x40
+	RISCV_HWPROBE_EXT_ZBC                = 0x80
+	RISCV_HWPROBE_EXT_ZBKB               = 0x100
+	RISCV_HWPROBE_EXT_ZBKC               = 0x200
+	RISCV_HWPROBE_EXT_ZBKX               = 0x400
+	RISCV_HWPROBE_EXT_ZKND               = 0x800
+	RISCV_HWPROBE_EXT_ZKNE               = 0x1000
+	RISCV_HWPROBE_EXT_ZKNH               = 0x2000
+	RISCV_HWPROBE_EXT_ZKSED              = 0x4000
+	RISCV_HWPROBE_EXT_ZKSH               = 0x8000
+	RISCV_HWPROBE_EXT_ZKT                = 0x10000
+	RISCV_HWPROBE_EXT_ZVBB               = 0x20000
+	RISCV_HWPROBE_EXT_ZVBC               = 0x40000
+	RISCV_HWPROBE_EXT_ZVKB               = 0x80000
+	RISCV_HWPROBE_EXT_ZVKG               = 0x100000
+	RISCV_HWPROBE_EXT_ZVKNED             = 0x200000
+	RISCV_HWPROBE_EXT_ZVKNHA             = 0x400000
+	RISCV_HWPROBE_EXT_ZVKNHB             = 0x800000
+	RISCV_HWPROBE_EXT_ZVKSED             = 0x1000000
+	RISCV_HWPROBE_EXT_ZVKSH              = 0x2000000
+	RISCV_HWPROBE_EXT_ZVKT               = 0x4000000
+	RISCV_HWPROBE_EXT_ZFH                = 0x8000000
+	RISCV_HWPROBE_EXT_ZFHMIN             = 0x10000000
+	RISCV_HWPROBE_EXT_ZIHINTNTL          = 0x20000000
+	RISCV_HWPROBE_EXT_ZVFH               = 0x40000000
+	RISCV_HWPROBE_EXT_ZVFHMIN            = 0x80000000
+	RISCV_HWPROBE_EXT_ZFA                = 0x100000000
+	RISCV_HWPROBE_EXT_ZTSO               = 0x200000000
+	RISCV_HWPROBE_EXT_ZACAS              = 0x400000000
+	RISCV_HWPROBE_EXT_ZICOND             = 0x800000000
+	RISCV_HWPROBE_EXT_ZIHINTPAUSE        = 0x1000000000
 	RISCV_HWPROBE_KEY_CPUPERF_0          = 0x5
 	RISCV_HWPROBE_MISALIGNED_UNKNOWN     = 0x0
 	RISCV_HWPROBE_MISALIGNED_EMULATED    = 0x1
@@ -734,4 +765,6 @@ const (
 	RISCV_HWPROBE_MISALIGNED_FAST        = 0x3
 	RISCV_HWPROBE_MISALIGNED_UNSUPPORTED = 0x4
 	RISCV_HWPROBE_MISALIGNED_MASK        = 0x7
+	RISCV_HWPROBE_KEY_ZICBOZ_BLOCK_SIZE  = 0x6
+	RISCV_HWPROBE_WHICH_CPUS             = 0x1
 )
diff --git a/vendor/golang.org/x/sys/unix/ztypes_zos_s390x.go b/vendor/golang.org/x/sys/unix/ztypes_zos_s390x.go
index d9a13af46..2e5d5a443 100644
--- a/vendor/golang.org/x/sys/unix/ztypes_zos_s390x.go
+++ b/vendor/golang.org/x/sys/unix/ztypes_zos_s390x.go
@@ -377,6 +377,12 @@ type Flock_t struct {
 	Pid    int32
 }
 
+type F_cnvrt struct {
+	Cvtcmd int32
+	Pccsid int16
+	Fccsid int16
+}
+
 type Termios struct {
 	Cflag uint32
 	Iflag uint32
diff --git a/vendor/golang.org/x/sys/windows/dll_windows.go b/vendor/golang.org/x/sys/windows/dll_windows.go
index 115341fba..4e613cf63 100644
--- a/vendor/golang.org/x/sys/windows/dll_windows.go
+++ b/vendor/golang.org/x/sys/windows/dll_windows.go
@@ -65,7 +65,7 @@ func LoadDLL(name string) (dll *DLL, err error) {
 	return d, nil
 }
 
-// MustLoadDLL is like LoadDLL but panics if load operation failes.
+// MustLoadDLL is like LoadDLL but panics if load operation fails.
 func MustLoadDLL(name string) *DLL {
 	d, e := LoadDLL(name)
 	if e != nil {
diff --git a/vendor/golang.org/x/sys/windows/security_windows.go b/vendor/golang.org/x/sys/windows/security_windows.go
index 97651b5bd..b6e1ab76f 100644
--- a/vendor/golang.org/x/sys/windows/security_windows.go
+++ b/vendor/golang.org/x/sys/windows/security_windows.go
@@ -1179,7 +1179,7 @@ type OBJECTS_AND_NAME struct {
 //sys	makeSelfRelativeSD(absoluteSD *SECURITY_DESCRIPTOR, selfRelativeSD *SECURITY_DESCRIPTOR, selfRelativeSDSize *uint32) (err error) = advapi32.MakeSelfRelativeSD
 
 //sys	setEntriesInAcl(countExplicitEntries uint32, explicitEntries *EXPLICIT_ACCESS, oldACL *ACL, newACL **ACL) (ret error) = advapi32.SetEntriesInAclW
-//sys	GetAce(acl *ACL, aceIndex uint32, pAce **ACCESS_ALLOWED_ACE) (ret error) = advapi32.GetAce
+//sys	GetAce(acl *ACL, aceIndex uint32, pAce **ACCESS_ALLOWED_ACE) (err error) = advapi32.GetAce
 
 // Control returns the security descriptor control bits.
 func (sd *SECURITY_DESCRIPTOR) Control() (control SECURITY_DESCRIPTOR_CONTROL, revision uint32, err error) {
diff --git a/vendor/golang.org/x/sys/windows/syscall_windows.go b/vendor/golang.org/x/sys/windows/syscall_windows.go
index 6525c62f3..4a3254386 100644
--- a/vendor/golang.org/x/sys/windows/syscall_windows.go
+++ b/vendor/golang.org/x/sys/windows/syscall_windows.go
@@ -17,8 +17,10 @@ import (
 	"unsafe"
 )
 
-type Handle uintptr
-type HWND uintptr
+type (
+	Handle uintptr
+	HWND   uintptr
+)
 
 const (
 	InvalidHandle = ^Handle(0)
@@ -166,6 +168,8 @@ func NewCallbackCDecl(fn interface{}) uintptr {
 //sys	CreateNamedPipe(name *uint16, flags uint32, pipeMode uint32, maxInstances uint32, outSize uint32, inSize uint32, defaultTimeout uint32, sa *SecurityAttributes) (handle Handle, err error)  [failretval==InvalidHandle] = CreateNamedPipeW
 //sys	ConnectNamedPipe(pipe Handle, overlapped *Overlapped) (err error)
 //sys	DisconnectNamedPipe(pipe Handle) (err error)
+//sys   GetNamedPipeClientProcessId(pipe Handle, clientProcessID *uint32) (err error)
+//sys   GetNamedPipeServerProcessId(pipe Handle, serverProcessID *uint32) (err error)
 //sys	GetNamedPipeInfo(pipe Handle, flags *uint32, outSize *uint32, inSize *uint32, maxInstances *uint32) (err error)
 //sys	GetNamedPipeHandleState(pipe Handle, state *uint32, curInstances *uint32, maxCollectionCount *uint32, collectDataTimeout *uint32, userName *uint16, maxUserNameSize uint32) (err error) = GetNamedPipeHandleStateW
 //sys	SetNamedPipeHandleState(pipe Handle, state *uint32, maxCollectionCount *uint32, collectDataTimeout *uint32) (err error) = SetNamedPipeHandleState
@@ -211,6 +215,10 @@ func NewCallbackCDecl(fn interface{}) uintptr {
 //sys	OpenProcess(desiredAccess uint32, inheritHandle bool, processId uint32) (handle Handle, err error)
 //sys	ShellExecute(hwnd Handle, verb *uint16, file *uint16, args *uint16, cwd *uint16, showCmd int32) (err error) [failretval<=32] = shell32.ShellExecuteW
 //sys	GetWindowThreadProcessId(hwnd HWND, pid *uint32) (tid uint32, err error) = user32.GetWindowThreadProcessId
+//sys	LoadKeyboardLayout(name *uint16, flags uint32) (hkl Handle, err error) [failretval==0] = user32.LoadKeyboardLayoutW
+//sys	UnloadKeyboardLayout(hkl Handle) (err error) = user32.UnloadKeyboardLayout
+//sys	GetKeyboardLayout(tid uint32) (hkl Handle) = user32.GetKeyboardLayout
+//sys	ToUnicodeEx(vkey uint32, scancode uint32, keystate *byte, pwszBuff *uint16, cchBuff int32, flags uint32, hkl Handle) (ret int32) = user32.ToUnicodeEx
 //sys	GetShellWindow() (shellWindow HWND) = user32.GetShellWindow
 //sys	MessageBox(hwnd HWND, text *uint16, caption *uint16, boxtype uint32) (ret int32, err error) [failretval==0] = user32.MessageBoxW
 //sys	ExitWindowsEx(flags uint32, reason uint32) (err error) = user32.ExitWindowsEx
@@ -307,6 +315,10 @@ func NewCallbackCDecl(fn interface{}) uintptr {
 //sys	SetConsoleMode(console Handle, mode uint32) (err error) = kernel32.SetConsoleMode
 //sys	GetConsoleScreenBufferInfo(console Handle, info *ConsoleScreenBufferInfo) (err error) = kernel32.GetConsoleScreenBufferInfo
 //sys	setConsoleCursorPosition(console Handle, position uint32) (err error) = kernel32.SetConsoleCursorPosition
+//sys	GetConsoleCP() (cp uint32, err error) = kernel32.GetConsoleCP
+//sys	GetConsoleOutputCP() (cp uint32, err error) = kernel32.GetConsoleOutputCP
+//sys	SetConsoleCP(cp uint32) (err error) = kernel32.SetConsoleCP
+//sys	SetConsoleOutputCP(cp uint32) (err error) = kernel32.SetConsoleOutputCP
 //sys	WriteConsole(console Handle, buf *uint16, towrite uint32, written *uint32, reserved *byte) (err error) = kernel32.WriteConsoleW
 //sys	ReadConsole(console Handle, buf *uint16, toread uint32, read *uint32, inputControl *byte) (err error) = kernel32.ReadConsoleW
 //sys	resizePseudoConsole(pconsole Handle, size uint32) (hr error) = kernel32.ResizePseudoConsole
@@ -715,20 +727,12 @@ func DurationSinceBoot() time.Duration {
 }
 
 func Ftruncate(fd Handle, length int64) (err error) {
-	curoffset, e := Seek(fd, 0, 1)
-	if e != nil {
-		return e
-	}
-	defer Seek(fd, curoffset, 0)
-	_, e = Seek(fd, length, 0)
-	if e != nil {
-		return e
+	type _FILE_END_OF_FILE_INFO struct {
+		EndOfFile int64
 	}
-	e = SetEndOfFile(fd)
-	if e != nil {
-		return e
-	}
-	return nil
+	var info _FILE_END_OF_FILE_INFO
+	info.EndOfFile = length
+	return SetFileInformationByHandle(fd, FileEndOfFileInfo, (*byte)(unsafe.Pointer(&info)), uint32(unsafe.Sizeof(info)))
 }
 
 func Gettimeofday(tv *Timeval) (err error) {
@@ -884,6 +888,11 @@ const socket_error = uintptr(^uint32(0))
 //sys	GetACP() (acp uint32) = kernel32.GetACP
 //sys	MultiByteToWideChar(codePage uint32, dwFlags uint32, str *byte, nstr int32, wchar *uint16, nwchar int32) (nwrite int32, err error) = kernel32.MultiByteToWideChar
 //sys	getBestInterfaceEx(sockaddr unsafe.Pointer, pdwBestIfIndex *uint32) (errcode error) = iphlpapi.GetBestInterfaceEx
+//sys   GetIfEntry2Ex(level uint32, row *MibIfRow2) (errcode error) = iphlpapi.GetIfEntry2Ex
+//sys   GetUnicastIpAddressEntry(row *MibUnicastIpAddressRow) (errcode error) = iphlpapi.GetUnicastIpAddressEntry
+//sys   NotifyIpInterfaceChange(family uint16, callback uintptr, callerContext unsafe.Pointer, initialNotification bool, notificationHandle *Handle) (errcode error) = iphlpapi.NotifyIpInterfaceChange
+//sys   NotifyUnicastIpAddressChange(family uint16, callback uintptr, callerContext unsafe.Pointer, initialNotification bool, notificationHandle *Handle) (errcode error) = iphlpapi.NotifyUnicastIpAddressChange
+//sys   CancelMibChangeNotify2(notificationHandle Handle) (errcode error) = iphlpapi.CancelMibChangeNotify2
 
 // For testing: clients can set this flag to force
 // creation of IPv6 sockets to return EAFNOSUPPORT.
@@ -1368,9 +1377,11 @@ func SetsockoptLinger(fd Handle, level, opt int, l *Linger) (err error) {
 func SetsockoptInet4Addr(fd Handle, level, opt int, value [4]byte) (err error) {
 	return Setsockopt(fd, int32(level), int32(opt), (*byte)(unsafe.Pointer(&value[0])), 4)
 }
+
 func SetsockoptIPMreq(fd Handle, level, opt int, mreq *IPMreq) (err error) {
 	return Setsockopt(fd, int32(level), int32(opt), (*byte)(unsafe.Pointer(mreq)), int32(unsafe.Sizeof(*mreq)))
 }
+
 func SetsockoptIPv6Mreq(fd Handle, level, opt int, mreq *IPv6Mreq) (err error) {
 	return syscall.EWINDOWS
 }
@@ -1673,13 +1684,16 @@ func (s NTStatus) Error() string {
 // do not use NTUnicodeString, and instead UTF16PtrFromString should be used for
 // the more common *uint16 string type.
 func NewNTUnicodeString(s string) (*NTUnicodeString, error) {
-	var u NTUnicodeString
-	s16, err := UTF16PtrFromString(s)
+	s16, err := UTF16FromString(s)
 	if err != nil {
 		return nil, err
 	}
-	RtlInitUnicodeString(&u, s16)
-	return &u, nil
+	n := uint16(len(s16) * 2)
+	return &NTUnicodeString{
+		Length:        n - 2, // subtract 2 bytes for the NULL terminator
+		MaximumLength: n,
+		Buffer:        &s16[0],
+	}, nil
 }
 
 // Slice returns a uint16 slice that aliases the data in the NTUnicodeString.
diff --git a/vendor/golang.org/x/sys/windows/types_windows.go b/vendor/golang.org/x/sys/windows/types_windows.go
index d8cb71db0..9d138de5f 100644
--- a/vendor/golang.org/x/sys/windows/types_windows.go
+++ b/vendor/golang.org/x/sys/windows/types_windows.go
@@ -176,6 +176,7 @@ const (
 	WAIT_FAILED    = 0xFFFFFFFF
 
 	// Access rights for process.
+	PROCESS_ALL_ACCESS                = 0xFFFF
 	PROCESS_CREATE_PROCESS            = 0x0080
 	PROCESS_CREATE_THREAD             = 0x0002
 	PROCESS_DUP_HANDLE                = 0x0040
@@ -1060,6 +1061,7 @@ const (
 	SIO_GET_EXTENSION_FUNCTION_POINTER = IOC_INOUT | IOC_WS2 | 6
 	SIO_KEEPALIVE_VALS                 = IOC_IN | IOC_VENDOR | 4
 	SIO_UDP_CONNRESET                  = IOC_IN | IOC_VENDOR | 12
+	SIO_UDP_NETRESET                   = IOC_IN | IOC_VENDOR | 15
 
 	// cf. http://support.microsoft.com/default.aspx?scid=kb;en-us;257460
 
@@ -2003,7 +2005,21 @@ const (
 	MOVEFILE_FAIL_IF_NOT_TRACKABLE = 0x20
 )
 
-const GAA_FLAG_INCLUDE_PREFIX = 0x00000010
+// Flags for GetAdaptersAddresses, see
+// https://learn.microsoft.com/en-us/windows/win32/api/iphlpapi/nf-iphlpapi-getadaptersaddresses.
+const (
+	GAA_FLAG_SKIP_UNICAST                = 0x1
+	GAA_FLAG_SKIP_ANYCAST                = 0x2
+	GAA_FLAG_SKIP_MULTICAST              = 0x4
+	GAA_FLAG_SKIP_DNS_SERVER             = 0x8
+	GAA_FLAG_INCLUDE_PREFIX              = 0x10
+	GAA_FLAG_SKIP_FRIENDLY_NAME          = 0x20
+	GAA_FLAG_INCLUDE_WINS_INFO           = 0x40
+	GAA_FLAG_INCLUDE_GATEWAYS            = 0x80
+	GAA_FLAG_INCLUDE_ALL_INTERFACES      = 0x100
+	GAA_FLAG_INCLUDE_ALL_COMPARTMENTS    = 0x200
+	GAA_FLAG_INCLUDE_TUNNEL_BINDINGORDER = 0x400
+)
 
 const (
 	IF_TYPE_OTHER              = 1
@@ -2017,6 +2033,50 @@ const (
 	IF_TYPE_IEEE1394           = 144
 )
 
+// Enum NL_PREFIX_ORIGIN for [IpAdapterUnicastAddress], see
+// https://learn.microsoft.com/en-us/windows/win32/api/nldef/ne-nldef-nl_prefix_origin
+const (
+	IpPrefixOriginOther               = 0
+	IpPrefixOriginManual              = 1
+	IpPrefixOriginWellKnown           = 2
+	IpPrefixOriginDhcp                = 3
+	IpPrefixOriginRouterAdvertisement = 4
+	IpPrefixOriginUnchanged           = 1 << 4
+)
+
+// Enum NL_SUFFIX_ORIGIN for [IpAdapterUnicastAddress], see
+// https://learn.microsoft.com/en-us/windows/win32/api/nldef/ne-nldef-nl_suffix_origin
+const (
+	NlsoOther                      = 0
+	NlsoManual                     = 1
+	NlsoWellKnown                  = 2
+	NlsoDhcp                       = 3
+	NlsoLinkLayerAddress           = 4
+	NlsoRandom                     = 5
+	IpSuffixOriginOther            = 0
+	IpSuffixOriginManual           = 1
+	IpSuffixOriginWellKnown        = 2
+	IpSuffixOriginDhcp             = 3
+	IpSuffixOriginLinkLayerAddress = 4
+	IpSuffixOriginRandom           = 5
+	IpSuffixOriginUnchanged        = 1 << 4
+)
+
+// Enum NL_DAD_STATE for [IpAdapterUnicastAddress], see
+// https://learn.microsoft.com/en-us/windows/win32/api/nldef/ne-nldef-nl_dad_state
+const (
+	NldsInvalid          = 0
+	NldsTentative        = 1
+	NldsDuplicate        = 2
+	NldsDeprecated       = 3
+	NldsPreferred        = 4
+	IpDadStateInvalid    = 0
+	IpDadStateTentative  = 1
+	IpDadStateDuplicate  = 2
+	IpDadStateDeprecated = 3
+	IpDadStatePreferred  = 4
+)
+
 type SocketAddress struct {
 	Sockaddr       *syscall.RawSockaddrAny
 	SockaddrLength int32
@@ -2144,6 +2204,132 @@ const (
 	IfOperStatusLowerLayerDown = 7
 )
 
+const (
+	IF_MAX_PHYS_ADDRESS_LENGTH = 32
+	IF_MAX_STRING_SIZE         = 256
+)
+
+// MIB_IF_ENTRY_LEVEL enumeration from netioapi.h or
+// https://learn.microsoft.com/en-us/windows/win32/api/netioapi/nf-netioapi-getifentry2ex.
+const (
+	MibIfEntryNormal                  = 0
+	MibIfEntryNormalWithoutStatistics = 2
+)
+
+// MIB_NOTIFICATION_TYPE enumeration from netioapi.h or
+// https://learn.microsoft.com/en-us/windows/win32/api/netioapi/ne-netioapi-mib_notification_type.
+const (
+	MibParameterNotification = 0
+	MibAddInstance           = 1
+	MibDeleteInstance        = 2
+	MibInitialNotification   = 3
+)
+
+// MibIfRow2 stores information about a particular interface. See
+// https://learn.microsoft.com/en-us/windows/win32/api/netioapi/ns-netioapi-mib_if_row2.
+type MibIfRow2 struct {
+	InterfaceLuid               uint64
+	InterfaceIndex              uint32
+	InterfaceGuid               GUID
+	Alias                       [IF_MAX_STRING_SIZE + 1]uint16
+	Description                 [IF_MAX_STRING_SIZE + 1]uint16
+	PhysicalAddressLength       uint32
+	PhysicalAddress             [IF_MAX_PHYS_ADDRESS_LENGTH]uint8
+	PermanentPhysicalAddress    [IF_MAX_PHYS_ADDRESS_LENGTH]uint8
+	Mtu                         uint32
+	Type                        uint32
+	TunnelType                  uint32
+	MediaType                   uint32
+	PhysicalMediumType          uint32
+	AccessType                  uint32
+	DirectionType               uint32
+	InterfaceAndOperStatusFlags uint8
+	OperStatus                  uint32
+	AdminStatus                 uint32
+	MediaConnectState           uint32
+	NetworkGuid                 GUID
+	ConnectionType              uint32
+	TransmitLinkSpeed           uint64
+	ReceiveLinkSpeed            uint64
+	InOctets                    uint64
+	InUcastPkts                 uint64
+	InNUcastPkts                uint64
+	InDiscards                  uint64
+	InErrors                    uint64
+	InUnknownProtos             uint64
+	InUcastOctets               uint64
+	InMulticastOctets           uint64
+	InBroadcastOctets           uint64
+	OutOctets                   uint64
+	OutUcastPkts                uint64
+	OutNUcastPkts               uint64
+	OutDiscards                 uint64
+	OutErrors                   uint64
+	OutUcastOctets              uint64
+	OutMulticastOctets          uint64
+	OutBroadcastOctets          uint64
+	OutQLen                     uint64
+}
+
+// MIB_UNICASTIPADDRESS_ROW stores information about a unicast IP address. See
+// https://learn.microsoft.com/en-us/windows/win32/api/netioapi/ns-netioapi-mib_unicastipaddress_row.
+type MibUnicastIpAddressRow struct {
+	Address            RawSockaddrInet6 // SOCKADDR_INET union
+	InterfaceLuid      uint64
+	InterfaceIndex     uint32
+	PrefixOrigin       uint32
+	SuffixOrigin       uint32
+	ValidLifetime      uint32
+	PreferredLifetime  uint32
+	OnLinkPrefixLength uint8
+	SkipAsSource       uint8
+	DadState           uint32
+	ScopeId            uint32
+	CreationTimeStamp  Filetime
+}
+
+const ScopeLevelCount = 16
+
+// MIB_IPINTERFACE_ROW stores interface management information for a particular IP address family on a network interface.
+// See https://learn.microsoft.com/en-us/windows/win32/api/netioapi/ns-netioapi-mib_ipinterface_row.
+type MibIpInterfaceRow struct {
+	Family                               uint16
+	InterfaceLuid                        uint64
+	InterfaceIndex                       uint32
+	MaxReassemblySize                    uint32
+	InterfaceIdentifier                  uint64
+	MinRouterAdvertisementInterval       uint32
+	MaxRouterAdvertisementInterval       uint32
+	AdvertisingEnabled                   uint8
+	ForwardingEnabled                    uint8
+	WeakHostSend                         uint8
+	WeakHostReceive                      uint8
+	UseAutomaticMetric                   uint8
+	UseNeighborUnreachabilityDetection   uint8
+	ManagedAddressConfigurationSupported uint8
+	OtherStatefulConfigurationSupported  uint8
+	AdvertiseDefaultRoute                uint8
+	RouterDiscoveryBehavior              uint32
+	DadTransmits                         uint32
+	BaseReachableTime                    uint32
+	RetransmitTime                       uint32
+	PathMtuDiscoveryTimeout              uint32
+	LinkLocalAddressBehavior             uint32
+	LinkLocalAddressTimeout              uint32
+	ZoneIndices                          [ScopeLevelCount]uint32
+	SitePrefixLength                     uint32
+	Metric                               uint32
+	NlMtu                                uint32
+	Connected                            uint8
+	SupportsWakeUpPatterns               uint8
+	SupportsNeighborDiscovery            uint8
+	SupportsRouterDiscovery              uint8
+	ReachableTime                        uint32
+	TransmitOffload                      uint32
+	ReceiveOffload                       uint32
+	DisableDefaultRoutes                 uint8
+}
+
 // Console related constants used for the mode parameter to SetConsoleMode. See
 // https://docs.microsoft.com/en-us/windows/console/setconsolemode for details.
 
@@ -3404,3 +3590,14 @@ type DCB struct {
 	EvtChar    byte
 	wReserved1 uint16
 }
+
+// Keyboard Layout Flags.
+// See https://learn.microsoft.com/en-us/windows/win32/api/winuser/nf-winuser-loadkeyboardlayoutw
+const (
+	KLF_ACTIVATE      = 0x00000001
+	KLF_SUBSTITUTE_OK = 0x00000002
+	KLF_REORDER       = 0x00000008
+	KLF_REPLACELANG   = 0x00000010
+	KLF_NOTELLSHELL   = 0x00000080
+	KLF_SETFORPROCESS = 0x00000100
+)
diff --git a/vendor/golang.org/x/sys/windows/zsyscall_windows.go b/vendor/golang.org/x/sys/windows/zsyscall_windows.go
index eba761018..01c0716c2 100644
--- a/vendor/golang.org/x/sys/windows/zsyscall_windows.go
+++ b/vendor/golang.org/x/sys/windows/zsyscall_windows.go
@@ -181,10 +181,15 @@ var (
 	procDnsRecordListFree                                    = moddnsapi.NewProc("DnsRecordListFree")
 	procDwmGetWindowAttribute                                = moddwmapi.NewProc("DwmGetWindowAttribute")
 	procDwmSetWindowAttribute                                = moddwmapi.NewProc("DwmSetWindowAttribute")
+	procCancelMibChangeNotify2                               = modiphlpapi.NewProc("CancelMibChangeNotify2")
 	procGetAdaptersAddresses                                 = modiphlpapi.NewProc("GetAdaptersAddresses")
 	procGetAdaptersInfo                                      = modiphlpapi.NewProc("GetAdaptersInfo")
 	procGetBestInterfaceEx                                   = modiphlpapi.NewProc("GetBestInterfaceEx")
 	procGetIfEntry                                           = modiphlpapi.NewProc("GetIfEntry")
+	procGetIfEntry2Ex                                        = modiphlpapi.NewProc("GetIfEntry2Ex")
+	procGetUnicastIpAddressEntry                             = modiphlpapi.NewProc("GetUnicastIpAddressEntry")
+	procNotifyIpInterfaceChange                              = modiphlpapi.NewProc("NotifyIpInterfaceChange")
+	procNotifyUnicastIpAddressChange                         = modiphlpapi.NewProc("NotifyUnicastIpAddressChange")
 	procAddDllDirectory                                      = modkernel32.NewProc("AddDllDirectory")
 	procAssignProcessToJobObject                             = modkernel32.NewProc("AssignProcessToJobObject")
 	procCancelIo                                             = modkernel32.NewProc("CancelIo")
@@ -247,7 +252,9 @@ var (
 	procGetCommandLineW                                      = modkernel32.NewProc("GetCommandLineW")
 	procGetComputerNameExW                                   = modkernel32.NewProc("GetComputerNameExW")
 	procGetComputerNameW                                     = modkernel32.NewProc("GetComputerNameW")
+	procGetConsoleCP                                         = modkernel32.NewProc("GetConsoleCP")
 	procGetConsoleMode                                       = modkernel32.NewProc("GetConsoleMode")
+	procGetConsoleOutputCP                                   = modkernel32.NewProc("GetConsoleOutputCP")
 	procGetConsoleScreenBufferInfo                           = modkernel32.NewProc("GetConsoleScreenBufferInfo")
 	procGetCurrentDirectoryW                                 = modkernel32.NewProc("GetCurrentDirectoryW")
 	procGetCurrentProcessId                                  = modkernel32.NewProc("GetCurrentProcessId")
@@ -273,8 +280,10 @@ var (
 	procGetMaximumProcessorCount                             = modkernel32.NewProc("GetMaximumProcessorCount")
 	procGetModuleFileNameW                                   = modkernel32.NewProc("GetModuleFileNameW")
 	procGetModuleHandleExW                                   = modkernel32.NewProc("GetModuleHandleExW")
+	procGetNamedPipeClientProcessId                          = modkernel32.NewProc("GetNamedPipeClientProcessId")
 	procGetNamedPipeHandleStateW                             = modkernel32.NewProc("GetNamedPipeHandleStateW")
 	procGetNamedPipeInfo                                     = modkernel32.NewProc("GetNamedPipeInfo")
+	procGetNamedPipeServerProcessId                          = modkernel32.NewProc("GetNamedPipeServerProcessId")
 	procGetOverlappedResult                                  = modkernel32.NewProc("GetOverlappedResult")
 	procGetPriorityClass                                     = modkernel32.NewProc("GetPriorityClass")
 	procGetProcAddress                                       = modkernel32.NewProc("GetProcAddress")
@@ -347,8 +356,10 @@ var (
 	procSetCommMask                                          = modkernel32.NewProc("SetCommMask")
 	procSetCommState                                         = modkernel32.NewProc("SetCommState")
 	procSetCommTimeouts                                      = modkernel32.NewProc("SetCommTimeouts")
+	procSetConsoleCP                                         = modkernel32.NewProc("SetConsoleCP")
 	procSetConsoleCursorPosition                             = modkernel32.NewProc("SetConsoleCursorPosition")
 	procSetConsoleMode                                       = modkernel32.NewProc("SetConsoleMode")
+	procSetConsoleOutputCP                                   = modkernel32.NewProc("SetConsoleOutputCP")
 	procSetCurrentDirectoryW                                 = modkernel32.NewProc("SetCurrentDirectoryW")
 	procSetDefaultDllDirectories                             = modkernel32.NewProc("SetDefaultDllDirectories")
 	procSetDllDirectoryW                                     = modkernel32.NewProc("SetDllDirectoryW")
@@ -478,12 +489,16 @@ var (
 	procGetDesktopWindow                                     = moduser32.NewProc("GetDesktopWindow")
 	procGetForegroundWindow                                  = moduser32.NewProc("GetForegroundWindow")
 	procGetGUIThreadInfo                                     = moduser32.NewProc("GetGUIThreadInfo")
+	procGetKeyboardLayout                                    = moduser32.NewProc("GetKeyboardLayout")
 	procGetShellWindow                                       = moduser32.NewProc("GetShellWindow")
 	procGetWindowThreadProcessId                             = moduser32.NewProc("GetWindowThreadProcessId")
 	procIsWindow                                             = moduser32.NewProc("IsWindow")
 	procIsWindowUnicode                                      = moduser32.NewProc("IsWindowUnicode")
 	procIsWindowVisible                                      = moduser32.NewProc("IsWindowVisible")
+	procLoadKeyboardLayoutW                                  = moduser32.NewProc("LoadKeyboardLayoutW")
 	procMessageBoxW                                          = moduser32.NewProc("MessageBoxW")
+	procToUnicodeEx                                          = moduser32.NewProc("ToUnicodeEx")
+	procUnloadKeyboardLayout                                 = moduser32.NewProc("UnloadKeyboardLayout")
 	procCreateEnvironmentBlock                               = moduserenv.NewProc("CreateEnvironmentBlock")
 	procDestroyEnvironmentBlock                              = moduserenv.NewProc("DestroyEnvironmentBlock")
 	procGetUserProfileDirectoryW                             = moduserenv.NewProc("GetUserProfileDirectoryW")
@@ -789,6 +804,14 @@ func FreeSid(sid *SID) (err error) {
 	return
 }
 
+func GetAce(acl *ACL, aceIndex uint32, pAce **ACCESS_ALLOWED_ACE) (err error) {
+	r1, _, e1 := syscall.Syscall(procGetAce.Addr(), 3, uintptr(unsafe.Pointer(acl)), uintptr(aceIndex), uintptr(unsafe.Pointer(pAce)))
+	if r1 == 0 {
+		err = errnoErr(e1)
+	}
+	return
+}
+
 func GetLengthSid(sid *SID) (len uint32) {
 	r0, _, _ := syscall.Syscall(procGetLengthSid.Addr(), 1, uintptr(unsafe.Pointer(sid)), 0, 0)
 	len = uint32(r0)
@@ -1225,14 +1248,6 @@ func setEntriesInAcl(countExplicitEntries uint32, explicitEntries *EXPLICIT_ACCE
 	return
 }
 
-func GetAce(acl *ACL, aceIndex uint32, pAce **ACCESS_ALLOWED_ACE) (ret error) {
-	r0, _, _ := syscall.Syscall(procGetAce.Addr(), 3, uintptr(unsafe.Pointer(acl)), uintptr(aceIndex), uintptr(unsafe.Pointer(pAce)))
-	if r0 == 0 {
-		ret = GetLastError()
-	}
-	return
-}
-
 func SetKernelObjectSecurity(handle Handle, securityInformation SECURITY_INFORMATION, securityDescriptor *SECURITY_DESCRIPTOR) (err error) {
 	r1, _, e1 := syscall.Syscall(procSetKernelObjectSecurity.Addr(), 3, uintptr(handle), uintptr(securityInformation), uintptr(unsafe.Pointer(securityDescriptor)))
 	if r1 == 0 {
@@ -1598,6 +1613,14 @@ func DwmSetWindowAttribute(hwnd HWND, attribute uint32, value unsafe.Pointer, si
 	return
 }
 
+func CancelMibChangeNotify2(notificationHandle Handle) (errcode error) {
+	r0, _, _ := syscall.Syscall(procCancelMibChangeNotify2.Addr(), 1, uintptr(notificationHandle), 0, 0)
+	if r0 != 0 {
+		errcode = syscall.Errno(r0)
+	}
+	return
+}
+
 func GetAdaptersAddresses(family uint32, flags uint32, reserved uintptr, adapterAddresses *IpAdapterAddresses, sizePointer *uint32) (errcode error) {
 	r0, _, _ := syscall.Syscall6(procGetAdaptersAddresses.Addr(), 5, uintptr(family), uintptr(flags), uintptr(reserved), uintptr(unsafe.Pointer(adapterAddresses)), uintptr(unsafe.Pointer(sizePointer)), 0)
 	if r0 != 0 {
@@ -1630,6 +1653,46 @@ func GetIfEntry(pIfRow *MibIfRow) (errcode error) {
 	return
 }
 
+func GetIfEntry2Ex(level uint32, row *MibIfRow2) (errcode error) {
+	r0, _, _ := syscall.Syscall(procGetIfEntry2Ex.Addr(), 2, uintptr(level), uintptr(unsafe.Pointer(row)), 0)
+	if r0 != 0 {
+		errcode = syscall.Errno(r0)
+	}
+	return
+}
+
+func GetUnicastIpAddressEntry(row *MibUnicastIpAddressRow) (errcode error) {
+	r0, _, _ := syscall.Syscall(procGetUnicastIpAddressEntry.Addr(), 1, uintptr(unsafe.Pointer(row)), 0, 0)
+	if r0 != 0 {
+		errcode = syscall.Errno(r0)
+	}
+	return
+}
+
+func NotifyIpInterfaceChange(family uint16, callback uintptr, callerContext unsafe.Pointer, initialNotification bool, notificationHandle *Handle) (errcode error) {
+	var _p0 uint32
+	if initialNotification {
+		_p0 = 1
+	}
+	r0, _, _ := syscall.Syscall6(procNotifyIpInterfaceChange.Addr(), 5, uintptr(family), uintptr(callback), uintptr(callerContext), uintptr(_p0), uintptr(unsafe.Pointer(notificationHandle)), 0)
+	if r0 != 0 {
+		errcode = syscall.Errno(r0)
+	}
+	return
+}
+
+func NotifyUnicastIpAddressChange(family uint16, callback uintptr, callerContext unsafe.Pointer, initialNotification bool, notificationHandle *Handle) (errcode error) {
+	var _p0 uint32
+	if initialNotification {
+		_p0 = 1
+	}
+	r0, _, _ := syscall.Syscall6(procNotifyUnicastIpAddressChange.Addr(), 5, uintptr(family), uintptr(callback), uintptr(callerContext), uintptr(_p0), uintptr(unsafe.Pointer(notificationHandle)), 0)
+	if r0 != 0 {
+		errcode = syscall.Errno(r0)
+	}
+	return
+}
+
 func AddDllDirectory(path *uint16) (cookie uintptr, err error) {
 	r0, _, e1 := syscall.Syscall(procAddDllDirectory.Addr(), 1, uintptr(unsafe.Pointer(path)), 0, 0)
 	cookie = uintptr(r0)
@@ -2158,6 +2221,15 @@ func GetComputerName(buf *uint16, n *uint32) (err error) {
 	return
 }
 
+func GetConsoleCP() (cp uint32, err error) {
+	r0, _, e1 := syscall.Syscall(procGetConsoleCP.Addr(), 0, 0, 0, 0)
+	cp = uint32(r0)
+	if cp == 0 {
+		err = errnoErr(e1)
+	}
+	return
+}
+
 func GetConsoleMode(console Handle, mode *uint32) (err error) {
 	r1, _, e1 := syscall.Syscall(procGetConsoleMode.Addr(), 2, uintptr(console), uintptr(unsafe.Pointer(mode)), 0)
 	if r1 == 0 {
@@ -2166,6 +2238,15 @@ func GetConsoleMode(console Handle, mode *uint32) (err error) {
 	return
 }
 
+func GetConsoleOutputCP() (cp uint32, err error) {
+	r0, _, e1 := syscall.Syscall(procGetConsoleOutputCP.Addr(), 0, 0, 0, 0)
+	cp = uint32(r0)
+	if cp == 0 {
+		err = errnoErr(e1)
+	}
+	return
+}
+
 func GetConsoleScreenBufferInfo(console Handle, info *ConsoleScreenBufferInfo) (err error) {
 	r1, _, e1 := syscall.Syscall(procGetConsoleScreenBufferInfo.Addr(), 2, uintptr(console), uintptr(unsafe.Pointer(info)), 0)
 	if r1 == 0 {
@@ -2367,6 +2448,14 @@ func GetModuleHandleEx(flags uint32, moduleName *uint16, module *Handle) (err er
 	return
 }
 
+func GetNamedPipeClientProcessId(pipe Handle, clientProcessID *uint32) (err error) {
+	r1, _, e1 := syscall.Syscall(procGetNamedPipeClientProcessId.Addr(), 2, uintptr(pipe), uintptr(unsafe.Pointer(clientProcessID)), 0)
+	if r1 == 0 {
+		err = errnoErr(e1)
+	}
+	return
+}
+
 func GetNamedPipeHandleState(pipe Handle, state *uint32, curInstances *uint32, maxCollectionCount *uint32, collectDataTimeout *uint32, userName *uint16, maxUserNameSize uint32) (err error) {
 	r1, _, e1 := syscall.Syscall9(procGetNamedPipeHandleStateW.Addr(), 7, uintptr(pipe), uintptr(unsafe.Pointer(state)), uintptr(unsafe.Pointer(curInstances)), uintptr(unsafe.Pointer(maxCollectionCount)), uintptr(unsafe.Pointer(collectDataTimeout)), uintptr(unsafe.Pointer(userName)), uintptr(maxUserNameSize), 0, 0)
 	if r1 == 0 {
@@ -2383,6 +2472,14 @@ func GetNamedPipeInfo(pipe Handle, flags *uint32, outSize *uint32, inSize *uint3
 	return
 }
 
+func GetNamedPipeServerProcessId(pipe Handle, serverProcessID *uint32) (err error) {
+	r1, _, e1 := syscall.Syscall(procGetNamedPipeServerProcessId.Addr(), 2, uintptr(pipe), uintptr(unsafe.Pointer(serverProcessID)), 0)
+	if r1 == 0 {
+		err = errnoErr(e1)
+	}
+	return
+}
+
 func GetOverlappedResult(handle Handle, overlapped *Overlapped, done *uint32, wait bool) (err error) {
 	var _p0 uint32
 	if wait {
@@ -3034,6 +3131,14 @@ func SetCommTimeouts(handle Handle, timeouts *CommTimeouts) (err error) {
 	return
 }
 
+func SetConsoleCP(cp uint32) (err error) {
+	r1, _, e1 := syscall.Syscall(procSetConsoleCP.Addr(), 1, uintptr(cp), 0, 0)
+	if r1 == 0 {
+		err = errnoErr(e1)
+	}
+	return
+}
+
 func setConsoleCursorPosition(console Handle, position uint32) (err error) {
 	r1, _, e1 := syscall.Syscall(procSetConsoleCursorPosition.Addr(), 2, uintptr(console), uintptr(position), 0)
 	if r1 == 0 {
@@ -3050,6 +3155,14 @@ func SetConsoleMode(console Handle, mode uint32) (err error) {
 	return
 }
 
+func SetConsoleOutputCP(cp uint32) (err error) {
+	r1, _, e1 := syscall.Syscall(procSetConsoleOutputCP.Addr(), 1, uintptr(cp), 0, 0)
+	if r1 == 0 {
+		err = errnoErr(e1)
+	}
+	return
+}
+
 func SetCurrentDirectory(path *uint16) (err error) {
 	r1, _, e1 := syscall.Syscall(procSetCurrentDirectoryW.Addr(), 1, uintptr(unsafe.Pointer(path)), 0, 0)
 	if r1 == 0 {
@@ -4082,6 +4195,12 @@ func GetGUIThreadInfo(thread uint32, info *GUIThreadInfo) (err error) {
 	return
 }
 
+func GetKeyboardLayout(tid uint32) (hkl Handle) {
+	r0, _, _ := syscall.Syscall(procGetKeyboardLayout.Addr(), 1, uintptr(tid), 0, 0)
+	hkl = Handle(r0)
+	return
+}
+
 func GetShellWindow() (shellWindow HWND) {
 	r0, _, _ := syscall.Syscall(procGetShellWindow.Addr(), 0, 0, 0, 0)
 	shellWindow = HWND(r0)
@@ -4115,6 +4234,15 @@ func IsWindowVisible(hwnd HWND) (isVisible bool) {
 	return
 }
 
+func LoadKeyboardLayout(name *uint16, flags uint32) (hkl Handle, err error) {
+	r0, _, e1 := syscall.Syscall(procLoadKeyboardLayoutW.Addr(), 2, uintptr(unsafe.Pointer(name)), uintptr(flags), 0)
+	hkl = Handle(r0)
+	if hkl == 0 {
+		err = errnoErr(e1)
+	}
+	return
+}
+
 func MessageBox(hwnd HWND, text *uint16, caption *uint16, boxtype uint32) (ret int32, err error) {
 	r0, _, e1 := syscall.Syscall6(procMessageBoxW.Addr(), 4, uintptr(hwnd), uintptr(unsafe.Pointer(text)), uintptr(unsafe.Pointer(caption)), uintptr(boxtype), 0, 0)
 	ret = int32(r0)
@@ -4124,6 +4252,20 @@ func MessageBox(hwnd HWND, text *uint16, caption *uint16, boxtype uint32) (ret i
 	return
 }
 
+func ToUnicodeEx(vkey uint32, scancode uint32, keystate *byte, pwszBuff *uint16, cchBuff int32, flags uint32, hkl Handle) (ret int32) {
+	r0, _, _ := syscall.Syscall9(procToUnicodeEx.Addr(), 7, uintptr(vkey), uintptr(scancode), uintptr(unsafe.Pointer(keystate)), uintptr(unsafe.Pointer(pwszBuff)), uintptr(cchBuff), uintptr(flags), uintptr(hkl), 0, 0)
+	ret = int32(r0)
+	return
+}
+
+func UnloadKeyboardLayout(hkl Handle) (err error) {
+	r1, _, e1 := syscall.Syscall(procUnloadKeyboardLayout.Addr(), 1, uintptr(hkl), 0, 0)
+	if r1 == 0 {
+		err = errnoErr(e1)
+	}
+	return
+}
+
 func CreateEnvironmentBlock(block **uint16, token Token, inheritExisting bool) (err error) {
 	var _p0 uint32
 	if inheritExisting {
diff --git a/vendor/golang.org/x/term/LICENSE b/vendor/golang.org/x/term/LICENSE
index 6a66aea5e..2a7cf70da 100644
--- a/vendor/golang.org/x/term/LICENSE
+++ b/vendor/golang.org/x/term/LICENSE
@@ -1,4 +1,4 @@
-Copyright (c) 2009 The Go Authors. All rights reserved.
+Copyright 2009 The Go Authors.
 
 Redistribution and use in source and binary forms, with or without
 modification, are permitted provided that the following conditions are
@@ -10,7 +10,7 @@ notice, this list of conditions and the following disclaimer.
 copyright notice, this list of conditions and the following disclaimer
 in the documentation and/or other materials provided with the
 distribution.
-   * Neither the name of Google Inc. nor the names of its
+   * Neither the name of Google LLC nor the names of its
 contributors may be used to endorse or promote products derived from
 this software without specific prior written permission.
 
diff --git a/vendor/golang.org/x/term/README.md b/vendor/golang.org/x/term/README.md
index d03d0aefe..05ff623f9 100644
--- a/vendor/golang.org/x/term/README.md
+++ b/vendor/golang.org/x/term/README.md
@@ -4,16 +4,13 @@
 
 This repository provides Go terminal and console support packages.
 
-## Download/Install
-
-The easiest way to install is to run `go get -u golang.org/x/term`. You can
-also manually git clone the repository to `$GOPATH/src/golang.org/x/term`.
-
 ## Report Issues / Send Patches
 
 This repository uses Gerrit for code changes. To learn how to submit changes to
-this repository, see https://golang.org/doc/contribute.html.
+this repository, see https://go.dev/doc/contribute.
+
+The git repository is https://go.googlesource.com/term.
 
 The main issue tracker for the term repository is located at
-https://github.com/golang/go/issues. Prefix your issue with "x/term:" in the
+https://go.dev/issues. Prefix your issue with "x/term:" in the
 subject line, so it is easy to find.
diff --git a/vendor/golang.org/x/term/term_windows.go b/vendor/golang.org/x/term/term_windows.go
index 465f56060..df6bf948e 100644
--- a/vendor/golang.org/x/term/term_windows.go
+++ b/vendor/golang.org/x/term/term_windows.go
@@ -26,6 +26,7 @@ func makeRaw(fd int) (*State, error) {
 		return nil, err
 	}
 	raw := st &^ (windows.ENABLE_ECHO_INPUT | windows.ENABLE_PROCESSED_INPUT | windows.ENABLE_LINE_INPUT | windows.ENABLE_PROCESSED_OUTPUT)
+	raw |= windows.ENABLE_VIRTUAL_TERMINAL_INPUT
 	if err := windows.SetConsoleMode(windows.Handle(fd), raw); err != nil {
 		return nil, err
 	}
diff --git a/vendor/golang.org/x/text/LICENSE b/vendor/golang.org/x/text/LICENSE
index 6a66aea5e..2a7cf70da 100644
--- a/vendor/golang.org/x/text/LICENSE
+++ b/vendor/golang.org/x/text/LICENSE
@@ -1,4 +1,4 @@
-Copyright (c) 2009 The Go Authors. All rights reserved.
+Copyright 2009 The Go Authors.
 
 Redistribution and use in source and binary forms, with or without
 modification, are permitted provided that the following conditions are
@@ -10,7 +10,7 @@ notice, this list of conditions and the following disclaimer.
 copyright notice, this list of conditions and the following disclaimer
 in the documentation and/or other materials provided with the
 distribution.
-   * Neither the name of Google Inc. nor the names of its
+   * Neither the name of Google LLC nor the names of its
 contributors may be used to endorse or promote products derived from
 this software without specific prior written permission.
 
diff --git a/vendor/golang.org/x/text/internal/catmsg/codec.go b/vendor/golang.org/x/text/internal/catmsg/codec.go
index 49c9fc978..547802b0f 100644
--- a/vendor/golang.org/x/text/internal/catmsg/codec.go
+++ b/vendor/golang.org/x/text/internal/catmsg/codec.go
@@ -257,7 +257,7 @@ func (d *Decoder) setError(err error) {
 // Language returns the language in which the message is being rendered.
 //
 // The destination language may be a child language of the language used for
-// encoding. For instance, a decoding language of "pt-PT"" is consistent with an
+// encoding. For instance, a decoding language of "pt-PT" is consistent with an
 // encoding language of "pt".
 func (d *Decoder) Language() language.Tag { return d.tag }
 
diff --git a/vendor/modules.txt b/vendor/modules.txt
index 7d9f4581e..4459b32cd 100644
--- a/vendor/modules.txt
+++ b/vendor/modules.txt
@@ -535,7 +535,7 @@ go.uber.org/zap/zapgrpc
 # go4.org v0.0.0-20230225012048-214862532bf5
 ## explicit; go 1.13
 go4.org/bytereplacer
-# golang.org/x/crypto v0.25.0
+# golang.org/x/crypto v0.31.0
 ## explicit; go 1.20
 golang.org/x/crypto/blowfish
 golang.org/x/crypto/chacha20
@@ -585,21 +585,21 @@ golang.org/x/oauth2/google/internal/stsexchange
 golang.org/x/oauth2/internal
 golang.org/x/oauth2/jws
 golang.org/x/oauth2/jwt
-# golang.org/x/sync v0.7.0
+# golang.org/x/sync v0.10.0
 ## explicit; go 1.18
 golang.org/x/sync/semaphore
 golang.org/x/sync/singleflight
-# golang.org/x/sys v0.22.0
+# golang.org/x/sys v0.28.0
 ## explicit; go 1.18
 golang.org/x/sys/cpu
 golang.org/x/sys/plan9
 golang.org/x/sys/unix
 golang.org/x/sys/windows
 golang.org/x/sys/windows/registry
-# golang.org/x/term v0.22.0
+# golang.org/x/term v0.27.0
 ## explicit; go 1.18
 golang.org/x/term
-# golang.org/x/text v0.16.0
+# golang.org/x/text v0.21.0
 ## explicit; go 1.18
 golang.org/x/text/encoding
 golang.org/x/text/encoding/charmap