diff --git a/cmd/gost/vendor/github.com/Yawning/chacha20/LICENSE b/cmd/gost/vendor/github.com/Yawning/chacha20/LICENSE
new file mode 100644
index 0000000..6ca207e
--- /dev/null
+++ b/cmd/gost/vendor/github.com/Yawning/chacha20/LICENSE
@@ -0,0 +1,122 @@
+Creative Commons Legal Code
+
+CC0 1.0 Universal
+
+ CREATIVE COMMONS CORPORATION IS NOT A LAW FIRM AND DOES NOT PROVIDE
+ LEGAL SERVICES. DISTRIBUTION OF THIS DOCUMENT DOES NOT CREATE AN
+ ATTORNEY-CLIENT RELATIONSHIP. CREATIVE COMMONS PROVIDES THIS
+ INFORMATION ON AN "AS-IS" BASIS. CREATIVE COMMONS MAKES NO WARRANTIES
+ REGARDING THE USE OF THIS DOCUMENT OR THE INFORMATION OR WORKS
+ PROVIDED HEREUNDER, AND DISCLAIMS LIABILITY FOR DAMAGES RESULTING FROM
+ THE USE OF THIS DOCUMENT OR THE INFORMATION OR WORKS PROVIDED
+ HEREUNDER.
+
+Statement of Purpose
+
+The laws of most jurisdictions throughout the world automatically confer
+exclusive Copyright and Related Rights (defined below) upon the creator
+and subsequent owner(s) (each and all, an "owner") of an original work of
+authorship and/or a database (each, a "Work").
+
+Certain owners wish to permanently relinquish those rights to a Work for
+the purpose of contributing to a commons of creative, cultural and
+scientific works ("Commons") that the public can reliably and without fear
+of later claims of infringement build upon, modify, incorporate in other
+works, reuse and redistribute as freely as possible in any form whatsoever
+and for any purposes, including without limitation commercial purposes.
+These owners may contribute to the Commons to promote the ideal of a free
+culture and the further production of creative, cultural and scientific
+works, or to gain reputation or greater distribution for their Work in
+part through the use and efforts of others.
+
+For these and/or other purposes and motivations, and without any
+expectation of additional consideration or compensation, the person
+associating CC0 with a Work (the "Affirmer"), to the extent that he or she
+is an owner of Copyright and Related Rights in the Work, voluntarily
+elects to apply CC0 to the Work and publicly distribute the Work under its
+terms, with knowledge of his or her Copyright and Related Rights in the
+Work and the meaning and intended legal effect of CC0 on those rights.
+
+1. Copyright and Related Rights. A Work made available under CC0 may be
+protected by copyright and related or neighboring rights ("Copyright and
+Related Rights"). Copyright and Related Rights include, but are not
+limited to, the following:
+
+ i. the right to reproduce, adapt, distribute, perform, display,
+ communicate, and translate a Work;
+ ii. moral rights retained by the original author(s) and/or performer(s);
+iii. publicity and privacy rights pertaining to a person's image or
+ likeness depicted in a Work;
+ iv. rights protecting against unfair competition in regards to a Work,
+ subject to the limitations in paragraph 4(a), below;
+ v. rights protecting the extraction, dissemination, use and reuse of data
+ in a Work;
+ vi. database rights (such as those arising under Directive 96/9/EC of the
+ European Parliament and of the Council of 11 March 1996 on the legal
+ protection of databases, and under any national implementation
+ thereof, including any amended or successor version of such
+ directive); and
+vii. other similar, equivalent or corresponding rights throughout the
+ world based on applicable law or treaty, and any national
+ implementations thereof.
+
+2. Waiver. To the greatest extent permitted by, but not in contravention
+of, applicable law, Affirmer hereby overtly, fully, permanently,
+irrevocably and unconditionally waives, abandons, and surrenders all of
+Affirmer's Copyright and Related Rights and associated claims and causes
+of action, whether now known or unknown (including existing as well as
+future claims and causes of action), in the Work (i) in all territories
+worldwide, (ii) for the maximum duration provided by applicable law or
+treaty (including future time extensions), (iii) in any current or future
+medium and for any number of copies, and (iv) for any purpose whatsoever,
+including without limitation commercial, advertising or promotional
+purposes (the "Waiver"). Affirmer makes the Waiver for the benefit of each
+member of the public at large and to the detriment of Affirmer's heirs and
+successors, fully intending that such Waiver shall not be subject to
+revocation, rescission, cancellation, termination, or any other legal or
+equitable action to disrupt the quiet enjoyment of the Work by the public
+as contemplated by Affirmer's express Statement of Purpose.
+
+3. Public License Fallback. Should any part of the Waiver for any reason
+be judged legally invalid or ineffective under applicable law, then the
+Waiver shall be preserved to the maximum extent permitted taking into
+account Affirmer's express Statement of Purpose. In addition, to the
+extent the Waiver is so judged Affirmer hereby grants to each affected
+person a royalty-free, non transferable, non sublicensable, non exclusive,
+irrevocable and unconditional license to exercise Affirmer's Copyright and
+Related Rights in the Work (i) in all territories worldwide, (ii) for the
+maximum duration provided by applicable law or treaty (including future
+time extensions), (iii) in any current or future medium and for any number
+of copies, and (iv) for any purpose whatsoever, including without
+limitation commercial, advertising or promotional purposes (the
+"License"). The License shall be deemed effective as of the date CC0 was
+applied by Affirmer to the Work. Should any part of the License for any
+reason be judged legally invalid or ineffective under applicable law, such
+partial invalidity or ineffectiveness shall not invalidate the remainder
+of the License, and in such case Affirmer hereby affirms that he or she
+will not (i) exercise any of his or her remaining Copyright and Related
+Rights in the Work or (ii) assert any associated claims and causes of
+action with respect to the Work, in either case contrary to Affirmer's
+express Statement of Purpose.
+
+4. Limitations and Disclaimers.
+
+ a. No trademark or patent rights held by Affirmer are waived, abandoned,
+ surrendered, licensed or otherwise affected by this document.
+ b. Affirmer offers the Work as-is and makes no representations or
+ warranties of any kind concerning the Work, express, implied,
+ statutory or otherwise, including without limitation warranties of
+ title, merchantability, fitness for a particular purpose, non
+ infringement, or the absence of latent or other defects, accuracy, or
+ the present or absence of errors, whether or not discoverable, all to
+ the greatest extent permissible under applicable law.
+ c. Affirmer disclaims responsibility for clearing rights of other persons
+ that may apply to the Work or any use thereof, including without
+ limitation any person's Copyright and Related Rights in the Work.
+ Further, Affirmer disclaims responsibility for obtaining any necessary
+ consents, permissions or other rights required for any use of the
+ Work.
+ d. Affirmer understands and acknowledges that Creative Commons is not a
+ party to this document and has no duty or obligation with respect to
+ this CC0 or use of the Work.
+
diff --git a/cmd/gost/vendor/github.com/Yawning/chacha20/README.md b/cmd/gost/vendor/github.com/Yawning/chacha20/README.md
new file mode 100644
index 0000000..9080a84
--- /dev/null
+++ b/cmd/gost/vendor/github.com/Yawning/chacha20/README.md
@@ -0,0 +1,14 @@
+### chacha20 - ChaCha20
+#### Yawning Angel (yawning at schwanenlied dot me)
+
+Yet another Go ChaCha20 implementation. Everything else I found was slow,
+didn't support all the variants I need to use, or relied on cgo to go fast.
+
+Features:
+
+ * 20 round, 256 bit key only. Everything else is pointless and stupid.
+ * IETF 96 bit nonce variant.
+ * XChaCha 24 byte nonce variant.
+ * SSE2 and AVX2 support on amd64 targets.
+ * Incremental encrypt/decrypt support, unlike golang.org/x/crypto/salsa20.
+
diff --git a/cmd/gost/vendor/github.com/Yawning/chacha20/chacha20.go b/cmd/gost/vendor/github.com/Yawning/chacha20/chacha20.go
new file mode 100644
index 0000000..07d5e4b
--- /dev/null
+++ b/cmd/gost/vendor/github.com/Yawning/chacha20/chacha20.go
@@ -0,0 +1,273 @@
+// chacha20.go - A ChaCha stream cipher implementation.
+//
+// To the extent possible under law, Yawning Angel has waived all copyright
+// and related or neighboring rights to chacha20, using the Creative
+// Commons "CC0" public domain dedication. See LICENSE or
+// for full details.
+
+package chacha20
+
+import (
+ "crypto/cipher"
+ "encoding/binary"
+ "errors"
+ "math"
+ "runtime"
+)
+
+const (
+ // KeySize is the ChaCha20 key size in bytes.
+ KeySize = 32
+
+ // NonceSize is the ChaCha20 nonce size in bytes.
+ NonceSize = 8
+
+ // INonceSize is the IETF ChaCha20 nonce size in bytes.
+ INonceSize = 12
+
+ // XNonceSize is the XChaCha20 nonce size in bytes.
+ XNonceSize = 24
+
+ // HNonceSize is the HChaCha20 nonce size in bytes.
+ HNonceSize = 16
+
+ // BlockSize is the ChaCha20 block size in bytes.
+ BlockSize = 64
+
+ stateSize = 16
+ chachaRounds = 20
+
+ // The constant "expand 32-byte k" as little endian uint32s.
+ sigma0 = uint32(0x61707865)
+ sigma1 = uint32(0x3320646e)
+ sigma2 = uint32(0x79622d32)
+ sigma3 = uint32(0x6b206574)
+)
+
+var (
+ // ErrInvalidKey is the error returned when the key is invalid.
+ ErrInvalidKey = errors.New("key length must be KeySize bytes")
+
+ // ErrInvalidNonce is the error returned when the nonce is invalid.
+ ErrInvalidNonce = errors.New("nonce length must be NonceSize/INonceSize/XNonceSize bytes")
+
+ // ErrInvalidCounter is the error returned when the counter is invalid.
+ ErrInvalidCounter = errors.New("block counter is invalid (out of range)")
+
+ useUnsafe = false
+ usingVectors = false
+ blocksFn = blocksRef
+)
+
+// A Cipher is an instance of ChaCha20/XChaCha20 using a particular key and
+// nonce.
+type Cipher struct {
+ state [stateSize]uint32
+
+ buf [BlockSize]byte
+ off int
+ ietf bool
+}
+
+// Reset zeros the key data so that it will no longer appear in the process's
+// memory.
+func (c *Cipher) Reset() {
+ for i := range c.state {
+ c.state[i] = 0
+ }
+ for i := range c.buf {
+ c.buf[i] = 0
+ }
+}
+
+// XORKeyStream sets dst to the result of XORing src with the key stream. Dst
+// and src may be the same slice but otherwise should not overlap.
+func (c *Cipher) XORKeyStream(dst, src []byte) {
+ if len(dst) < len(src) {
+ src = src[:len(dst)]
+ }
+
+ for remaining := len(src); remaining > 0; {
+ // Process multiple blocks at once.
+ if c.off == BlockSize {
+ nrBlocks := remaining / BlockSize
+ directBytes := nrBlocks * BlockSize
+ if nrBlocks > 0 {
+ blocksFn(&c.state, src, dst, nrBlocks, c.ietf)
+ remaining -= directBytes
+ if remaining == 0 {
+ return
+ }
+ dst = dst[directBytes:]
+ src = src[directBytes:]
+ }
+
+ // If there's a partial block, generate 1 block of keystream into
+ // the internal buffer.
+ blocksFn(&c.state, nil, c.buf[:], 1, c.ietf)
+ c.off = 0
+ }
+
+ // Process partial blocks from the buffered keystream.
+ toXor := BlockSize - c.off
+ if remaining < toXor {
+ toXor = remaining
+ }
+ if toXor > 0 {
+ for i, v := range src[:toXor] {
+ dst[i] = v ^ c.buf[c.off+i]
+ }
+ dst = dst[toXor:]
+ src = src[toXor:]
+
+ remaining -= toXor
+ c.off += toXor
+ }
+ }
+}
+
+// KeyStream sets dst to the raw keystream.
+func (c *Cipher) KeyStream(dst []byte) {
+ for remaining := len(dst); remaining > 0; {
+ // Process multiple blocks at once.
+ if c.off == BlockSize {
+ nrBlocks := remaining / BlockSize
+ directBytes := nrBlocks * BlockSize
+ if nrBlocks > 0 {
+ blocksFn(&c.state, nil, dst, nrBlocks, c.ietf)
+ remaining -= directBytes
+ if remaining == 0 {
+ return
+ }
+ dst = dst[directBytes:]
+ }
+
+ // If there's a partial block, generate 1 block of keystream into
+ // the internal buffer.
+ blocksFn(&c.state, nil, c.buf[:], 1, c.ietf)
+ c.off = 0
+ }
+
+ // Process partial blocks from the buffered keystream.
+ toCopy := BlockSize - c.off
+ if remaining < toCopy {
+ toCopy = remaining
+ }
+ if toCopy > 0 {
+ copy(dst[:toCopy], c.buf[c.off:c.off+toCopy])
+ dst = dst[toCopy:]
+ remaining -= toCopy
+ c.off += toCopy
+ }
+ }
+}
+
+// ReKey reinitializes the ChaCha20/XChaCha20 instance with the provided key
+// and nonce.
+func (c *Cipher) ReKey(key, nonce []byte) error {
+ if len(key) != KeySize {
+ return ErrInvalidKey
+ }
+
+ switch len(nonce) {
+ case NonceSize:
+ case INonceSize:
+ case XNonceSize:
+ var subkey [KeySize]byte
+ var subnonce [HNonceSize]byte
+ copy(subnonce[:], nonce[0:16])
+ HChaCha(key, &subnonce, &subkey)
+ key = subkey[:]
+ nonce = nonce[16:24]
+ defer func() {
+ for i := range subkey {
+ subkey[i] = 0
+ }
+ }()
+ default:
+ return ErrInvalidNonce
+ }
+
+ c.Reset()
+ c.state[0] = sigma0
+ c.state[1] = sigma1
+ c.state[2] = sigma2
+ c.state[3] = sigma3
+ c.state[4] = binary.LittleEndian.Uint32(key[0:4])
+ c.state[5] = binary.LittleEndian.Uint32(key[4:8])
+ c.state[6] = binary.LittleEndian.Uint32(key[8:12])
+ c.state[7] = binary.LittleEndian.Uint32(key[12:16])
+ c.state[8] = binary.LittleEndian.Uint32(key[16:20])
+ c.state[9] = binary.LittleEndian.Uint32(key[20:24])
+ c.state[10] = binary.LittleEndian.Uint32(key[24:28])
+ c.state[11] = binary.LittleEndian.Uint32(key[28:32])
+ c.state[12] = 0
+ if len(nonce) == INonceSize {
+ c.state[13] = binary.LittleEndian.Uint32(nonce[0:4])
+ c.state[14] = binary.LittleEndian.Uint32(nonce[4:8])
+ c.state[15] = binary.LittleEndian.Uint32(nonce[8:12])
+ c.ietf = true
+ } else {
+ c.state[13] = 0
+ c.state[14] = binary.LittleEndian.Uint32(nonce[0:4])
+ c.state[15] = binary.LittleEndian.Uint32(nonce[4:8])
+ c.ietf = false
+ }
+ c.off = BlockSize
+ return nil
+
+}
+
+// Seek sets the block counter to a given offset.
+func (c *Cipher) Seek(blockCounter uint64) error {
+ if c.ietf {
+ if blockCounter > math.MaxUint32 {
+ return ErrInvalidCounter
+ }
+ c.state[12] = uint32(blockCounter)
+ } else {
+ c.state[12] = uint32(blockCounter)
+ c.state[13] = uint32(blockCounter >> 32)
+ }
+ c.off = BlockSize
+ return nil
+}
+
+// NewCipher returns a new ChaCha20/XChaCha20 instance.
+func NewCipher(key, nonce []byte) (*Cipher, error) {
+ c := new(Cipher)
+ if err := c.ReKey(key, nonce); err != nil {
+ return nil, err
+ }
+ return c, nil
+}
+
+// HChaCha is the HChaCha20 hash function used to make XChaCha.
+func HChaCha(key []byte, nonce *[HNonceSize]byte, out *[32]byte) {
+ var x [stateSize]uint32 // Last 4 slots unused, sigma hardcoded.
+ x[0] = binary.LittleEndian.Uint32(key[0:4])
+ x[1] = binary.LittleEndian.Uint32(key[4:8])
+ x[2] = binary.LittleEndian.Uint32(key[8:12])
+ x[3] = binary.LittleEndian.Uint32(key[12:16])
+ x[4] = binary.LittleEndian.Uint32(key[16:20])
+ x[5] = binary.LittleEndian.Uint32(key[20:24])
+ x[6] = binary.LittleEndian.Uint32(key[24:28])
+ x[7] = binary.LittleEndian.Uint32(key[28:32])
+ x[8] = binary.LittleEndian.Uint32(nonce[0:4])
+ x[9] = binary.LittleEndian.Uint32(nonce[4:8])
+ x[10] = binary.LittleEndian.Uint32(nonce[8:12])
+ x[11] = binary.LittleEndian.Uint32(nonce[12:16])
+ hChaChaRef(&x, out)
+}
+
+func init() {
+ switch runtime.GOARCH {
+ case "386", "amd64":
+ // Abuse unsafe to skip calling binary.LittleEndian.PutUint32
+ // in the critical path. This is a big boost on systems that are
+ // little endian and not overly picky about alignment.
+ useUnsafe = true
+ }
+}
+
+var _ cipher.Stream = (*Cipher)(nil)
diff --git a/cmd/gost/vendor/github.com/Yawning/chacha20/chacha20_amd64.go b/cmd/gost/vendor/github.com/Yawning/chacha20/chacha20_amd64.go
new file mode 100644
index 0000000..b2c8623
--- /dev/null
+++ b/cmd/gost/vendor/github.com/Yawning/chacha20/chacha20_amd64.go
@@ -0,0 +1,95 @@
+// chacha20_amd64.go - AMD64 optimized chacha20.
+//
+// To the extent possible under law, Yawning Angel has waived all copyright
+// and related or neighboring rights to chacha20, using the Creative
+// Commons "CC0" public domain dedication. See LICENSE or
+// for full details.
+
+// +build amd64,!gccgo,!appengine
+
+package chacha20
+
+import (
+ "math"
+)
+
+var usingAVX2 = false
+
+func blocksAmd64SSE2(x *uint32, inp, outp *byte, nrBlocks uint)
+
+func blocksAmd64AVX2(x *uint32, inp, outp *byte, nrBlocks uint)
+
+func cpuidAmd64(cpuidParams *uint32)
+
+func xgetbv0Amd64(xcrVec *uint32)
+
+func blocksAmd64(x *[stateSize]uint32, in []byte, out []byte, nrBlocks int, isIetf bool) {
+ // Probably unneeded, but stating this explicitly simplifies the assembly.
+ if nrBlocks == 0 {
+ return
+ }
+
+ if isIetf {
+ var totalBlocks uint64
+ totalBlocks = uint64(x[8]) + uint64(nrBlocks)
+ if totalBlocks > math.MaxUint32 {
+ panic("chacha20: Exceeded keystream per nonce limit")
+ }
+ }
+
+ if in == nil {
+ for i := range out {
+ out[i] = 0
+ }
+ in = out
+ }
+
+ // Pointless to call the AVX2 code for just a single block, since half of
+ // the output gets discarded...
+ if usingAVX2 && nrBlocks > 1 {
+ blocksAmd64AVX2(&x[0], &in[0], &out[0], uint(nrBlocks))
+ } else {
+ blocksAmd64SSE2(&x[0], &in[0], &out[0], uint(nrBlocks))
+ }
+}
+
+func supportsAVX2() bool {
+ // https://software.intel.com/en-us/articles/how-to-detect-new-instruction-support-in-the-4th-generation-intel-core-processor-family
+ const (
+ osXsaveBit = 1 << 27
+ avx2Bit = 1 << 5
+ )
+
+ // Check to see if CPUID actually supports the leaf that indicates AVX2.
+ // CPUID.(EAX=0H, ECX=0H) >= 7
+ regs := [4]uint32{0x00}
+ cpuidAmd64(®s[0])
+ if regs[0] < 7 {
+ return false
+ }
+
+ // Check to see if the OS knows how to save/restore XMM/YMM state.
+ // CPUID.(EAX=01H, ECX=0H):ECX.OSXSAVE[bit 27]==1
+ regs = [4]uint32{0x01}
+ cpuidAmd64(®s[0])
+ if regs[2]&osXsaveBit == 0 {
+ return false
+ }
+ xcrRegs := [2]uint32{}
+ xgetbv0Amd64(&xcrRegs[0])
+ if xcrRegs[0]&6 != 6 {
+ return false
+ }
+
+ // Check for AVX2 support.
+ // CPUID.(EAX=07H, ECX=0H):EBX.AVX2[bit 5]==1
+ regs = [4]uint32{0x07}
+ cpuidAmd64(®s[0])
+ return regs[1]&avx2Bit != 0
+}
+
+func init() {
+ blocksFn = blocksAmd64
+ usingVectors = true
+ usingAVX2 = supportsAVX2()
+}
diff --git a/cmd/gost/vendor/github.com/Yawning/chacha20/chacha20_amd64.py b/cmd/gost/vendor/github.com/Yawning/chacha20/chacha20_amd64.py
new file mode 100644
index 0000000..5b689d1
--- /dev/null
+++ b/cmd/gost/vendor/github.com/Yawning/chacha20/chacha20_amd64.py
@@ -0,0 +1,1303 @@
+#!/usr/bin/env python3
+#
+# To the extent possible under law, Yawning Angel has waived all copyright
+# and related or neighboring rights to chacha20, using the Creative
+# Commons "CC0" public domain dedication. See LICENSE or
+# for full details.
+
+#
+# cgo sucks. Plan 9 assembly sucks. Real languages have SIMD intrinsics.
+# The least terrible/retarded option is to use a Python code generator, so
+# that's what I did.
+#
+# Code based on Ted Krovetz's vec128 C implementation, with corrections
+# to use a 64 bit counter instead of 32 bit, and to allow unaligned input and
+# output pointers.
+#
+# Dependencies: https://github.com/Maratyszcza/PeachPy
+#
+# python3 -m peachpy.x86_64 -mabi=goasm -S -o chacha20_amd64.s chacha20_amd64.py
+#
+
+from peachpy import *
+from peachpy.x86_64 import *
+
+x = Argument(ptr(uint32_t))
+inp = Argument(ptr(const_uint8_t))
+outp = Argument(ptr(uint8_t))
+nrBlocks = Argument(ptr(size_t))
+
+#
+# SSE2 helper functions. A temporary register is explicitly passed in because
+# the main fast loop uses every single register (and even spills) so manual
+# control is needed.
+#
+# This used to also have a DQROUNDS helper that did 2 rounds of ChaCha like
+# in the C code, but the C code has the luxury of an optimizer reordering
+# everything, while this does not.
+#
+
+def ROTW16_sse2(tmp, d):
+ MOVDQA(tmp, d)
+ PSLLD(tmp, 16)
+ PSRLD(d, 16)
+ PXOR(d, tmp)
+
+def ROTW12_sse2(tmp, b):
+ MOVDQA(tmp, b)
+ PSLLD(tmp, 12)
+ PSRLD(b, 20)
+ PXOR(b, tmp)
+
+def ROTW8_sse2(tmp, d):
+ MOVDQA(tmp, d)
+ PSLLD(tmp, 8)
+ PSRLD(d, 24)
+ PXOR(d, tmp)
+
+def ROTW7_sse2(tmp, b):
+ MOVDQA(tmp, b)
+ PSLLD(tmp, 7)
+ PSRLD(b, 25)
+ PXOR(b, tmp)
+
+def WriteXor_sse2(tmp, inp, outp, d, v0, v1, v2, v3):
+ MOVDQU(tmp, [inp+d])
+ PXOR(tmp, v0)
+ MOVDQU([outp+d], tmp)
+ MOVDQU(tmp, [inp+d+16])
+ PXOR(tmp, v1)
+ MOVDQU([outp+d+16], tmp)
+ MOVDQU(tmp, [inp+d+32])
+ PXOR(tmp, v2)
+ MOVDQU([outp+d+32], tmp)
+ MOVDQU(tmp, [inp+d+48])
+ PXOR(tmp, v3)
+ MOVDQU([outp+d+48], tmp)
+
+# SSE2 ChaCha20 (aka vec128). Does not handle partial blocks, and will
+# process 4/2/1 blocks at a time. x (the ChaCha20 state) must be 16 byte
+# aligned.
+with Function("blocksAmd64SSE2", (x, inp, outp, nrBlocks)):
+ reg_x = GeneralPurposeRegister64()
+ reg_inp = GeneralPurposeRegister64()
+ reg_outp = GeneralPurposeRegister64()
+ reg_blocks = GeneralPurposeRegister64()
+ reg_sp_save = GeneralPurposeRegister64()
+
+ LOAD.ARGUMENT(reg_x, x)
+ LOAD.ARGUMENT(reg_inp, inp)
+ LOAD.ARGUMENT(reg_outp, outp)
+ LOAD.ARGUMENT(reg_blocks, nrBlocks)
+
+ # Align the stack to a 32 byte boundary.
+ reg_align = GeneralPurposeRegister64()
+ MOV(reg_sp_save, registers.rsp)
+ MOV(reg_align, 0x1f)
+ NOT(reg_align)
+ AND(registers.rsp, reg_align)
+ SUB(registers.rsp, 0x20)
+
+ # Build the counter increment vector on the stack, and allocate the scratch
+ # space
+ xmm_v0 = XMMRegister()
+ PXOR(xmm_v0, xmm_v0)
+ SUB(registers.rsp, 16+16)
+ MOVDQA([registers.rsp], xmm_v0)
+ reg_tmp = GeneralPurposeRegister32()
+ MOV(reg_tmp, 0x00000001)
+ MOV([registers.rsp], reg_tmp)
+ mem_one = [registers.rsp] # (Stack) Counter increment vector
+ mem_tmp0 = [registers.rsp+16] # (Stack) Scratch space.
+
+ mem_s0 = [reg_x] # (Memory) Cipher state [0..3]
+ mem_s1 = [reg_x+16] # (Memory) Cipher state [4..7]
+ mem_s2 = [reg_x+32] # (Memory) Cipher state [8..11]
+ mem_s3 = [reg_x+48] # (Memory) Cipher state [12..15]
+
+ # xmm_v0 allocated above...
+ xmm_v1 = XMMRegister()
+ xmm_v2 = XMMRegister()
+ xmm_v3 = XMMRegister()
+
+ xmm_v4 = XMMRegister()
+ xmm_v5 = XMMRegister()
+ xmm_v6 = XMMRegister()
+ xmm_v7 = XMMRegister()
+
+ xmm_v8 = XMMRegister()
+ xmm_v9 = XMMRegister()
+ xmm_v10 = XMMRegister()
+ xmm_v11 = XMMRegister()
+
+ xmm_v12 = XMMRegister()
+ xmm_v13 = XMMRegister()
+ xmm_v14 = XMMRegister()
+ xmm_v15 = XMMRegister()
+
+ xmm_tmp = xmm_v12
+
+ #
+ # 4 blocks at a time.
+ #
+
+ vector_loop4 = Loop()
+ SUB(reg_blocks, 4)
+ JB(vector_loop4.end)
+ with vector_loop4:
+ MOVDQA(xmm_v0, mem_s0)
+ MOVDQA(xmm_v1, mem_s1)
+ MOVDQA(xmm_v2, mem_s2)
+ MOVDQA(xmm_v3, mem_s3)
+
+ MOVDQA(xmm_v4, xmm_v0)
+ MOVDQA(xmm_v5, xmm_v1)
+ MOVDQA(xmm_v6, xmm_v2)
+ MOVDQA(xmm_v7, xmm_v3)
+ PADDQ(xmm_v7, mem_one)
+
+ MOVDQA(xmm_v8, xmm_v0)
+ MOVDQA(xmm_v9, xmm_v1)
+ MOVDQA(xmm_v10, xmm_v2)
+ MOVDQA(xmm_v11, xmm_v7)
+ PADDQ(xmm_v11, mem_one)
+
+ MOVDQA(xmm_v12, xmm_v0)
+ MOVDQA(xmm_v13, xmm_v1)
+ MOVDQA(xmm_v14, xmm_v2)
+ MOVDQA(xmm_v15, xmm_v11)
+ PADDQ(xmm_v15, mem_one)
+
+ reg_rounds = GeneralPurposeRegister64()
+ MOV(reg_rounds, 20)
+ rounds_loop4 = Loop()
+ with rounds_loop4:
+ # a += b; d ^= a; d = ROTW16(d);
+ PADDD(xmm_v0, xmm_v1)
+ PADDD(xmm_v4, xmm_v5)
+ PADDD(xmm_v8, xmm_v9)
+ PADDD(xmm_v12, xmm_v13)
+ PXOR(xmm_v3, xmm_v0)
+ PXOR(xmm_v7, xmm_v4)
+ PXOR(xmm_v11, xmm_v8)
+ PXOR(xmm_v15, xmm_v12)
+
+ MOVDQA(mem_tmp0, xmm_tmp) # Save
+
+ ROTW16_sse2(xmm_tmp, xmm_v3)
+ ROTW16_sse2(xmm_tmp, xmm_v7)
+ ROTW16_sse2(xmm_tmp, xmm_v11)
+ ROTW16_sse2(xmm_tmp, xmm_v15)
+
+ # c += d; b ^= c; b = ROTW12(b);
+ PADDD(xmm_v2, xmm_v3)
+ PADDD(xmm_v6, xmm_v7)
+ PADDD(xmm_v10, xmm_v11)
+ PADDD(xmm_v14, xmm_v15)
+ PXOR(xmm_v1, xmm_v2)
+ PXOR(xmm_v5, xmm_v6)
+ PXOR(xmm_v9, xmm_v10)
+ PXOR(xmm_v13, xmm_v14)
+ ROTW12_sse2(xmm_tmp, xmm_v1)
+ ROTW12_sse2(xmm_tmp, xmm_v5)
+ ROTW12_sse2(xmm_tmp, xmm_v9)
+ ROTW12_sse2(xmm_tmp, xmm_v13)
+
+ # a += b; d ^= a; d = ROTW8(d);
+ MOVDQA(xmm_tmp, mem_tmp0) # Restore
+
+ PADDD(xmm_v0, xmm_v1)
+ PADDD(xmm_v4, xmm_v5)
+ PADDD(xmm_v8, xmm_v9)
+ PADDD(xmm_v12, xmm_v13)
+ PXOR(xmm_v3, xmm_v0)
+ PXOR(xmm_v7, xmm_v4)
+ PXOR(xmm_v11, xmm_v8)
+ PXOR(xmm_v15, xmm_v12)
+
+ MOVDQA(mem_tmp0, xmm_tmp) # Save
+
+ ROTW8_sse2(xmm_tmp, xmm_v3)
+ ROTW8_sse2(xmm_tmp, xmm_v7)
+ ROTW8_sse2(xmm_tmp, xmm_v11)
+ ROTW8_sse2(xmm_tmp, xmm_v15)
+
+ # c += d; b ^= c; b = ROTW7(b)
+ PADDD(xmm_v2, xmm_v3)
+ PADDD(xmm_v6, xmm_v7)
+ PADDD(xmm_v10, xmm_v11)
+ PADDD(xmm_v14, xmm_v15)
+ PXOR(xmm_v1, xmm_v2)
+ PXOR(xmm_v5, xmm_v6)
+ PXOR(xmm_v9, xmm_v10)
+ PXOR(xmm_v13, xmm_v14)
+ ROTW7_sse2(xmm_tmp, xmm_v1)
+ ROTW7_sse2(xmm_tmp, xmm_v5)
+ ROTW7_sse2(xmm_tmp, xmm_v9)
+ ROTW7_sse2(xmm_tmp, xmm_v13)
+
+ # b = ROTV1(b); c = ROTV2(c); d = ROTV3(d);
+ PSHUFD(xmm_v1, xmm_v1, 0x39)
+ PSHUFD(xmm_v5, xmm_v5, 0x39)
+ PSHUFD(xmm_v9, xmm_v9, 0x39)
+ PSHUFD(xmm_v13, xmm_v13, 0x39)
+ PSHUFD(xmm_v2, xmm_v2, 0x4e)
+ PSHUFD(xmm_v6, xmm_v6, 0x4e)
+ PSHUFD(xmm_v10, xmm_v10, 0x4e)
+ PSHUFD(xmm_v14, xmm_v14, 0x4e)
+ PSHUFD(xmm_v3, xmm_v3, 0x93)
+ PSHUFD(xmm_v7, xmm_v7, 0x93)
+ PSHUFD(xmm_v11, xmm_v11, 0x93)
+ PSHUFD(xmm_v15, xmm_v15, 0x93)
+
+ MOVDQA(xmm_tmp, mem_tmp0) # Restore
+
+ # a += b; d ^= a; d = ROTW16(d);
+ PADDD(xmm_v0, xmm_v1)
+ PADDD(xmm_v4, xmm_v5)
+ PADDD(xmm_v8, xmm_v9)
+ PADDD(xmm_v12, xmm_v13)
+ PXOR(xmm_v3, xmm_v0)
+ PXOR(xmm_v7, xmm_v4)
+ PXOR(xmm_v11, xmm_v8)
+ PXOR(xmm_v15, xmm_v12)
+
+ MOVDQA(mem_tmp0, xmm_tmp) # Save
+
+ ROTW16_sse2(xmm_tmp, xmm_v3)
+ ROTW16_sse2(xmm_tmp, xmm_v7)
+ ROTW16_sse2(xmm_tmp, xmm_v11)
+ ROTW16_sse2(xmm_tmp, xmm_v15)
+
+ # c += d; b ^= c; b = ROTW12(b);
+ PADDD(xmm_v2, xmm_v3)
+ PADDD(xmm_v6, xmm_v7)
+ PADDD(xmm_v10, xmm_v11)
+ PADDD(xmm_v14, xmm_v15)
+ PXOR(xmm_v1, xmm_v2)
+ PXOR(xmm_v5, xmm_v6)
+ PXOR(xmm_v9, xmm_v10)
+ PXOR(xmm_v13, xmm_v14)
+ ROTW12_sse2(xmm_tmp, xmm_v1)
+ ROTW12_sse2(xmm_tmp, xmm_v5)
+ ROTW12_sse2(xmm_tmp, xmm_v9)
+ ROTW12_sse2(xmm_tmp, xmm_v13)
+
+ # a += b; d ^= a; d = ROTW8(d);
+ MOVDQA(xmm_tmp, mem_tmp0) # Restore
+
+ PADDD(xmm_v0, xmm_v1)
+ PADDD(xmm_v4, xmm_v5)
+ PADDD(xmm_v8, xmm_v9)
+ PADDD(xmm_v12, xmm_v13)
+ PXOR(xmm_v3, xmm_v0)
+ PXOR(xmm_v7, xmm_v4)
+ PXOR(xmm_v11, xmm_v8)
+ PXOR(xmm_v15, xmm_v12)
+
+ MOVDQA(mem_tmp0, xmm_tmp) # Save
+
+ ROTW8_sse2(xmm_tmp, xmm_v3)
+ ROTW8_sse2(xmm_tmp, xmm_v7)
+ ROTW8_sse2(xmm_tmp, xmm_v11)
+ ROTW8_sse2(xmm_tmp, xmm_v15)
+
+ # c += d; b ^= c; b = ROTW7(b)
+ PADDD(xmm_v2, xmm_v3)
+ PADDD(xmm_v6, xmm_v7)
+ PADDD(xmm_v10, xmm_v11)
+ PADDD(xmm_v14, xmm_v15)
+ PXOR(xmm_v1, xmm_v2)
+ PXOR(xmm_v5, xmm_v6)
+ PXOR(xmm_v9, xmm_v10)
+ PXOR(xmm_v13, xmm_v14)
+ ROTW7_sse2(xmm_tmp, xmm_v1)
+ ROTW7_sse2(xmm_tmp, xmm_v5)
+ ROTW7_sse2(xmm_tmp, xmm_v9)
+ ROTW7_sse2(xmm_tmp, xmm_v13)
+
+ # b = ROTV1(b); c = ROTV2(c); d = ROTV3(d);
+ PSHUFD(xmm_v1, xmm_v1, 0x93)
+ PSHUFD(xmm_v5, xmm_v5, 0x93)
+ PSHUFD(xmm_v9, xmm_v9, 0x93)
+ PSHUFD(xmm_v13, xmm_v13, 0x93)
+ PSHUFD(xmm_v2, xmm_v2, 0x4e)
+ PSHUFD(xmm_v6, xmm_v6, 0x4e)
+ PSHUFD(xmm_v10, xmm_v10, 0x4e)
+ PSHUFD(xmm_v14, xmm_v14, 0x4e)
+ PSHUFD(xmm_v3, xmm_v3, 0x39)
+ PSHUFD(xmm_v7, xmm_v7, 0x39)
+ PSHUFD(xmm_v11, xmm_v11, 0x39)
+ PSHUFD(xmm_v15, xmm_v15, 0x39)
+
+ MOVDQA(xmm_tmp, mem_tmp0) # Restore
+
+ SUB(reg_rounds, 2)
+ JNZ(rounds_loop4.begin)
+
+ MOVDQA(mem_tmp0, xmm_tmp)
+
+ PADDD(xmm_v0, mem_s0)
+ PADDD(xmm_v1, mem_s1)
+ PADDD(xmm_v2, mem_s2)
+ PADDD(xmm_v3, mem_s3)
+ WriteXor_sse2(xmm_tmp, reg_inp, reg_outp, 0, xmm_v0, xmm_v1, xmm_v2, xmm_v3)
+ MOVDQA(xmm_v3, mem_s3)
+ PADDQ(xmm_v3, mem_one)
+
+ PADDD(xmm_v4, mem_s0)
+ PADDD(xmm_v5, mem_s1)
+ PADDD(xmm_v6, mem_s2)
+ PADDD(xmm_v7, xmm_v3)
+ WriteXor_sse2(xmm_tmp, reg_inp, reg_outp, 64, xmm_v4, xmm_v5, xmm_v6, xmm_v7)
+ PADDQ(xmm_v3, mem_one)
+
+ PADDD(xmm_v8, mem_s0)
+ PADDD(xmm_v9, mem_s1)
+ PADDD(xmm_v10, mem_s2)
+ PADDD(xmm_v11, xmm_v3)
+ WriteXor_sse2(xmm_tmp, reg_inp, reg_outp, 128, xmm_v8, xmm_v9, xmm_v10, xmm_v11)
+ PADDQ(xmm_v3, mem_one)
+
+ MOVDQA(xmm_tmp, mem_tmp0)
+
+ PADDD(xmm_v12, mem_s0)
+ PADDD(xmm_v13, mem_s1)
+ PADDD(xmm_v14, mem_s2)
+ PADDD(xmm_v15, xmm_v3)
+ WriteXor_sse2(xmm_v0, reg_inp, reg_outp, 192, xmm_v12, xmm_v13, xmm_v14, xmm_v15)
+ PADDQ(xmm_v3, mem_one)
+
+ MOVDQA(mem_s3, xmm_v3)
+
+ ADD(reg_inp, 4 * 64)
+ ADD(reg_outp, 4 * 64)
+
+ SUB(reg_blocks, 4)
+ JAE(vector_loop4.begin)
+
+ ADD(reg_blocks, 4)
+ out = Label()
+ JZ(out)
+
+ # Past this point, we no longer need to use every single register to hold
+ # the in progress state.
+
+ xmm_s0 = xmm_v8
+ xmm_s1 = xmm_v9
+ xmm_s2 = xmm_v10
+ xmm_s3 = xmm_v11
+ xmm_one = xmm_v13
+ MOVDQA(xmm_s0, mem_s0)
+ MOVDQA(xmm_s1, mem_s1)
+ MOVDQA(xmm_s2, mem_s2)
+ MOVDQA(xmm_s3, mem_s3)
+ MOVDQA(xmm_one, mem_one)
+
+ #
+ # 2 blocks at a time.
+ #
+
+ SUB(reg_blocks, 2)
+ vector_loop2 = Loop()
+ JB(vector_loop2.end)
+ with vector_loop2:
+ MOVDQA(xmm_v0, xmm_s0)
+ MOVDQA(xmm_v1, xmm_s1)
+ MOVDQA(xmm_v2, xmm_s2)
+ MOVDQA(xmm_v3, xmm_s3)
+
+ MOVDQA(xmm_v4, xmm_v0)
+ MOVDQA(xmm_v5, xmm_v1)
+ MOVDQA(xmm_v6, xmm_v2)
+ MOVDQA(xmm_v7, xmm_v3)
+ PADDQ(xmm_v7, xmm_one)
+
+ reg_rounds = GeneralPurposeRegister64()
+ MOV(reg_rounds, 20)
+ rounds_loop2 = Loop()
+ with rounds_loop2:
+ # a += b; d ^= a; d = ROTW16(d);
+ PADDD(xmm_v0, xmm_v1)
+ PADDD(xmm_v4, xmm_v5)
+ PXOR(xmm_v3, xmm_v0)
+ PXOR(xmm_v7, xmm_v4)
+ ROTW16_sse2(xmm_tmp, xmm_v3)
+ ROTW16_sse2(xmm_tmp, xmm_v7)
+
+ # c += d; b ^= c; b = ROTW12(b);
+ PADDD(xmm_v2, xmm_v3)
+ PADDD(xmm_v6, xmm_v7)
+ PXOR(xmm_v1, xmm_v2)
+ PXOR(xmm_v5, xmm_v6)
+ ROTW12_sse2(xmm_tmp, xmm_v1)
+ ROTW12_sse2(xmm_tmp, xmm_v5)
+
+ # a += b; d ^= a; d = ROTW8(d);
+ PADDD(xmm_v0, xmm_v1)
+ PADDD(xmm_v4, xmm_v5)
+ PXOR(xmm_v3, xmm_v0)
+ PXOR(xmm_v7, xmm_v4)
+ ROTW8_sse2(xmm_tmp, xmm_v3)
+ ROTW8_sse2(xmm_tmp, xmm_v7)
+
+ # c += d; b ^= c; b = ROTW7(b)
+ PADDD(xmm_v2, xmm_v3)
+ PADDD(xmm_v6, xmm_v7)
+ PXOR(xmm_v1, xmm_v2)
+ PXOR(xmm_v5, xmm_v6)
+ ROTW7_sse2(xmm_tmp, xmm_v1)
+ ROTW7_sse2(xmm_tmp, xmm_v5)
+
+ # b = ROTV1(b); c = ROTV2(c); d = ROTV3(d);
+ PSHUFD(xmm_v1, xmm_v1, 0x39)
+ PSHUFD(xmm_v5, xmm_v5, 0x39)
+ PSHUFD(xmm_v2, xmm_v2, 0x4e)
+ PSHUFD(xmm_v6, xmm_v6, 0x4e)
+ PSHUFD(xmm_v3, xmm_v3, 0x93)
+ PSHUFD(xmm_v7, xmm_v7, 0x93)
+
+ # a += b; d ^= a; d = ROTW16(d);
+ PADDD(xmm_v0, xmm_v1)
+ PADDD(xmm_v4, xmm_v5)
+ PXOR(xmm_v3, xmm_v0)
+ PXOR(xmm_v7, xmm_v4)
+ ROTW16_sse2(xmm_tmp, xmm_v3)
+ ROTW16_sse2(xmm_tmp, xmm_v7)
+
+ # c += d; b ^= c; b = ROTW12(b);
+ PADDD(xmm_v2, xmm_v3)
+ PADDD(xmm_v6, xmm_v7)
+ PXOR(xmm_v1, xmm_v2)
+ PXOR(xmm_v5, xmm_v6)
+ ROTW12_sse2(xmm_tmp, xmm_v1)
+ ROTW12_sse2(xmm_tmp, xmm_v5)
+
+ # a += b; d ^= a; d = ROTW8(d);
+ PADDD(xmm_v0, xmm_v1)
+ PADDD(xmm_v4, xmm_v5)
+ PXOR(xmm_v3, xmm_v0)
+ PXOR(xmm_v7, xmm_v4)
+ ROTW8_sse2(xmm_tmp, xmm_v3)
+ ROTW8_sse2(xmm_tmp, xmm_v7)
+
+ # c += d; b ^= c; b = ROTW7(b)
+ PADDD(xmm_v2, xmm_v3)
+ PADDD(xmm_v6, xmm_v7)
+ PXOR(xmm_v1, xmm_v2)
+ PXOR(xmm_v5, xmm_v6)
+ ROTW7_sse2(xmm_tmp, xmm_v1)
+ ROTW7_sse2(xmm_tmp, xmm_v5)
+
+ # b = ROTV1(b); c = ROTV2(c); d = ROTV3(d);
+ PSHUFD(xmm_v1, xmm_v1, 0x93)
+ PSHUFD(xmm_v5, xmm_v5, 0x93)
+ PSHUFD(xmm_v2, xmm_v2, 0x4e)
+ PSHUFD(xmm_v6, xmm_v6, 0x4e)
+ PSHUFD(xmm_v3, xmm_v3, 0x39)
+ PSHUFD(xmm_v7, xmm_v7, 0x39)
+
+ SUB(reg_rounds, 2)
+ JNZ(rounds_loop2.begin)
+
+ PADDD(xmm_v0, xmm_s0)
+ PADDD(xmm_v1, xmm_s1)
+ PADDD(xmm_v2, xmm_s2)
+ PADDD(xmm_v3, xmm_s3)
+ WriteXor_sse2(xmm_tmp, reg_inp, reg_outp, 0, xmm_v0, xmm_v1, xmm_v2, xmm_v3)
+ PADDQ(xmm_s3, xmm_one)
+
+ PADDD(xmm_v4, xmm_s0)
+ PADDD(xmm_v5, xmm_s1)
+ PADDD(xmm_v6, xmm_s2)
+ PADDD(xmm_v7, xmm_s3)
+ WriteXor_sse2(xmm_tmp, reg_inp, reg_outp, 64, xmm_v4, xmm_v5, xmm_v6, xmm_v7)
+ PADDQ(xmm_s3, xmm_one)
+
+ ADD(reg_inp, 2 * 64)
+ ADD(reg_outp, 2 * 64)
+
+ SUB(reg_blocks, 2)
+ JAE(vector_loop2.begin)
+
+ ADD(reg_blocks, 2)
+ out_serial = Label()
+ JZ(out_serial)
+
+ #
+ # 1 block at a time. Only executed once, because if there was > 1,
+ # the parallel code would have processed it already.
+ #
+
+ MOVDQA(xmm_v0, xmm_s0)
+ MOVDQA(xmm_v1, xmm_s1)
+ MOVDQA(xmm_v2, xmm_s2)
+ MOVDQA(xmm_v3, xmm_s3)
+
+ reg_rounds = GeneralPurposeRegister64()
+ MOV(reg_rounds, 20)
+ rounds_loop1 = Loop()
+ with rounds_loop1:
+ # a += b; d ^= a; d = ROTW16(d);
+ PADDD(xmm_v0, xmm_v1)
+ PXOR(xmm_v3, xmm_v0)
+ ROTW16_sse2(xmm_tmp, xmm_v3)
+
+ # c += d; b ^= c; b = ROTW12(b);
+ PADDD(xmm_v2, xmm_v3)
+ PXOR(xmm_v1, xmm_v2)
+ ROTW12_sse2(xmm_tmp, xmm_v1)
+
+ # a += b; d ^= a; d = ROTW8(d);
+ PADDD(xmm_v0, xmm_v1)
+ PXOR(xmm_v3, xmm_v0)
+ ROTW8_sse2(xmm_tmp, xmm_v3)
+
+ # c += d; b ^= c; b = ROTW7(b)
+ PADDD(xmm_v2, xmm_v3)
+ PXOR(xmm_v1, xmm_v2)
+ ROTW7_sse2(xmm_tmp, xmm_v1)
+
+ # b = ROTV1(b); c = ROTV2(c); d = ROTV3(d);
+ PSHUFD(xmm_v1, xmm_v1, 0x39)
+ PSHUFD(xmm_v2, xmm_v2, 0x4e)
+ PSHUFD(xmm_v3, xmm_v3, 0x93)
+
+ # a += b; d ^= a; d = ROTW16(d);
+ PADDD(xmm_v0, xmm_v1)
+ PXOR(xmm_v3, xmm_v0)
+ ROTW16_sse2(xmm_tmp, xmm_v3)
+
+ # c += d; b ^= c; b = ROTW12(b);
+ PADDD(xmm_v2, xmm_v3)
+ PXOR(xmm_v1, xmm_v2)
+ ROTW12_sse2(xmm_tmp, xmm_v1)
+
+ # a += b; d ^= a; d = ROTW8(d);
+ PADDD(xmm_v0, xmm_v1)
+ PXOR(xmm_v3, xmm_v0)
+ ROTW8_sse2(xmm_tmp, xmm_v3)
+
+ # c += d; b ^= c; b = ROTW7(b)
+ PADDD(xmm_v2, xmm_v3)
+ PXOR(xmm_v1, xmm_v2)
+ ROTW7_sse2(xmm_tmp, xmm_v1)
+
+ # b = ROTV1(b); c = ROTV2(c); d = ROTV3(d);
+ PSHUFD(xmm_v1, xmm_v1, 0x93)
+ PSHUFD(xmm_v2, xmm_v2, 0x4e)
+ PSHUFD(xmm_v3, xmm_v3, 0x39)
+
+ SUB(reg_rounds, 2)
+ JNZ(rounds_loop1.begin)
+
+ PADDD(xmm_v0, xmm_s0)
+ PADDD(xmm_v1, xmm_s1)
+ PADDD(xmm_v2, xmm_s2)
+ PADDD(xmm_v3, xmm_s3)
+ WriteXor_sse2(xmm_tmp, reg_inp, reg_outp, 0, xmm_v0, xmm_v1, xmm_v2, xmm_v3)
+ PADDQ(xmm_s3, xmm_one)
+
+ LABEL(out_serial)
+
+ # Write back the updated counter. Stoping at 2^70 bytes is the user's
+ # problem, not mine. (Skipped if there's exactly a multiple of 4 blocks
+ # because the counter is incremented in memory while looping.)
+ MOVDQA(mem_s3, xmm_s3)
+
+ LABEL(out)
+
+ # Paranoia, cleanse the scratch space.
+ PXOR(xmm_v0, xmm_v0)
+ MOVDQA(mem_tmp0, xmm_v0)
+
+ # Remove our stack allocation.
+ MOV(registers.rsp, reg_sp_save)
+
+ RETURN()
+
+#
+# AVX2 helpers. Like the SSE2 equivalents, the scratch register is explicit,
+# and more helpers are used to increase readability for destructive operations.
+#
+# XXX/Performance: ROTW16_avx2/ROTW8_avx2 both can use VPSHUFFB.
+#
+
+def ADD_avx2(dst, src):
+ VPADDD(dst, dst, src)
+
+def XOR_avx2(dst, src):
+ VPXOR(dst, dst, src)
+
+def ROTW16_avx2(tmp, d):
+ VPSLLD(tmp, d, 16)
+ VPSRLD(d, d, 16)
+ XOR_avx2(d, tmp)
+
+def ROTW12_avx2(tmp, b):
+ VPSLLD(tmp, b, 12)
+ VPSRLD(b, b, 20)
+ XOR_avx2(b, tmp)
+
+def ROTW8_avx2(tmp, d):
+ VPSLLD(tmp, d, 8)
+ VPSRLD(d, d, 24)
+ XOR_avx2(d, tmp)
+
+def ROTW7_avx2(tmp, b):
+ VPSLLD(tmp, b, 7)
+ VPSRLD(b, b, 25)
+ XOR_avx2(b, tmp)
+
+def WriteXor_avx2(tmp, inp, outp, d, v0, v1, v2, v3):
+ # XOR_WRITE(out+ 0, in+ 0, _mm256_permute2x128_si256(v0,v1,0x20));
+ VPERM2I128(tmp, v0, v1, 0x20)
+ VPXOR(tmp, tmp, [inp+d])
+ VMOVDQU([outp+d], tmp)
+
+ # XOR_WRITE(out+32, in+32, _mm256_permute2x128_si256(v2,v3,0x20));
+ VPERM2I128(tmp, v2, v3, 0x20)
+ VPXOR(tmp, tmp, [inp+d+32])
+ VMOVDQU([outp+d+32], tmp)
+
+ # XOR_WRITE(out+64, in+64, _mm256_permute2x128_si256(v0,v1,0x31));
+ VPERM2I128(tmp, v0, v1, 0x31)
+ VPXOR(tmp, tmp, [inp+d+64])
+ VMOVDQU([outp+d+64], tmp)
+
+ # XOR_WRITE(out+96, in+96, _mm256_permute2x128_si256(v2,v3,0x31));
+ VPERM2I128(tmp, v2, v3, 0x31)
+ VPXOR(tmp, tmp, [inp+d+96])
+ VMOVDQU([outp+d+96], tmp)
+
+# AVX2 ChaCha20 (aka avx2). Does not handle partial blocks, will process
+# 8/4/2 blocks at a time. Alignment blah blah blah fuck you.
+with Function("blocksAmd64AVX2", (x, inp, outp, nrBlocks), target=uarch.broadwell):
+ reg_x = GeneralPurposeRegister64()
+ reg_inp = GeneralPurposeRegister64()
+ reg_outp = GeneralPurposeRegister64()
+ reg_blocks = GeneralPurposeRegister64()
+ reg_sp_save = GeneralPurposeRegister64()
+
+ LOAD.ARGUMENT(reg_x, x)
+ LOAD.ARGUMENT(reg_inp, inp)
+ LOAD.ARGUMENT(reg_outp, outp)
+ LOAD.ARGUMENT(reg_blocks, nrBlocks)
+
+ # Align the stack to a 32 byte boundary.
+ reg_align = GeneralPurposeRegister64()
+ MOV(reg_sp_save, registers.rsp)
+ MOV(reg_align, 0x1f)
+ NOT(reg_align)
+ AND(registers.rsp, reg_align)
+ SUB(registers.rsp, 0x20)
+
+ x_s0 = [reg_x] # (Memory) Cipher state [0..3]
+ x_s1 = [reg_x+16] # (Memory) Cipher state [4..7]
+ x_s2 = [reg_x+32] # (Memory) Cipher state [8..11]
+ x_s3 = [reg_x+48] # (Memory) Cipher state [12..15]
+
+ ymm_v0 = YMMRegister()
+ ymm_v1 = YMMRegister()
+ ymm_v2 = YMMRegister()
+ ymm_v3 = YMMRegister()
+
+ ymm_v4 = YMMRegister()
+ ymm_v5 = YMMRegister()
+ ymm_v6 = YMMRegister()
+ ymm_v7 = YMMRegister()
+
+ ymm_v8 = YMMRegister()
+ ymm_v9 = YMMRegister()
+ ymm_v10 = YMMRegister()
+ ymm_v11 = YMMRegister()
+
+ ymm_v12 = YMMRegister()
+ ymm_v13 = YMMRegister()
+ ymm_v14 = YMMRegister()
+ ymm_v15 = YMMRegister()
+
+ ymm_tmp0 = ymm_v12
+
+ # Allocate the neccecary stack space for the counter vector and two ymm
+ # registers that we will spill.
+ SUB(registers.rsp, 96)
+ mem_tmp0 = [registers.rsp+64] # (Stack) Scratch space.
+ mem_s3 = [registers.rsp+32] # (Stack) Working copy of s3. (8x)
+ mem_inc = [registers.rsp] # (Stack) Counter increment vector.
+
+ # Increment the counter for one side of the state vector.
+ VPXOR(ymm_tmp0, ymm_tmp0, ymm_tmp0)
+ VMOVDQU(mem_inc, ymm_tmp0)
+ reg_tmp = GeneralPurposeRegister32()
+ MOV(reg_tmp, 0x00000001)
+ MOV([registers.rsp+16], reg_tmp)
+ VBROADCASTI128(ymm_v3, x_s3)
+ VPADDQ(ymm_v3, ymm_v3, [registers.rsp])
+ VMOVDQA(mem_s3, ymm_v3)
+
+ # As we process 2xN blocks at a time, so the counter increment for both
+ # sides of the state vector is 2.
+ MOV(reg_tmp, 0x00000002)
+ MOV([registers.rsp], reg_tmp)
+ MOV([registers.rsp+16], reg_tmp)
+
+ out_write_even = Label()
+ out_write_odd = Label()
+
+ #
+ # 8 blocks at a time. Ted Krovetz's avx2 code does not do this, but it's
+ # a decent gain despite all the pain...
+ #
+
+ vector_loop8 = Loop()
+ SUB(reg_blocks, 8)
+ JB(vector_loop8.end)
+ with vector_loop8:
+ VBROADCASTI128(ymm_v0, x_s0)
+ VBROADCASTI128(ymm_v1, x_s1)
+ VBROADCASTI128(ymm_v2, x_s2)
+ VMOVDQA(ymm_v3, mem_s3)
+
+ VMOVDQA(ymm_v4, ymm_v0)
+ VMOVDQA(ymm_v5, ymm_v1)
+ VMOVDQA(ymm_v6, ymm_v2)
+ VPADDQ(ymm_v7, ymm_v3, mem_inc)
+
+ VMOVDQA(ymm_v8, ymm_v0)
+ VMOVDQA(ymm_v9, ymm_v1)
+ VMOVDQA(ymm_v10, ymm_v2)
+ VPADDQ(ymm_v11, ymm_v7, mem_inc)
+
+ VMOVDQA(ymm_v12, ymm_v0)
+ VMOVDQA(ymm_v13, ymm_v1)
+ VMOVDQA(ymm_v14, ymm_v2)
+ VPADDQ(ymm_v15, ymm_v11, mem_inc)
+
+ reg_rounds = GeneralPurposeRegister64()
+ MOV(reg_rounds, 20)
+ rounds_loop8 = Loop()
+ with rounds_loop8:
+ # a += b; d ^= a; d = ROTW16(d);
+ ADD_avx2(ymm_v0, ymm_v1)
+ ADD_avx2(ymm_v4, ymm_v5)
+ ADD_avx2(ymm_v8, ymm_v9)
+ ADD_avx2(ymm_v12, ymm_v13)
+ XOR_avx2(ymm_v3, ymm_v0)
+ XOR_avx2(ymm_v7, ymm_v4)
+ XOR_avx2(ymm_v11, ymm_v8)
+ XOR_avx2(ymm_v15, ymm_v12)
+
+ VMOVDQA(mem_tmp0, ymm_tmp0) # Save
+
+ ROTW16_avx2(ymm_tmp0, ymm_v3)
+ ROTW16_avx2(ymm_tmp0, ymm_v7)
+ ROTW16_avx2(ymm_tmp0, ymm_v11)
+ ROTW16_avx2(ymm_tmp0, ymm_v15)
+
+ # c += d; b ^= c; b = ROTW12(b);
+ ADD_avx2(ymm_v2, ymm_v3)
+ ADD_avx2(ymm_v6, ymm_v7)
+ ADD_avx2(ymm_v10, ymm_v11)
+ ADD_avx2(ymm_v14, ymm_v15)
+ XOR_avx2(ymm_v1, ymm_v2)
+ XOR_avx2(ymm_v5, ymm_v6)
+ XOR_avx2(ymm_v9, ymm_v10)
+ XOR_avx2(ymm_v13, ymm_v14)
+ ROTW12_avx2(ymm_tmp0, ymm_v1)
+ ROTW12_avx2(ymm_tmp0, ymm_v5)
+ ROTW12_avx2(ymm_tmp0, ymm_v9)
+ ROTW12_avx2(ymm_tmp0, ymm_v13)
+
+ # a += b; d ^= a; d = ROTW8(d);
+ VMOVDQA(ymm_tmp0, mem_tmp0) # Restore
+
+ ADD_avx2(ymm_v0, ymm_v1)
+ ADD_avx2(ymm_v4, ymm_v5)
+ ADD_avx2(ymm_v8, ymm_v9)
+ ADD_avx2(ymm_v12, ymm_v13)
+ XOR_avx2(ymm_v3, ymm_v0)
+ XOR_avx2(ymm_v7, ymm_v4)
+ XOR_avx2(ymm_v11, ymm_v8)
+ XOR_avx2(ymm_v15, ymm_v12)
+
+ VMOVDQA(mem_tmp0, ymm_tmp0) # Save
+
+ ROTW8_avx2(ymm_tmp0, ymm_v3)
+ ROTW8_avx2(ymm_tmp0, ymm_v7)
+ ROTW8_avx2(ymm_tmp0, ymm_v11)
+ ROTW8_avx2(ymm_tmp0, ymm_v15)
+
+ # c += d; b ^= c; b = ROTW7(b)
+ ADD_avx2(ymm_v2, ymm_v3)
+ ADD_avx2(ymm_v6, ymm_v7)
+ ADD_avx2(ymm_v10, ymm_v11)
+ ADD_avx2(ymm_v14, ymm_v15)
+ XOR_avx2(ymm_v1, ymm_v2)
+ XOR_avx2(ymm_v5, ymm_v6)
+ XOR_avx2(ymm_v9, ymm_v10)
+ XOR_avx2(ymm_v13, ymm_v14)
+ ROTW7_avx2(ymm_tmp0, ymm_v1)
+ ROTW7_avx2(ymm_tmp0, ymm_v5)
+ ROTW7_avx2(ymm_tmp0, ymm_v9)
+ ROTW7_avx2(ymm_tmp0, ymm_v13)
+
+ # b = ROTV1(b); c = ROTV2(c); d = ROTV3(d);
+ VPSHUFD(ymm_v1, ymm_v1, 0x39)
+ VPSHUFD(ymm_v5, ymm_v5, 0x39)
+ VPSHUFD(ymm_v9, ymm_v9, 0x39)
+ VPSHUFD(ymm_v13, ymm_v13, 0x39)
+ VPSHUFD(ymm_v2, ymm_v2, 0x4e)
+ VPSHUFD(ymm_v6, ymm_v6, 0x4e)
+ VPSHUFD(ymm_v10, ymm_v10, 0x4e)
+ VPSHUFD(ymm_v14, ymm_v14, 0x4e)
+ VPSHUFD(ymm_v3, ymm_v3, 0x93)
+ VPSHUFD(ymm_v7, ymm_v7, 0x93)
+ VPSHUFD(ymm_v11, ymm_v11, 0x93)
+ VPSHUFD(ymm_v15, ymm_v15, 0x93)
+
+ # a += b; d ^= a; d = ROTW16(d);
+ VMOVDQA(ymm_tmp0, mem_tmp0) # Restore
+
+ ADD_avx2(ymm_v0, ymm_v1)
+ ADD_avx2(ymm_v4, ymm_v5)
+ ADD_avx2(ymm_v8, ymm_v9)
+ ADD_avx2(ymm_v12, ymm_v13)
+ XOR_avx2(ymm_v3, ymm_v0)
+ XOR_avx2(ymm_v7, ymm_v4)
+ XOR_avx2(ymm_v11, ymm_v8)
+ XOR_avx2(ymm_v15, ymm_v12)
+
+ VMOVDQA(mem_tmp0, ymm_tmp0) # Save
+
+ ROTW16_avx2(ymm_tmp0, ymm_v3)
+ ROTW16_avx2(ymm_tmp0, ymm_v7)
+ ROTW16_avx2(ymm_tmp0, ymm_v11)
+ ROTW16_avx2(ymm_tmp0, ymm_v15)
+
+ # c += d; b ^= c; b = ROTW12(b);
+ ADD_avx2(ymm_v2, ymm_v3)
+ ADD_avx2(ymm_v6, ymm_v7)
+ ADD_avx2(ymm_v10, ymm_v11)
+ ADD_avx2(ymm_v14, ymm_v15)
+ XOR_avx2(ymm_v1, ymm_v2)
+ XOR_avx2(ymm_v5, ymm_v6)
+ XOR_avx2(ymm_v9, ymm_v10)
+ XOR_avx2(ymm_v13, ymm_v14)
+ ROTW12_avx2(ymm_tmp0, ymm_v1)
+ ROTW12_avx2(ymm_tmp0, ymm_v5)
+ ROTW12_avx2(ymm_tmp0, ymm_v9)
+ ROTW12_avx2(ymm_tmp0, ymm_v13)
+
+ # a += b; d ^= a; d = ROTW8(d);
+ VMOVDQA(ymm_tmp0, mem_tmp0) # Restore
+
+ ADD_avx2(ymm_v0, ymm_v1)
+ ADD_avx2(ymm_v4, ymm_v5)
+ ADD_avx2(ymm_v8, ymm_v9)
+ ADD_avx2(ymm_v12, ymm_v13)
+ XOR_avx2(ymm_v3, ymm_v0)
+ XOR_avx2(ymm_v7, ymm_v4)
+ XOR_avx2(ymm_v11, ymm_v8)
+ XOR_avx2(ymm_v15, ymm_v12)
+
+ VMOVDQA(mem_tmp0, ymm_tmp0) # Save
+
+ ROTW8_avx2(ymm_tmp0, ymm_v3)
+ ROTW8_avx2(ymm_tmp0, ymm_v7)
+ ROTW8_avx2(ymm_tmp0, ymm_v11)
+ ROTW8_avx2(ymm_tmp0, ymm_v15)
+
+ # c += d; b ^= c; b = ROTW7(b)
+ ADD_avx2(ymm_v2, ymm_v3)
+ ADD_avx2(ymm_v6, ymm_v7)
+ ADD_avx2(ymm_v10, ymm_v11)
+ ADD_avx2(ymm_v14, ymm_v15)
+ XOR_avx2(ymm_v1, ymm_v2)
+ XOR_avx2(ymm_v5, ymm_v6)
+ XOR_avx2(ymm_v9, ymm_v10)
+ XOR_avx2(ymm_v13, ymm_v14)
+ ROTW7_avx2(ymm_tmp0, ymm_v1)
+ ROTW7_avx2(ymm_tmp0, ymm_v5)
+ ROTW7_avx2(ymm_tmp0, ymm_v9)
+ ROTW7_avx2(ymm_tmp0, ymm_v13)
+
+ # b = ROTV1(b); c = ROTV2(c); d = ROTV3(d);
+ VPSHUFD(ymm_v1, ymm_v1, 0x93)
+ VPSHUFD(ymm_v5, ymm_v5, 0x93)
+ VPSHUFD(ymm_v9, ymm_v9, 0x93)
+ VPSHUFD(ymm_v13, ymm_v13, 0x93)
+ VPSHUFD(ymm_v2, ymm_v2, 0x4e)
+ VPSHUFD(ymm_v6, ymm_v6, 0x4e)
+ VPSHUFD(ymm_v10, ymm_v10, 0x4e)
+ VPSHUFD(ymm_v14, ymm_v14, 0x4e)
+ VPSHUFD(ymm_v3, ymm_v3, 0x39)
+ VPSHUFD(ymm_v7, ymm_v7, 0x39)
+ VPSHUFD(ymm_v11, ymm_v11, 0x39)
+ VPSHUFD(ymm_v15, ymm_v15, 0x39)
+
+ VMOVDQA(ymm_tmp0, mem_tmp0) # Restore
+
+ SUB(reg_rounds, 2)
+ JNZ(rounds_loop8.begin)
+
+ # ymm_v12 is in mem_tmp0 and is current....
+
+ # XXX: I assume VBROADCASTI128 is about as fast as VMOVDQA....
+ VBROADCASTI128(ymm_tmp0, x_s0)
+ ADD_avx2(ymm_v0, ymm_tmp0)
+ ADD_avx2(ymm_v4, ymm_tmp0)
+ ADD_avx2(ymm_v8, ymm_tmp0)
+ ADD_avx2(ymm_tmp0, mem_tmp0)
+ VMOVDQA(mem_tmp0, ymm_tmp0)
+
+ VBROADCASTI128(ymm_tmp0, x_s1)
+ ADD_avx2(ymm_v1, ymm_tmp0)
+ ADD_avx2(ymm_v5, ymm_tmp0)
+ ADD_avx2(ymm_v9, ymm_tmp0)
+ ADD_avx2(ymm_v13, ymm_tmp0)
+
+ VBROADCASTI128(ymm_tmp0, x_s2)
+ ADD_avx2(ymm_v2, ymm_tmp0)
+ ADD_avx2(ymm_v6, ymm_tmp0)
+ ADD_avx2(ymm_v10, ymm_tmp0)
+ ADD_avx2(ymm_v14, ymm_tmp0)
+
+ ADD_avx2(ymm_v3, mem_s3)
+ WriteXor_avx2(ymm_tmp0, reg_inp, reg_outp, 0, ymm_v0, ymm_v1, ymm_v2, ymm_v3)
+ VMOVDQA(ymm_v3, mem_s3)
+ ADD_avx2(ymm_v3, mem_inc)
+
+ ADD_avx2(ymm_v7, ymm_v3)
+ WriteXor_avx2(ymm_tmp0, reg_inp, reg_outp, 128, ymm_v4, ymm_v5, ymm_v6, ymm_v7)
+ ADD_avx2(ymm_v3, mem_inc)
+
+ ADD_avx2(ymm_v11, ymm_v3)
+ WriteXor_avx2(ymm_tmp0, reg_inp, reg_outp, 256, ymm_v8, ymm_v9, ymm_v10, ymm_v11)
+ ADD_avx2(ymm_v3, mem_inc)
+
+ VMOVDQA(ymm_v12, mem_tmp0)
+ ADD_avx2(ymm_v15, ymm_v3)
+ WriteXor_avx2(ymm_v0, reg_inp, reg_outp, 384, ymm_v12, ymm_v13, ymm_v14, ymm_v15)
+ ADD_avx2(ymm_v3, mem_inc)
+
+ VMOVDQA(mem_s3, ymm_v3)
+
+ ADD(reg_inp, 8 * 64)
+ ADD(reg_outp, 8 * 64)
+
+ SUB(reg_blocks, 8)
+ JAE(vector_loop8.begin)
+
+ # ymm_v3 contains a current copy of mem_s3 either from when it was built,
+ # or because the loop updates it. Copy this before we mess with the block
+ # counter in case we need to write it back and return.
+ ymm_s3 = ymm_v11
+ VMOVDQA(ymm_s3, ymm_v3)
+
+ ADD(reg_blocks, 8)
+ JZ(out_write_even)
+
+ # We now actually can do everything in registers.
+ ymm_s0 = ymm_v8
+ VBROADCASTI128(ymm_s0, x_s0)
+ ymm_s1 = ymm_v9
+ VBROADCASTI128(ymm_s1, x_s1)
+ ymm_s2 = ymm_v10
+ VBROADCASTI128(ymm_s2, x_s2)
+ ymm_inc = ymm_v14
+ VMOVDQA(ymm_inc, mem_inc)
+
+ #
+ # 4 blocks at a time.
+ #
+
+ SUB(reg_blocks, 4)
+ vector_loop4 = Loop()
+ JB(vector_loop4.end)
+ with vector_loop4:
+ VMOVDQA(ymm_v0, ymm_s0)
+ VMOVDQA(ymm_v1, ymm_s1)
+ VMOVDQA(ymm_v2, ymm_s2)
+ VMOVDQA(ymm_v3, ymm_s3)
+
+ VMOVDQA(ymm_v4, ymm_v0)
+ VMOVDQA(ymm_v5, ymm_v1)
+ VMOVDQA(ymm_v6, ymm_v2)
+ VPADDQ(ymm_v7, ymm_v3, ymm_inc)
+
+ reg_rounds = GeneralPurposeRegister64()
+ MOV(reg_rounds, 20)
+ rounds_loop4 = Loop()
+ with rounds_loop4:
+ # a += b; d ^= a; d = ROTW16(d);
+ ADD_avx2(ymm_v0, ymm_v1)
+ ADD_avx2(ymm_v4, ymm_v5)
+ XOR_avx2(ymm_v3, ymm_v0)
+ XOR_avx2(ymm_v7, ymm_v4)
+ ROTW16_avx2(ymm_tmp0, ymm_v3)
+ ROTW16_avx2(ymm_tmp0, ymm_v7)
+
+ # c += d; b ^= c; b = ROTW12(b);
+ ADD_avx2(ymm_v2, ymm_v3)
+ ADD_avx2(ymm_v6, ymm_v7)
+ XOR_avx2(ymm_v1, ymm_v2)
+ XOR_avx2(ymm_v5, ymm_v6)
+ ROTW12_avx2(ymm_tmp0, ymm_v1)
+ ROTW12_avx2(ymm_tmp0, ymm_v5)
+
+ # a += b; d ^= a; d = ROTW8(d);
+ ADD_avx2(ymm_v0, ymm_v1)
+ ADD_avx2(ymm_v4, ymm_v5)
+ XOR_avx2(ymm_v3, ymm_v0)
+ XOR_avx2(ymm_v7, ymm_v4)
+ ROTW8_avx2(ymm_tmp0, ymm_v3)
+ ROTW8_avx2(ymm_tmp0, ymm_v7)
+
+ # c += d; b ^= c; b = ROTW7(b)
+ ADD_avx2(ymm_v2, ymm_v3)
+ ADD_avx2(ymm_v6, ymm_v7)
+ XOR_avx2(ymm_v1, ymm_v2)
+ XOR_avx2(ymm_v5, ymm_v6)
+ ROTW7_avx2(ymm_tmp0, ymm_v1)
+ ROTW7_avx2(ymm_tmp0, ymm_v5)
+
+ # b = ROTV1(b); c = ROTV2(c); d = ROTV3(d);
+ VPSHUFD(ymm_v1, ymm_v1, 0x39)
+ VPSHUFD(ymm_v5, ymm_v5, 0x39)
+ VPSHUFD(ymm_v2, ymm_v2, 0x4e)
+ VPSHUFD(ymm_v6, ymm_v6, 0x4e)
+ VPSHUFD(ymm_v3, ymm_v3, 0x93)
+ VPSHUFD(ymm_v7, ymm_v7, 0x93)
+
+ # a += b; d ^= a; d = ROTW16(d);
+ ADD_avx2(ymm_v0, ymm_v1)
+ ADD_avx2(ymm_v4, ymm_v5)
+ XOR_avx2(ymm_v3, ymm_v0)
+ XOR_avx2(ymm_v7, ymm_v4)
+ ROTW16_avx2(ymm_tmp0, ymm_v3)
+ ROTW16_avx2(ymm_tmp0, ymm_v7)
+
+ # c += d; b ^= c; b = ROTW12(b);
+ ADD_avx2(ymm_v2, ymm_v3)
+ ADD_avx2(ymm_v6, ymm_v7)
+ XOR_avx2(ymm_v1, ymm_v2)
+ XOR_avx2(ymm_v5, ymm_v6)
+ ROTW12_avx2(ymm_tmp0, ymm_v1)
+ ROTW12_avx2(ymm_tmp0, ymm_v5)
+
+ # a += b; d ^= a; d = ROTW8(d);
+ ADD_avx2(ymm_v0, ymm_v1)
+ ADD_avx2(ymm_v4, ymm_v5)
+ XOR_avx2(ymm_v3, ymm_v0)
+ XOR_avx2(ymm_v7, ymm_v4)
+ ROTW8_avx2(ymm_tmp0, ymm_v3)
+ ROTW8_avx2(ymm_tmp0, ymm_v7)
+
+ # c += d; b ^= c; b = ROTW7(b)
+ ADD_avx2(ymm_v2, ymm_v3)
+ ADD_avx2(ymm_v6, ymm_v7)
+ XOR_avx2(ymm_v1, ymm_v2)
+ XOR_avx2(ymm_v5, ymm_v6)
+ ROTW7_avx2(ymm_tmp0, ymm_v1)
+ ROTW7_avx2(ymm_tmp0, ymm_v5)
+
+ # b = ROTV1(b); c = ROTV2(c); d = ROTV3(d);
+ VPSHUFD(ymm_v1, ymm_v1, 0x93)
+ VPSHUFD(ymm_v5, ymm_v5, 0x93)
+ VPSHUFD(ymm_v2, ymm_v2, 0x4e)
+ VPSHUFD(ymm_v6, ymm_v6, 0x4e)
+ VPSHUFD(ymm_v3, ymm_v3, 0x39)
+ VPSHUFD(ymm_v7, ymm_v7, 0x39)
+
+ SUB(reg_rounds, 2)
+ JNZ(rounds_loop4.begin)
+
+ ADD_avx2(ymm_v0, ymm_s0)
+ ADD_avx2(ymm_v1, ymm_s1)
+ ADD_avx2(ymm_v2, ymm_s2)
+ ADD_avx2(ymm_v3, ymm_s3)
+ WriteXor_avx2(ymm_tmp0, reg_inp, reg_outp, 0, ymm_v0, ymm_v1, ymm_v2, ymm_v3)
+ ADD_avx2(ymm_s3, ymm_inc)
+
+ ADD_avx2(ymm_v4, ymm_s0)
+ ADD_avx2(ymm_v5, ymm_s1)
+ ADD_avx2(ymm_v6, ymm_s2)
+ ADD_avx2(ymm_v7, ymm_s3)
+ WriteXor_avx2(ymm_tmp0, reg_inp, reg_outp, 128, ymm_v4, ymm_v5, ymm_v6, ymm_v7)
+ ADD_avx2(ymm_s3, ymm_inc)
+
+ ADD(reg_inp, 4 * 64)
+ ADD(reg_outp, 4 * 64)
+
+ SUB(reg_blocks, 4)
+ JAE(vector_loop4.begin)
+
+ ADD(reg_blocks, 4)
+ JZ(out_write_even)
+
+ #
+ # 2/1 blocks at a time. The two codepaths are unified because
+ # with AVX2 we do 2 blocks at a time anyway, and this only gets called
+ # if 3/2/1 blocks are remaining, so the extra branches don't hurt that
+ # much.
+ #
+
+ vector_loop2 = Loop()
+ with vector_loop2:
+ VMOVDQA(ymm_v0, ymm_s0)
+ VMOVDQA(ymm_v1, ymm_s1)
+ VMOVDQA(ymm_v2, ymm_s2)
+ VMOVDQA(ymm_v3, ymm_s3)
+
+ reg_rounds = GeneralPurposeRegister64()
+ MOV(reg_rounds, 20)
+ rounds_loop2 = Loop()
+ with rounds_loop2:
+ # a += b; d ^= a; d = ROTW16(d);
+ ADD_avx2(ymm_v0, ymm_v1)
+ XOR_avx2(ymm_v3, ymm_v0)
+ ROTW16_avx2(ymm_tmp0, ymm_v3)
+
+ # c += d; b ^= c; b = ROTW12(b);
+ ADD_avx2(ymm_v2, ymm_v3)
+ XOR_avx2(ymm_v1, ymm_v2)
+ ROTW12_avx2(ymm_tmp0, ymm_v1)
+
+ # a += b; d ^= a; d = ROTW8(d);
+ ADD_avx2(ymm_v0, ymm_v1)
+ XOR_avx2(ymm_v3, ymm_v0)
+ ROTW8_avx2(ymm_tmp0, ymm_v3)
+
+ # c += d; b ^= c; b = ROTW7(b)
+ ADD_avx2(ymm_v2, ymm_v3)
+ XOR_avx2(ymm_v1, ymm_v2)
+ ROTW7_avx2(ymm_tmp0, ymm_v1)
+
+ # b = ROTV1(b); c = ROTV2(c); d = ROTV3(d);
+ VPSHUFD(ymm_v1, ymm_v1, 0x39)
+ VPSHUFD(ymm_v2, ymm_v2, 0x4e)
+ VPSHUFD(ymm_v3, ymm_v3, 0x93)
+
+ # a += b; d ^= a; d = ROTW16(d);
+ ADD_avx2(ymm_v0, ymm_v1)
+ XOR_avx2(ymm_v3, ymm_v0)
+ ROTW16_avx2(ymm_tmp0, ymm_v3)
+
+ # c += d; b ^= c; b = ROTW12(b);
+ ADD_avx2(ymm_v2, ymm_v3)
+ XOR_avx2(ymm_v1, ymm_v2)
+ ROTW12_avx2(ymm_tmp0, ymm_v1)
+
+ # a += b; d ^= a; d = ROTW8(d);
+ ADD_avx2(ymm_v0, ymm_v1)
+ XOR_avx2(ymm_v3, ymm_v0)
+ ROTW8_avx2(ymm_tmp0, ymm_v3)
+
+ # c += d; b ^= c; b = ROTW7(b)
+ ADD_avx2(ymm_v2, ymm_v3)
+ XOR_avx2(ymm_v1, ymm_v2)
+ ROTW7_avx2(ymm_tmp0, ymm_v1)
+
+ # b = ROTV1(b); c = ROTV2(c); d = ROTV3(d);
+ VPSHUFD(ymm_v1, ymm_v1, 0x93)
+ VPSHUFD(ymm_v2, ymm_v2, 0x4e)
+ VPSHUFD(ymm_v3, ymm_v3, 0x39)
+
+ SUB(reg_rounds, 2)
+ JNZ(rounds_loop2.begin)
+
+ ADD_avx2(ymm_v0, ymm_s0)
+ ADD_avx2(ymm_v1, ymm_s1)
+ ADD_avx2(ymm_v2, ymm_s2)
+ ADD_avx2(ymm_v3, ymm_s3)
+
+ # XOR_WRITE(out+ 0, in+ 0, _mm256_permute2x128_si256(v0,v1,0x20));
+ VPERM2I128(ymm_tmp0, ymm_v0, ymm_v1, 0x20)
+ VPXOR(ymm_tmp0, ymm_tmp0, [reg_inp])
+ VMOVDQU([reg_outp], ymm_tmp0)
+
+ # XOR_WRITE(out+32, in+32, _mm256_permute2x128_si256(v2,v3,0x20));
+ VPERM2I128(ymm_tmp0, ymm_v2, ymm_v3, 0x20)
+ VPXOR(ymm_tmp0, ymm_tmp0, [reg_inp+32])
+ VMOVDQU([reg_outp+32], ymm_tmp0)
+
+ SUB(reg_blocks, 1)
+ JZ(out_write_odd)
+
+ ADD_avx2(ymm_s3, ymm_inc)
+
+ # XOR_WRITE(out+64, in+64, _mm256_permute2x128_si256(v0,v1,0x31));
+ VPERM2I128(ymm_tmp0, ymm_v0, ymm_v1, 0x31)
+ VPXOR(ymm_tmp0, ymm_tmp0, [reg_inp+64])
+ VMOVDQU([reg_outp+64], ymm_tmp0)
+
+ # XOR_WRITE(out+96, in+96, _mm256_permute2x128_si256(v2,v3,0x31));
+ VPERM2I128(ymm_tmp0, ymm_v2, ymm_v3, 0x31)
+ VPXOR(ymm_tmp0, ymm_tmp0, [reg_inp+96])
+ VMOVDQU([reg_outp+96], ymm_tmp0)
+
+ SUB(reg_blocks, 1)
+ JZ(out_write_even)
+
+ ADD(reg_inp, 2 * 64)
+ ADD(reg_outp, 2 * 64)
+ JMP(vector_loop2.begin)
+
+ LABEL(out_write_odd)
+ VPERM2I128(ymm_s3, ymm_s3, ymm_s3, 0x01) # Odd number of blocks.
+
+ LABEL(out_write_even)
+ VMOVDQA(x_s3, ymm_s3.as_xmm) # Write back ymm_s3 to x_v3
+
+ # Paranoia, cleanse the scratch space.
+ VPXOR(ymm_v0, ymm_v0, ymm_v0)
+ VMOVDQA(mem_tmp0, ymm_v0)
+ VMOVDQA(mem_s3, ymm_v0)
+
+ # Remove our stack allocation.
+ MOV(registers.rsp, reg_sp_save)
+
+ RETURN()
+
+#
+# CPUID
+#
+
+cpuidParams = Argument(ptr(uint32_t))
+
+with Function("cpuidAmd64", (cpuidParams,)):
+ reg_params = registers.r15
+ LOAD.ARGUMENT(reg_params, cpuidParams)
+
+ MOV(registers.eax, [reg_params])
+ MOV(registers.ecx, [reg_params+4])
+
+ CPUID()
+
+ MOV([reg_params], registers.eax)
+ MOV([reg_params+4], registers.ebx)
+ MOV([reg_params+8], registers.ecx)
+ MOV([reg_params+12], registers.edx)
+
+ RETURN()
+
+#
+# XGETBV (ECX = 0)
+#
+
+xcrVec = Argument(ptr(uint32_t))
+
+with Function("xgetbv0Amd64", (xcrVec,)):
+ reg_vec = GeneralPurposeRegister64()
+
+ LOAD.ARGUMENT(reg_vec, xcrVec)
+
+ XOR(registers.ecx, registers.ecx)
+
+ XGETBV()
+
+ MOV([reg_vec], registers.eax)
+ MOV([reg_vec+4], registers.edx)
+
+ RETURN()
diff --git a/cmd/gost/vendor/github.com/Yawning/chacha20/chacha20_amd64.s b/cmd/gost/vendor/github.com/Yawning/chacha20/chacha20_amd64.s
new file mode 100644
index 0000000..4970397
--- /dev/null
+++ b/cmd/gost/vendor/github.com/Yawning/chacha20/chacha20_amd64.s
@@ -0,0 +1,1187 @@
+// Generated by PeachPy 0.2.0 from chacha20_amd64.py
+
+
+// func blocksAmd64SSE2(x *uint32, inp *uint8, outp *uint8, nrBlocks *uint)
+TEXT ·blocksAmd64SSE2(SB),4,$0-32
+ MOVQ x+0(FP), AX
+ MOVQ inp+8(FP), BX
+ MOVQ outp+16(FP), CX
+ MOVQ nrBlocks+24(FP), DX
+ MOVQ SP, DI
+ MOVQ $31, SI
+ NOTQ SI
+ ANDQ SI, SP
+ SUBQ $32, SP
+ PXOR X0, X0
+ SUBQ $32, SP
+ MOVO X0, 0(SP)
+ MOVL $1, SI
+ MOVL SI, 0(SP)
+ SUBQ $4, DX
+ JCS vector_loop4_end
+vector_loop4_begin:
+ MOVO 0(AX), X0
+ MOVO 16(AX), X1
+ MOVO 32(AX), X2
+ MOVO 48(AX), X3
+ MOVO X0, X4
+ MOVO X1, X5
+ MOVO X2, X6
+ MOVO X3, X7
+ PADDQ 0(SP), X7
+ MOVO X0, X8
+ MOVO X1, X9
+ MOVO X2, X10
+ MOVO X7, X11
+ PADDQ 0(SP), X11
+ MOVO X0, X12
+ MOVO X1, X13
+ MOVO X2, X14
+ MOVO X11, X15
+ PADDQ 0(SP), X15
+ MOVQ $20, SI
+rounds_loop4_begin:
+ PADDL X1, X0
+ PADDL X5, X4
+ PADDL X9, X8
+ PADDL X13, X12
+ PXOR X0, X3
+ PXOR X4, X7
+ PXOR X8, X11
+ PXOR X12, X15
+ MOVO X12, 16(SP)
+ MOVO X3, X12
+ PSLLL $16, X12
+ PSRLL $16, X3
+ PXOR X12, X3
+ MOVO X7, X12
+ PSLLL $16, X12
+ PSRLL $16, X7
+ PXOR X12, X7
+ MOVO X11, X12
+ PSLLL $16, X12
+ PSRLL $16, X11
+ PXOR X12, X11
+ MOVO X15, X12
+ PSLLL $16, X12
+ PSRLL $16, X15
+ PXOR X12, X15
+ PADDL X3, X2
+ PADDL X7, X6
+ PADDL X11, X10
+ PADDL X15, X14
+ PXOR X2, X1
+ PXOR X6, X5
+ PXOR X10, X9
+ PXOR X14, X13
+ MOVO X1, X12
+ PSLLL $12, X12
+ PSRLL $20, X1
+ PXOR X12, X1
+ MOVO X5, X12
+ PSLLL $12, X12
+ PSRLL $20, X5
+ PXOR X12, X5
+ MOVO X9, X12
+ PSLLL $12, X12
+ PSRLL $20, X9
+ PXOR X12, X9
+ MOVO X13, X12
+ PSLLL $12, X12
+ PSRLL $20, X13
+ PXOR X12, X13
+ MOVO 16(SP), X12
+ PADDL X1, X0
+ PADDL X5, X4
+ PADDL X9, X8
+ PADDL X13, X12
+ PXOR X0, X3
+ PXOR X4, X7
+ PXOR X8, X11
+ PXOR X12, X15
+ MOVO X12, 16(SP)
+ MOVO X3, X12
+ PSLLL $8, X12
+ PSRLL $24, X3
+ PXOR X12, X3
+ MOVO X7, X12
+ PSLLL $8, X12
+ PSRLL $24, X7
+ PXOR X12, X7
+ MOVO X11, X12
+ PSLLL $8, X12
+ PSRLL $24, X11
+ PXOR X12, X11
+ MOVO X15, X12
+ PSLLL $8, X12
+ PSRLL $24, X15
+ PXOR X12, X15
+ PADDL X3, X2
+ PADDL X7, X6
+ PADDL X11, X10
+ PADDL X15, X14
+ PXOR X2, X1
+ PXOR X6, X5
+ PXOR X10, X9
+ PXOR X14, X13
+ MOVO X1, X12
+ PSLLL $7, X12
+ PSRLL $25, X1
+ PXOR X12, X1
+ MOVO X5, X12
+ PSLLL $7, X12
+ PSRLL $25, X5
+ PXOR X12, X5
+ MOVO X9, X12
+ PSLLL $7, X12
+ PSRLL $25, X9
+ PXOR X12, X9
+ MOVO X13, X12
+ PSLLL $7, X12
+ PSRLL $25, X13
+ PXOR X12, X13
+ PSHUFL $57, X1, X1
+ PSHUFL $57, X5, X5
+ PSHUFL $57, X9, X9
+ PSHUFL $57, X13, X13
+ PSHUFL $78, X2, X2
+ PSHUFL $78, X6, X6
+ PSHUFL $78, X10, X10
+ PSHUFL $78, X14, X14
+ PSHUFL $147, X3, X3
+ PSHUFL $147, X7, X7
+ PSHUFL $147, X11, X11
+ PSHUFL $147, X15, X15
+ MOVO 16(SP), X12
+ PADDL X1, X0
+ PADDL X5, X4
+ PADDL X9, X8
+ PADDL X13, X12
+ PXOR X0, X3
+ PXOR X4, X7
+ PXOR X8, X11
+ PXOR X12, X15
+ MOVO X12, 16(SP)
+ MOVO X3, X12
+ PSLLL $16, X12
+ PSRLL $16, X3
+ PXOR X12, X3
+ MOVO X7, X12
+ PSLLL $16, X12
+ PSRLL $16, X7
+ PXOR X12, X7
+ MOVO X11, X12
+ PSLLL $16, X12
+ PSRLL $16, X11
+ PXOR X12, X11
+ MOVO X15, X12
+ PSLLL $16, X12
+ PSRLL $16, X15
+ PXOR X12, X15
+ PADDL X3, X2
+ PADDL X7, X6
+ PADDL X11, X10
+ PADDL X15, X14
+ PXOR X2, X1
+ PXOR X6, X5
+ PXOR X10, X9
+ PXOR X14, X13
+ MOVO X1, X12
+ PSLLL $12, X12
+ PSRLL $20, X1
+ PXOR X12, X1
+ MOVO X5, X12
+ PSLLL $12, X12
+ PSRLL $20, X5
+ PXOR X12, X5
+ MOVO X9, X12
+ PSLLL $12, X12
+ PSRLL $20, X9
+ PXOR X12, X9
+ MOVO X13, X12
+ PSLLL $12, X12
+ PSRLL $20, X13
+ PXOR X12, X13
+ MOVO 16(SP), X12
+ PADDL X1, X0
+ PADDL X5, X4
+ PADDL X9, X8
+ PADDL X13, X12
+ PXOR X0, X3
+ PXOR X4, X7
+ PXOR X8, X11
+ PXOR X12, X15
+ MOVO X12, 16(SP)
+ MOVO X3, X12
+ PSLLL $8, X12
+ PSRLL $24, X3
+ PXOR X12, X3
+ MOVO X7, X12
+ PSLLL $8, X12
+ PSRLL $24, X7
+ PXOR X12, X7
+ MOVO X11, X12
+ PSLLL $8, X12
+ PSRLL $24, X11
+ PXOR X12, X11
+ MOVO X15, X12
+ PSLLL $8, X12
+ PSRLL $24, X15
+ PXOR X12, X15
+ PADDL X3, X2
+ PADDL X7, X6
+ PADDL X11, X10
+ PADDL X15, X14
+ PXOR X2, X1
+ PXOR X6, X5
+ PXOR X10, X9
+ PXOR X14, X13
+ MOVO X1, X12
+ PSLLL $7, X12
+ PSRLL $25, X1
+ PXOR X12, X1
+ MOVO X5, X12
+ PSLLL $7, X12
+ PSRLL $25, X5
+ PXOR X12, X5
+ MOVO X9, X12
+ PSLLL $7, X12
+ PSRLL $25, X9
+ PXOR X12, X9
+ MOVO X13, X12
+ PSLLL $7, X12
+ PSRLL $25, X13
+ PXOR X12, X13
+ PSHUFL $147, X1, X1
+ PSHUFL $147, X5, X5
+ PSHUFL $147, X9, X9
+ PSHUFL $147, X13, X13
+ PSHUFL $78, X2, X2
+ PSHUFL $78, X6, X6
+ PSHUFL $78, X10, X10
+ PSHUFL $78, X14, X14
+ PSHUFL $57, X3, X3
+ PSHUFL $57, X7, X7
+ PSHUFL $57, X11, X11
+ PSHUFL $57, X15, X15
+ MOVO 16(SP), X12
+ SUBQ $2, SI
+ JNE rounds_loop4_begin
+ MOVO X12, 16(SP)
+ PADDL 0(AX), X0
+ PADDL 16(AX), X1
+ PADDL 32(AX), X2
+ PADDL 48(AX), X3
+ MOVOU 0(BX), X12
+ PXOR X0, X12
+ MOVOU X12, 0(CX)
+ MOVOU 16(BX), X12
+ PXOR X1, X12
+ MOVOU X12, 16(CX)
+ MOVOU 32(BX), X12
+ PXOR X2, X12
+ MOVOU X12, 32(CX)
+ MOVOU 48(BX), X12
+ PXOR X3, X12
+ MOVOU X12, 48(CX)
+ MOVO 48(AX), X3
+ PADDQ 0(SP), X3
+ PADDL 0(AX), X4
+ PADDL 16(AX), X5
+ PADDL 32(AX), X6
+ PADDL X3, X7
+ MOVOU 64(BX), X12
+ PXOR X4, X12
+ MOVOU X12, 64(CX)
+ MOVOU 80(BX), X12
+ PXOR X5, X12
+ MOVOU X12, 80(CX)
+ MOVOU 96(BX), X12
+ PXOR X6, X12
+ MOVOU X12, 96(CX)
+ MOVOU 112(BX), X12
+ PXOR X7, X12
+ MOVOU X12, 112(CX)
+ PADDQ 0(SP), X3
+ PADDL 0(AX), X8
+ PADDL 16(AX), X9
+ PADDL 32(AX), X10
+ PADDL X3, X11
+ MOVOU 128(BX), X12
+ PXOR X8, X12
+ MOVOU X12, 128(CX)
+ MOVOU 144(BX), X12
+ PXOR X9, X12
+ MOVOU X12, 144(CX)
+ MOVOU 160(BX), X12
+ PXOR X10, X12
+ MOVOU X12, 160(CX)
+ MOVOU 176(BX), X12
+ PXOR X11, X12
+ MOVOU X12, 176(CX)
+ PADDQ 0(SP), X3
+ MOVO 16(SP), X12
+ PADDL 0(AX), X12
+ PADDL 16(AX), X13
+ PADDL 32(AX), X14
+ PADDL X3, X15
+ MOVOU 192(BX), X0
+ PXOR X12, X0
+ MOVOU X0, 192(CX)
+ MOVOU 208(BX), X0
+ PXOR X13, X0
+ MOVOU X0, 208(CX)
+ MOVOU 224(BX), X0
+ PXOR X14, X0
+ MOVOU X0, 224(CX)
+ MOVOU 240(BX), X0
+ PXOR X15, X0
+ MOVOU X0, 240(CX)
+ PADDQ 0(SP), X3
+ MOVO X3, 48(AX)
+ ADDQ $256, BX
+ ADDQ $256, CX
+ SUBQ $4, DX
+ JCC vector_loop4_begin
+vector_loop4_end:
+ ADDQ $4, DX
+ JEQ out
+ MOVO 0(AX), X8
+ MOVO 16(AX), X9
+ MOVO 32(AX), X10
+ MOVO 48(AX), X11
+ MOVO 0(SP), X13
+ SUBQ $2, DX
+ JCS vector_loop2_end
+vector_loop2_begin:
+ MOVO X8, X0
+ MOVO X9, X1
+ MOVO X10, X2
+ MOVO X11, X3
+ MOVO X0, X4
+ MOVO X1, X5
+ MOVO X2, X6
+ MOVO X3, X7
+ PADDQ X13, X7
+ MOVQ $20, SI
+rounds_loop2_begin:
+ PADDL X1, X0
+ PADDL X5, X4
+ PXOR X0, X3
+ PXOR X4, X7
+ MOVO X3, X12
+ PSLLL $16, X12
+ PSRLL $16, X3
+ PXOR X12, X3
+ MOVO X7, X12
+ PSLLL $16, X12
+ PSRLL $16, X7
+ PXOR X12, X7
+ PADDL X3, X2
+ PADDL X7, X6
+ PXOR X2, X1
+ PXOR X6, X5
+ MOVO X1, X12
+ PSLLL $12, X12
+ PSRLL $20, X1
+ PXOR X12, X1
+ MOVO X5, X12
+ PSLLL $12, X12
+ PSRLL $20, X5
+ PXOR X12, X5
+ PADDL X1, X0
+ PADDL X5, X4
+ PXOR X0, X3
+ PXOR X4, X7
+ MOVO X3, X12
+ PSLLL $8, X12
+ PSRLL $24, X3
+ PXOR X12, X3
+ MOVO X7, X12
+ PSLLL $8, X12
+ PSRLL $24, X7
+ PXOR X12, X7
+ PADDL X3, X2
+ PADDL X7, X6
+ PXOR X2, X1
+ PXOR X6, X5
+ MOVO X1, X12
+ PSLLL $7, X12
+ PSRLL $25, X1
+ PXOR X12, X1
+ MOVO X5, X12
+ PSLLL $7, X12
+ PSRLL $25, X5
+ PXOR X12, X5
+ PSHUFL $57, X1, X1
+ PSHUFL $57, X5, X5
+ PSHUFL $78, X2, X2
+ PSHUFL $78, X6, X6
+ PSHUFL $147, X3, X3
+ PSHUFL $147, X7, X7
+ PADDL X1, X0
+ PADDL X5, X4
+ PXOR X0, X3
+ PXOR X4, X7
+ MOVO X3, X12
+ PSLLL $16, X12
+ PSRLL $16, X3
+ PXOR X12, X3
+ MOVO X7, X12
+ PSLLL $16, X12
+ PSRLL $16, X7
+ PXOR X12, X7
+ PADDL X3, X2
+ PADDL X7, X6
+ PXOR X2, X1
+ PXOR X6, X5
+ MOVO X1, X12
+ PSLLL $12, X12
+ PSRLL $20, X1
+ PXOR X12, X1
+ MOVO X5, X12
+ PSLLL $12, X12
+ PSRLL $20, X5
+ PXOR X12, X5
+ PADDL X1, X0
+ PADDL X5, X4
+ PXOR X0, X3
+ PXOR X4, X7
+ MOVO X3, X12
+ PSLLL $8, X12
+ PSRLL $24, X3
+ PXOR X12, X3
+ MOVO X7, X12
+ PSLLL $8, X12
+ PSRLL $24, X7
+ PXOR X12, X7
+ PADDL X3, X2
+ PADDL X7, X6
+ PXOR X2, X1
+ PXOR X6, X5
+ MOVO X1, X12
+ PSLLL $7, X12
+ PSRLL $25, X1
+ PXOR X12, X1
+ MOVO X5, X12
+ PSLLL $7, X12
+ PSRLL $25, X5
+ PXOR X12, X5
+ PSHUFL $147, X1, X1
+ PSHUFL $147, X5, X5
+ PSHUFL $78, X2, X2
+ PSHUFL $78, X6, X6
+ PSHUFL $57, X3, X3
+ PSHUFL $57, X7, X7
+ SUBQ $2, SI
+ JNE rounds_loop2_begin
+ PADDL X8, X0
+ PADDL X9, X1
+ PADDL X10, X2
+ PADDL X11, X3
+ MOVOU 0(BX), X12
+ PXOR X0, X12
+ MOVOU X12, 0(CX)
+ MOVOU 16(BX), X12
+ PXOR X1, X12
+ MOVOU X12, 16(CX)
+ MOVOU 32(BX), X12
+ PXOR X2, X12
+ MOVOU X12, 32(CX)
+ MOVOU 48(BX), X12
+ PXOR X3, X12
+ MOVOU X12, 48(CX)
+ PADDQ X13, X11
+ PADDL X8, X4
+ PADDL X9, X5
+ PADDL X10, X6
+ PADDL X11, X7
+ MOVOU 64(BX), X12
+ PXOR X4, X12
+ MOVOU X12, 64(CX)
+ MOVOU 80(BX), X12
+ PXOR X5, X12
+ MOVOU X12, 80(CX)
+ MOVOU 96(BX), X12
+ PXOR X6, X12
+ MOVOU X12, 96(CX)
+ MOVOU 112(BX), X12
+ PXOR X7, X12
+ MOVOU X12, 112(CX)
+ PADDQ X13, X11
+ ADDQ $128, BX
+ ADDQ $128, CX
+ SUBQ $2, DX
+ JCC vector_loop2_begin
+vector_loop2_end:
+ ADDQ $2, DX
+ JEQ out_serial
+ MOVO X8, X0
+ MOVO X9, X1
+ MOVO X10, X2
+ MOVO X11, X3
+ MOVQ $20, DX
+rounds_loop1_begin:
+ PADDL X1, X0
+ PXOR X0, X3
+ MOVO X3, X12
+ PSLLL $16, X12
+ PSRLL $16, X3
+ PXOR X12, X3
+ PADDL X3, X2
+ PXOR X2, X1
+ MOVO X1, X12
+ PSLLL $12, X12
+ PSRLL $20, X1
+ PXOR X12, X1
+ PADDL X1, X0
+ PXOR X0, X3
+ MOVO X3, X12
+ PSLLL $8, X12
+ PSRLL $24, X3
+ PXOR X12, X3
+ PADDL X3, X2
+ PXOR X2, X1
+ MOVO X1, X12
+ PSLLL $7, X12
+ PSRLL $25, X1
+ PXOR X12, X1
+ PSHUFL $57, X1, X1
+ PSHUFL $78, X2, X2
+ PSHUFL $147, X3, X3
+ PADDL X1, X0
+ PXOR X0, X3
+ MOVO X3, X12
+ PSLLL $16, X12
+ PSRLL $16, X3
+ PXOR X12, X3
+ PADDL X3, X2
+ PXOR X2, X1
+ MOVO X1, X12
+ PSLLL $12, X12
+ PSRLL $20, X1
+ PXOR X12, X1
+ PADDL X1, X0
+ PXOR X0, X3
+ MOVO X3, X12
+ PSLLL $8, X12
+ PSRLL $24, X3
+ PXOR X12, X3
+ PADDL X3, X2
+ PXOR X2, X1
+ MOVO X1, X12
+ PSLLL $7, X12
+ PSRLL $25, X1
+ PXOR X12, X1
+ PSHUFL $147, X1, X1
+ PSHUFL $78, X2, X2
+ PSHUFL $57, X3, X3
+ SUBQ $2, DX
+ JNE rounds_loop1_begin
+ PADDL X8, X0
+ PADDL X9, X1
+ PADDL X10, X2
+ PADDL X11, X3
+ MOVOU 0(BX), X12
+ PXOR X0, X12
+ MOVOU X12, 0(CX)
+ MOVOU 16(BX), X12
+ PXOR X1, X12
+ MOVOU X12, 16(CX)
+ MOVOU 32(BX), X12
+ PXOR X2, X12
+ MOVOU X12, 32(CX)
+ MOVOU 48(BX), X12
+ PXOR X3, X12
+ MOVOU X12, 48(CX)
+ PADDQ X13, X11
+out_serial:
+ MOVO X11, 48(AX)
+out:
+ PXOR X0, X0
+ MOVO X0, 16(SP)
+ MOVQ DI, SP
+ RET
+
+// func blocksAmd64AVX2(x *uint32, inp *uint8, outp *uint8, nrBlocks *uint)
+TEXT ·blocksAmd64AVX2(SB),4,$0-32
+ MOVQ x+0(FP), AX
+ MOVQ inp+8(FP), BX
+ MOVQ outp+16(FP), CX
+ MOVQ nrBlocks+24(FP), DX
+ MOVQ SP, DI
+ MOVQ $31, SI
+ NOTQ SI
+ ANDQ SI, SP
+ SUBQ $32, SP
+ SUBQ $96, SP
+ BYTE $0xC4; BYTE $0x41; BYTE $0x1D; BYTE $0xEF; BYTE $0xE4 // VPXOR ymm12, ymm12, ymm12
+ BYTE $0xC5; BYTE $0x7E; BYTE $0x7F; BYTE $0x24; BYTE $0x24 // VMOVDQU [rsp], ymm12
+ MOVL $1, SI
+ MOVL SI, 16(SP)
+ BYTE $0xC4; BYTE $0xE2; BYTE $0x7D; BYTE $0x5A; BYTE $0x58; BYTE $0x30 // VBROADCASTI128 ymm3, [rax + 48]
+ BYTE $0xC5; BYTE $0xE5; BYTE $0xD4; BYTE $0x1C; BYTE $0x24 // VPADDQ ymm3, ymm3, [rsp]
+ BYTE $0xC5; BYTE $0xFD; BYTE $0x7F; BYTE $0x5C; BYTE $0x24; BYTE $0x20 // VMOVDQA [rsp + 32], ymm3
+ MOVL $2, SI
+ MOVL SI, 0(SP)
+ MOVL SI, 16(SP)
+ SUBQ $8, DX
+ JCS vector_loop8_end
+vector_loop8_begin:
+ BYTE $0xC4; BYTE $0xE2; BYTE $0x7D; BYTE $0x5A; BYTE $0x00 // VBROADCASTI128 ymm0, [rax]
+ BYTE $0xC4; BYTE $0xE2; BYTE $0x7D; BYTE $0x5A; BYTE $0x48; BYTE $0x10 // VBROADCASTI128 ymm1, [rax + 16]
+ BYTE $0xC4; BYTE $0xE2; BYTE $0x7D; BYTE $0x5A; BYTE $0x50; BYTE $0x20 // VBROADCASTI128 ymm2, [rax + 32]
+ BYTE $0xC5; BYTE $0xFD; BYTE $0x6F; BYTE $0x5C; BYTE $0x24; BYTE $0x20 // VMOVDQA ymm3, [rsp + 32]
+ BYTE $0xC5; BYTE $0xFD; BYTE $0x6F; BYTE $0xE0 // VMOVDQA ymm4, ymm0
+ BYTE $0xC5; BYTE $0xFD; BYTE $0x6F; BYTE $0xE9 // VMOVDQA ymm5, ymm1
+ BYTE $0xC5; BYTE $0xFD; BYTE $0x6F; BYTE $0xF2 // VMOVDQA ymm6, ymm2
+ BYTE $0xC5; BYTE $0xE5; BYTE $0xD4; BYTE $0x3C; BYTE $0x24 // VPADDQ ymm7, ymm3, [rsp]
+ BYTE $0xC5; BYTE $0x7D; BYTE $0x6F; BYTE $0xC0 // VMOVDQA ymm8, ymm0
+ BYTE $0xC5; BYTE $0x7D; BYTE $0x6F; BYTE $0xC9 // VMOVDQA ymm9, ymm1
+ BYTE $0xC5; BYTE $0x7D; BYTE $0x6F; BYTE $0xD2 // VMOVDQA ymm10, ymm2
+ BYTE $0xC5; BYTE $0x45; BYTE $0xD4; BYTE $0x1C; BYTE $0x24 // VPADDQ ymm11, ymm7, [rsp]
+ BYTE $0xC5; BYTE $0x7D; BYTE $0x6F; BYTE $0xE0 // VMOVDQA ymm12, ymm0
+ BYTE $0xC5; BYTE $0x7D; BYTE $0x6F; BYTE $0xE9 // VMOVDQA ymm13, ymm1
+ BYTE $0xC5; BYTE $0x7D; BYTE $0x6F; BYTE $0xF2 // VMOVDQA ymm14, ymm2
+ BYTE $0xC5; BYTE $0x25; BYTE $0xD4; BYTE $0x3C; BYTE $0x24 // VPADDQ ymm15, ymm11, [rsp]
+ MOVQ $20, SI
+rounds_loop8_begin:
+ BYTE $0xC5; BYTE $0xFD; BYTE $0xFE; BYTE $0xC1 // VPADDD ymm0, ymm0, ymm1
+ BYTE $0xC5; BYTE $0xDD; BYTE $0xFE; BYTE $0xE5 // VPADDD ymm4, ymm4, ymm5
+ BYTE $0xC4; BYTE $0x41; BYTE $0x3D; BYTE $0xFE; BYTE $0xC1 // VPADDD ymm8, ymm8, ymm9
+ BYTE $0xC4; BYTE $0x41; BYTE $0x1D; BYTE $0xFE; BYTE $0xE5 // VPADDD ymm12, ymm12, ymm13
+ BYTE $0xC5; BYTE $0xE5; BYTE $0xEF; BYTE $0xD8 // VPXOR ymm3, ymm3, ymm0
+ BYTE $0xC5; BYTE $0xC5; BYTE $0xEF; BYTE $0xFC // VPXOR ymm7, ymm7, ymm4
+ BYTE $0xC4; BYTE $0x41; BYTE $0x25; BYTE $0xEF; BYTE $0xD8 // VPXOR ymm11, ymm11, ymm8
+ BYTE $0xC4; BYTE $0x41; BYTE $0x05; BYTE $0xEF; BYTE $0xFC // VPXOR ymm15, ymm15, ymm12
+ BYTE $0xC5; BYTE $0x7D; BYTE $0x7F; BYTE $0x64; BYTE $0x24; BYTE $0x40 // VMOVDQA [rsp + 64], ymm12
+ BYTE $0xC5; BYTE $0x9D; BYTE $0x72; BYTE $0xF3; BYTE $0x10 // VPSLLD ymm12, ymm3, 16
+ BYTE $0xC5; BYTE $0xE5; BYTE $0x72; BYTE $0xD3; BYTE $0x10 // VPSRLD ymm3, ymm3, 16
+ BYTE $0xC4; BYTE $0xC1; BYTE $0x65; BYTE $0xEF; BYTE $0xDC // VPXOR ymm3, ymm3, ymm12
+ BYTE $0xC5; BYTE $0x9D; BYTE $0x72; BYTE $0xF7; BYTE $0x10 // VPSLLD ymm12, ymm7, 16
+ BYTE $0xC5; BYTE $0xC5; BYTE $0x72; BYTE $0xD7; BYTE $0x10 // VPSRLD ymm7, ymm7, 16
+ BYTE $0xC4; BYTE $0xC1; BYTE $0x45; BYTE $0xEF; BYTE $0xFC // VPXOR ymm7, ymm7, ymm12
+ BYTE $0xC4; BYTE $0xC1; BYTE $0x1D; BYTE $0x72; BYTE $0xF3; BYTE $0x10 // VPSLLD ymm12, ymm11, 16
+ BYTE $0xC4; BYTE $0xC1; BYTE $0x25; BYTE $0x72; BYTE $0xD3; BYTE $0x10 // VPSRLD ymm11, ymm11, 16
+ BYTE $0xC4; BYTE $0x41; BYTE $0x25; BYTE $0xEF; BYTE $0xDC // VPXOR ymm11, ymm11, ymm12
+ BYTE $0xC4; BYTE $0xC1; BYTE $0x1D; BYTE $0x72; BYTE $0xF7; BYTE $0x10 // VPSLLD ymm12, ymm15, 16
+ BYTE $0xC4; BYTE $0xC1; BYTE $0x05; BYTE $0x72; BYTE $0xD7; BYTE $0x10 // VPSRLD ymm15, ymm15, 16
+ BYTE $0xC4; BYTE $0x41; BYTE $0x05; BYTE $0xEF; BYTE $0xFC // VPXOR ymm15, ymm15, ymm12
+ BYTE $0xC5; BYTE $0xED; BYTE $0xFE; BYTE $0xD3 // VPADDD ymm2, ymm2, ymm3
+ BYTE $0xC5; BYTE $0xCD; BYTE $0xFE; BYTE $0xF7 // VPADDD ymm6, ymm6, ymm7
+ BYTE $0xC4; BYTE $0x41; BYTE $0x2D; BYTE $0xFE; BYTE $0xD3 // VPADDD ymm10, ymm10, ymm11
+ BYTE $0xC4; BYTE $0x41; BYTE $0x0D; BYTE $0xFE; BYTE $0xF7 // VPADDD ymm14, ymm14, ymm15
+ BYTE $0xC5; BYTE $0xF5; BYTE $0xEF; BYTE $0xCA // VPXOR ymm1, ymm1, ymm2
+ BYTE $0xC5; BYTE $0xD5; BYTE $0xEF; BYTE $0xEE // VPXOR ymm5, ymm5, ymm6
+ BYTE $0xC4; BYTE $0x41; BYTE $0x35; BYTE $0xEF; BYTE $0xCA // VPXOR ymm9, ymm9, ymm10
+ BYTE $0xC4; BYTE $0x41; BYTE $0x15; BYTE $0xEF; BYTE $0xEE // VPXOR ymm13, ymm13, ymm14
+ BYTE $0xC5; BYTE $0x9D; BYTE $0x72; BYTE $0xF1; BYTE $0x0C // VPSLLD ymm12, ymm1, 12
+ BYTE $0xC5; BYTE $0xF5; BYTE $0x72; BYTE $0xD1; BYTE $0x14 // VPSRLD ymm1, ymm1, 20
+ BYTE $0xC4; BYTE $0xC1; BYTE $0x75; BYTE $0xEF; BYTE $0xCC // VPXOR ymm1, ymm1, ymm12
+ BYTE $0xC5; BYTE $0x9D; BYTE $0x72; BYTE $0xF5; BYTE $0x0C // VPSLLD ymm12, ymm5, 12
+ BYTE $0xC5; BYTE $0xD5; BYTE $0x72; BYTE $0xD5; BYTE $0x14 // VPSRLD ymm5, ymm5, 20
+ BYTE $0xC4; BYTE $0xC1; BYTE $0x55; BYTE $0xEF; BYTE $0xEC // VPXOR ymm5, ymm5, ymm12
+ BYTE $0xC4; BYTE $0xC1; BYTE $0x1D; BYTE $0x72; BYTE $0xF1; BYTE $0x0C // VPSLLD ymm12, ymm9, 12
+ BYTE $0xC4; BYTE $0xC1; BYTE $0x35; BYTE $0x72; BYTE $0xD1; BYTE $0x14 // VPSRLD ymm9, ymm9, 20
+ BYTE $0xC4; BYTE $0x41; BYTE $0x35; BYTE $0xEF; BYTE $0xCC // VPXOR ymm9, ymm9, ymm12
+ BYTE $0xC4; BYTE $0xC1; BYTE $0x1D; BYTE $0x72; BYTE $0xF5; BYTE $0x0C // VPSLLD ymm12, ymm13, 12
+ BYTE $0xC4; BYTE $0xC1; BYTE $0x15; BYTE $0x72; BYTE $0xD5; BYTE $0x14 // VPSRLD ymm13, ymm13, 20
+ BYTE $0xC4; BYTE $0x41; BYTE $0x15; BYTE $0xEF; BYTE $0xEC // VPXOR ymm13, ymm13, ymm12
+ BYTE $0xC5; BYTE $0x7D; BYTE $0x6F; BYTE $0x64; BYTE $0x24; BYTE $0x40 // VMOVDQA ymm12, [rsp + 64]
+ BYTE $0xC5; BYTE $0xFD; BYTE $0xFE; BYTE $0xC1 // VPADDD ymm0, ymm0, ymm1
+ BYTE $0xC5; BYTE $0xDD; BYTE $0xFE; BYTE $0xE5 // VPADDD ymm4, ymm4, ymm5
+ BYTE $0xC4; BYTE $0x41; BYTE $0x3D; BYTE $0xFE; BYTE $0xC1 // VPADDD ymm8, ymm8, ymm9
+ BYTE $0xC4; BYTE $0x41; BYTE $0x1D; BYTE $0xFE; BYTE $0xE5 // VPADDD ymm12, ymm12, ymm13
+ BYTE $0xC5; BYTE $0xE5; BYTE $0xEF; BYTE $0xD8 // VPXOR ymm3, ymm3, ymm0
+ BYTE $0xC5; BYTE $0xC5; BYTE $0xEF; BYTE $0xFC // VPXOR ymm7, ymm7, ymm4
+ BYTE $0xC4; BYTE $0x41; BYTE $0x25; BYTE $0xEF; BYTE $0xD8 // VPXOR ymm11, ymm11, ymm8
+ BYTE $0xC4; BYTE $0x41; BYTE $0x05; BYTE $0xEF; BYTE $0xFC // VPXOR ymm15, ymm15, ymm12
+ BYTE $0xC5; BYTE $0x7D; BYTE $0x7F; BYTE $0x64; BYTE $0x24; BYTE $0x40 // VMOVDQA [rsp + 64], ymm12
+ BYTE $0xC5; BYTE $0x9D; BYTE $0x72; BYTE $0xF3; BYTE $0x08 // VPSLLD ymm12, ymm3, 8
+ BYTE $0xC5; BYTE $0xE5; BYTE $0x72; BYTE $0xD3; BYTE $0x18 // VPSRLD ymm3, ymm3, 24
+ BYTE $0xC4; BYTE $0xC1; BYTE $0x65; BYTE $0xEF; BYTE $0xDC // VPXOR ymm3, ymm3, ymm12
+ BYTE $0xC5; BYTE $0x9D; BYTE $0x72; BYTE $0xF7; BYTE $0x08 // VPSLLD ymm12, ymm7, 8
+ BYTE $0xC5; BYTE $0xC5; BYTE $0x72; BYTE $0xD7; BYTE $0x18 // VPSRLD ymm7, ymm7, 24
+ BYTE $0xC4; BYTE $0xC1; BYTE $0x45; BYTE $0xEF; BYTE $0xFC // VPXOR ymm7, ymm7, ymm12
+ BYTE $0xC4; BYTE $0xC1; BYTE $0x1D; BYTE $0x72; BYTE $0xF3; BYTE $0x08 // VPSLLD ymm12, ymm11, 8
+ BYTE $0xC4; BYTE $0xC1; BYTE $0x25; BYTE $0x72; BYTE $0xD3; BYTE $0x18 // VPSRLD ymm11, ymm11, 24
+ BYTE $0xC4; BYTE $0x41; BYTE $0x25; BYTE $0xEF; BYTE $0xDC // VPXOR ymm11, ymm11, ymm12
+ BYTE $0xC4; BYTE $0xC1; BYTE $0x1D; BYTE $0x72; BYTE $0xF7; BYTE $0x08 // VPSLLD ymm12, ymm15, 8
+ BYTE $0xC4; BYTE $0xC1; BYTE $0x05; BYTE $0x72; BYTE $0xD7; BYTE $0x18 // VPSRLD ymm15, ymm15, 24
+ BYTE $0xC4; BYTE $0x41; BYTE $0x05; BYTE $0xEF; BYTE $0xFC // VPXOR ymm15, ymm15, ymm12
+ BYTE $0xC5; BYTE $0xED; BYTE $0xFE; BYTE $0xD3 // VPADDD ymm2, ymm2, ymm3
+ BYTE $0xC5; BYTE $0xCD; BYTE $0xFE; BYTE $0xF7 // VPADDD ymm6, ymm6, ymm7
+ BYTE $0xC4; BYTE $0x41; BYTE $0x2D; BYTE $0xFE; BYTE $0xD3 // VPADDD ymm10, ymm10, ymm11
+ BYTE $0xC4; BYTE $0x41; BYTE $0x0D; BYTE $0xFE; BYTE $0xF7 // VPADDD ymm14, ymm14, ymm15
+ BYTE $0xC5; BYTE $0xF5; BYTE $0xEF; BYTE $0xCA // VPXOR ymm1, ymm1, ymm2
+ BYTE $0xC5; BYTE $0xD5; BYTE $0xEF; BYTE $0xEE // VPXOR ymm5, ymm5, ymm6
+ BYTE $0xC4; BYTE $0x41; BYTE $0x35; BYTE $0xEF; BYTE $0xCA // VPXOR ymm9, ymm9, ymm10
+ BYTE $0xC4; BYTE $0x41; BYTE $0x15; BYTE $0xEF; BYTE $0xEE // VPXOR ymm13, ymm13, ymm14
+ BYTE $0xC5; BYTE $0x9D; BYTE $0x72; BYTE $0xF1; BYTE $0x07 // VPSLLD ymm12, ymm1, 7
+ BYTE $0xC5; BYTE $0xF5; BYTE $0x72; BYTE $0xD1; BYTE $0x19 // VPSRLD ymm1, ymm1, 25
+ BYTE $0xC4; BYTE $0xC1; BYTE $0x75; BYTE $0xEF; BYTE $0xCC // VPXOR ymm1, ymm1, ymm12
+ BYTE $0xC5; BYTE $0x9D; BYTE $0x72; BYTE $0xF5; BYTE $0x07 // VPSLLD ymm12, ymm5, 7
+ BYTE $0xC5; BYTE $0xD5; BYTE $0x72; BYTE $0xD5; BYTE $0x19 // VPSRLD ymm5, ymm5, 25
+ BYTE $0xC4; BYTE $0xC1; BYTE $0x55; BYTE $0xEF; BYTE $0xEC // VPXOR ymm5, ymm5, ymm12
+ BYTE $0xC4; BYTE $0xC1; BYTE $0x1D; BYTE $0x72; BYTE $0xF1; BYTE $0x07 // VPSLLD ymm12, ymm9, 7
+ BYTE $0xC4; BYTE $0xC1; BYTE $0x35; BYTE $0x72; BYTE $0xD1; BYTE $0x19 // VPSRLD ymm9, ymm9, 25
+ BYTE $0xC4; BYTE $0x41; BYTE $0x35; BYTE $0xEF; BYTE $0xCC // VPXOR ymm9, ymm9, ymm12
+ BYTE $0xC4; BYTE $0xC1; BYTE $0x1D; BYTE $0x72; BYTE $0xF5; BYTE $0x07 // VPSLLD ymm12, ymm13, 7
+ BYTE $0xC4; BYTE $0xC1; BYTE $0x15; BYTE $0x72; BYTE $0xD5; BYTE $0x19 // VPSRLD ymm13, ymm13, 25
+ BYTE $0xC4; BYTE $0x41; BYTE $0x15; BYTE $0xEF; BYTE $0xEC // VPXOR ymm13, ymm13, ymm12
+ BYTE $0xC5; BYTE $0xFD; BYTE $0x70; BYTE $0xC9; BYTE $0x39 // VPSHUFD ymm1, ymm1, 57
+ BYTE $0xC5; BYTE $0xFD; BYTE $0x70; BYTE $0xED; BYTE $0x39 // VPSHUFD ymm5, ymm5, 57
+ BYTE $0xC4; BYTE $0x41; BYTE $0x7D; BYTE $0x70; BYTE $0xC9; BYTE $0x39 // VPSHUFD ymm9, ymm9, 57
+ BYTE $0xC4; BYTE $0x41; BYTE $0x7D; BYTE $0x70; BYTE $0xED; BYTE $0x39 // VPSHUFD ymm13, ymm13, 57
+ BYTE $0xC5; BYTE $0xFD; BYTE $0x70; BYTE $0xD2; BYTE $0x4E // VPSHUFD ymm2, ymm2, 78
+ BYTE $0xC5; BYTE $0xFD; BYTE $0x70; BYTE $0xF6; BYTE $0x4E // VPSHUFD ymm6, ymm6, 78
+ BYTE $0xC4; BYTE $0x41; BYTE $0x7D; BYTE $0x70; BYTE $0xD2; BYTE $0x4E // VPSHUFD ymm10, ymm10, 78
+ BYTE $0xC4; BYTE $0x41; BYTE $0x7D; BYTE $0x70; BYTE $0xF6; BYTE $0x4E // VPSHUFD ymm14, ymm14, 78
+ BYTE $0xC5; BYTE $0xFD; BYTE $0x70; BYTE $0xDB; BYTE $0x93 // VPSHUFD ymm3, ymm3, 147
+ BYTE $0xC5; BYTE $0xFD; BYTE $0x70; BYTE $0xFF; BYTE $0x93 // VPSHUFD ymm7, ymm7, 147
+ BYTE $0xC4; BYTE $0x41; BYTE $0x7D; BYTE $0x70; BYTE $0xDB; BYTE $0x93 // VPSHUFD ymm11, ymm11, 147
+ BYTE $0xC4; BYTE $0x41; BYTE $0x7D; BYTE $0x70; BYTE $0xFF; BYTE $0x93 // VPSHUFD ymm15, ymm15, 147
+ BYTE $0xC5; BYTE $0x7D; BYTE $0x6F; BYTE $0x64; BYTE $0x24; BYTE $0x40 // VMOVDQA ymm12, [rsp + 64]
+ BYTE $0xC5; BYTE $0xFD; BYTE $0xFE; BYTE $0xC1 // VPADDD ymm0, ymm0, ymm1
+ BYTE $0xC5; BYTE $0xDD; BYTE $0xFE; BYTE $0xE5 // VPADDD ymm4, ymm4, ymm5
+ BYTE $0xC4; BYTE $0x41; BYTE $0x3D; BYTE $0xFE; BYTE $0xC1 // VPADDD ymm8, ymm8, ymm9
+ BYTE $0xC4; BYTE $0x41; BYTE $0x1D; BYTE $0xFE; BYTE $0xE5 // VPADDD ymm12, ymm12, ymm13
+ BYTE $0xC5; BYTE $0xE5; BYTE $0xEF; BYTE $0xD8 // VPXOR ymm3, ymm3, ymm0
+ BYTE $0xC5; BYTE $0xC5; BYTE $0xEF; BYTE $0xFC // VPXOR ymm7, ymm7, ymm4
+ BYTE $0xC4; BYTE $0x41; BYTE $0x25; BYTE $0xEF; BYTE $0xD8 // VPXOR ymm11, ymm11, ymm8
+ BYTE $0xC4; BYTE $0x41; BYTE $0x05; BYTE $0xEF; BYTE $0xFC // VPXOR ymm15, ymm15, ymm12
+ BYTE $0xC5; BYTE $0x7D; BYTE $0x7F; BYTE $0x64; BYTE $0x24; BYTE $0x40 // VMOVDQA [rsp + 64], ymm12
+ BYTE $0xC5; BYTE $0x9D; BYTE $0x72; BYTE $0xF3; BYTE $0x10 // VPSLLD ymm12, ymm3, 16
+ BYTE $0xC5; BYTE $0xE5; BYTE $0x72; BYTE $0xD3; BYTE $0x10 // VPSRLD ymm3, ymm3, 16
+ BYTE $0xC4; BYTE $0xC1; BYTE $0x65; BYTE $0xEF; BYTE $0xDC // VPXOR ymm3, ymm3, ymm12
+ BYTE $0xC5; BYTE $0x9D; BYTE $0x72; BYTE $0xF7; BYTE $0x10 // VPSLLD ymm12, ymm7, 16
+ BYTE $0xC5; BYTE $0xC5; BYTE $0x72; BYTE $0xD7; BYTE $0x10 // VPSRLD ymm7, ymm7, 16
+ BYTE $0xC4; BYTE $0xC1; BYTE $0x45; BYTE $0xEF; BYTE $0xFC // VPXOR ymm7, ymm7, ymm12
+ BYTE $0xC4; BYTE $0xC1; BYTE $0x1D; BYTE $0x72; BYTE $0xF3; BYTE $0x10 // VPSLLD ymm12, ymm11, 16
+ BYTE $0xC4; BYTE $0xC1; BYTE $0x25; BYTE $0x72; BYTE $0xD3; BYTE $0x10 // VPSRLD ymm11, ymm11, 16
+ BYTE $0xC4; BYTE $0x41; BYTE $0x25; BYTE $0xEF; BYTE $0xDC // VPXOR ymm11, ymm11, ymm12
+ BYTE $0xC4; BYTE $0xC1; BYTE $0x1D; BYTE $0x72; BYTE $0xF7; BYTE $0x10 // VPSLLD ymm12, ymm15, 16
+ BYTE $0xC4; BYTE $0xC1; BYTE $0x05; BYTE $0x72; BYTE $0xD7; BYTE $0x10 // VPSRLD ymm15, ymm15, 16
+ BYTE $0xC4; BYTE $0x41; BYTE $0x05; BYTE $0xEF; BYTE $0xFC // VPXOR ymm15, ymm15, ymm12
+ BYTE $0xC5; BYTE $0xED; BYTE $0xFE; BYTE $0xD3 // VPADDD ymm2, ymm2, ymm3
+ BYTE $0xC5; BYTE $0xCD; BYTE $0xFE; BYTE $0xF7 // VPADDD ymm6, ymm6, ymm7
+ BYTE $0xC4; BYTE $0x41; BYTE $0x2D; BYTE $0xFE; BYTE $0xD3 // VPADDD ymm10, ymm10, ymm11
+ BYTE $0xC4; BYTE $0x41; BYTE $0x0D; BYTE $0xFE; BYTE $0xF7 // VPADDD ymm14, ymm14, ymm15
+ BYTE $0xC5; BYTE $0xF5; BYTE $0xEF; BYTE $0xCA // VPXOR ymm1, ymm1, ymm2
+ BYTE $0xC5; BYTE $0xD5; BYTE $0xEF; BYTE $0xEE // VPXOR ymm5, ymm5, ymm6
+ BYTE $0xC4; BYTE $0x41; BYTE $0x35; BYTE $0xEF; BYTE $0xCA // VPXOR ymm9, ymm9, ymm10
+ BYTE $0xC4; BYTE $0x41; BYTE $0x15; BYTE $0xEF; BYTE $0xEE // VPXOR ymm13, ymm13, ymm14
+ BYTE $0xC5; BYTE $0x9D; BYTE $0x72; BYTE $0xF1; BYTE $0x0C // VPSLLD ymm12, ymm1, 12
+ BYTE $0xC5; BYTE $0xF5; BYTE $0x72; BYTE $0xD1; BYTE $0x14 // VPSRLD ymm1, ymm1, 20
+ BYTE $0xC4; BYTE $0xC1; BYTE $0x75; BYTE $0xEF; BYTE $0xCC // VPXOR ymm1, ymm1, ymm12
+ BYTE $0xC5; BYTE $0x9D; BYTE $0x72; BYTE $0xF5; BYTE $0x0C // VPSLLD ymm12, ymm5, 12
+ BYTE $0xC5; BYTE $0xD5; BYTE $0x72; BYTE $0xD5; BYTE $0x14 // VPSRLD ymm5, ymm5, 20
+ BYTE $0xC4; BYTE $0xC1; BYTE $0x55; BYTE $0xEF; BYTE $0xEC // VPXOR ymm5, ymm5, ymm12
+ BYTE $0xC4; BYTE $0xC1; BYTE $0x1D; BYTE $0x72; BYTE $0xF1; BYTE $0x0C // VPSLLD ymm12, ymm9, 12
+ BYTE $0xC4; BYTE $0xC1; BYTE $0x35; BYTE $0x72; BYTE $0xD1; BYTE $0x14 // VPSRLD ymm9, ymm9, 20
+ BYTE $0xC4; BYTE $0x41; BYTE $0x35; BYTE $0xEF; BYTE $0xCC // VPXOR ymm9, ymm9, ymm12
+ BYTE $0xC4; BYTE $0xC1; BYTE $0x1D; BYTE $0x72; BYTE $0xF5; BYTE $0x0C // VPSLLD ymm12, ymm13, 12
+ BYTE $0xC4; BYTE $0xC1; BYTE $0x15; BYTE $0x72; BYTE $0xD5; BYTE $0x14 // VPSRLD ymm13, ymm13, 20
+ BYTE $0xC4; BYTE $0x41; BYTE $0x15; BYTE $0xEF; BYTE $0xEC // VPXOR ymm13, ymm13, ymm12
+ BYTE $0xC5; BYTE $0x7D; BYTE $0x6F; BYTE $0x64; BYTE $0x24; BYTE $0x40 // VMOVDQA ymm12, [rsp + 64]
+ BYTE $0xC5; BYTE $0xFD; BYTE $0xFE; BYTE $0xC1 // VPADDD ymm0, ymm0, ymm1
+ BYTE $0xC5; BYTE $0xDD; BYTE $0xFE; BYTE $0xE5 // VPADDD ymm4, ymm4, ymm5
+ BYTE $0xC4; BYTE $0x41; BYTE $0x3D; BYTE $0xFE; BYTE $0xC1 // VPADDD ymm8, ymm8, ymm9
+ BYTE $0xC4; BYTE $0x41; BYTE $0x1D; BYTE $0xFE; BYTE $0xE5 // VPADDD ymm12, ymm12, ymm13
+ BYTE $0xC5; BYTE $0xE5; BYTE $0xEF; BYTE $0xD8 // VPXOR ymm3, ymm3, ymm0
+ BYTE $0xC5; BYTE $0xC5; BYTE $0xEF; BYTE $0xFC // VPXOR ymm7, ymm7, ymm4
+ BYTE $0xC4; BYTE $0x41; BYTE $0x25; BYTE $0xEF; BYTE $0xD8 // VPXOR ymm11, ymm11, ymm8
+ BYTE $0xC4; BYTE $0x41; BYTE $0x05; BYTE $0xEF; BYTE $0xFC // VPXOR ymm15, ymm15, ymm12
+ BYTE $0xC5; BYTE $0x7D; BYTE $0x7F; BYTE $0x64; BYTE $0x24; BYTE $0x40 // VMOVDQA [rsp + 64], ymm12
+ BYTE $0xC5; BYTE $0x9D; BYTE $0x72; BYTE $0xF3; BYTE $0x08 // VPSLLD ymm12, ymm3, 8
+ BYTE $0xC5; BYTE $0xE5; BYTE $0x72; BYTE $0xD3; BYTE $0x18 // VPSRLD ymm3, ymm3, 24
+ BYTE $0xC4; BYTE $0xC1; BYTE $0x65; BYTE $0xEF; BYTE $0xDC // VPXOR ymm3, ymm3, ymm12
+ BYTE $0xC5; BYTE $0x9D; BYTE $0x72; BYTE $0xF7; BYTE $0x08 // VPSLLD ymm12, ymm7, 8
+ BYTE $0xC5; BYTE $0xC5; BYTE $0x72; BYTE $0xD7; BYTE $0x18 // VPSRLD ymm7, ymm7, 24
+ BYTE $0xC4; BYTE $0xC1; BYTE $0x45; BYTE $0xEF; BYTE $0xFC // VPXOR ymm7, ymm7, ymm12
+ BYTE $0xC4; BYTE $0xC1; BYTE $0x1D; BYTE $0x72; BYTE $0xF3; BYTE $0x08 // VPSLLD ymm12, ymm11, 8
+ BYTE $0xC4; BYTE $0xC1; BYTE $0x25; BYTE $0x72; BYTE $0xD3; BYTE $0x18 // VPSRLD ymm11, ymm11, 24
+ BYTE $0xC4; BYTE $0x41; BYTE $0x25; BYTE $0xEF; BYTE $0xDC // VPXOR ymm11, ymm11, ymm12
+ BYTE $0xC4; BYTE $0xC1; BYTE $0x1D; BYTE $0x72; BYTE $0xF7; BYTE $0x08 // VPSLLD ymm12, ymm15, 8
+ BYTE $0xC4; BYTE $0xC1; BYTE $0x05; BYTE $0x72; BYTE $0xD7; BYTE $0x18 // VPSRLD ymm15, ymm15, 24
+ BYTE $0xC4; BYTE $0x41; BYTE $0x05; BYTE $0xEF; BYTE $0xFC // VPXOR ymm15, ymm15, ymm12
+ BYTE $0xC5; BYTE $0xED; BYTE $0xFE; BYTE $0xD3 // VPADDD ymm2, ymm2, ymm3
+ BYTE $0xC5; BYTE $0xCD; BYTE $0xFE; BYTE $0xF7 // VPADDD ymm6, ymm6, ymm7
+ BYTE $0xC4; BYTE $0x41; BYTE $0x2D; BYTE $0xFE; BYTE $0xD3 // VPADDD ymm10, ymm10, ymm11
+ BYTE $0xC4; BYTE $0x41; BYTE $0x0D; BYTE $0xFE; BYTE $0xF7 // VPADDD ymm14, ymm14, ymm15
+ BYTE $0xC5; BYTE $0xF5; BYTE $0xEF; BYTE $0xCA // VPXOR ymm1, ymm1, ymm2
+ BYTE $0xC5; BYTE $0xD5; BYTE $0xEF; BYTE $0xEE // VPXOR ymm5, ymm5, ymm6
+ BYTE $0xC4; BYTE $0x41; BYTE $0x35; BYTE $0xEF; BYTE $0xCA // VPXOR ymm9, ymm9, ymm10
+ BYTE $0xC4; BYTE $0x41; BYTE $0x15; BYTE $0xEF; BYTE $0xEE // VPXOR ymm13, ymm13, ymm14
+ BYTE $0xC5; BYTE $0x9D; BYTE $0x72; BYTE $0xF1; BYTE $0x07 // VPSLLD ymm12, ymm1, 7
+ BYTE $0xC5; BYTE $0xF5; BYTE $0x72; BYTE $0xD1; BYTE $0x19 // VPSRLD ymm1, ymm1, 25
+ BYTE $0xC4; BYTE $0xC1; BYTE $0x75; BYTE $0xEF; BYTE $0xCC // VPXOR ymm1, ymm1, ymm12
+ BYTE $0xC5; BYTE $0x9D; BYTE $0x72; BYTE $0xF5; BYTE $0x07 // VPSLLD ymm12, ymm5, 7
+ BYTE $0xC5; BYTE $0xD5; BYTE $0x72; BYTE $0xD5; BYTE $0x19 // VPSRLD ymm5, ymm5, 25
+ BYTE $0xC4; BYTE $0xC1; BYTE $0x55; BYTE $0xEF; BYTE $0xEC // VPXOR ymm5, ymm5, ymm12
+ BYTE $0xC4; BYTE $0xC1; BYTE $0x1D; BYTE $0x72; BYTE $0xF1; BYTE $0x07 // VPSLLD ymm12, ymm9, 7
+ BYTE $0xC4; BYTE $0xC1; BYTE $0x35; BYTE $0x72; BYTE $0xD1; BYTE $0x19 // VPSRLD ymm9, ymm9, 25
+ BYTE $0xC4; BYTE $0x41; BYTE $0x35; BYTE $0xEF; BYTE $0xCC // VPXOR ymm9, ymm9, ymm12
+ BYTE $0xC4; BYTE $0xC1; BYTE $0x1D; BYTE $0x72; BYTE $0xF5; BYTE $0x07 // VPSLLD ymm12, ymm13, 7
+ BYTE $0xC4; BYTE $0xC1; BYTE $0x15; BYTE $0x72; BYTE $0xD5; BYTE $0x19 // VPSRLD ymm13, ymm13, 25
+ BYTE $0xC4; BYTE $0x41; BYTE $0x15; BYTE $0xEF; BYTE $0xEC // VPXOR ymm13, ymm13, ymm12
+ BYTE $0xC5; BYTE $0xFD; BYTE $0x70; BYTE $0xC9; BYTE $0x93 // VPSHUFD ymm1, ymm1, 147
+ BYTE $0xC5; BYTE $0xFD; BYTE $0x70; BYTE $0xED; BYTE $0x93 // VPSHUFD ymm5, ymm5, 147
+ BYTE $0xC4; BYTE $0x41; BYTE $0x7D; BYTE $0x70; BYTE $0xC9; BYTE $0x93 // VPSHUFD ymm9, ymm9, 147
+ BYTE $0xC4; BYTE $0x41; BYTE $0x7D; BYTE $0x70; BYTE $0xED; BYTE $0x93 // VPSHUFD ymm13, ymm13, 147
+ BYTE $0xC5; BYTE $0xFD; BYTE $0x70; BYTE $0xD2; BYTE $0x4E // VPSHUFD ymm2, ymm2, 78
+ BYTE $0xC5; BYTE $0xFD; BYTE $0x70; BYTE $0xF6; BYTE $0x4E // VPSHUFD ymm6, ymm6, 78
+ BYTE $0xC4; BYTE $0x41; BYTE $0x7D; BYTE $0x70; BYTE $0xD2; BYTE $0x4E // VPSHUFD ymm10, ymm10, 78
+ BYTE $0xC4; BYTE $0x41; BYTE $0x7D; BYTE $0x70; BYTE $0xF6; BYTE $0x4E // VPSHUFD ymm14, ymm14, 78
+ BYTE $0xC5; BYTE $0xFD; BYTE $0x70; BYTE $0xDB; BYTE $0x39 // VPSHUFD ymm3, ymm3, 57
+ BYTE $0xC5; BYTE $0xFD; BYTE $0x70; BYTE $0xFF; BYTE $0x39 // VPSHUFD ymm7, ymm7, 57
+ BYTE $0xC4; BYTE $0x41; BYTE $0x7D; BYTE $0x70; BYTE $0xDB; BYTE $0x39 // VPSHUFD ymm11, ymm11, 57
+ BYTE $0xC4; BYTE $0x41; BYTE $0x7D; BYTE $0x70; BYTE $0xFF; BYTE $0x39 // VPSHUFD ymm15, ymm15, 57
+ BYTE $0xC5; BYTE $0x7D; BYTE $0x6F; BYTE $0x64; BYTE $0x24; BYTE $0x40 // VMOVDQA ymm12, [rsp + 64]
+ SUBQ $2, SI
+ JNE rounds_loop8_begin
+ BYTE $0xC4; BYTE $0x62; BYTE $0x7D; BYTE $0x5A; BYTE $0x20 // VBROADCASTI128 ymm12, [rax]
+ BYTE $0xC4; BYTE $0xC1; BYTE $0x7D; BYTE $0xFE; BYTE $0xC4 // VPADDD ymm0, ymm0, ymm12
+ BYTE $0xC4; BYTE $0xC1; BYTE $0x5D; BYTE $0xFE; BYTE $0xE4 // VPADDD ymm4, ymm4, ymm12
+ BYTE $0xC4; BYTE $0x41; BYTE $0x3D; BYTE $0xFE; BYTE $0xC4 // VPADDD ymm8, ymm8, ymm12
+ BYTE $0xC5; BYTE $0x1D; BYTE $0xFE; BYTE $0x64; BYTE $0x24; BYTE $0x40 // VPADDD ymm12, ymm12, [rsp + 64]
+ BYTE $0xC5; BYTE $0x7D; BYTE $0x7F; BYTE $0x64; BYTE $0x24; BYTE $0x40 // VMOVDQA [rsp + 64], ymm12
+ BYTE $0xC4; BYTE $0x62; BYTE $0x7D; BYTE $0x5A; BYTE $0x60; BYTE $0x10 // VBROADCASTI128 ymm12, [rax + 16]
+ BYTE $0xC4; BYTE $0xC1; BYTE $0x75; BYTE $0xFE; BYTE $0xCC // VPADDD ymm1, ymm1, ymm12
+ BYTE $0xC4; BYTE $0xC1; BYTE $0x55; BYTE $0xFE; BYTE $0xEC // VPADDD ymm5, ymm5, ymm12
+ BYTE $0xC4; BYTE $0x41; BYTE $0x35; BYTE $0xFE; BYTE $0xCC // VPADDD ymm9, ymm9, ymm12
+ BYTE $0xC4; BYTE $0x41; BYTE $0x15; BYTE $0xFE; BYTE $0xEC // VPADDD ymm13, ymm13, ymm12
+ BYTE $0xC4; BYTE $0x62; BYTE $0x7D; BYTE $0x5A; BYTE $0x60; BYTE $0x20 // VBROADCASTI128 ymm12, [rax + 32]
+ BYTE $0xC4; BYTE $0xC1; BYTE $0x6D; BYTE $0xFE; BYTE $0xD4 // VPADDD ymm2, ymm2, ymm12
+ BYTE $0xC4; BYTE $0xC1; BYTE $0x4D; BYTE $0xFE; BYTE $0xF4 // VPADDD ymm6, ymm6, ymm12
+ BYTE $0xC4; BYTE $0x41; BYTE $0x2D; BYTE $0xFE; BYTE $0xD4 // VPADDD ymm10, ymm10, ymm12
+ BYTE $0xC4; BYTE $0x41; BYTE $0x0D; BYTE $0xFE; BYTE $0xF4 // VPADDD ymm14, ymm14, ymm12
+ BYTE $0xC5; BYTE $0xE5; BYTE $0xFE; BYTE $0x5C; BYTE $0x24; BYTE $0x20 // VPADDD ymm3, ymm3, [rsp + 32]
+ BYTE $0xC4; BYTE $0x63; BYTE $0x7D; BYTE $0x46; BYTE $0xE1; BYTE $0x20 // VPERM2I128 ymm12, ymm0, ymm1, 32
+ BYTE $0xC5; BYTE $0x1D; BYTE $0xEF; BYTE $0x23 // VPXOR ymm12, ymm12, [rbx]
+ BYTE $0xC5; BYTE $0x7E; BYTE $0x7F; BYTE $0x21 // VMOVDQU [rcx], ymm12
+ BYTE $0xC4; BYTE $0x63; BYTE $0x6D; BYTE $0x46; BYTE $0xE3; BYTE $0x20 // VPERM2I128 ymm12, ymm2, ymm3, 32
+ BYTE $0xC5; BYTE $0x1D; BYTE $0xEF; BYTE $0x63; BYTE $0x20 // VPXOR ymm12, ymm12, [rbx + 32]
+ BYTE $0xC5; BYTE $0x7E; BYTE $0x7F; BYTE $0x61; BYTE $0x20 // VMOVDQU [rcx + 32], ymm12
+ BYTE $0xC4; BYTE $0x63; BYTE $0x7D; BYTE $0x46; BYTE $0xE1; BYTE $0x31 // VPERM2I128 ymm12, ymm0, ymm1, 49
+ BYTE $0xC5; BYTE $0x1D; BYTE $0xEF; BYTE $0x63; BYTE $0x40 // VPXOR ymm12, ymm12, [rbx + 64]
+ BYTE $0xC5; BYTE $0x7E; BYTE $0x7F; BYTE $0x61; BYTE $0x40 // VMOVDQU [rcx + 64], ymm12
+ BYTE $0xC4; BYTE $0x63; BYTE $0x6D; BYTE $0x46; BYTE $0xE3; BYTE $0x31 // VPERM2I128 ymm12, ymm2, ymm3, 49
+ BYTE $0xC5; BYTE $0x1D; BYTE $0xEF; BYTE $0x63; BYTE $0x60 // VPXOR ymm12, ymm12, [rbx + 96]
+ BYTE $0xC5; BYTE $0x7E; BYTE $0x7F; BYTE $0x61; BYTE $0x60 // VMOVDQU [rcx + 96], ymm12
+ BYTE $0xC5; BYTE $0xFD; BYTE $0x6F; BYTE $0x5C; BYTE $0x24; BYTE $0x20 // VMOVDQA ymm3, [rsp + 32]
+ BYTE $0xC5; BYTE $0xE5; BYTE $0xFE; BYTE $0x1C; BYTE $0x24 // VPADDD ymm3, ymm3, [rsp]
+ BYTE $0xC5; BYTE $0xC5; BYTE $0xFE; BYTE $0xFB // VPADDD ymm7, ymm7, ymm3
+ BYTE $0xC4; BYTE $0x63; BYTE $0x5D; BYTE $0x46; BYTE $0xE5; BYTE $0x20 // VPERM2I128 ymm12, ymm4, ymm5, 32
+ BYTE $0xC5; BYTE $0x1D; BYTE $0xEF; BYTE $0xA3; BYTE $0x80; BYTE $0x00; BYTE $0x00; BYTE $0x00 // VPXOR ymm12, ymm12, [rbx + 128]
+ BYTE $0xC5; BYTE $0x7E; BYTE $0x7F; BYTE $0xA1; BYTE $0x80; BYTE $0x00; BYTE $0x00; BYTE $0x00 // VMOVDQU [rcx + 128], ymm12
+ BYTE $0xC4; BYTE $0x63; BYTE $0x4D; BYTE $0x46; BYTE $0xE7; BYTE $0x20 // VPERM2I128 ymm12, ymm6, ymm7, 32
+ BYTE $0xC5; BYTE $0x1D; BYTE $0xEF; BYTE $0xA3; BYTE $0xA0; BYTE $0x00; BYTE $0x00; BYTE $0x00 // VPXOR ymm12, ymm12, [rbx + 160]
+ BYTE $0xC5; BYTE $0x7E; BYTE $0x7F; BYTE $0xA1; BYTE $0xA0; BYTE $0x00; BYTE $0x00; BYTE $0x00 // VMOVDQU [rcx + 160], ymm12
+ BYTE $0xC4; BYTE $0x63; BYTE $0x5D; BYTE $0x46; BYTE $0xE5; BYTE $0x31 // VPERM2I128 ymm12, ymm4, ymm5, 49
+ BYTE $0xC5; BYTE $0x1D; BYTE $0xEF; BYTE $0xA3; BYTE $0xC0; BYTE $0x00; BYTE $0x00; BYTE $0x00 // VPXOR ymm12, ymm12, [rbx + 192]
+ BYTE $0xC5; BYTE $0x7E; BYTE $0x7F; BYTE $0xA1; BYTE $0xC0; BYTE $0x00; BYTE $0x00; BYTE $0x00 // VMOVDQU [rcx + 192], ymm12
+ BYTE $0xC4; BYTE $0x63; BYTE $0x4D; BYTE $0x46; BYTE $0xE7; BYTE $0x31 // VPERM2I128 ymm12, ymm6, ymm7, 49
+ BYTE $0xC5; BYTE $0x1D; BYTE $0xEF; BYTE $0xA3; BYTE $0xE0; BYTE $0x00; BYTE $0x00; BYTE $0x00 // VPXOR ymm12, ymm12, [rbx + 224]
+ BYTE $0xC5; BYTE $0x7E; BYTE $0x7F; BYTE $0xA1; BYTE $0xE0; BYTE $0x00; BYTE $0x00; BYTE $0x00 // VMOVDQU [rcx + 224], ymm12
+ BYTE $0xC5; BYTE $0xE5; BYTE $0xFE; BYTE $0x1C; BYTE $0x24 // VPADDD ymm3, ymm3, [rsp]
+ BYTE $0xC5; BYTE $0x25; BYTE $0xFE; BYTE $0xDB // VPADDD ymm11, ymm11, ymm3
+ BYTE $0xC4; BYTE $0x43; BYTE $0x3D; BYTE $0x46; BYTE $0xE1; BYTE $0x20 // VPERM2I128 ymm12, ymm8, ymm9, 32
+ BYTE $0xC5; BYTE $0x1D; BYTE $0xEF; BYTE $0xA3; BYTE $0x00; BYTE $0x01; BYTE $0x00; BYTE $0x00 // VPXOR ymm12, ymm12, [rbx + 256]
+ BYTE $0xC5; BYTE $0x7E; BYTE $0x7F; BYTE $0xA1; BYTE $0x00; BYTE $0x01; BYTE $0x00; BYTE $0x00 // VMOVDQU [rcx + 256], ymm12
+ BYTE $0xC4; BYTE $0x43; BYTE $0x2D; BYTE $0x46; BYTE $0xE3; BYTE $0x20 // VPERM2I128 ymm12, ymm10, ymm11, 32
+ BYTE $0xC5; BYTE $0x1D; BYTE $0xEF; BYTE $0xA3; BYTE $0x20; BYTE $0x01; BYTE $0x00; BYTE $0x00 // VPXOR ymm12, ymm12, [rbx + 288]
+ BYTE $0xC5; BYTE $0x7E; BYTE $0x7F; BYTE $0xA1; BYTE $0x20; BYTE $0x01; BYTE $0x00; BYTE $0x00 // VMOVDQU [rcx + 288], ymm12
+ BYTE $0xC4; BYTE $0x43; BYTE $0x3D; BYTE $0x46; BYTE $0xE1; BYTE $0x31 // VPERM2I128 ymm12, ymm8, ymm9, 49
+ BYTE $0xC5; BYTE $0x1D; BYTE $0xEF; BYTE $0xA3; BYTE $0x40; BYTE $0x01; BYTE $0x00; BYTE $0x00 // VPXOR ymm12, ymm12, [rbx + 320]
+ BYTE $0xC5; BYTE $0x7E; BYTE $0x7F; BYTE $0xA1; BYTE $0x40; BYTE $0x01; BYTE $0x00; BYTE $0x00 // VMOVDQU [rcx + 320], ymm12
+ BYTE $0xC4; BYTE $0x43; BYTE $0x2D; BYTE $0x46; BYTE $0xE3; BYTE $0x31 // VPERM2I128 ymm12, ymm10, ymm11, 49
+ BYTE $0xC5; BYTE $0x1D; BYTE $0xEF; BYTE $0xA3; BYTE $0x60; BYTE $0x01; BYTE $0x00; BYTE $0x00 // VPXOR ymm12, ymm12, [rbx + 352]
+ BYTE $0xC5; BYTE $0x7E; BYTE $0x7F; BYTE $0xA1; BYTE $0x60; BYTE $0x01; BYTE $0x00; BYTE $0x00 // VMOVDQU [rcx + 352], ymm12
+ BYTE $0xC5; BYTE $0xE5; BYTE $0xFE; BYTE $0x1C; BYTE $0x24 // VPADDD ymm3, ymm3, [rsp]
+ BYTE $0xC5; BYTE $0x7D; BYTE $0x6F; BYTE $0x64; BYTE $0x24; BYTE $0x40 // VMOVDQA ymm12, [rsp + 64]
+ BYTE $0xC5; BYTE $0x05; BYTE $0xFE; BYTE $0xFB // VPADDD ymm15, ymm15, ymm3
+ BYTE $0xC4; BYTE $0xC3; BYTE $0x1D; BYTE $0x46; BYTE $0xC5; BYTE $0x20 // VPERM2I128 ymm0, ymm12, ymm13, 32
+ BYTE $0xC5; BYTE $0xFD; BYTE $0xEF; BYTE $0x83; BYTE $0x80; BYTE $0x01; BYTE $0x00; BYTE $0x00 // VPXOR ymm0, ymm0, [rbx + 384]
+ BYTE $0xC5; BYTE $0xFE; BYTE $0x7F; BYTE $0x81; BYTE $0x80; BYTE $0x01; BYTE $0x00; BYTE $0x00 // VMOVDQU [rcx + 384], ymm0
+ BYTE $0xC4; BYTE $0xC3; BYTE $0x0D; BYTE $0x46; BYTE $0xC7; BYTE $0x20 // VPERM2I128 ymm0, ymm14, ymm15, 32
+ BYTE $0xC5; BYTE $0xFD; BYTE $0xEF; BYTE $0x83; BYTE $0xA0; BYTE $0x01; BYTE $0x00; BYTE $0x00 // VPXOR ymm0, ymm0, [rbx + 416]
+ BYTE $0xC5; BYTE $0xFE; BYTE $0x7F; BYTE $0x81; BYTE $0xA0; BYTE $0x01; BYTE $0x00; BYTE $0x00 // VMOVDQU [rcx + 416], ymm0
+ BYTE $0xC4; BYTE $0xC3; BYTE $0x1D; BYTE $0x46; BYTE $0xC5; BYTE $0x31 // VPERM2I128 ymm0, ymm12, ymm13, 49
+ BYTE $0xC5; BYTE $0xFD; BYTE $0xEF; BYTE $0x83; BYTE $0xC0; BYTE $0x01; BYTE $0x00; BYTE $0x00 // VPXOR ymm0, ymm0, [rbx + 448]
+ BYTE $0xC5; BYTE $0xFE; BYTE $0x7F; BYTE $0x81; BYTE $0xC0; BYTE $0x01; BYTE $0x00; BYTE $0x00 // VMOVDQU [rcx + 448], ymm0
+ BYTE $0xC4; BYTE $0xC3; BYTE $0x0D; BYTE $0x46; BYTE $0xC7; BYTE $0x31 // VPERM2I128 ymm0, ymm14, ymm15, 49
+ BYTE $0xC5; BYTE $0xFD; BYTE $0xEF; BYTE $0x83; BYTE $0xE0; BYTE $0x01; BYTE $0x00; BYTE $0x00 // VPXOR ymm0, ymm0, [rbx + 480]
+ BYTE $0xC5; BYTE $0xFE; BYTE $0x7F; BYTE $0x81; BYTE $0xE0; BYTE $0x01; BYTE $0x00; BYTE $0x00 // VMOVDQU [rcx + 480], ymm0
+ BYTE $0xC5; BYTE $0xE5; BYTE $0xFE; BYTE $0x1C; BYTE $0x24 // VPADDD ymm3, ymm3, [rsp]
+ BYTE $0xC5; BYTE $0xFD; BYTE $0x7F; BYTE $0x5C; BYTE $0x24; BYTE $0x20 // VMOVDQA [rsp + 32], ymm3
+ ADDQ $512, BX
+ ADDQ $512, CX
+ SUBQ $8, DX
+ JCC vector_loop8_begin
+vector_loop8_end:
+ BYTE $0xC5; BYTE $0x7D; BYTE $0x6F; BYTE $0xDB // VMOVDQA ymm11, ymm3
+ ADDQ $8, DX
+ JEQ out_write_even
+ BYTE $0xC4; BYTE $0x62; BYTE $0x7D; BYTE $0x5A; BYTE $0x00 // VBROADCASTI128 ymm8, [rax]
+ BYTE $0xC4; BYTE $0x62; BYTE $0x7D; BYTE $0x5A; BYTE $0x48; BYTE $0x10 // VBROADCASTI128 ymm9, [rax + 16]
+ BYTE $0xC4; BYTE $0x62; BYTE $0x7D; BYTE $0x5A; BYTE $0x50; BYTE $0x20 // VBROADCASTI128 ymm10, [rax + 32]
+ BYTE $0xC5; BYTE $0x7D; BYTE $0x6F; BYTE $0x34; BYTE $0x24 // VMOVDQA ymm14, [rsp]
+ SUBQ $4, DX
+ JCS vector_loop4_end
+vector_loop4_begin:
+ BYTE $0xC5; BYTE $0x7D; BYTE $0x7F; BYTE $0xC0 // VMOVDQA ymm0, ymm8
+ BYTE $0xC5; BYTE $0x7D; BYTE $0x7F; BYTE $0xC9 // VMOVDQA ymm1, ymm9
+ BYTE $0xC5; BYTE $0x7D; BYTE $0x7F; BYTE $0xD2 // VMOVDQA ymm2, ymm10
+ BYTE $0xC5; BYTE $0x7D; BYTE $0x7F; BYTE $0xDB // VMOVDQA ymm3, ymm11
+ BYTE $0xC5; BYTE $0xFD; BYTE $0x6F; BYTE $0xE0 // VMOVDQA ymm4, ymm0
+ BYTE $0xC5; BYTE $0xFD; BYTE $0x6F; BYTE $0xE9 // VMOVDQA ymm5, ymm1
+ BYTE $0xC5; BYTE $0xFD; BYTE $0x6F; BYTE $0xF2 // VMOVDQA ymm6, ymm2
+ BYTE $0xC4; BYTE $0xC1; BYTE $0x65; BYTE $0xD4; BYTE $0xFE // VPADDQ ymm7, ymm3, ymm14
+ MOVQ $20, SI
+rounds_loop4_begin:
+ BYTE $0xC5; BYTE $0xFD; BYTE $0xFE; BYTE $0xC1 // VPADDD ymm0, ymm0, ymm1
+ BYTE $0xC5; BYTE $0xDD; BYTE $0xFE; BYTE $0xE5 // VPADDD ymm4, ymm4, ymm5
+ BYTE $0xC5; BYTE $0xE5; BYTE $0xEF; BYTE $0xD8 // VPXOR ymm3, ymm3, ymm0
+ BYTE $0xC5; BYTE $0xC5; BYTE $0xEF; BYTE $0xFC // VPXOR ymm7, ymm7, ymm4
+ BYTE $0xC5; BYTE $0x9D; BYTE $0x72; BYTE $0xF3; BYTE $0x10 // VPSLLD ymm12, ymm3, 16
+ BYTE $0xC5; BYTE $0xE5; BYTE $0x72; BYTE $0xD3; BYTE $0x10 // VPSRLD ymm3, ymm3, 16
+ BYTE $0xC4; BYTE $0xC1; BYTE $0x65; BYTE $0xEF; BYTE $0xDC // VPXOR ymm3, ymm3, ymm12
+ BYTE $0xC5; BYTE $0x9D; BYTE $0x72; BYTE $0xF7; BYTE $0x10 // VPSLLD ymm12, ymm7, 16
+ BYTE $0xC5; BYTE $0xC5; BYTE $0x72; BYTE $0xD7; BYTE $0x10 // VPSRLD ymm7, ymm7, 16
+ BYTE $0xC4; BYTE $0xC1; BYTE $0x45; BYTE $0xEF; BYTE $0xFC // VPXOR ymm7, ymm7, ymm12
+ BYTE $0xC5; BYTE $0xED; BYTE $0xFE; BYTE $0xD3 // VPADDD ymm2, ymm2, ymm3
+ BYTE $0xC5; BYTE $0xCD; BYTE $0xFE; BYTE $0xF7 // VPADDD ymm6, ymm6, ymm7
+ BYTE $0xC5; BYTE $0xF5; BYTE $0xEF; BYTE $0xCA // VPXOR ymm1, ymm1, ymm2
+ BYTE $0xC5; BYTE $0xD5; BYTE $0xEF; BYTE $0xEE // VPXOR ymm5, ymm5, ymm6
+ BYTE $0xC5; BYTE $0x9D; BYTE $0x72; BYTE $0xF1; BYTE $0x0C // VPSLLD ymm12, ymm1, 12
+ BYTE $0xC5; BYTE $0xF5; BYTE $0x72; BYTE $0xD1; BYTE $0x14 // VPSRLD ymm1, ymm1, 20
+ BYTE $0xC4; BYTE $0xC1; BYTE $0x75; BYTE $0xEF; BYTE $0xCC // VPXOR ymm1, ymm1, ymm12
+ BYTE $0xC5; BYTE $0x9D; BYTE $0x72; BYTE $0xF5; BYTE $0x0C // VPSLLD ymm12, ymm5, 12
+ BYTE $0xC5; BYTE $0xD5; BYTE $0x72; BYTE $0xD5; BYTE $0x14 // VPSRLD ymm5, ymm5, 20
+ BYTE $0xC4; BYTE $0xC1; BYTE $0x55; BYTE $0xEF; BYTE $0xEC // VPXOR ymm5, ymm5, ymm12
+ BYTE $0xC5; BYTE $0xFD; BYTE $0xFE; BYTE $0xC1 // VPADDD ymm0, ymm0, ymm1
+ BYTE $0xC5; BYTE $0xDD; BYTE $0xFE; BYTE $0xE5 // VPADDD ymm4, ymm4, ymm5
+ BYTE $0xC5; BYTE $0xE5; BYTE $0xEF; BYTE $0xD8 // VPXOR ymm3, ymm3, ymm0
+ BYTE $0xC5; BYTE $0xC5; BYTE $0xEF; BYTE $0xFC // VPXOR ymm7, ymm7, ymm4
+ BYTE $0xC5; BYTE $0x9D; BYTE $0x72; BYTE $0xF3; BYTE $0x08 // VPSLLD ymm12, ymm3, 8
+ BYTE $0xC5; BYTE $0xE5; BYTE $0x72; BYTE $0xD3; BYTE $0x18 // VPSRLD ymm3, ymm3, 24
+ BYTE $0xC4; BYTE $0xC1; BYTE $0x65; BYTE $0xEF; BYTE $0xDC // VPXOR ymm3, ymm3, ymm12
+ BYTE $0xC5; BYTE $0x9D; BYTE $0x72; BYTE $0xF7; BYTE $0x08 // VPSLLD ymm12, ymm7, 8
+ BYTE $0xC5; BYTE $0xC5; BYTE $0x72; BYTE $0xD7; BYTE $0x18 // VPSRLD ymm7, ymm7, 24
+ BYTE $0xC4; BYTE $0xC1; BYTE $0x45; BYTE $0xEF; BYTE $0xFC // VPXOR ymm7, ymm7, ymm12
+ BYTE $0xC5; BYTE $0xED; BYTE $0xFE; BYTE $0xD3 // VPADDD ymm2, ymm2, ymm3
+ BYTE $0xC5; BYTE $0xCD; BYTE $0xFE; BYTE $0xF7 // VPADDD ymm6, ymm6, ymm7
+ BYTE $0xC5; BYTE $0xF5; BYTE $0xEF; BYTE $0xCA // VPXOR ymm1, ymm1, ymm2
+ BYTE $0xC5; BYTE $0xD5; BYTE $0xEF; BYTE $0xEE // VPXOR ymm5, ymm5, ymm6
+ BYTE $0xC5; BYTE $0x9D; BYTE $0x72; BYTE $0xF1; BYTE $0x07 // VPSLLD ymm12, ymm1, 7
+ BYTE $0xC5; BYTE $0xF5; BYTE $0x72; BYTE $0xD1; BYTE $0x19 // VPSRLD ymm1, ymm1, 25
+ BYTE $0xC4; BYTE $0xC1; BYTE $0x75; BYTE $0xEF; BYTE $0xCC // VPXOR ymm1, ymm1, ymm12
+ BYTE $0xC5; BYTE $0x9D; BYTE $0x72; BYTE $0xF5; BYTE $0x07 // VPSLLD ymm12, ymm5, 7
+ BYTE $0xC5; BYTE $0xD5; BYTE $0x72; BYTE $0xD5; BYTE $0x19 // VPSRLD ymm5, ymm5, 25
+ BYTE $0xC4; BYTE $0xC1; BYTE $0x55; BYTE $0xEF; BYTE $0xEC // VPXOR ymm5, ymm5, ymm12
+ BYTE $0xC5; BYTE $0xFD; BYTE $0x70; BYTE $0xC9; BYTE $0x39 // VPSHUFD ymm1, ymm1, 57
+ BYTE $0xC5; BYTE $0xFD; BYTE $0x70; BYTE $0xED; BYTE $0x39 // VPSHUFD ymm5, ymm5, 57
+ BYTE $0xC5; BYTE $0xFD; BYTE $0x70; BYTE $0xD2; BYTE $0x4E // VPSHUFD ymm2, ymm2, 78
+ BYTE $0xC5; BYTE $0xFD; BYTE $0x70; BYTE $0xF6; BYTE $0x4E // VPSHUFD ymm6, ymm6, 78
+ BYTE $0xC5; BYTE $0xFD; BYTE $0x70; BYTE $0xDB; BYTE $0x93 // VPSHUFD ymm3, ymm3, 147
+ BYTE $0xC5; BYTE $0xFD; BYTE $0x70; BYTE $0xFF; BYTE $0x93 // VPSHUFD ymm7, ymm7, 147
+ BYTE $0xC5; BYTE $0xFD; BYTE $0xFE; BYTE $0xC1 // VPADDD ymm0, ymm0, ymm1
+ BYTE $0xC5; BYTE $0xDD; BYTE $0xFE; BYTE $0xE5 // VPADDD ymm4, ymm4, ymm5
+ BYTE $0xC5; BYTE $0xE5; BYTE $0xEF; BYTE $0xD8 // VPXOR ymm3, ymm3, ymm0
+ BYTE $0xC5; BYTE $0xC5; BYTE $0xEF; BYTE $0xFC // VPXOR ymm7, ymm7, ymm4
+ BYTE $0xC5; BYTE $0x9D; BYTE $0x72; BYTE $0xF3; BYTE $0x10 // VPSLLD ymm12, ymm3, 16
+ BYTE $0xC5; BYTE $0xE5; BYTE $0x72; BYTE $0xD3; BYTE $0x10 // VPSRLD ymm3, ymm3, 16
+ BYTE $0xC4; BYTE $0xC1; BYTE $0x65; BYTE $0xEF; BYTE $0xDC // VPXOR ymm3, ymm3, ymm12
+ BYTE $0xC5; BYTE $0x9D; BYTE $0x72; BYTE $0xF7; BYTE $0x10 // VPSLLD ymm12, ymm7, 16
+ BYTE $0xC5; BYTE $0xC5; BYTE $0x72; BYTE $0xD7; BYTE $0x10 // VPSRLD ymm7, ymm7, 16
+ BYTE $0xC4; BYTE $0xC1; BYTE $0x45; BYTE $0xEF; BYTE $0xFC // VPXOR ymm7, ymm7, ymm12
+ BYTE $0xC5; BYTE $0xED; BYTE $0xFE; BYTE $0xD3 // VPADDD ymm2, ymm2, ymm3
+ BYTE $0xC5; BYTE $0xCD; BYTE $0xFE; BYTE $0xF7 // VPADDD ymm6, ymm6, ymm7
+ BYTE $0xC5; BYTE $0xF5; BYTE $0xEF; BYTE $0xCA // VPXOR ymm1, ymm1, ymm2
+ BYTE $0xC5; BYTE $0xD5; BYTE $0xEF; BYTE $0xEE // VPXOR ymm5, ymm5, ymm6
+ BYTE $0xC5; BYTE $0x9D; BYTE $0x72; BYTE $0xF1; BYTE $0x0C // VPSLLD ymm12, ymm1, 12
+ BYTE $0xC5; BYTE $0xF5; BYTE $0x72; BYTE $0xD1; BYTE $0x14 // VPSRLD ymm1, ymm1, 20
+ BYTE $0xC4; BYTE $0xC1; BYTE $0x75; BYTE $0xEF; BYTE $0xCC // VPXOR ymm1, ymm1, ymm12
+ BYTE $0xC5; BYTE $0x9D; BYTE $0x72; BYTE $0xF5; BYTE $0x0C // VPSLLD ymm12, ymm5, 12
+ BYTE $0xC5; BYTE $0xD5; BYTE $0x72; BYTE $0xD5; BYTE $0x14 // VPSRLD ymm5, ymm5, 20
+ BYTE $0xC4; BYTE $0xC1; BYTE $0x55; BYTE $0xEF; BYTE $0xEC // VPXOR ymm5, ymm5, ymm12
+ BYTE $0xC5; BYTE $0xFD; BYTE $0xFE; BYTE $0xC1 // VPADDD ymm0, ymm0, ymm1
+ BYTE $0xC5; BYTE $0xDD; BYTE $0xFE; BYTE $0xE5 // VPADDD ymm4, ymm4, ymm5
+ BYTE $0xC5; BYTE $0xE5; BYTE $0xEF; BYTE $0xD8 // VPXOR ymm3, ymm3, ymm0
+ BYTE $0xC5; BYTE $0xC5; BYTE $0xEF; BYTE $0xFC // VPXOR ymm7, ymm7, ymm4
+ BYTE $0xC5; BYTE $0x9D; BYTE $0x72; BYTE $0xF3; BYTE $0x08 // VPSLLD ymm12, ymm3, 8
+ BYTE $0xC5; BYTE $0xE5; BYTE $0x72; BYTE $0xD3; BYTE $0x18 // VPSRLD ymm3, ymm3, 24
+ BYTE $0xC4; BYTE $0xC1; BYTE $0x65; BYTE $0xEF; BYTE $0xDC // VPXOR ymm3, ymm3, ymm12
+ BYTE $0xC5; BYTE $0x9D; BYTE $0x72; BYTE $0xF7; BYTE $0x08 // VPSLLD ymm12, ymm7, 8
+ BYTE $0xC5; BYTE $0xC5; BYTE $0x72; BYTE $0xD7; BYTE $0x18 // VPSRLD ymm7, ymm7, 24
+ BYTE $0xC4; BYTE $0xC1; BYTE $0x45; BYTE $0xEF; BYTE $0xFC // VPXOR ymm7, ymm7, ymm12
+ BYTE $0xC5; BYTE $0xED; BYTE $0xFE; BYTE $0xD3 // VPADDD ymm2, ymm2, ymm3
+ BYTE $0xC5; BYTE $0xCD; BYTE $0xFE; BYTE $0xF7 // VPADDD ymm6, ymm6, ymm7
+ BYTE $0xC5; BYTE $0xF5; BYTE $0xEF; BYTE $0xCA // VPXOR ymm1, ymm1, ymm2
+ BYTE $0xC5; BYTE $0xD5; BYTE $0xEF; BYTE $0xEE // VPXOR ymm5, ymm5, ymm6
+ BYTE $0xC5; BYTE $0x9D; BYTE $0x72; BYTE $0xF1; BYTE $0x07 // VPSLLD ymm12, ymm1, 7
+ BYTE $0xC5; BYTE $0xF5; BYTE $0x72; BYTE $0xD1; BYTE $0x19 // VPSRLD ymm1, ymm1, 25
+ BYTE $0xC4; BYTE $0xC1; BYTE $0x75; BYTE $0xEF; BYTE $0xCC // VPXOR ymm1, ymm1, ymm12
+ BYTE $0xC5; BYTE $0x9D; BYTE $0x72; BYTE $0xF5; BYTE $0x07 // VPSLLD ymm12, ymm5, 7
+ BYTE $0xC5; BYTE $0xD5; BYTE $0x72; BYTE $0xD5; BYTE $0x19 // VPSRLD ymm5, ymm5, 25
+ BYTE $0xC4; BYTE $0xC1; BYTE $0x55; BYTE $0xEF; BYTE $0xEC // VPXOR ymm5, ymm5, ymm12
+ BYTE $0xC5; BYTE $0xFD; BYTE $0x70; BYTE $0xC9; BYTE $0x93 // VPSHUFD ymm1, ymm1, 147
+ BYTE $0xC5; BYTE $0xFD; BYTE $0x70; BYTE $0xED; BYTE $0x93 // VPSHUFD ymm5, ymm5, 147
+ BYTE $0xC5; BYTE $0xFD; BYTE $0x70; BYTE $0xD2; BYTE $0x4E // VPSHUFD ymm2, ymm2, 78
+ BYTE $0xC5; BYTE $0xFD; BYTE $0x70; BYTE $0xF6; BYTE $0x4E // VPSHUFD ymm6, ymm6, 78
+ BYTE $0xC5; BYTE $0xFD; BYTE $0x70; BYTE $0xDB; BYTE $0x39 // VPSHUFD ymm3, ymm3, 57
+ BYTE $0xC5; BYTE $0xFD; BYTE $0x70; BYTE $0xFF; BYTE $0x39 // VPSHUFD ymm7, ymm7, 57
+ SUBQ $2, SI
+ JNE rounds_loop4_begin
+ BYTE $0xC4; BYTE $0xC1; BYTE $0x7D; BYTE $0xFE; BYTE $0xC0 // VPADDD ymm0, ymm0, ymm8
+ BYTE $0xC4; BYTE $0xC1; BYTE $0x75; BYTE $0xFE; BYTE $0xC9 // VPADDD ymm1, ymm1, ymm9
+ BYTE $0xC4; BYTE $0xC1; BYTE $0x6D; BYTE $0xFE; BYTE $0xD2 // VPADDD ymm2, ymm2, ymm10
+ BYTE $0xC4; BYTE $0xC1; BYTE $0x65; BYTE $0xFE; BYTE $0xDB // VPADDD ymm3, ymm3, ymm11
+ BYTE $0xC4; BYTE $0x63; BYTE $0x7D; BYTE $0x46; BYTE $0xE1; BYTE $0x20 // VPERM2I128 ymm12, ymm0, ymm1, 32
+ BYTE $0xC5; BYTE $0x1D; BYTE $0xEF; BYTE $0x23 // VPXOR ymm12, ymm12, [rbx]
+ BYTE $0xC5; BYTE $0x7E; BYTE $0x7F; BYTE $0x21 // VMOVDQU [rcx], ymm12
+ BYTE $0xC4; BYTE $0x63; BYTE $0x6D; BYTE $0x46; BYTE $0xE3; BYTE $0x20 // VPERM2I128 ymm12, ymm2, ymm3, 32
+ BYTE $0xC5; BYTE $0x1D; BYTE $0xEF; BYTE $0x63; BYTE $0x20 // VPXOR ymm12, ymm12, [rbx + 32]
+ BYTE $0xC5; BYTE $0x7E; BYTE $0x7F; BYTE $0x61; BYTE $0x20 // VMOVDQU [rcx + 32], ymm12
+ BYTE $0xC4; BYTE $0x63; BYTE $0x7D; BYTE $0x46; BYTE $0xE1; BYTE $0x31 // VPERM2I128 ymm12, ymm0, ymm1, 49
+ BYTE $0xC5; BYTE $0x1D; BYTE $0xEF; BYTE $0x63; BYTE $0x40 // VPXOR ymm12, ymm12, [rbx + 64]
+ BYTE $0xC5; BYTE $0x7E; BYTE $0x7F; BYTE $0x61; BYTE $0x40 // VMOVDQU [rcx + 64], ymm12
+ BYTE $0xC4; BYTE $0x63; BYTE $0x6D; BYTE $0x46; BYTE $0xE3; BYTE $0x31 // VPERM2I128 ymm12, ymm2, ymm3, 49
+ BYTE $0xC5; BYTE $0x1D; BYTE $0xEF; BYTE $0x63; BYTE $0x60 // VPXOR ymm12, ymm12, [rbx + 96]
+ BYTE $0xC5; BYTE $0x7E; BYTE $0x7F; BYTE $0x61; BYTE $0x60 // VMOVDQU [rcx + 96], ymm12
+ BYTE $0xC4; BYTE $0x41; BYTE $0x25; BYTE $0xFE; BYTE $0xDE // VPADDD ymm11, ymm11, ymm14
+ BYTE $0xC4; BYTE $0xC1; BYTE $0x5D; BYTE $0xFE; BYTE $0xE0 // VPADDD ymm4, ymm4, ymm8
+ BYTE $0xC4; BYTE $0xC1; BYTE $0x55; BYTE $0xFE; BYTE $0xE9 // VPADDD ymm5, ymm5, ymm9
+ BYTE $0xC4; BYTE $0xC1; BYTE $0x4D; BYTE $0xFE; BYTE $0xF2 // VPADDD ymm6, ymm6, ymm10
+ BYTE $0xC4; BYTE $0xC1; BYTE $0x45; BYTE $0xFE; BYTE $0xFB // VPADDD ymm7, ymm7, ymm11
+ BYTE $0xC4; BYTE $0x63; BYTE $0x5D; BYTE $0x46; BYTE $0xE5; BYTE $0x20 // VPERM2I128 ymm12, ymm4, ymm5, 32
+ BYTE $0xC5; BYTE $0x1D; BYTE $0xEF; BYTE $0xA3; BYTE $0x80; BYTE $0x00; BYTE $0x00; BYTE $0x00 // VPXOR ymm12, ymm12, [rbx + 128]
+ BYTE $0xC5; BYTE $0x7E; BYTE $0x7F; BYTE $0xA1; BYTE $0x80; BYTE $0x00; BYTE $0x00; BYTE $0x00 // VMOVDQU [rcx + 128], ymm12
+ BYTE $0xC4; BYTE $0x63; BYTE $0x4D; BYTE $0x46; BYTE $0xE7; BYTE $0x20 // VPERM2I128 ymm12, ymm6, ymm7, 32
+ BYTE $0xC5; BYTE $0x1D; BYTE $0xEF; BYTE $0xA3; BYTE $0xA0; BYTE $0x00; BYTE $0x00; BYTE $0x00 // VPXOR ymm12, ymm12, [rbx + 160]
+ BYTE $0xC5; BYTE $0x7E; BYTE $0x7F; BYTE $0xA1; BYTE $0xA0; BYTE $0x00; BYTE $0x00; BYTE $0x00 // VMOVDQU [rcx + 160], ymm12
+ BYTE $0xC4; BYTE $0x63; BYTE $0x5D; BYTE $0x46; BYTE $0xE5; BYTE $0x31 // VPERM2I128 ymm12, ymm4, ymm5, 49
+ BYTE $0xC5; BYTE $0x1D; BYTE $0xEF; BYTE $0xA3; BYTE $0xC0; BYTE $0x00; BYTE $0x00; BYTE $0x00 // VPXOR ymm12, ymm12, [rbx + 192]
+ BYTE $0xC5; BYTE $0x7E; BYTE $0x7F; BYTE $0xA1; BYTE $0xC0; BYTE $0x00; BYTE $0x00; BYTE $0x00 // VMOVDQU [rcx + 192], ymm12
+ BYTE $0xC4; BYTE $0x63; BYTE $0x4D; BYTE $0x46; BYTE $0xE7; BYTE $0x31 // VPERM2I128 ymm12, ymm6, ymm7, 49
+ BYTE $0xC5; BYTE $0x1D; BYTE $0xEF; BYTE $0xA3; BYTE $0xE0; BYTE $0x00; BYTE $0x00; BYTE $0x00 // VPXOR ymm12, ymm12, [rbx + 224]
+ BYTE $0xC5; BYTE $0x7E; BYTE $0x7F; BYTE $0xA1; BYTE $0xE0; BYTE $0x00; BYTE $0x00; BYTE $0x00 // VMOVDQU [rcx + 224], ymm12
+ BYTE $0xC4; BYTE $0x41; BYTE $0x25; BYTE $0xFE; BYTE $0xDE // VPADDD ymm11, ymm11, ymm14
+ ADDQ $256, BX
+ ADDQ $256, CX
+ SUBQ $4, DX
+ JCC vector_loop4_begin
+vector_loop4_end:
+ ADDQ $4, DX
+ JEQ out_write_even
+vector_loop2_begin:
+ BYTE $0xC5; BYTE $0x7D; BYTE $0x7F; BYTE $0xC0 // VMOVDQA ymm0, ymm8
+ BYTE $0xC5; BYTE $0x7D; BYTE $0x7F; BYTE $0xC9 // VMOVDQA ymm1, ymm9
+ BYTE $0xC5; BYTE $0x7D; BYTE $0x7F; BYTE $0xD2 // VMOVDQA ymm2, ymm10
+ BYTE $0xC5; BYTE $0x7D; BYTE $0x7F; BYTE $0xDB // VMOVDQA ymm3, ymm11
+ MOVQ $20, SI
+rounds_loop2_begin:
+ BYTE $0xC5; BYTE $0xFD; BYTE $0xFE; BYTE $0xC1 // VPADDD ymm0, ymm0, ymm1
+ BYTE $0xC5; BYTE $0xE5; BYTE $0xEF; BYTE $0xD8 // VPXOR ymm3, ymm3, ymm0
+ BYTE $0xC5; BYTE $0x9D; BYTE $0x72; BYTE $0xF3; BYTE $0x10 // VPSLLD ymm12, ymm3, 16
+ BYTE $0xC5; BYTE $0xE5; BYTE $0x72; BYTE $0xD3; BYTE $0x10 // VPSRLD ymm3, ymm3, 16
+ BYTE $0xC4; BYTE $0xC1; BYTE $0x65; BYTE $0xEF; BYTE $0xDC // VPXOR ymm3, ymm3, ymm12
+ BYTE $0xC5; BYTE $0xED; BYTE $0xFE; BYTE $0xD3 // VPADDD ymm2, ymm2, ymm3
+ BYTE $0xC5; BYTE $0xF5; BYTE $0xEF; BYTE $0xCA // VPXOR ymm1, ymm1, ymm2
+ BYTE $0xC5; BYTE $0x9D; BYTE $0x72; BYTE $0xF1; BYTE $0x0C // VPSLLD ymm12, ymm1, 12
+ BYTE $0xC5; BYTE $0xF5; BYTE $0x72; BYTE $0xD1; BYTE $0x14 // VPSRLD ymm1, ymm1, 20
+ BYTE $0xC4; BYTE $0xC1; BYTE $0x75; BYTE $0xEF; BYTE $0xCC // VPXOR ymm1, ymm1, ymm12
+ BYTE $0xC5; BYTE $0xFD; BYTE $0xFE; BYTE $0xC1 // VPADDD ymm0, ymm0, ymm1
+ BYTE $0xC5; BYTE $0xE5; BYTE $0xEF; BYTE $0xD8 // VPXOR ymm3, ymm3, ymm0
+ BYTE $0xC5; BYTE $0x9D; BYTE $0x72; BYTE $0xF3; BYTE $0x08 // VPSLLD ymm12, ymm3, 8
+ BYTE $0xC5; BYTE $0xE5; BYTE $0x72; BYTE $0xD3; BYTE $0x18 // VPSRLD ymm3, ymm3, 24
+ BYTE $0xC4; BYTE $0xC1; BYTE $0x65; BYTE $0xEF; BYTE $0xDC // VPXOR ymm3, ymm3, ymm12
+ BYTE $0xC5; BYTE $0xED; BYTE $0xFE; BYTE $0xD3 // VPADDD ymm2, ymm2, ymm3
+ BYTE $0xC5; BYTE $0xF5; BYTE $0xEF; BYTE $0xCA // VPXOR ymm1, ymm1, ymm2
+ BYTE $0xC5; BYTE $0x9D; BYTE $0x72; BYTE $0xF1; BYTE $0x07 // VPSLLD ymm12, ymm1, 7
+ BYTE $0xC5; BYTE $0xF5; BYTE $0x72; BYTE $0xD1; BYTE $0x19 // VPSRLD ymm1, ymm1, 25
+ BYTE $0xC4; BYTE $0xC1; BYTE $0x75; BYTE $0xEF; BYTE $0xCC // VPXOR ymm1, ymm1, ymm12
+ BYTE $0xC5; BYTE $0xFD; BYTE $0x70; BYTE $0xC9; BYTE $0x39 // VPSHUFD ymm1, ymm1, 57
+ BYTE $0xC5; BYTE $0xFD; BYTE $0x70; BYTE $0xD2; BYTE $0x4E // VPSHUFD ymm2, ymm2, 78
+ BYTE $0xC5; BYTE $0xFD; BYTE $0x70; BYTE $0xDB; BYTE $0x93 // VPSHUFD ymm3, ymm3, 147
+ BYTE $0xC5; BYTE $0xFD; BYTE $0xFE; BYTE $0xC1 // VPADDD ymm0, ymm0, ymm1
+ BYTE $0xC5; BYTE $0xE5; BYTE $0xEF; BYTE $0xD8 // VPXOR ymm3, ymm3, ymm0
+ BYTE $0xC5; BYTE $0x9D; BYTE $0x72; BYTE $0xF3; BYTE $0x10 // VPSLLD ymm12, ymm3, 16
+ BYTE $0xC5; BYTE $0xE5; BYTE $0x72; BYTE $0xD3; BYTE $0x10 // VPSRLD ymm3, ymm3, 16
+ BYTE $0xC4; BYTE $0xC1; BYTE $0x65; BYTE $0xEF; BYTE $0xDC // VPXOR ymm3, ymm3, ymm12
+ BYTE $0xC5; BYTE $0xED; BYTE $0xFE; BYTE $0xD3 // VPADDD ymm2, ymm2, ymm3
+ BYTE $0xC5; BYTE $0xF5; BYTE $0xEF; BYTE $0xCA // VPXOR ymm1, ymm1, ymm2
+ BYTE $0xC5; BYTE $0x9D; BYTE $0x72; BYTE $0xF1; BYTE $0x0C // VPSLLD ymm12, ymm1, 12
+ BYTE $0xC5; BYTE $0xF5; BYTE $0x72; BYTE $0xD1; BYTE $0x14 // VPSRLD ymm1, ymm1, 20
+ BYTE $0xC4; BYTE $0xC1; BYTE $0x75; BYTE $0xEF; BYTE $0xCC // VPXOR ymm1, ymm1, ymm12
+ BYTE $0xC5; BYTE $0xFD; BYTE $0xFE; BYTE $0xC1 // VPADDD ymm0, ymm0, ymm1
+ BYTE $0xC5; BYTE $0xE5; BYTE $0xEF; BYTE $0xD8 // VPXOR ymm3, ymm3, ymm0
+ BYTE $0xC5; BYTE $0x9D; BYTE $0x72; BYTE $0xF3; BYTE $0x08 // VPSLLD ymm12, ymm3, 8
+ BYTE $0xC5; BYTE $0xE5; BYTE $0x72; BYTE $0xD3; BYTE $0x18 // VPSRLD ymm3, ymm3, 24
+ BYTE $0xC4; BYTE $0xC1; BYTE $0x65; BYTE $0xEF; BYTE $0xDC // VPXOR ymm3, ymm3, ymm12
+ BYTE $0xC5; BYTE $0xED; BYTE $0xFE; BYTE $0xD3 // VPADDD ymm2, ymm2, ymm3
+ BYTE $0xC5; BYTE $0xF5; BYTE $0xEF; BYTE $0xCA // VPXOR ymm1, ymm1, ymm2
+ BYTE $0xC5; BYTE $0x9D; BYTE $0x72; BYTE $0xF1; BYTE $0x07 // VPSLLD ymm12, ymm1, 7
+ BYTE $0xC5; BYTE $0xF5; BYTE $0x72; BYTE $0xD1; BYTE $0x19 // VPSRLD ymm1, ymm1, 25
+ BYTE $0xC4; BYTE $0xC1; BYTE $0x75; BYTE $0xEF; BYTE $0xCC // VPXOR ymm1, ymm1, ymm12
+ BYTE $0xC5; BYTE $0xFD; BYTE $0x70; BYTE $0xC9; BYTE $0x93 // VPSHUFD ymm1, ymm1, 147
+ BYTE $0xC5; BYTE $0xFD; BYTE $0x70; BYTE $0xD2; BYTE $0x4E // VPSHUFD ymm2, ymm2, 78
+ BYTE $0xC5; BYTE $0xFD; BYTE $0x70; BYTE $0xDB; BYTE $0x39 // VPSHUFD ymm3, ymm3, 57
+ SUBQ $2, SI
+ JNE rounds_loop2_begin
+ BYTE $0xC4; BYTE $0xC1; BYTE $0x7D; BYTE $0xFE; BYTE $0xC0 // VPADDD ymm0, ymm0, ymm8
+ BYTE $0xC4; BYTE $0xC1; BYTE $0x75; BYTE $0xFE; BYTE $0xC9 // VPADDD ymm1, ymm1, ymm9
+ BYTE $0xC4; BYTE $0xC1; BYTE $0x6D; BYTE $0xFE; BYTE $0xD2 // VPADDD ymm2, ymm2, ymm10
+ BYTE $0xC4; BYTE $0xC1; BYTE $0x65; BYTE $0xFE; BYTE $0xDB // VPADDD ymm3, ymm3, ymm11
+ BYTE $0xC4; BYTE $0x63; BYTE $0x7D; BYTE $0x46; BYTE $0xE1; BYTE $0x20 // VPERM2I128 ymm12, ymm0, ymm1, 32
+ BYTE $0xC5; BYTE $0x1D; BYTE $0xEF; BYTE $0x23 // VPXOR ymm12, ymm12, [rbx]
+ BYTE $0xC5; BYTE $0x7E; BYTE $0x7F; BYTE $0x21 // VMOVDQU [rcx], ymm12
+ BYTE $0xC4; BYTE $0x63; BYTE $0x6D; BYTE $0x46; BYTE $0xE3; BYTE $0x20 // VPERM2I128 ymm12, ymm2, ymm3, 32
+ BYTE $0xC5; BYTE $0x1D; BYTE $0xEF; BYTE $0x63; BYTE $0x20 // VPXOR ymm12, ymm12, [rbx + 32]
+ BYTE $0xC5; BYTE $0x7E; BYTE $0x7F; BYTE $0x61; BYTE $0x20 // VMOVDQU [rcx + 32], ymm12
+ SUBQ $1, DX
+ JEQ out_write_odd
+ BYTE $0xC4; BYTE $0x41; BYTE $0x25; BYTE $0xFE; BYTE $0xDE // VPADDD ymm11, ymm11, ymm14
+ BYTE $0xC4; BYTE $0x63; BYTE $0x7D; BYTE $0x46; BYTE $0xE1; BYTE $0x31 // VPERM2I128 ymm12, ymm0, ymm1, 49
+ BYTE $0xC5; BYTE $0x1D; BYTE $0xEF; BYTE $0x63; BYTE $0x40 // VPXOR ymm12, ymm12, [rbx + 64]
+ BYTE $0xC5; BYTE $0x7E; BYTE $0x7F; BYTE $0x61; BYTE $0x40 // VMOVDQU [rcx + 64], ymm12
+ BYTE $0xC4; BYTE $0x63; BYTE $0x6D; BYTE $0x46; BYTE $0xE3; BYTE $0x31 // VPERM2I128 ymm12, ymm2, ymm3, 49
+ BYTE $0xC5; BYTE $0x1D; BYTE $0xEF; BYTE $0x63; BYTE $0x60 // VPXOR ymm12, ymm12, [rbx + 96]
+ BYTE $0xC5; BYTE $0x7E; BYTE $0x7F; BYTE $0x61; BYTE $0x60 // VMOVDQU [rcx + 96], ymm12
+ SUBQ $1, DX
+ JEQ out_write_even
+ ADDQ $128, BX
+ ADDQ $128, CX
+ JMP vector_loop2_begin
+out_write_odd:
+ BYTE $0xC4; BYTE $0x43; BYTE $0x25; BYTE $0x46; BYTE $0xDB; BYTE $0x01 // VPERM2I128 ymm11, ymm11, ymm11, 1
+out_write_even:
+ BYTE $0xC5; BYTE $0x79; BYTE $0x7F; BYTE $0x58; BYTE $0x30 // VMOVDQA [rax + 48], xmm11
+ BYTE $0xC5; BYTE $0xFD; BYTE $0xEF; BYTE $0xC0 // VPXOR ymm0, ymm0, ymm0
+ BYTE $0xC5; BYTE $0xFD; BYTE $0x7F; BYTE $0x44; BYTE $0x24; BYTE $0x40 // VMOVDQA [rsp + 64], ymm0
+ BYTE $0xC5; BYTE $0xFD; BYTE $0x7F; BYTE $0x44; BYTE $0x24; BYTE $0x20 // VMOVDQA [rsp + 32], ymm0
+ MOVQ DI, SP
+ BYTE $0xC5; BYTE $0xF8; BYTE $0x77 // VZEROUPPER
+ RET
+
+// func cpuidAmd64(cpuidParams *uint32)
+TEXT ·cpuidAmd64(SB),4,$0-8
+ MOVQ cpuidParams+0(FP), R15
+ MOVL 0(R15), AX
+ MOVL 4(R15), CX
+ CPUID
+ MOVL AX, 0(R15)
+ MOVL BX, 4(R15)
+ MOVL CX, 8(R15)
+ MOVL DX, 12(R15)
+ RET
+
+// func xgetbv0Amd64(xcrVec *uint32)
+TEXT ·xgetbv0Amd64(SB),4,$0-8
+ MOVQ xcrVec+0(FP), BX
+ XORL CX, CX
+ BYTE $0x0F; BYTE $0x01; BYTE $0xD0 // XGETBV
+ MOVL AX, 0(BX)
+ MOVL DX, 4(BX)
+ RET
diff --git a/cmd/gost/vendor/github.com/Yawning/chacha20/chacha20_ref.go b/cmd/gost/vendor/github.com/Yawning/chacha20/chacha20_ref.go
new file mode 100644
index 0000000..694c937
--- /dev/null
+++ b/cmd/gost/vendor/github.com/Yawning/chacha20/chacha20_ref.go
@@ -0,0 +1,392 @@
+// chacha20_ref.go - Reference ChaCha20.
+//
+// To the extent possible under law, Yawning Angel has waived all copyright
+// and related or neighboring rights to chacha20, using the Creative
+// Commons "CC0" public domain dedication. See LICENSE or
+// for full details.
+
+package chacha20
+
+import (
+ "encoding/binary"
+ "math"
+ "unsafe"
+)
+
+func blocksRef(x *[stateSize]uint32, in []byte, out []byte, nrBlocks int, isIetf bool) {
+ if isIetf {
+ var totalBlocks uint64
+ totalBlocks = uint64(x[8]) + uint64(nrBlocks)
+ if totalBlocks > math.MaxUint32 {
+ panic("chacha20: Exceeded keystream per nonce limit")
+ }
+ }
+
+ // This routine ignores x[0]...x[4] in favor the const values since it's
+ // ever so slightly faster.
+
+ for n := 0; n < nrBlocks; n++ {
+ x0, x1, x2, x3 := sigma0, sigma1, sigma2, sigma3
+ x4, x5, x6, x7, x8, x9, x10, x11, x12, x13, x14, x15 := x[4], x[5], x[6], x[7], x[8], x[9], x[10], x[11], x[12], x[13], x[14], x[15]
+
+ for i := chachaRounds; i > 0; i -= 2 {
+ // quarterround(x, 0, 4, 8, 12)
+ x0 += x4
+ x12 ^= x0
+ x12 = (x12 << 16) | (x12 >> 16)
+ x8 += x12
+ x4 ^= x8
+ x4 = (x4 << 12) | (x4 >> 20)
+ x0 += x4
+ x12 ^= x0
+ x12 = (x12 << 8) | (x12 >> 24)
+ x8 += x12
+ x4 ^= x8
+ x4 = (x4 << 7) | (x4 >> 25)
+
+ // quarterround(x, 1, 5, 9, 13)
+ x1 += x5
+ x13 ^= x1
+ x13 = (x13 << 16) | (x13 >> 16)
+ x9 += x13
+ x5 ^= x9
+ x5 = (x5 << 12) | (x5 >> 20)
+ x1 += x5
+ x13 ^= x1
+ x13 = (x13 << 8) | (x13 >> 24)
+ x9 += x13
+ x5 ^= x9
+ x5 = (x5 << 7) | (x5 >> 25)
+
+ // quarterround(x, 2, 6, 10, 14)
+ x2 += x6
+ x14 ^= x2
+ x14 = (x14 << 16) | (x14 >> 16)
+ x10 += x14
+ x6 ^= x10
+ x6 = (x6 << 12) | (x6 >> 20)
+ x2 += x6
+ x14 ^= x2
+ x14 = (x14 << 8) | (x14 >> 24)
+ x10 += x14
+ x6 ^= x10
+ x6 = (x6 << 7) | (x6 >> 25)
+
+ // quarterround(x, 3, 7, 11, 15)
+ x3 += x7
+ x15 ^= x3
+ x15 = (x15 << 16) | (x15 >> 16)
+ x11 += x15
+ x7 ^= x11
+ x7 = (x7 << 12) | (x7 >> 20)
+ x3 += x7
+ x15 ^= x3
+ x15 = (x15 << 8) | (x15 >> 24)
+ x11 += x15
+ x7 ^= x11
+ x7 = (x7 << 7) | (x7 >> 25)
+
+ // quarterround(x, 0, 5, 10, 15)
+ x0 += x5
+ x15 ^= x0
+ x15 = (x15 << 16) | (x15 >> 16)
+ x10 += x15
+ x5 ^= x10
+ x5 = (x5 << 12) | (x5 >> 20)
+ x0 += x5
+ x15 ^= x0
+ x15 = (x15 << 8) | (x15 >> 24)
+ x10 += x15
+ x5 ^= x10
+ x5 = (x5 << 7) | (x5 >> 25)
+
+ // quarterround(x, 1, 6, 11, 12)
+ x1 += x6
+ x12 ^= x1
+ x12 = (x12 << 16) | (x12 >> 16)
+ x11 += x12
+ x6 ^= x11
+ x6 = (x6 << 12) | (x6 >> 20)
+ x1 += x6
+ x12 ^= x1
+ x12 = (x12 << 8) | (x12 >> 24)
+ x11 += x12
+ x6 ^= x11
+ x6 = (x6 << 7) | (x6 >> 25)
+
+ // quarterround(x, 2, 7, 8, 13)
+ x2 += x7
+ x13 ^= x2
+ x13 = (x13 << 16) | (x13 >> 16)
+ x8 += x13
+ x7 ^= x8
+ x7 = (x7 << 12) | (x7 >> 20)
+ x2 += x7
+ x13 ^= x2
+ x13 = (x13 << 8) | (x13 >> 24)
+ x8 += x13
+ x7 ^= x8
+ x7 = (x7 << 7) | (x7 >> 25)
+
+ // quarterround(x, 3, 4, 9, 14)
+ x3 += x4
+ x14 ^= x3
+ x14 = (x14 << 16) | (x14 >> 16)
+ x9 += x14
+ x4 ^= x9
+ x4 = (x4 << 12) | (x4 >> 20)
+ x3 += x4
+ x14 ^= x3
+ x14 = (x14 << 8) | (x14 >> 24)
+ x9 += x14
+ x4 ^= x9
+ x4 = (x4 << 7) | (x4 >> 25)
+ }
+
+ // On amd64 at least, this is a rather big boost.
+ if useUnsafe {
+ if in != nil {
+ inArr := (*[16]uint32)(unsafe.Pointer(&in[n*BlockSize]))
+ outArr := (*[16]uint32)(unsafe.Pointer(&out[n*BlockSize]))
+ outArr[0] = inArr[0] ^ (x0 + sigma0)
+ outArr[1] = inArr[1] ^ (x1 + sigma1)
+ outArr[2] = inArr[2] ^ (x2 + sigma2)
+ outArr[3] = inArr[3] ^ (x3 + sigma3)
+ outArr[4] = inArr[4] ^ (x4 + x[4])
+ outArr[5] = inArr[5] ^ (x5 + x[5])
+ outArr[6] = inArr[6] ^ (x6 + x[6])
+ outArr[7] = inArr[7] ^ (x7 + x[7])
+ outArr[8] = inArr[8] ^ (x8 + x[8])
+ outArr[9] = inArr[9] ^ (x9 + x[9])
+ outArr[10] = inArr[10] ^ (x10 + x[10])
+ outArr[11] = inArr[11] ^ (x11 + x[11])
+ outArr[12] = inArr[12] ^ (x12 + x[12])
+ outArr[13] = inArr[13] ^ (x13 + x[13])
+ outArr[14] = inArr[14] ^ (x14 + x[14])
+ outArr[15] = inArr[15] ^ (x15 + x[15])
+ } else {
+ outArr := (*[16]uint32)(unsafe.Pointer(&out[n*BlockSize]))
+ outArr[0] = x0 + sigma0
+ outArr[1] = x1 + sigma1
+ outArr[2] = x2 + sigma2
+ outArr[3] = x3 + sigma3
+ outArr[4] = x4 + x[4]
+ outArr[5] = x5 + x[5]
+ outArr[6] = x6 + x[6]
+ outArr[7] = x7 + x[7]
+ outArr[8] = x8 + x[8]
+ outArr[9] = x9 + x[9]
+ outArr[10] = x10 + x[10]
+ outArr[11] = x11 + x[11]
+ outArr[12] = x12 + x[12]
+ outArr[13] = x13 + x[13]
+ outArr[14] = x14 + x[14]
+ outArr[15] = x15 + x[15]
+ }
+ } else {
+ // Slow path, either the architecture cares about alignment, or is not little endian.
+ x0 += sigma0
+ x1 += sigma1
+ x2 += sigma2
+ x3 += sigma3
+ x4 += x[4]
+ x5 += x[5]
+ x6 += x[6]
+ x7 += x[7]
+ x8 += x[8]
+ x9 += x[9]
+ x10 += x[10]
+ x11 += x[11]
+ x12 += x[12]
+ x13 += x[13]
+ x14 += x[14]
+ x15 += x[15]
+ if in != nil {
+ binary.LittleEndian.PutUint32(out[0:4], binary.LittleEndian.Uint32(in[0:4])^x0)
+ binary.LittleEndian.PutUint32(out[4:8], binary.LittleEndian.Uint32(in[4:8])^x1)
+ binary.LittleEndian.PutUint32(out[8:12], binary.LittleEndian.Uint32(in[8:12])^x2)
+ binary.LittleEndian.PutUint32(out[12:16], binary.LittleEndian.Uint32(in[12:16])^x3)
+ binary.LittleEndian.PutUint32(out[16:20], binary.LittleEndian.Uint32(in[16:20])^x4)
+ binary.LittleEndian.PutUint32(out[20:24], binary.LittleEndian.Uint32(in[20:24])^x5)
+ binary.LittleEndian.PutUint32(out[24:28], binary.LittleEndian.Uint32(in[24:28])^x6)
+ binary.LittleEndian.PutUint32(out[28:32], binary.LittleEndian.Uint32(in[28:32])^x7)
+ binary.LittleEndian.PutUint32(out[32:36], binary.LittleEndian.Uint32(in[32:36])^x8)
+ binary.LittleEndian.PutUint32(out[36:40], binary.LittleEndian.Uint32(in[36:40])^x9)
+ binary.LittleEndian.PutUint32(out[40:44], binary.LittleEndian.Uint32(in[40:44])^x10)
+ binary.LittleEndian.PutUint32(out[44:48], binary.LittleEndian.Uint32(in[44:48])^x11)
+ binary.LittleEndian.PutUint32(out[48:52], binary.LittleEndian.Uint32(in[48:52])^x12)
+ binary.LittleEndian.PutUint32(out[52:56], binary.LittleEndian.Uint32(in[52:56])^x13)
+ binary.LittleEndian.PutUint32(out[56:60], binary.LittleEndian.Uint32(in[56:60])^x14)
+ binary.LittleEndian.PutUint32(out[60:64], binary.LittleEndian.Uint32(in[60:64])^x15)
+ in = in[BlockSize:]
+ } else {
+ binary.LittleEndian.PutUint32(out[0:4], x0)
+ binary.LittleEndian.PutUint32(out[4:8], x1)
+ binary.LittleEndian.PutUint32(out[8:12], x2)
+ binary.LittleEndian.PutUint32(out[12:16], x3)
+ binary.LittleEndian.PutUint32(out[16:20], x4)
+ binary.LittleEndian.PutUint32(out[20:24], x5)
+ binary.LittleEndian.PutUint32(out[24:28], x6)
+ binary.LittleEndian.PutUint32(out[28:32], x7)
+ binary.LittleEndian.PutUint32(out[32:36], x8)
+ binary.LittleEndian.PutUint32(out[36:40], x9)
+ binary.LittleEndian.PutUint32(out[40:44], x10)
+ binary.LittleEndian.PutUint32(out[44:48], x11)
+ binary.LittleEndian.PutUint32(out[48:52], x12)
+ binary.LittleEndian.PutUint32(out[52:56], x13)
+ binary.LittleEndian.PutUint32(out[56:60], x14)
+ binary.LittleEndian.PutUint32(out[60:64], x15)
+ }
+ out = out[BlockSize:]
+ }
+
+ // Stoping at 2^70 bytes per nonce is the user's responsibility.
+ ctr := uint64(x[13])<<32 | uint64(x[12])
+ ctr++
+ x[12] = uint32(ctr)
+ x[13] = uint32(ctr >> 32)
+ }
+}
+
+func hChaChaRef(x *[stateSize]uint32, out *[32]byte) {
+ x0, x1, x2, x3 := sigma0, sigma1, sigma2, sigma3
+ x4, x5, x6, x7, x8, x9, x10, x11, x12, x13, x14, x15 := x[0], x[1], x[2], x[3], x[4], x[5], x[6], x[7], x[8], x[9], x[10], x[11]
+
+ for i := chachaRounds; i > 0; i -= 2 {
+ // quarterround(x, 0, 4, 8, 12)
+ x0 += x4
+ x12 ^= x0
+ x12 = (x12 << 16) | (x12 >> 16)
+ x8 += x12
+ x4 ^= x8
+ x4 = (x4 << 12) | (x4 >> 20)
+ x0 += x4
+ x12 ^= x0
+ x12 = (x12 << 8) | (x12 >> 24)
+ x8 += x12
+ x4 ^= x8
+ x4 = (x4 << 7) | (x4 >> 25)
+
+ // quarterround(x, 1, 5, 9, 13)
+ x1 += x5
+ x13 ^= x1
+ x13 = (x13 << 16) | (x13 >> 16)
+ x9 += x13
+ x5 ^= x9
+ x5 = (x5 << 12) | (x5 >> 20)
+ x1 += x5
+ x13 ^= x1
+ x13 = (x13 << 8) | (x13 >> 24)
+ x9 += x13
+ x5 ^= x9
+ x5 = (x5 << 7) | (x5 >> 25)
+
+ // quarterround(x, 2, 6, 10, 14)
+ x2 += x6
+ x14 ^= x2
+ x14 = (x14 << 16) | (x14 >> 16)
+ x10 += x14
+ x6 ^= x10
+ x6 = (x6 << 12) | (x6 >> 20)
+ x2 += x6
+ x14 ^= x2
+ x14 = (x14 << 8) | (x14 >> 24)
+ x10 += x14
+ x6 ^= x10
+ x6 = (x6 << 7) | (x6 >> 25)
+
+ // quarterround(x, 3, 7, 11, 15)
+ x3 += x7
+ x15 ^= x3
+ x15 = (x15 << 16) | (x15 >> 16)
+ x11 += x15
+ x7 ^= x11
+ x7 = (x7 << 12) | (x7 >> 20)
+ x3 += x7
+ x15 ^= x3
+ x15 = (x15 << 8) | (x15 >> 24)
+ x11 += x15
+ x7 ^= x11
+ x7 = (x7 << 7) | (x7 >> 25)
+
+ // quarterround(x, 0, 5, 10, 15)
+ x0 += x5
+ x15 ^= x0
+ x15 = (x15 << 16) | (x15 >> 16)
+ x10 += x15
+ x5 ^= x10
+ x5 = (x5 << 12) | (x5 >> 20)
+ x0 += x5
+ x15 ^= x0
+ x15 = (x15 << 8) | (x15 >> 24)
+ x10 += x15
+ x5 ^= x10
+ x5 = (x5 << 7) | (x5 >> 25)
+
+ // quarterround(x, 1, 6, 11, 12)
+ x1 += x6
+ x12 ^= x1
+ x12 = (x12 << 16) | (x12 >> 16)
+ x11 += x12
+ x6 ^= x11
+ x6 = (x6 << 12) | (x6 >> 20)
+ x1 += x6
+ x12 ^= x1
+ x12 = (x12 << 8) | (x12 >> 24)
+ x11 += x12
+ x6 ^= x11
+ x6 = (x6 << 7) | (x6 >> 25)
+
+ // quarterround(x, 2, 7, 8, 13)
+ x2 += x7
+ x13 ^= x2
+ x13 = (x13 << 16) | (x13 >> 16)
+ x8 += x13
+ x7 ^= x8
+ x7 = (x7 << 12) | (x7 >> 20)
+ x2 += x7
+ x13 ^= x2
+ x13 = (x13 << 8) | (x13 >> 24)
+ x8 += x13
+ x7 ^= x8
+ x7 = (x7 << 7) | (x7 >> 25)
+
+ // quarterround(x, 3, 4, 9, 14)
+ x3 += x4
+ x14 ^= x3
+ x14 = (x14 << 16) | (x14 >> 16)
+ x9 += x14
+ x4 ^= x9
+ x4 = (x4 << 12) | (x4 >> 20)
+ x3 += x4
+ x14 ^= x3
+ x14 = (x14 << 8) | (x14 >> 24)
+ x9 += x14
+ x4 ^= x9
+ x4 = (x4 << 7) | (x4 >> 25)
+ }
+
+ // HChaCha returns x0...x3 | x12...x15, which corresponds to the
+ // indexes of the ChaCha constant and the indexes of the IV.
+ if useUnsafe {
+ outArr := (*[16]uint32)(unsafe.Pointer(&out[0]))
+ outArr[0] = x0
+ outArr[1] = x1
+ outArr[2] = x2
+ outArr[3] = x3
+ outArr[4] = x12
+ outArr[5] = x13
+ outArr[6] = x14
+ outArr[7] = x15
+ } else {
+ binary.LittleEndian.PutUint32(out[0:4], x0)
+ binary.LittleEndian.PutUint32(out[4:8], x1)
+ binary.LittleEndian.PutUint32(out[8:12], x2)
+ binary.LittleEndian.PutUint32(out[12:16], x3)
+ binary.LittleEndian.PutUint32(out[16:20], x12)
+ binary.LittleEndian.PutUint32(out[20:24], x13)
+ binary.LittleEndian.PutUint32(out[24:28], x14)
+ binary.LittleEndian.PutUint32(out[28:32], x15)
+ }
+ return
+}
diff --git a/cmd/gost/vendor/github.com/ginuerzh/gost/forward.go b/cmd/gost/vendor/github.com/ginuerzh/gost/forward.go
index 750cb50..ca76904 100644
--- a/cmd/gost/vendor/github.com/ginuerzh/gost/forward.go
+++ b/cmd/gost/vendor/github.com/ginuerzh/gost/forward.go
@@ -63,15 +63,15 @@ func (s *TcpForwardServer) handleTcpForward(conn net.Conn, raddr net.Addr) {
}
type packet struct {
- srcAddr *net.UDPAddr // src address
- dstAddr *net.UDPAddr // dest address
+ srcAddr string // src address
+ dstAddr string // dest address
data []byte
}
type cnode struct {
chain *ProxyChain
conn net.Conn
- srcAddr, dstAddr *net.UDPAddr
+ srcAddr, dstAddr string
rChan, wChan chan *packet
err error
ttl time.Duration
@@ -146,13 +146,9 @@ func (node *cnode) run() {
timer.Reset(node.ttl)
glog.V(LDEBUG).Infof("[udp] %s <<< %s : length %d", node.srcAddr, addr, n)
- if node.dstAddr.String() != addr.String() {
- glog.V(LWARNING).Infof("[udp] %s <- %s : dst-addr mismatch (%s)", node.srcAddr, node.dstAddr, addr)
- break
- }
select {
// swap srcAddr with dstAddr
- case node.rChan <- &packet{srcAddr: node.dstAddr, dstAddr: node.srcAddr, data: b[:n]}:
+ case node.rChan <- &packet{srcAddr: addr.String(), dstAddr: node.srcAddr, data: b[:n]}:
case <-time.After(time.Second * 3):
glog.V(LWARNING).Infof("[udp] %s <- %s : %s", node.srcAddr, node.dstAddr, "recv queue is full, discard")
}
@@ -169,13 +165,9 @@ func (node *cnode) run() {
timer.Reset(node.ttl)
glog.V(LDEBUG).Infof("[udp-tun] %s <<< %s : length %d", node.srcAddr, dgram.Header.Addr.String(), len(dgram.Data))
- if dgram.Header.Addr.String() != node.dstAddr.String() {
- glog.V(LWARNING).Infof("[udp-tun] %s <- %s : dst-addr mismatch (%s)", node.srcAddr, node.dstAddr, dgram.Header.Addr)
- break
- }
select {
// swap srcAddr with dstAddr
- case node.rChan <- &packet{srcAddr: node.dstAddr, dstAddr: node.srcAddr, data: dgram.Data}:
+ case node.rChan <- &packet{srcAddr: dgram.Header.Addr.String(), dstAddr: node.srcAddr, data: dgram.Data}:
case <-time.After(time.Second * 3):
glog.V(LWARNING).Infof("[udp-tun] %s <- %s : %s", node.srcAddr, node.dstAddr, "recv queue is full, discard")
}
@@ -187,9 +179,15 @@ func (node *cnode) run() {
for pkt := range node.wChan {
timer.Reset(node.ttl)
+ dstAddr, err := net.ResolveUDPAddr("udp", pkt.dstAddr)
+ if err != nil {
+ glog.V(LWARNING).Infof("[udp] %s -> %s : %s", pkt.srcAddr, pkt.dstAddr, err)
+ continue
+ }
+
switch c := node.conn.(type) {
case *net.UDPConn:
- if _, err := c.WriteToUDP(pkt.data, pkt.dstAddr); err != nil {
+ if _, err := c.WriteToUDP(pkt.data, dstAddr); err != nil {
glog.V(LWARNING).Infof("[udp] %s -> %s : %s", pkt.srcAddr, pkt.dstAddr, err)
node.err = err
errChan <- err
@@ -198,7 +196,7 @@ func (node *cnode) run() {
glog.V(LDEBUG).Infof("[udp] %s >>> %s : length %d", pkt.srcAddr, pkt.dstAddr, len(pkt.data))
default:
- dgram := gosocks5.NewUDPDatagram(gosocks5.NewUDPHeader(uint16(len(pkt.data)), 0, ToSocksAddr(pkt.dstAddr)), pkt.data)
+ dgram := gosocks5.NewUDPDatagram(gosocks5.NewUDPHeader(uint16(len(pkt.data)), 0, ToSocksAddr(dstAddr)), pkt.data)
if err := dgram.Write(c); err != nil {
glog.V(LWARNING).Infof("[udp-tun] %s -> %s : %s", pkt.srcAddr, pkt.dstAddr, err)
node.err = err
@@ -255,7 +253,7 @@ func (s *UdpForwardServer) ListenAndServe() error {
}
select {
- case ch <- &packet{srcAddr: addr, dstAddr: raddr, data: b[:n]}:
+ case ch <- &packet{srcAddr: addr.String(), dstAddr: raddr.String(), data: b[:n]}:
case <-time.After(time.Second * 3):
glog.V(LWARNING).Infof("[udp] %s -> %s : %s", addr, raddr, "send queue is full, discard")
}
@@ -264,7 +262,12 @@ func (s *UdpForwardServer) ListenAndServe() error {
// start recv queue
go func(ch <-chan *packet) {
for pkt := range ch {
- if _, err := conn.WriteToUDP(pkt.data, pkt.dstAddr); err != nil {
+ dstAddr, err := net.ResolveUDPAddr("udp", pkt.dstAddr)
+ if err != nil {
+ glog.V(LWARNING).Infof("[udp] %s <- %s : %s", pkt.dstAddr, pkt.srcAddr, err)
+ continue
+ }
+ if _, err := conn.WriteToUDP(pkt.data, dstAddr); err != nil {
glog.V(LWARNING).Infof("[udp] %s <- %s : %s", pkt.dstAddr, pkt.srcAddr, err)
return
}
@@ -285,7 +288,7 @@ func (s *UdpForwardServer) ListenAndServe() error {
}
}
- node, ok := m[pkt.srcAddr.String()]
+ node, ok := m[pkt.srcAddr]
if !ok {
node = &cnode{
chain: s.Base.Chain,
@@ -295,7 +298,7 @@ func (s *UdpForwardServer) ListenAndServe() error {
wChan: make(chan *packet, 32),
ttl: time.Duration(s.TTL) * time.Second,
}
- m[pkt.srcAddr.String()] = node
+ m[pkt.srcAddr] = node
go node.run()
glog.V(LINFO).Infof("[udp] %s -> %s : new client (%d)", pkt.srcAddr, pkt.dstAddr, len(m))
}
diff --git a/cmd/gost/vendor/github.com/ginuerzh/gost/gost.go b/cmd/gost/vendor/github.com/ginuerzh/gost/gost.go
index 895d939..6c043e7 100644
--- a/cmd/gost/vendor/github.com/ginuerzh/gost/gost.go
+++ b/cmd/gost/vendor/github.com/ginuerzh/gost/gost.go
@@ -11,7 +11,7 @@ import (
)
const (
- Version = "2.3"
+ Version = "2.4-dev"
)
// Log level for glog
diff --git a/cmd/gost/vendor/github.com/ginuerzh/gost/node.go b/cmd/gost/vendor/github.com/ginuerzh/gost/node.go
index 88d7153..e72d0bb 100644
--- a/cmd/gost/vendor/github.com/ginuerzh/gost/node.go
+++ b/cmd/gost/vendor/github.com/ginuerzh/gost/node.go
@@ -71,7 +71,7 @@ func ParseProxyNode(s string) (node ProxyNode, err error) {
}
switch node.Transport {
- case "ws", "wss", "tls", "http2", "ssu", "quic", "kcp", "redirect":
+ case "ws", "wss", "tls", "http2", "quic", "kcp", "redirect", "ssu":
case "https":
node.Protocol = "http"
node.Transport = "tls"
diff --git a/cmd/gost/vendor/github.com/ginuerzh/gost/server.go b/cmd/gost/vendor/github.com/ginuerzh/gost/server.go
index 74b3861..63ff9a0 100644
--- a/cmd/gost/vendor/github.com/ginuerzh/gost/server.go
+++ b/cmd/gost/vendor/github.com/ginuerzh/gost/server.go
@@ -32,7 +32,7 @@ func NewProxyServer(node ProxyNode, chain *ProxyChain, config *tls.Config) *Prox
var cipher *ss.Cipher
var ota bool
- if node.Protocol == "ss" {
+ if node.Protocol == "ss" || node.Transport == "ssu" {
var err error
var method, password string
@@ -98,8 +98,6 @@ func (s *ProxyServer) Serve() error {
return NewRTcpForwardServer(s).Serve()
case "rudp": // Remote UDP port forwarding
return NewRUdpForwardServer(s).Serve()
- case "ssu": // TODO: shadowsocks udp relay
- return NewShadowUdpServer(s).ListenAndServe()
case "quic":
return NewQuicServer(s).ListenAndServeTLS(s.TLSConfig)
case "kcp":
@@ -118,6 +116,12 @@ func (s *ProxyServer) Serve() error {
return NewKCPServer(s, config).ListenAndServe()
case "redirect":
return NewRedsocksTCPServer(s).ListenAndServe()
+ case "ssu": // shadowsocks udp relay
+ ttl, _ := strconv.Atoi(s.Node.Get("ttl"))
+ if ttl <= 0 {
+ ttl = DefaultTTL
+ }
+ return NewShadowUdpServer(s, ttl).ListenAndServe()
default:
ln, err = net.Listen("tcp", node.Addr)
}
diff --git a/cmd/gost/vendor/github.com/ginuerzh/gost/ss.go b/cmd/gost/vendor/github.com/ginuerzh/gost/ss.go
index 12bc167..81d48af 100644
--- a/cmd/gost/vendor/github.com/ginuerzh/gost/ss.go
+++ b/cmd/gost/vendor/github.com/ginuerzh/gost/ss.go
@@ -5,6 +5,7 @@ import (
"encoding/binary"
"errors"
"fmt"
+ "github.com/ginuerzh/gosocks5"
"github.com/golang/glog"
ss "github.com/shadowsocks/shadowsocks-go/shadowsocks"
"io"
@@ -65,47 +66,6 @@ func (s *ShadowServer) Serve() {
glog.V(LINFO).Infof("[ss] %s >-< %s", s.conn.RemoteAddr(), addr)
}
-type ShadowUdpServer struct {
- Base *ProxyServer
- Handler func(conn *net.UDPConn, addr *net.UDPAddr, data []byte)
-}
-
-func NewShadowUdpServer(base *ProxyServer) *ShadowUdpServer {
- return &ShadowUdpServer{Base: base}
-}
-
-func (s *ShadowUdpServer) ListenAndServe() error {
- laddr, err := net.ResolveUDPAddr("udp", s.Base.Node.Addr)
- if err != nil {
- return err
- }
- lconn, err := net.ListenUDP("udp", laddr)
- if err != nil {
- return err
- }
- defer lconn.Close()
-
- if s.Handler == nil {
- s.Handler = s.HandleConn
- }
-
- for {
- b := make([]byte, LargeBufferSize)
- n, addr, err := lconn.ReadFromUDP(b)
- if err != nil {
- glog.V(LWARNING).Infoln(err)
- continue
- }
-
- go s.Handler(lconn, addr, b[:n])
- }
-}
-
-// TODO: shadowsocks udp relay handler
-func (s *ShadowUdpServer) HandleConn(conn *net.UDPConn, addr *net.UDPAddr, data []byte) {
-
-}
-
// This function is copied from shadowsocks library with some modification.
func (s *ShadowServer) getRequest() (host string, ota bool, err error) {
// buf size should at least have the same size with the largest possible
@@ -276,3 +236,109 @@ func (c *shadowConn) SetReadDeadline(t time.Time) error {
func (c *shadowConn) SetWriteDeadline(t time.Time) error {
return c.conn.SetWriteDeadline(t)
}
+
+type ShadowUdpServer struct {
+ Base *ProxyServer
+ TTL int
+}
+
+func NewShadowUdpServer(base *ProxyServer, ttl int) *ShadowUdpServer {
+ return &ShadowUdpServer{Base: base, TTL: ttl}
+}
+
+func (s *ShadowUdpServer) ListenAndServe() error {
+ laddr, err := net.ResolveUDPAddr("udp", s.Base.Node.Addr)
+ if err != nil {
+ return err
+ }
+ lconn, err := net.ListenUDP("udp", laddr)
+ if err != nil {
+ return err
+ }
+ defer lconn.Close()
+
+ conn := ss.NewSecurePacketConn(lconn, s.Base.cipher.Copy(), true) // force OTA on
+
+ rChan, wChan := make(chan *packet, 128), make(chan *packet, 128)
+ // start send queue
+ go func(ch chan<- *packet) {
+ for {
+ b := make([]byte, MediumBufferSize)
+ n, addr, err := conn.ReadFrom(b[3:]) // add rsv and frag fields to make it the standard SOCKS5 UDP datagram
+ if err != nil {
+ glog.V(LWARNING).Infof("[ssu] %s -> %s : %s", addr, laddr, err)
+ continue
+ }
+
+ if b[3]&ss.OneTimeAuthMask > 0 {
+ glog.V(LWARNING).Infof("[ssu] %s -> %s : client does not support OTA", addr, laddr)
+ continue
+ }
+ b[3] &= ss.AddrMask
+
+ dgram, err := gosocks5.ReadUDPDatagram(bytes.NewReader(b[:n+3]))
+ if err != nil {
+ glog.V(LWARNING).Infof("[ssu] %s -> %s : %s", addr, laddr, err)
+ continue
+ }
+
+ select {
+ case ch <- &packet{srcAddr: addr.String(), dstAddr: dgram.Header.Addr.String(), data: b[:n+3]}:
+ case <-time.After(time.Second * 3):
+ glog.V(LWARNING).Infof("[ssu] %s -> %s : %s", addr, dgram.Header.Addr.String(), "send queue is full, discard")
+ }
+ }
+ }(wChan)
+ // start recv queue
+ go func(ch <-chan *packet) {
+ for pkt := range ch {
+ dstAddr, err := net.ResolveUDPAddr("udp", pkt.dstAddr)
+ if err != nil {
+ glog.V(LWARNING).Infof("[ssu] %s <- %s : %s", pkt.dstAddr, pkt.srcAddr, err)
+ continue
+ }
+ if _, err := conn.WriteTo(pkt.data, dstAddr); err != nil {
+ glog.V(LWARNING).Infof("[ssu] %s <- %s : %s", pkt.dstAddr, pkt.srcAddr, err)
+ return
+ }
+ }
+ }(rChan)
+
+ // mapping client to node
+ m := make(map[string]*cnode)
+
+ // start dispatcher
+ for pkt := range wChan {
+ // clear obsolete nodes
+ for k, node := range m {
+ if node != nil && node.err != nil {
+ close(node.wChan)
+ delete(m, k)
+ glog.V(LINFO).Infof("[ssu] clear node %s", k)
+ }
+ }
+
+ node, ok := m[pkt.srcAddr]
+ if !ok {
+ node = &cnode{
+ chain: s.Base.Chain,
+ srcAddr: pkt.srcAddr,
+ dstAddr: pkt.dstAddr,
+ rChan: rChan,
+ wChan: make(chan *packet, 32),
+ ttl: time.Duration(s.TTL) * time.Second,
+ }
+ m[pkt.srcAddr] = node
+ go node.run()
+ glog.V(LINFO).Infof("[ssu] %s -> %s : new client (%d)", pkt.srcAddr, pkt.dstAddr, len(m))
+ }
+
+ select {
+ case node.wChan <- pkt:
+ case <-time.After(time.Second * 3):
+ glog.V(LWARNING).Infof("[ssu] %s -> %s : %s", pkt.srcAddr, pkt.dstAddr, "node send queue is full, discard")
+ }
+ }
+
+ return nil
+}
diff --git a/cmd/gost/vendor/github.com/shadowsocks/shadowsocks-go/shadowsocks/encrypt.go b/cmd/gost/vendor/github.com/shadowsocks/shadowsocks-go/shadowsocks/encrypt.go
index 146947d..44d96a7 100644
--- a/cmd/gost/vendor/github.com/shadowsocks/shadowsocks-go/shadowsocks/encrypt.go
+++ b/cmd/gost/vendor/github.com/shadowsocks/shadowsocks-go/shadowsocks/encrypt.go
@@ -12,7 +12,7 @@ import (
"io"
"strings"
- "github.com/codahale/chacha20"
+ "github.com/Yawning/chacha20"
"golang.org/x/crypto/blowfish"
"golang.org/x/crypto/cast5"
"golang.org/x/crypto/salsa20/salsa"
@@ -65,11 +65,19 @@ func newStream(block cipher.Block, err error, key, iv []byte,
}
}
-func newAESStream(key, iv []byte, doe DecOrEnc) (cipher.Stream, error) {
+func newAESCFBStream(key, iv []byte, doe DecOrEnc) (cipher.Stream, error) {
block, err := aes.NewCipher(key)
return newStream(block, err, key, iv, doe)
}
+func newAESCTRStream(key, iv []byte, doe DecOrEnc) (cipher.Stream, error) {
+ block, err := aes.NewCipher(key)
+ if err != nil {
+ return nil, err
+ }
+ return cipher.NewCTR(block, iv), nil
+}
+
func newDESStream(key, iv []byte, doe DecOrEnc) (cipher.Stream, error) {
block, err := des.NewCipher(key)
return newStream(block, err, key, iv, doe)
@@ -95,7 +103,11 @@ func newRC4MD5Stream(key, iv []byte, _ DecOrEnc) (cipher.Stream, error) {
}
func newChaCha20Stream(key, iv []byte, _ DecOrEnc) (cipher.Stream, error) {
- return chacha20.New(key, iv)
+ return chacha20.NewCipher(key, iv)
+}
+
+func newChaCha20IETFStream(key, iv []byte, _ DecOrEnc) (cipher.Stream, error) {
+ return chacha20.NewCipher(key, iv)
}
type salsaStreamCipher struct {
@@ -145,15 +157,19 @@ type cipherInfo struct {
}
var cipherMethod = map[string]*cipherInfo{
- "aes-128-cfb": {16, 16, newAESStream},
- "aes-192-cfb": {24, 16, newAESStream},
- "aes-256-cfb": {32, 16, newAESStream},
- "des-cfb": {8, 8, newDESStream},
- "bf-cfb": {16, 8, newBlowFishStream},
- "cast5-cfb": {16, 8, newCast5Stream},
- "rc4-md5": {16, 16, newRC4MD5Stream},
- "chacha20": {32, 8, newChaCha20Stream},
- "salsa20": {32, 8, newSalsa20Stream},
+ "aes-128-cfb": {16, 16, newAESCFBStream},
+ "aes-192-cfb": {24, 16, newAESCFBStream},
+ "aes-256-cfb": {32, 16, newAESCFBStream},
+ "aes-128-ctr": {16, 16, newAESCTRStream},
+ "aes-192-ctr": {24, 16, newAESCTRStream},
+ "aes-256-ctr": {32, 16, newAESCTRStream},
+ "des-cfb": {8, 8, newDESStream},
+ "bf-cfb": {16, 8, newBlowFishStream},
+ "cast5-cfb": {16, 8, newCast5Stream},
+ "rc4-md5": {16, 16, newRC4MD5Stream},
+ "chacha20": {32, 8, newChaCha20Stream},
+ "chacha20-ietf": {32, 12, newChaCha20IETFStream},
+ "salsa20": {32, 8, newSalsa20Stream},
}
func CheckCipherMethod(method string) error {
diff --git a/cmd/gost/vendor/github.com/shadowsocks/shadowsocks-go/shadowsocks/udp.go b/cmd/gost/vendor/github.com/shadowsocks/shadowsocks-go/shadowsocks/udp.go
new file mode 100644
index 0000000..62a55d8
--- /dev/null
+++ b/cmd/gost/vendor/github.com/shadowsocks/shadowsocks-go/shadowsocks/udp.go
@@ -0,0 +1,135 @@
+package shadowsocks
+
+import (
+ "bytes"
+ "fmt"
+ "net"
+ "time"
+)
+
+const (
+ maxPacketSize = 4096 // increase it if error occurs
+)
+
+var (
+ errPacketTooSmall = fmt.Errorf("[udp]read error: cannot decrypt, received packet is smaller than ivLen")
+ errPacketTooLarge = fmt.Errorf("[udp]read error: received packet is latger than maxPacketSize(%d)", maxPacketSize)
+ errBufferTooSmall = fmt.Errorf("[udp]read error: given buffer is too small to hold data")
+ errPacketOtaFailed = fmt.Errorf("[udp]read error: received packet has invalid ota")
+)
+
+type SecurePacketConn struct {
+ net.PacketConn
+ *Cipher
+ ota bool
+}
+
+func NewSecurePacketConn(c net.PacketConn, cipher *Cipher, ota bool) *SecurePacketConn {
+ return &SecurePacketConn{
+ PacketConn: c,
+ Cipher: cipher,
+ ota: ota,
+ }
+}
+
+func (c *SecurePacketConn) Close() error {
+ return c.PacketConn.Close()
+}
+
+func (c *SecurePacketConn) ReadFrom(b []byte) (n int, src net.Addr, err error) {
+ ota := false
+ cipher := c.Copy()
+ buf := make([]byte, 4096)
+ n, src, err = c.PacketConn.ReadFrom(buf)
+ if err != nil {
+ return
+ }
+
+ if n < c.info.ivLen {
+ return 0, nil, errPacketTooSmall
+ }
+
+ if len(b) < n-c.info.ivLen {
+ err = errBufferTooSmall // just a warning
+ }
+
+ iv := make([]byte, c.info.ivLen)
+ copy(iv, buf[:c.info.ivLen])
+
+ if err = cipher.initDecrypt(iv); err != nil {
+ return
+ }
+
+ cipher.decrypt(b[0:], buf[c.info.ivLen:n])
+ n -= c.info.ivLen
+ if b[idType]&OneTimeAuthMask > 0 {
+ ota = true
+ }
+
+ if c.ota && !ota {
+ return 0, src, errPacketOtaFailed
+ }
+
+ if ota {
+ key := cipher.key
+ actualHmacSha1Buf := HmacSha1(append(iv, key...), b[:n-lenHmacSha1])
+ if !bytes.Equal(b[n-lenHmacSha1:n], actualHmacSha1Buf) {
+ Debug.Printf("verify one time auth failed, iv=%v key=%v data=%v", iv, key, b)
+ return 0, src, errPacketOtaFailed
+ }
+ n -= lenHmacSha1
+ }
+
+ return
+}
+
+func (c *SecurePacketConn) WriteTo(b []byte, dst net.Addr) (n int, err error) {
+ cipher := c.Copy()
+ iv, err := cipher.initEncrypt()
+ if err != nil {
+ return
+ }
+ packetLen := len(b) + len(iv)
+
+ if c.ota {
+ b[idType] |= OneTimeAuthMask
+ packetLen += lenHmacSha1
+ key := cipher.key
+ actualHmacSha1Buf := HmacSha1(append(iv, key...), b)
+ b = append(b, actualHmacSha1Buf...)
+ }
+
+ cipherData := make([]byte, packetLen)
+ copy(cipherData, iv)
+
+ cipher.encrypt(cipherData[len(iv):], b)
+ n, err = c.PacketConn.WriteTo(cipherData, dst)
+ if c.ota {
+ n -= lenHmacSha1
+ }
+ return
+}
+
+func (c *SecurePacketConn) LocalAddr() net.Addr {
+ return c.PacketConn.LocalAddr()
+}
+
+func (c *SecurePacketConn) SetDeadline(t time.Time) error {
+ return c.PacketConn.SetDeadline(t)
+}
+
+func (c *SecurePacketConn) SetReadDeadline(t time.Time) error {
+ return c.PacketConn.SetReadDeadline(t)
+}
+
+func (c *SecurePacketConn) SetWriteDeadline(t time.Time) error {
+ return c.PacketConn.SetWriteDeadline(t)
+}
+
+func (c *SecurePacketConn) IsOta() bool {
+ return c.ota
+}
+
+func (c *SecurePacketConn) ForceOTA() net.PacketConn {
+ return NewSecurePacketConn(c.PacketConn, c.Cipher.Copy(), true)
+}
diff --git a/cmd/gost/vendor/github.com/shadowsocks/shadowsocks-go/shadowsocks/udprelay.go b/cmd/gost/vendor/github.com/shadowsocks/shadowsocks-go/shadowsocks/udprelay.go
new file mode 100644
index 0000000..727efb6
--- /dev/null
+++ b/cmd/gost/vendor/github.com/shadowsocks/shadowsocks-go/shadowsocks/udprelay.go
@@ -0,0 +1,265 @@
+package shadowsocks
+
+import (
+ "encoding/binary"
+ "fmt"
+ "net"
+ "strconv"
+ "strings"
+ "sync"
+ "syscall"
+ "time"
+)
+
+const (
+ idType = 0 // address type index
+ idIP0 = 1 // ip addres start index
+ idDmLen = 1 // domain address length index
+ idDm0 = 2 // domain address start index
+
+ typeIPv4 = 1 // type is ipv4 address
+ typeDm = 3 // type is domain address
+ typeIPv6 = 4 // type is ipv6 address
+
+ lenIPv4 = 1 + net.IPv4len + 2 // 1addrType + ipv4 + 2port
+ lenIPv6 = 1 + net.IPv6len + 2 // 1addrType + ipv6 + 2port
+ lenDmBase = 1 + 1 + 2 // 1addrType + 1addrLen + 2port, plus addrLen
+ lenHmacSha1 = 10
+)
+
+var (
+ reqList = newReqList()
+ natlist = newNatTable()
+ udpTimeout = 30 * time.Second
+ reqListRefreshTime = 5 * time.Minute
+)
+
+type natTable struct {
+ sync.Mutex
+ conns map[string]net.PacketConn
+}
+
+func newNatTable() *natTable {
+ return &natTable{conns: map[string]net.PacketConn{}}
+}
+
+func (table *natTable) Delete(index string) net.PacketConn {
+ table.Lock()
+ defer table.Unlock()
+ c, ok := table.conns[index]
+ if ok {
+ delete(table.conns, index)
+ return c
+ }
+ return nil
+}
+
+func (table *natTable) Get(index string) (c net.PacketConn, ok bool, err error) {
+ table.Lock()
+ defer table.Unlock()
+ c, ok = table.conns[index]
+ if !ok {
+ c, err = net.ListenPacket("udp", "")
+ if err != nil {
+ return nil, false, err
+ }
+ table.conns[index] = c
+ }
+ return
+}
+
+type requestHeaderList struct {
+ sync.Mutex
+ List map[string]([]byte)
+}
+
+func newReqList() *requestHeaderList {
+ ret := &requestHeaderList{List: map[string]([]byte){}}
+ go func() {
+ for {
+ time.Sleep(reqListRefreshTime)
+ ret.Refresh()
+ }
+ }()
+ return ret
+}
+
+func (r *requestHeaderList) Refresh() {
+ r.Lock()
+ defer r.Unlock()
+ for k, _ := range r.List {
+ delete(r.List, k)
+ }
+}
+
+func (r *requestHeaderList) Get(dstaddr string) (req []byte, ok bool) {
+ r.Lock()
+ defer r.Unlock()
+ req, ok = r.List[dstaddr]
+ return
+}
+
+func (r *requestHeaderList) Put(dstaddr string, req []byte) {
+ r.Lock()
+ defer r.Unlock()
+ r.List[dstaddr] = req
+ return
+}
+
+func parseHeaderFromAddr(addr net.Addr) ([]byte, int) {
+ // if the request address type is domain, it cannot be reverselookuped
+ ip, port, err := net.SplitHostPort(addr.String())
+ if err != nil {
+ return nil, 0
+ }
+ buf := make([]byte, 20)
+ IP := net.ParseIP(ip)
+ b1 := IP.To4()
+ iplen := 0
+ if b1 == nil { //ipv6
+ b1 = IP.To16()
+ buf[0] = typeIPv6
+ iplen = net.IPv6len
+ } else { //ipv4
+ buf[0] = typeIPv4
+ iplen = net.IPv4len
+ }
+ copy(buf[1:], b1)
+ port_i, _ := strconv.Atoi(port)
+ binary.BigEndian.PutUint16(buf[1+iplen:], uint16(port_i))
+ return buf[:1+iplen+2], 1 + iplen + 2
+}
+
+func Pipeloop(write net.PacketConn, writeAddr net.Addr, readClose net.PacketConn) {
+ buf := leakyBuf.Get()
+ defer leakyBuf.Put(buf)
+ defer readClose.Close()
+ for {
+ readClose.SetDeadline(time.Now().Add(udpTimeout))
+ n, raddr, err := readClose.ReadFrom(buf)
+ if err != nil {
+ if ne, ok := err.(*net.OpError); ok {
+ if ne.Err == syscall.EMFILE || ne.Err == syscall.ENFILE {
+ // log too many open file error
+ // EMFILE is process reaches open file limits, ENFILE is system limit
+ Debug.Println("[udp]read error:", err)
+ }
+ }
+ Debug.Printf("[udp]closed pipe %s<-%s\n", writeAddr, readClose.LocalAddr())
+ return
+ }
+ // need improvement here
+ if req, ok := reqList.Get(raddr.String()); ok {
+ write.WriteTo(append(req, buf[:n]...), writeAddr)
+ } else {
+ header, hlen := parseHeaderFromAddr(raddr)
+ write.WriteTo(append(header[:hlen], buf[:n]...), writeAddr)
+ }
+ }
+}
+
+func handleUDPConnection(handle *SecurePacketConn, n int, src net.Addr, receive []byte) {
+ var dstIP net.IP
+ var reqLen int
+ var ota bool
+ addrType := receive[idType]
+ defer leakyBuf.Put(receive)
+
+ if addrType&OneTimeAuthMask > 0 {
+ ota = true
+ }
+ receive[idType] &= ^OneTimeAuthMask
+ compatiblemode := !handle.IsOta() && ota
+
+ switch addrType & AddrMask {
+ case typeIPv4:
+ reqLen = lenIPv4
+ if len(receive) < reqLen {
+ Debug.Println("[udp]invalid received message.")
+ }
+ dstIP = net.IP(receive[idIP0 : idIP0+net.IPv4len])
+ case typeIPv6:
+ reqLen = lenIPv6
+ if len(receive) < reqLen {
+ Debug.Println("[udp]invalid received message.")
+ }
+ dstIP = net.IP(receive[idIP0 : idIP0+net.IPv6len])
+ case typeDm:
+ reqLen = int(receive[idDmLen]) + lenDmBase
+ if len(receive) < reqLen {
+ Debug.Println("[udp]invalid received message.")
+ }
+ name := string(receive[idDm0 : idDm0+int(receive[idDmLen])])
+ // avoid panic: syscall: string with NUL passed to StringToUTF16 on windows.
+ if strings.ContainsRune(name, 0x00) {
+ fmt.Println("[udp]invalid domain name.")
+ return
+ }
+ dIP, err := net.ResolveIPAddr("ip", name) // carefully with const type
+ if err != nil {
+ Debug.Printf("[udp]failed to resolve domain name: %s\n", string(receive[idDm0:idDm0+receive[idDmLen]]))
+ return
+ }
+ dstIP = dIP.IP
+ default:
+ Debug.Printf("[udp]addrType %d not supported", addrType)
+ return
+ }
+ dst := &net.UDPAddr{
+ IP: dstIP,
+ Port: int(binary.BigEndian.Uint16(receive[reqLen-2 : reqLen])),
+ }
+ if _, ok := reqList.Get(dst.String()); !ok {
+ req := make([]byte, reqLen)
+ copy(req, receive)
+ reqList.Put(dst.String(), req)
+ }
+
+ remote, exist, err := natlist.Get(src.String())
+ if err != nil {
+ return
+ }
+ if !exist {
+ Debug.Printf("[udp]new client %s->%s via %s ota=%v\n", src, dst, remote.LocalAddr(), ota)
+ go func() {
+ if compatiblemode {
+ Pipeloop(handle.ForceOTA(), src, remote)
+ } else {
+ Pipeloop(handle, src, remote)
+ }
+
+ natlist.Delete(src.String())
+ }()
+ } else {
+ Debug.Printf("[udp]using cached client %s->%s via %s ota=%v\n", src, dst, remote.LocalAddr(), ota)
+ }
+ if remote == nil {
+ fmt.Println("WTF")
+ }
+ remote.SetDeadline(time.Now().Add(udpTimeout))
+ _, err = remote.WriteTo(receive[reqLen:n], dst)
+ if err != nil {
+ if ne, ok := err.(*net.OpError); ok && (ne.Err == syscall.EMFILE || ne.Err == syscall.ENFILE) {
+ // log too many open file error
+ // EMFILE is process reaches open file limits, ENFILE is system limit
+ Debug.Println("[udp]write error:", err)
+ } else {
+ Debug.Println("[udp]error connecting to:", dst, err)
+ }
+ if conn := natlist.Delete(src.String()); conn != nil {
+ conn.Close()
+ }
+ }
+ // Pipeloop
+ return
+}
+
+func ReadAndHandleUDPReq(c *SecurePacketConn) error {
+ buf := leakyBuf.Get()
+ n, src, err := c.ReadFrom(buf[0:])
+ if err != nil {
+ return err
+ }
+ go handleUDPConnection(c, n, src, buf)
+ return nil
+}
diff --git a/cmd/gost/vendor/github.com/shadowsocks/shadowsocks-go/shadowsocks/util.go b/cmd/gost/vendor/github.com/shadowsocks/shadowsocks-go/shadowsocks/util.go
index 21c01c6..671fe6c 100644
--- a/cmd/gost/vendor/github.com/shadowsocks/shadowsocks-go/shadowsocks/util.go
+++ b/cmd/gost/vendor/github.com/shadowsocks/shadowsocks-go/shadowsocks/util.go
@@ -1,16 +1,16 @@
package shadowsocks
import (
- "errors"
- "fmt"
- "os"
"crypto/hmac"
"crypto/sha1"
"encoding/binary"
+ "errors"
+ "fmt"
+ "os"
)
func PrintVersion() {
- const version = "1.1.5"
+ const version = "1.2.0"
fmt.Println("shadowsocks-go version", version)
}
@@ -57,4 +57,4 @@ func (flag *ClosedFlag) SetClosed() {
func (flag *ClosedFlag) IsClosed() bool {
return flag.flag
-}
\ No newline at end of file
+}
diff --git a/cmd/gost/vendor/vendor.json b/cmd/gost/vendor/vendor.json
index d380a97..25bb639 100644
--- a/cmd/gost/vendor/vendor.json
+++ b/cmd/gost/vendor/vendor.json
@@ -2,6 +2,12 @@
"comment": "",
"ignore": "test",
"package": [
+ {
+ "checksumSHA1": "IFJyJgPCjumDG37lEb0lyRBBGZE=",
+ "path": "github.com/Yawning/chacha20",
+ "revision": "c91e78db502ff629614837aacb7aa4efa61c651a",
+ "revisionTime": "2016-04-30T09:49:23Z"
+ },
{
"checksumSHA1": "QPs3L3mjPoi+a9GJCjW8HhyJczM=",
"path": "github.com/codahale/chacha20",
@@ -15,10 +21,10 @@
"revisionTime": "2017-01-19T05:34:58Z"
},
{
- "checksumSHA1": "b0uHAM/lCGCJ9GeKfClvrMMWXQM=",
+ "checksumSHA1": "idpL1fpHpfntk74IVfWtkP1PMZs=",
"path": "github.com/ginuerzh/gost",
- "revision": "358f57add6087d77b1d978e92e2f7c8073c2f544",
- "revisionTime": "2017-01-21T03:14:59Z"
+ "revision": "321b03712af504981d35a47c50c2cfe4dd788a9d",
+ "revisionTime": "2017-01-21T03:16:33Z"
},
{
"checksumSHA1": "URsJa4y/sUUw/STmbeYx9EKqaYE=",
@@ -153,10 +159,10 @@
"revisionTime": "2016-10-02T05:25:12Z"
},
{
- "checksumSHA1": "o0WHRL8mNIhfsoWlzhdJ8du6+C8=",
+ "checksumSHA1": "MRsfMrdZwnnCTfIzT3czcj0lb0s=",
"path": "github.com/shadowsocks/shadowsocks-go/shadowsocks",
- "revision": "5c9897ecdf623f385ccb8c2c78e32c5256961b41",
- "revisionTime": "2016-06-15T15:25:08Z"
+ "revision": "97a5c71f80ba5f5b3e549f14a619fe557ff4f3c9",
+ "revisionTime": "2017-01-21T20:35:16Z"
},
{
"checksumSHA1": "JsJdKXhz87gWenMwBeejTOeNE7k=",
diff --git a/forward.go b/forward.go
index 750cb50..ca76904 100644
--- a/forward.go
+++ b/forward.go
@@ -63,15 +63,15 @@ func (s *TcpForwardServer) handleTcpForward(conn net.Conn, raddr net.Addr) {
}
type packet struct {
- srcAddr *net.UDPAddr // src address
- dstAddr *net.UDPAddr // dest address
+ srcAddr string // src address
+ dstAddr string // dest address
data []byte
}
type cnode struct {
chain *ProxyChain
conn net.Conn
- srcAddr, dstAddr *net.UDPAddr
+ srcAddr, dstAddr string
rChan, wChan chan *packet
err error
ttl time.Duration
@@ -146,13 +146,9 @@ func (node *cnode) run() {
timer.Reset(node.ttl)
glog.V(LDEBUG).Infof("[udp] %s <<< %s : length %d", node.srcAddr, addr, n)
- if node.dstAddr.String() != addr.String() {
- glog.V(LWARNING).Infof("[udp] %s <- %s : dst-addr mismatch (%s)", node.srcAddr, node.dstAddr, addr)
- break
- }
select {
// swap srcAddr with dstAddr
- case node.rChan <- &packet{srcAddr: node.dstAddr, dstAddr: node.srcAddr, data: b[:n]}:
+ case node.rChan <- &packet{srcAddr: addr.String(), dstAddr: node.srcAddr, data: b[:n]}:
case <-time.After(time.Second * 3):
glog.V(LWARNING).Infof("[udp] %s <- %s : %s", node.srcAddr, node.dstAddr, "recv queue is full, discard")
}
@@ -169,13 +165,9 @@ func (node *cnode) run() {
timer.Reset(node.ttl)
glog.V(LDEBUG).Infof("[udp-tun] %s <<< %s : length %d", node.srcAddr, dgram.Header.Addr.String(), len(dgram.Data))
- if dgram.Header.Addr.String() != node.dstAddr.String() {
- glog.V(LWARNING).Infof("[udp-tun] %s <- %s : dst-addr mismatch (%s)", node.srcAddr, node.dstAddr, dgram.Header.Addr)
- break
- }
select {
// swap srcAddr with dstAddr
- case node.rChan <- &packet{srcAddr: node.dstAddr, dstAddr: node.srcAddr, data: dgram.Data}:
+ case node.rChan <- &packet{srcAddr: dgram.Header.Addr.String(), dstAddr: node.srcAddr, data: dgram.Data}:
case <-time.After(time.Second * 3):
glog.V(LWARNING).Infof("[udp-tun] %s <- %s : %s", node.srcAddr, node.dstAddr, "recv queue is full, discard")
}
@@ -187,9 +179,15 @@ func (node *cnode) run() {
for pkt := range node.wChan {
timer.Reset(node.ttl)
+ dstAddr, err := net.ResolveUDPAddr("udp", pkt.dstAddr)
+ if err != nil {
+ glog.V(LWARNING).Infof("[udp] %s -> %s : %s", pkt.srcAddr, pkt.dstAddr, err)
+ continue
+ }
+
switch c := node.conn.(type) {
case *net.UDPConn:
- if _, err := c.WriteToUDP(pkt.data, pkt.dstAddr); err != nil {
+ if _, err := c.WriteToUDP(pkt.data, dstAddr); err != nil {
glog.V(LWARNING).Infof("[udp] %s -> %s : %s", pkt.srcAddr, pkt.dstAddr, err)
node.err = err
errChan <- err
@@ -198,7 +196,7 @@ func (node *cnode) run() {
glog.V(LDEBUG).Infof("[udp] %s >>> %s : length %d", pkt.srcAddr, pkt.dstAddr, len(pkt.data))
default:
- dgram := gosocks5.NewUDPDatagram(gosocks5.NewUDPHeader(uint16(len(pkt.data)), 0, ToSocksAddr(pkt.dstAddr)), pkt.data)
+ dgram := gosocks5.NewUDPDatagram(gosocks5.NewUDPHeader(uint16(len(pkt.data)), 0, ToSocksAddr(dstAddr)), pkt.data)
if err := dgram.Write(c); err != nil {
glog.V(LWARNING).Infof("[udp-tun] %s -> %s : %s", pkt.srcAddr, pkt.dstAddr, err)
node.err = err
@@ -255,7 +253,7 @@ func (s *UdpForwardServer) ListenAndServe() error {
}
select {
- case ch <- &packet{srcAddr: addr, dstAddr: raddr, data: b[:n]}:
+ case ch <- &packet{srcAddr: addr.String(), dstAddr: raddr.String(), data: b[:n]}:
case <-time.After(time.Second * 3):
glog.V(LWARNING).Infof("[udp] %s -> %s : %s", addr, raddr, "send queue is full, discard")
}
@@ -264,7 +262,12 @@ func (s *UdpForwardServer) ListenAndServe() error {
// start recv queue
go func(ch <-chan *packet) {
for pkt := range ch {
- if _, err := conn.WriteToUDP(pkt.data, pkt.dstAddr); err != nil {
+ dstAddr, err := net.ResolveUDPAddr("udp", pkt.dstAddr)
+ if err != nil {
+ glog.V(LWARNING).Infof("[udp] %s <- %s : %s", pkt.dstAddr, pkt.srcAddr, err)
+ continue
+ }
+ if _, err := conn.WriteToUDP(pkt.data, dstAddr); err != nil {
glog.V(LWARNING).Infof("[udp] %s <- %s : %s", pkt.dstAddr, pkt.srcAddr, err)
return
}
@@ -285,7 +288,7 @@ func (s *UdpForwardServer) ListenAndServe() error {
}
}
- node, ok := m[pkt.srcAddr.String()]
+ node, ok := m[pkt.srcAddr]
if !ok {
node = &cnode{
chain: s.Base.Chain,
@@ -295,7 +298,7 @@ func (s *UdpForwardServer) ListenAndServe() error {
wChan: make(chan *packet, 32),
ttl: time.Duration(s.TTL) * time.Second,
}
- m[pkt.srcAddr.String()] = node
+ m[pkt.srcAddr] = node
go node.run()
glog.V(LINFO).Infof("[udp] %s -> %s : new client (%d)", pkt.srcAddr, pkt.dstAddr, len(m))
}
diff --git a/gost.go b/gost.go
index 895d939..6c043e7 100644
--- a/gost.go
+++ b/gost.go
@@ -11,7 +11,7 @@ import (
)
const (
- Version = "2.3"
+ Version = "2.4-dev"
)
// Log level for glog
diff --git a/node.go b/node.go
index 88d7153..e72d0bb 100644
--- a/node.go
+++ b/node.go
@@ -71,7 +71,7 @@ func ParseProxyNode(s string) (node ProxyNode, err error) {
}
switch node.Transport {
- case "ws", "wss", "tls", "http2", "ssu", "quic", "kcp", "redirect":
+ case "ws", "wss", "tls", "http2", "quic", "kcp", "redirect", "ssu":
case "https":
node.Protocol = "http"
node.Transport = "tls"
diff --git a/server.go b/server.go
index 74b3861..63ff9a0 100644
--- a/server.go
+++ b/server.go
@@ -32,7 +32,7 @@ func NewProxyServer(node ProxyNode, chain *ProxyChain, config *tls.Config) *Prox
var cipher *ss.Cipher
var ota bool
- if node.Protocol == "ss" {
+ if node.Protocol == "ss" || node.Transport == "ssu" {
var err error
var method, password string
@@ -98,8 +98,6 @@ func (s *ProxyServer) Serve() error {
return NewRTcpForwardServer(s).Serve()
case "rudp": // Remote UDP port forwarding
return NewRUdpForwardServer(s).Serve()
- case "ssu": // TODO: shadowsocks udp relay
- return NewShadowUdpServer(s).ListenAndServe()
case "quic":
return NewQuicServer(s).ListenAndServeTLS(s.TLSConfig)
case "kcp":
@@ -118,6 +116,12 @@ func (s *ProxyServer) Serve() error {
return NewKCPServer(s, config).ListenAndServe()
case "redirect":
return NewRedsocksTCPServer(s).ListenAndServe()
+ case "ssu": // shadowsocks udp relay
+ ttl, _ := strconv.Atoi(s.Node.Get("ttl"))
+ if ttl <= 0 {
+ ttl = DefaultTTL
+ }
+ return NewShadowUdpServer(s, ttl).ListenAndServe()
default:
ln, err = net.Listen("tcp", node.Addr)
}
diff --git a/ss.go b/ss.go
index 12bc167..81d48af 100644
--- a/ss.go
+++ b/ss.go
@@ -5,6 +5,7 @@ import (
"encoding/binary"
"errors"
"fmt"
+ "github.com/ginuerzh/gosocks5"
"github.com/golang/glog"
ss "github.com/shadowsocks/shadowsocks-go/shadowsocks"
"io"
@@ -65,47 +66,6 @@ func (s *ShadowServer) Serve() {
glog.V(LINFO).Infof("[ss] %s >-< %s", s.conn.RemoteAddr(), addr)
}
-type ShadowUdpServer struct {
- Base *ProxyServer
- Handler func(conn *net.UDPConn, addr *net.UDPAddr, data []byte)
-}
-
-func NewShadowUdpServer(base *ProxyServer) *ShadowUdpServer {
- return &ShadowUdpServer{Base: base}
-}
-
-func (s *ShadowUdpServer) ListenAndServe() error {
- laddr, err := net.ResolveUDPAddr("udp", s.Base.Node.Addr)
- if err != nil {
- return err
- }
- lconn, err := net.ListenUDP("udp", laddr)
- if err != nil {
- return err
- }
- defer lconn.Close()
-
- if s.Handler == nil {
- s.Handler = s.HandleConn
- }
-
- for {
- b := make([]byte, LargeBufferSize)
- n, addr, err := lconn.ReadFromUDP(b)
- if err != nil {
- glog.V(LWARNING).Infoln(err)
- continue
- }
-
- go s.Handler(lconn, addr, b[:n])
- }
-}
-
-// TODO: shadowsocks udp relay handler
-func (s *ShadowUdpServer) HandleConn(conn *net.UDPConn, addr *net.UDPAddr, data []byte) {
-
-}
-
// This function is copied from shadowsocks library with some modification.
func (s *ShadowServer) getRequest() (host string, ota bool, err error) {
// buf size should at least have the same size with the largest possible
@@ -276,3 +236,109 @@ func (c *shadowConn) SetReadDeadline(t time.Time) error {
func (c *shadowConn) SetWriteDeadline(t time.Time) error {
return c.conn.SetWriteDeadline(t)
}
+
+type ShadowUdpServer struct {
+ Base *ProxyServer
+ TTL int
+}
+
+func NewShadowUdpServer(base *ProxyServer, ttl int) *ShadowUdpServer {
+ return &ShadowUdpServer{Base: base, TTL: ttl}
+}
+
+func (s *ShadowUdpServer) ListenAndServe() error {
+ laddr, err := net.ResolveUDPAddr("udp", s.Base.Node.Addr)
+ if err != nil {
+ return err
+ }
+ lconn, err := net.ListenUDP("udp", laddr)
+ if err != nil {
+ return err
+ }
+ defer lconn.Close()
+
+ conn := ss.NewSecurePacketConn(lconn, s.Base.cipher.Copy(), true) // force OTA on
+
+ rChan, wChan := make(chan *packet, 128), make(chan *packet, 128)
+ // start send queue
+ go func(ch chan<- *packet) {
+ for {
+ b := make([]byte, MediumBufferSize)
+ n, addr, err := conn.ReadFrom(b[3:]) // add rsv and frag fields to make it the standard SOCKS5 UDP datagram
+ if err != nil {
+ glog.V(LWARNING).Infof("[ssu] %s -> %s : %s", addr, laddr, err)
+ continue
+ }
+
+ if b[3]&ss.OneTimeAuthMask > 0 {
+ glog.V(LWARNING).Infof("[ssu] %s -> %s : client does not support OTA", addr, laddr)
+ continue
+ }
+ b[3] &= ss.AddrMask
+
+ dgram, err := gosocks5.ReadUDPDatagram(bytes.NewReader(b[:n+3]))
+ if err != nil {
+ glog.V(LWARNING).Infof("[ssu] %s -> %s : %s", addr, laddr, err)
+ continue
+ }
+
+ select {
+ case ch <- &packet{srcAddr: addr.String(), dstAddr: dgram.Header.Addr.String(), data: b[:n+3]}:
+ case <-time.After(time.Second * 3):
+ glog.V(LWARNING).Infof("[ssu] %s -> %s : %s", addr, dgram.Header.Addr.String(), "send queue is full, discard")
+ }
+ }
+ }(wChan)
+ // start recv queue
+ go func(ch <-chan *packet) {
+ for pkt := range ch {
+ dstAddr, err := net.ResolveUDPAddr("udp", pkt.dstAddr)
+ if err != nil {
+ glog.V(LWARNING).Infof("[ssu] %s <- %s : %s", pkt.dstAddr, pkt.srcAddr, err)
+ continue
+ }
+ if _, err := conn.WriteTo(pkt.data, dstAddr); err != nil {
+ glog.V(LWARNING).Infof("[ssu] %s <- %s : %s", pkt.dstAddr, pkt.srcAddr, err)
+ return
+ }
+ }
+ }(rChan)
+
+ // mapping client to node
+ m := make(map[string]*cnode)
+
+ // start dispatcher
+ for pkt := range wChan {
+ // clear obsolete nodes
+ for k, node := range m {
+ if node != nil && node.err != nil {
+ close(node.wChan)
+ delete(m, k)
+ glog.V(LINFO).Infof("[ssu] clear node %s", k)
+ }
+ }
+
+ node, ok := m[pkt.srcAddr]
+ if !ok {
+ node = &cnode{
+ chain: s.Base.Chain,
+ srcAddr: pkt.srcAddr,
+ dstAddr: pkt.dstAddr,
+ rChan: rChan,
+ wChan: make(chan *packet, 32),
+ ttl: time.Duration(s.TTL) * time.Second,
+ }
+ m[pkt.srcAddr] = node
+ go node.run()
+ glog.V(LINFO).Infof("[ssu] %s -> %s : new client (%d)", pkt.srcAddr, pkt.dstAddr, len(m))
+ }
+
+ select {
+ case node.wChan <- pkt:
+ case <-time.After(time.Second * 3):
+ glog.V(LWARNING).Infof("[ssu] %s -> %s : %s", pkt.srcAddr, pkt.dstAddr, "node send queue is full, discard")
+ }
+ }
+
+ return nil
+}