// // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. // kernel_SSE.h: a collection of Intel SSE optimized kernels. // Check in kernel_default.h which one(s) are actually used by default. // Others are mere experiments; they are still covered by tests // in case they might be useful some day. // #ifndef GEMMLOWP_INTERNAL_KERNEL_AVX_H_ #define GEMMLOWP_INTERNAL_KERNEL_AVX_H_ #include "kernel.h" #include #include namespace gemmlowp { #ifdef GEMMLOWP_AVX2_64 struct AVX2_64_Kernel24x8Depth2 : KernelBase { typedef KernelFormat, 3>, KernelSideFormat, 1>> Format; const char *Name() const override { return "AVX, 24x8, depth 2"; } void Run(std::int32_t *dst_ptr, std::size_t dst_row_stride, std::size_t dst_col_stride, const std::uint8_t *lhs_ptr, const std::uint8_t *rhs_ptr, std::size_t start_depth, std::size_t run_depth) const override { ScopedProfilingLabel label("optimized kernel"); assert(dst_row_stride == 1); const std::int64_t run_depth_cells = run_depth / Format::kDepth; const std::int64_t dst_col_stride_q = dst_col_stride; /* Main loop */ // A 2x8 cell of Rhs is stored in 16bit in ymm1 . // A 24x2 block of 3 8x2 cells Lhs is stored in 16bit in ymm0, replaced // every Iteration. // A 8x8 block of accumulators is stored in 32bit in xmm4--xmm15. // // +-------+-------+-------+-------+ // |ymm1[0] |ymm2[2] | // Rhs +-------+---------------+-------+ // |ymm1[1] |ymm1[4] | // +-------+-------+-------+-------+ // // | | | | | // // Lhs | | | | | // // +--+--+ - - - - +-------+-------+-------+-------+ // |ymm0 | | ymm4 | ymm5 | ymm6 | ymm7 | // |ymm0 | (Iter1) | ymm4 | ymm5 | ymm6 | ymm7 | // |ymm0 | | ymm4 | ymm5 | ymm6 | ymm7 | // |ymm0 | | ymm4 | ymm5 | ymm6 | ymm7 | // +--+--+ - - - - +-------+-------+-------+-------+ // |ymm0 | | ymm8 | ymm9 | ymm10 | ymm11 | // |ymm0 | (Iter2) | ymm8 | ymm9 | ymm10 | ymm11 | // |ymm0 | | ymm8 | ymm9 | ymm10 | ymm11 | // |ymm0 | | ymm8 | ymm9 | ymm10 | ymm11 | // +--+--+ - - - - +-------+-------+-------+-------+ // |ymm0 | | ymm12 | ymm13 | ymm14 | ymm15 | // |ymm0 | (Iter3) | ymm12 | ymm13 | ymm14 | ymm15 | // |ymm0 | | ymm12 | ymm13 | ymm14 | ymm15 | // |ymm0 | | ymm12 | ymm13 | ymm14 | ymm15 | // +--+--+ - - - - +-------+-------+-------+-------+ // // Accumulator asm volatile( // Set registers for destination "movq %[dst_col_stride_q], %%r12\n\t" // stride is r12 "shlq $2, %%r12\n\t" // set stride dword "leaq (%%r12,%%r12,0x2), %%r13\n\t" // load stride aligned r13 // Set accumulators to zero. "vpxor %%ymm4, %%ymm4, %%ymm4 \n\t" // zero accumulators "vpxor %%ymm5, %%ymm5, %%ymm5 \n\t" // zero accumulators "vpxor %%ymm6, %%ymm6, %%ymm6 \n\t" // zero accumulators "vpxor %%ymm7, %%ymm7, %%ymm7 \n\t" // zero accumulators "vpxor %%ymm8, %%ymm8, %%ymm8 \n\t" // zero accumulators "vpxor %%ymm9, %%ymm9, %%ymm9 \n\t" // zero accumulators "vpxor %%ymm10, %%ymm10, %%ymm10\n\t" // zero accumulators "vpxor %%ymm11, %%ymm11, %%ymm11\n\t" // zero accumulators "vpxor %%ymm12, %%ymm12, %%ymm12\n\t" // zero accumulators "vpxor %%ymm13, %%ymm13, %%ymm13\n\t" // zero accumulators "vpxor %%ymm14, %%ymm14, %%ymm14\n\t" // zero accumulators "vpxor %%ymm15, %%ymm15, %%ymm15\n\t" // zero accumulators "movq %[run_depth_cells], %%r14 \n\t" // load cell depth r14 "subq $2, %%r14 \n\t" // cell depth is 2 "js outerLoop1%= \n\t" // outerloop for matrix // Loop for K unrolled by 4 "outerLoop2%=: \n\t" // outer loop unroll // K = 0,1,2,3 // RHS cell to ymm1 // lower half "vpmovzxbw (%[rhs_ptr]), %%ymm1 \n\t" // mov rhs to ymm1 "vpermq $0x44,%%ymm1, %%ymm1 \n\t" // LHS cell elements 0 and 1 "vpmovzxbw 0x00(%[lhs_ptr]), %%ymm0\n\t" // mov lhs to ymm0 "vpshufd $0x00,%%ymm1,%%ymm2 \n\t" // move rhs 0 element to all ymm2 "vpshufd $0x55,%%ymm1,%%ymm3 \n\t" // move rhs 1 element to all ymm3 "vpmaddwd %%ymm0, %%ymm2, %%ymm2 \n\t" // mul add lhs rhs0 into ymm2 "vpmaddwd %%ymm0, %%ymm3, %%ymm3 \n\t" // mul add lhs rhs1 into ymm3 "vpaddd %%ymm2, %%ymm4, %%ymm4 \n\t" // add muladd lhs + rhs0 into ymm4 "vpaddd %%ymm3, %%ymm5, %%ymm5 \n\t" // add muladd lhs + rhs1 into ymm5 // LHS cell elements 2 and 3 "vpshufd $0xaa, %%ymm1, %%ymm2 \n\t" // move rhs 2 element to all ymm2 "vpmaddwd %%ymm0, %%ymm2, %%ymm2 \n\t" // mul add lhs rh3 into ymm2 "vpshufd $0xff,%%ymm1,%%ymm3 \n\t" // mov rhs 3 element into all ymm3 "vpmaddwd %%ymm0, %%ymm3, %%ymm3 \n\t" // mul add lhs rh4 into ymm3 "vpaddd %%ymm2, %%ymm6, %%ymm6 \n\t" // add muladd lhs + rhs2 into ymm6 "vpaddd %%ymm3, %%ymm7, %%ymm7 \n\t" // add muladd lhs + rhs3 into ymm7 // cache prefect lhs //see if it works better? //"prefetcht0 0x80(%[lhs_ptr]) \n\t" //prefetch cache lines "vpmovzxbw (%[rhs_ptr]), %%ymm1 \n\t" // mov rhs to ymm1 "vpermq $0x44,%%ymm1, %%ymm1 \n\t" // K = 5,6,7,8 // next LHS cell elements 0 and 1 "vpmovzxbw 0x10(%[lhs_ptr]), %%ymm0 \n\t" // mov lhs to ymm0 "vpshufd $0x00,%%ymm1,%%ymm2 \n\t" // mov rhs 0 element to all ymm2 "vpshufd $0x55,%%ymm1,%%ymm3 \n\t" // mov rhs 1 element to all ymm3 "vpmaddwd %%ymm0, %%ymm2, %%ymm2 \n\t" // mul add lhs rhs0 into ymm2 "vpmaddwd %%ymm0, %%ymm3, %%ymm3 \n\t" // mul add lhs rhs1 into ymm3 "vpaddd %%ymm2, %%ymm8, %%ymm8 \n\t" // add muladd lhs + rhs0 into ymm8 "vpaddd %%ymm3, %%ymm9, %%ymm9 \n\t" // add muladd lhs + rhs1 into ymm9 // next LHS cell elements 2 and 3 "vpshufd $0xaa,%%ymm1,%%ymm2 \n\t" // mov rhs 2 element to all ymm2 "vpshufd $0xff,%%ymm1,%%ymm3 \n\t" // mov rhs 3 element to all ymm3 "vpmaddwd %%ymm0, %%ymm2, %%ymm2 \n\t" // mul add lhs rhs2 into ymm2 "vpmaddwd %%ymm0, %%ymm3, %%ymm3 \n\t" // mul add lhs rhs3 into ymm3 "vpaddd %%ymm2, %%ymm10, %%ymm10 \n\t" // add muladd lhs + rhs2 into ymm10 "vpaddd %%ymm3, %%ymm11, %%ymm11 \n\t" // add muladd lhs + rhs3 into ymm11 // rhs lower half "vpmovzxbw (%[rhs_ptr]), %%ymm1 \n\t" // mov rhs to ymm1 "vpermq $0x44,%%ymm1, %%ymm1 \n\t" // duplcate lower 16 // next LHS cell elements 0 and 1 "vpmovzxbw 0x20(%[lhs_ptr]), %%ymm0 \n\t" // mov lhs to ymm0 "vpshufd $0x00,%%ymm1,%%ymm2 \n\t" // mov rhs 0 element to all ymm2 "vpshufd $0x55,%%ymm1,%%ymm3 \n\t" // mov rhs 1 element to all ymm3 "vpmaddwd %%ymm0, %%ymm2, %%ymm2 \n\t" // mul add lhs rhs0 into ymm2 "vpmaddwd %%ymm0, %%ymm3, %%ymm3 \n\t" // mul add lhs rhs1 into ymm3 "vpaddd %%ymm2, %%ymm12, %%ymm12 \n\t" // add muladd lhs + rhs0 into ymm8 "vpaddd %%ymm3, %%ymm13, %%ymm13 \n\t" // add muladd lhs + rhs1 into ymm9 // cache prefetch rhs //see if it works better? //"prefetcht0 0x80(%[rhs_ptr]) \n\t" // next LHS cell elements 2 and 3 "vpshufd $0xaa,%%ymm1,%%ymm2 \n\t" // mov rhs 2 element to all ymm2 "vpshufd $0xff,%%ymm1,%%ymm3 \n\t" // mov rhs 3 element to all ymm3 "vpmaddwd %%ymm0, %%ymm2, %%ymm2 \n\t" // mul add lhs rhs2 into ymm2 "vpmaddwd %%ymm0, %%ymm3, %%ymm3 \n\t" // mul add lhs rhs3 into ymm3 "vpaddd %%ymm2, %%ymm14, %%ymm14 \n\t" // add muladd lhs + rhs2 into ymm10 "vpaddd %%ymm3, %%ymm15, %%ymm15 \n\t" // add muladd lhs + rhs3 into ymm11 // current result in ymm4, ymm5, ymm6, ymm7, ymm8, ymm9, ymm10 ymm11 ymm12 ymm13 ymm14 ymm15 // rhs+10 lower half "vpmovzxbw 0x08(%[rhs_ptr]), %%ymm1 \n\t" // mov rhs to ymm1 "vpermq $0x44,%%ymm1, %%ymm1 \n\t" // next LHS cell elements 0 and 1 "vpmovzxbw 0x30(%[lhs_ptr]), %%ymm0 \n\t" // mov lhs to ymm0 "vpshufd $0x00,%%ymm1,%%ymm2 \n\t" // move rhs 0 element to ymm2 "vpshufd $0x55,%%ymm1,%%ymm3 \n\t" // move rhs 1 element to ymm3 "vpmaddwd %%ymm0, %%ymm2, %%ymm2 \n\t" // muladd lhs rhs0 into ymm2 "vpmaddwd %%ymm0, %%ymm3, %%ymm3 \n\t" // muladd lhs rhs1 into ymm3 "vpaddd %%ymm2, %%ymm4, %%ymm4 \n\t" // accumulate to ymm4 "vpaddd %%ymm3, %%ymm5, %%ymm5 \n\t" // accumulate to ymm5 // next LHS cell elements 2 and 3 "vpshufd $0xaa,%%ymm1,%%ymm2 \n\t" // mov rhs 2 element to ymm2 "vpshufd $0xff,%%ymm1,%%ymm3 \n\t" // mov rhs 3 element to ymm2 "vpmaddwd %%ymm0, %%ymm2, %%ymm2 \n\t" // mul add lhs rhs2 into ymm2 "vpmaddwd %%ymm0, %%ymm3, %%ymm3 \n\t" // mull add lhs rhs3 into ymm3 "vpaddd %%ymm2, %%ymm6, %%ymm6 \n\t" // add lhs rhs2 to ymm6 "vpaddd %%ymm3, %%ymm7, %%ymm7 \n\t" // add lhs rhs3 to ymm7 // rhs+10 lower half "vpmovzxbw 0x08(%[rhs_ptr]), %%ymm1 \n\t" // mov rhs to ymm1 "vpermq $0x44,%%ymm1, %%ymm1 \n\t" // next LHS cell elements 4 and 5 "vpmovzxbw 0x40(%[lhs_ptr]), %%ymm0 \n\t" // mov lhs to ymm0 "vpshufd $0x00,%%ymm1,%%ymm2 \n\t" // move rhs 0 element to ymm2 "vpshufd $0x55,%%ymm1,%%ymm3 \n\t" // move rhs 1 element to ymm3 "vpmaddwd %%ymm0, %%ymm2, %%ymm2 \n\t" // muladd lhs rhs0 into ymm2 "vpmaddwd %%ymm0, %%ymm3, %%ymm3 \n\t" // muladd lhs rhs1 into ymm3 "vpaddd %%ymm2, %%ymm8, %%ymm8 \n\t" // accumulate to ymm8 "vpaddd %%ymm3, %%ymm9, %%ymm9 \n\t" // accumulate to ymm9 // next LHS cell elements 6 and 7 "vpshufd $0xaa,%%ymm1,%%ymm2 \n\t" // mov rhs 2 element to ymm2 "vpshufd $0xff,%%ymm1,%%ymm3 \n\t" // mov rhs 3 element to ymm2 "vpmaddwd %%ymm0, %%ymm2, %%ymm2 \n\t" // mul add lhs rhs2 into ymm2 "vpmaddwd %%ymm0, %%ymm3, %%ymm3 \n\t" // mull add lhs rhs3 into ymm3 "vpaddd %%ymm2, %%ymm10, %%ymm10 \n\t" // add lhs rhs2 to ymm10 "vpaddd %%ymm3, %%ymm11, %%ymm11 \n\t" // add lhs rhs3 to ymm11 "vpmovzxbw 0x08(%[rhs_ptr]), %%ymm1 \n\t" // mov rhs to ymm1 "vpermq $0x44,%%ymm1, %%ymm1 \n\t" // next LHS cell elements 9 and 10 "vpmovzxbw 0x50(%[lhs_ptr]), %%ymm0 \n\t" // mov lhs to ymm0 "vpshufd $0x00,%%ymm1,%%ymm2 \n\t" // move rhs 0 element to ymm2 "vpshufd $0x55,%%ymm1,%%ymm3 \n\t" // move rhs 1 element to ymm3 "vpmaddwd %%ymm0, %%ymm2, %%ymm2 \n\t" // muladd lhs rhs0 into ymm2 "vpmaddwd %%ymm0, %%ymm3, %%ymm3 \n\t" // muladd lhs rhs1 into ymm3 "vpaddd %%ymm2, %%ymm12, %%ymm12 \n\t" // accumulate to ymm12 "vpaddd %%ymm3, %%ymm13, %%ymm13 \n\t" // accumulate to ymm13 // next LHS cell elements 11 and 12 "vpshufd $0xaa,%%ymm1,%%ymm2 \n\t" // mov rhs 2 element to ymm2 "vpshufd $0xff,%%ymm1,%%ymm3 \n\t" // mov rhs 3 element to ymm2 "vpmaddwd %%ymm0, %%ymm2, %%ymm2 \n\t" // mul add lhs rhs2 into ymm2 "vpmaddwd %%ymm0, %%ymm3, %%ymm3 \n\t" // mull add lhs rhs3 into ymm3 "vpaddd %%ymm2, %%ymm14, %%ymm14 \n\t" // add lhs rhs2 to ymm14 "vpaddd %%ymm3, %%ymm15, %%ymm15 \n\t" // add lhs rhs3 to ymm15 // completed rhs+10 "addq $0x60, %[lhs_ptr] \n\t" // increment stride lhs "addq $0x10, %[rhs_ptr] \n\t" // increment stride rhs "subq $2, %[run_depth_cells] \n\t" "ja outerLoop2%= \n\t" "movq %[run_depth_cells], %%r14 \n\t" "decq %%r14 \n\t" "js finish%= \n\t" // Loop for K unrolled by 2 "outerLoop1%=: \n\t" // rhs lower "vpmovzxbw (%[rhs_ptr]), %%ymm1 \n\t" // get rhs into ymm1 "vpermq $0x44,%%ymm1, %%ymm1 \n\t" // LHS cell "vpmovzxbw (%[lhs_ptr]), %%ymm0 \n\t" // lhs in into ymm0 "vpshufd $0x00,%%ymm1,%%ymm2 \n\t" // rhs element 0 into ymm2 "vpshufd $0x55,%%ymm1,%%ymm3 \n\t" // rhs element 1 into ymm3 "vpmaddwd %%ymm0, %%ymm2, %%ymm2 \n\t" // muladd lhs rhs element 0 ymm2 "vpmaddwd %%ymm0, %%ymm3, %%ymm3 \n\t" // muladd lhs rhs element 1 ymm3 "vpaddd %%ymm2, %%ymm4, %%ymm4 \n\t" // acc element 0 ymm4 "vpaddd %%ymm3, %%ymm5, %%ymm5 \n\t" // acc element 1 ymm5 "vpshufd $0xaa,%%ymm1,%%ymm2 \n\t" // rhs element 2 into ymm2 "vpshufd $0xff,%%ymm1,%%ymm3 \n\t" // rhs element 3 into ymm3 "vpmaddwd %%ymm0, %%ymm2, %%ymm2 \n\t" // muladd lhs rhs element 2 ymm2 "vpmaddwd %%ymm0, %%ymm3, %%ymm3 \n\t" // muladd lhs rhs element 3 ymm3 "vpaddd %%ymm2, %%ymm6, %%ymm6 \n\t" // acc element 2 into ymm6 "vpaddd %%ymm3, %%ymm7, %%ymm7 \n\t" // acc element 3 into ymm7 // lhs+10 "vpmovzxbw 0x10(%[lhs_ptr]), %%ymm0 \n\t" // lhs in into ymm0 "vpshufd $0x00, %%ymm1, %%ymm2 \n\t" // rhs element 0 into ymm2 "vpshufd $0x55, %%ymm1, %%ymm3 \n\t" // rhs element 1 into ymm3 "vpmaddwd %%ymm0, %%ymm2, %%ymm2 \n\t" // muladd lhs rhs element 0 ymm2 "vpmaddwd %%ymm0, %%ymm3, %%ymm3 \n\t" // muladd lhs rhs element 1 ymm3 "vpaddd %%ymm2, %%ymm8, %%ymm8 \n\t" // acc element 0 ymm8 "vpaddd %%ymm3, %%ymm9, %%ymm9 \n\t" // acc element 1 ymm9 "vpshufd $0xaa,%%ymm1,%%ymm2 \n\t" // rhs element 2 into ymm2 "vpshufd $0xff,%%ymm1,%%ymm3 \n\t" // rhs element 3 into ymm3 "vpmaddwd %%ymm0, %%ymm2, %%ymm2 \n\t" // muladd lhs rhs element 2 ymm2 "vpmaddwd %%ymm0, %%ymm3, %%ymm3 \n\t" // muladd lhs rhs element 3 ymm3 "vpaddd %%ymm2, %%ymm10, %%ymm10 \n\t" // acc element 2 into ymm10 "vpaddd %%ymm3, %%ymm11, %%ymm11 \n\t" // acc element 3 into ymm11 "vpmovzxbw 0x20(%[lhs_ptr]), %%ymm0 \n\t" "vpshufd $0x00, %%ymm1, %%ymm2 \n\t" // rhs element 0 into ymm2 "vpshufd $0x55, %%ymm1, %%ymm3 \n\t" // rhs element 1 into ymm3 "vpmaddwd %%ymm0, %%ymm2, %%ymm2 \n\t" // muladd lhs rhs element 0 ymm2 "vpmaddwd %%ymm0, %%ymm3, %%ymm3 \n\t" // muladd lhs rhs element 1 ymm3 "vpaddd %%ymm2, %%ymm12, %%ymm12 \n\t" // acc element 0 ymm12 "vpaddd %%ymm3, %%ymm13, %%ymm13 \n\t" // acc element 1 ymm13 "vpshufd $0xaa,%%ymm1,%%ymm2 \n\t" // rhs element 2 into ymm2 "vpshufd $0xff,%%ymm1,%%ymm3 \n\t" // rhs element 3 into ymm3 "vpmaddwd %%ymm0, %%ymm2, %%ymm2 \n\t" // muladd lhs rhs element 2 ymm2 "vpmaddwd %%ymm0, %%ymm3, %%ymm3 \n\t" // muladd lhs rhs element 3 ymm3 "vpaddd %%ymm2, %%ymm14, %%ymm14 \n\t" // acc element 2 into ymm14 "vpaddd %%ymm3, %%ymm15, %%ymm15 \n\t" // acc element 3 into ymm15 // update matrix pointers "addq $0x30, %[lhs_ptr] \n\t" "addq $0x08, %[rhs_ptr] \n\t" "decq %[run_depth_cells] \n\t" "jnz outerLoop1%= \n\t" "finish%=:\n\t" "test %[start_depth], %[start_depth] \n\t" "jz storeDst%= \n\t" "vpaddd 0x00(%[dst_ptr]), %%ymm4, %%ymm4 \n\t" // rhs0 "vpaddd 0x20(%[dst_ptr]), %%ymm8, %%ymm8 \n\t" // rhs0 "vpaddd 0x40(%[dst_ptr]), %%ymm12, %%ymm12 \n\t" // rhs0 "vpaddd 0x00(%[dst_ptr], %%r12, 1) , %%ymm5, %%ymm5 \n\t" // rhs1 "vpaddd 0x20(%[dst_ptr], %%r12, 1) , %%ymm9, %%ymm9 \n\t" // rhs1 "vpaddd 0x40(%[dst_ptr], %%r12, 1) , %%ymm13, %%ymm13 \n\t" // rhs1 "vpaddd 0x00(%[dst_ptr], %%r12, 2) , %%ymm6, %%ymm6 \n\t" // rhs2 "vpaddd 0x20(%[dst_ptr], %%r12, 2) , %%ymm10, %%ymm10 \n\t" // rhs2 "vpaddd 0x40(%[dst_ptr], %%r12, 2) , %%ymm14, %%ymm14 \n\t" // rhs2 "vpaddd 0x00(%[dst_ptr], %%r13, 1) , %%ymm7, %%ymm7 \n\t" // rhs3 "vpaddd 0x20(%[dst_ptr], %%r13, 1) , %%ymm11, %%ymm11 \n\t" // rhs3 "vpaddd 0x40(%[dst_ptr], %%r13, 1) , %%ymm15, %%ymm15 \n\t" // rhs3 "storeDst%=:\n\t" "vmovdqu %%ymm4, 0x00(%[dst_ptr]) \n\t" // rhs0 "vmovdqu %%ymm8, 0x20(%[dst_ptr]) \n\t" // rhs0 "vmovdqu %%ymm12, 0x40(%[dst_ptr]) \n\t" // rhs0 "vmovdqu %%ymm5, 0x00(%[dst_ptr], %%r12, 1) \n\t" // rhs1 "vmovdqu %%ymm9, 0x20(%[dst_ptr], %%r12, 1) \n\t" // rhs1 "vmovdqu %%ymm13, 0x40(%[dst_ptr], %%r12, 1) \n\t" // rhs1 "vmovdqu %%ymm6, 0x00(%[dst_ptr], %%r12, 2) \n\t" // rhs2 "vmovdqu %%ymm10, 0x20(%[dst_ptr], %%r12, 2) \n\t" // rhs2 "vmovdqu %%ymm14, 0x40(%[dst_ptr], %%r12, 2) \n\t" // rhs2 "vmovdqu %%ymm7, 0x00(%[dst_ptr], %%r13, 1) \n\t" // rhs3 "vmovdqu %%ymm11, 0x20(%[dst_ptr], %%r13, 1) \n\t" // rhs3 "vmovdqu %%ymm15, 0x40(%[dst_ptr], %%r13, 1) \n\t" // rhs3 : // outputs [lhs_ptr] "+r"(lhs_ptr), [rhs_ptr] "+r"(rhs_ptr), [dst_ptr] "+r"(dst_ptr) : // inputs [start_depth] "r"(start_depth), [dst_col_stride_q] "r"(dst_col_stride_q), [run_depth_cells] "r"(run_depth_cells) : // clobbers "cc", "memory", "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "%r12", "%r13", "%r14"); } }; #endif } // namespace gemmlowp #endif // GEMMLOWP_INTERNAL_KERNEL_AVX_H_