2 Copyright (C) 2001-2002 Michael Niedermayer (michaelni@gmx.at)
4 This program is free software; you can redistribute it and/or modify
5 it under the terms of the GNU General Public License as published by
6 the Free Software Foundation; either version 2 of the License, or
7 (at your option) any later version.
9 This program is distributed in the hope that it will be useful,
10 but WITHOUT ANY WARRANTY; without even the implied warranty of
11 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 GNU General Public License for more details.
14 You should have received a copy of the GNU General Public License
15 along with this program; if not, write to the Free Software
16 Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
20 #include "../dsputil.h"
21 #include "../mangle.h"
32 #define C0 23170 //cos(i*M_PI/16)*sqrt(2)*(1<<14) + 0.5
33 #define C1 22725 //cos(i*M_PI/16)*sqrt(2)*(1<<14) + 0.5
34 #define C2 21407 //cos(i*M_PI/16)*sqrt(2)*(1<<14) + 0.5
35 #define C3 19266 //cos(i*M_PI/16)*sqrt(2)*(1<<14) + 0.5
37 #define C4 16384 //cos(i*M_PI/16)*sqrt(2)*(1<<14) + 0.5
39 #define C4 16383 //cos(i*M_PI/16)*sqrt(2)*(1<<14) - 0.5
41 #define C5 12873 //cos(i*M_PI/16)*sqrt(2)*(1<<14) + 0.5
42 #define C6 8867 //cos(i*M_PI/16)*sqrt(2)*(1<<14) + 0.5
43 #define C7 4520 //cos(i*M_PI/16)*sqrt(2)*(1<<14) + 0.5
46 #define COL_SHIFT 20 // 6
48 static const uint64_t __attribute__((aligned(8))) wm1010= 0xFFFF0000FFFF0000ULL;
49 static const uint64_t __attribute__((aligned(8))) d40000= 0x0000000000040000ULL;
50 static int16_t __attribute__((aligned(8))) temp[64];
51 static int16_t __attribute__((aligned(8))) coeffs[]= {
52 1<<(ROW_SHIFT-1), 0, 1<<(ROW_SHIFT-1), 0,
53 // 1<<(COL_SHIFT-1), 0, 1<<(COL_SHIFT-1), 0,
54 // 0, 1<<(COL_SHIFT-1-16), 0, 1<<(COL_SHIFT-1-16),
55 1<<(ROW_SHIFT-1), 1, 1<<(ROW_SHIFT-1), 0,
56 // the 1 = ((1<<(COL_SHIFT-1))/C4)<<ROW_SHIFT :)
80 static void unused_var_killer(){
81 int a= wm1010 + d40000;
85 static void inline idctCol (int16_t * col, int16_t *input)
95 int a0, a1, a2, a3, b0, b1, b2, b3;
96 const int C0 = 23170; //cos(i*M_PI/16)*sqrt(2)*(1<<14) + 0.5
97 const int C1 = 22725; //cos(i*M_PI/16)*sqrt(2)*(1<<14) + 0.5
98 const int C2 = 21407; //cos(i*M_PI/16)*sqrt(2)*(1<<14) + 0.5
99 const int C3 = 19266; //cos(i*M_PI/16)*sqrt(2)*(1<<14) + 0.5
100 const int C4 = 16383; //cos(i*M_PI/16)*sqrt(2)*(1<<14) + 0.5
101 const int C5 = 12873; //cos(i*M_PI/16)*sqrt(2)*(1<<14) + 0.5
102 const int C6 = 8867; //cos(i*M_PI/16)*sqrt(2)*(1<<14) + 0.5
103 const int C7 = 4520; //cos(i*M_PI/16)*sqrt(2)*(1<<14) + 0.5
105 if( !(col[8*1] | col[8*2] |col[8*3] |col[8*4] |col[8*5] |col[8*6] | col[8*7])) {
106 col[8*0] = col[8*1] = col[8*2] = col[8*3] = col[8*4] =
107 col[8*5] = col[8*6] = col[8*7] = col[8*0]<<3;
111 col[8*0] = input[8*0 + 0];
112 col[8*1] = input[8*2 + 0];
113 col[8*2] = input[8*0 + 1];
114 col[8*3] = input[8*2 + 1];
115 col[8*4] = input[8*4 + 0];
116 col[8*5] = input[8*6 + 0];
117 col[8*6] = input[8*4 + 1];
118 col[8*7] = input[8*6 + 1];
120 a0 = C4*col[8*0] + C2*col[8*2] + C4*col[8*4] + C6*col[8*6] + (1<<(COL_SHIFT-1));
121 a1 = C4*col[8*0] + C6*col[8*2] - C4*col[8*4] - C2*col[8*6] + (1<<(COL_SHIFT-1));
122 a2 = C4*col[8*0] - C6*col[8*2] - C4*col[8*4] + C2*col[8*6] + (1<<(COL_SHIFT-1));
123 a3 = C4*col[8*0] - C2*col[8*2] + C4*col[8*4] - C6*col[8*6] + (1<<(COL_SHIFT-1));
125 b0 = C1*col[8*1] + C3*col[8*3] + C5*col[8*5] + C7*col[8*7];
126 b1 = C3*col[8*1] - C7*col[8*3] - C1*col[8*5] - C5*col[8*7];
127 b2 = C5*col[8*1] - C1*col[8*3] + C7*col[8*5] + C3*col[8*7];
128 b3 = C7*col[8*1] - C5*col[8*3] + C3*col[8*5] - C1*col[8*7];
130 col[8*0] = (a0 + b0) >> COL_SHIFT;
131 col[8*1] = (a1 + b1) >> COL_SHIFT;
132 col[8*2] = (a2 + b2) >> COL_SHIFT;
133 col[8*3] = (a3 + b3) >> COL_SHIFT;
134 col[8*4] = (a3 - b3) >> COL_SHIFT;
135 col[8*5] = (a2 - b2) >> COL_SHIFT;
136 col[8*6] = (a1 - b1) >> COL_SHIFT;
137 col[8*7] = (a0 - b0) >> COL_SHIFT;
140 static void inline idctRow (int16_t * output, int16_t * input)
144 int a0, a1, a2, a3, b0, b1, b2, b3;
145 const int C0 = 23170; //cos(i*M_PI/16)*sqrt(2)*(1<<14) + 0.5
146 const int C1 = 22725; //cos(i*M_PI/16)*sqrt(2)*(1<<14) + 0.5
147 const int C2 = 21407; //cos(i*M_PI/16)*sqrt(2)*(1<<14) + 0.5
148 const int C3 = 19266; //cos(i*M_PI/16)*sqrt(2)*(1<<14) + 0.5
149 const int C4 = 16383; //cos(i*M_PI/16)*sqrt(2)*(1<<14) + 0.5
150 const int C5 = 12873; //cos(i*M_PI/16)*sqrt(2)*(1<<14) + 0.5
151 const int C6 = 8867; //cos(i*M_PI/16)*sqrt(2)*(1<<14) + 0.5
152 const int C7 = 4520; //cos(i*M_PI/16)*sqrt(2)*(1<<14) + 0.5
163 if( !(row[1] | row[2] |row[3] |row[4] |row[5] |row[6] | row[7]) ) {
164 row[0] = row[1] = row[2] = row[3] = row[4] =
165 row[5] = row[6] = row[7] = row[0]<<3;
177 a0 = C4*row[0] + C2*row[2] + C4*row[4] + C6*row[6] + (1<<(ROW_SHIFT-1));
178 a1 = C4*row[0] + C6*row[2] - C4*row[4] - C2*row[6] + (1<<(ROW_SHIFT-1));
179 a2 = C4*row[0] - C6*row[2] - C4*row[4] + C2*row[6] + (1<<(ROW_SHIFT-1));
180 a3 = C4*row[0] - C2*row[2] + C4*row[4] - C6*row[6] + (1<<(ROW_SHIFT-1));
182 b0 = C1*row[1] + C3*row[3] + C5*row[5] + C7*row[7];
183 b1 = C3*row[1] - C7*row[3] - C1*row[5] - C5*row[7];
184 b2 = C5*row[1] - C1*row[3] + C7*row[5] + C3*row[7];
185 b3 = C7*row[1] - C5*row[3] + C3*row[5] - C1*row[7];
187 row[0] = (a0 + b0) >> ROW_SHIFT;
188 row[1] = (a1 + b1) >> ROW_SHIFT;
189 row[2] = (a2 + b2) >> ROW_SHIFT;
190 row[3] = (a3 + b3) >> ROW_SHIFT;
191 row[4] = (a3 - b3) >> ROW_SHIFT;
192 row[5] = (a2 - b2) >> ROW_SHIFT;
193 row[6] = (a1 - b1) >> ROW_SHIFT;
194 row[7] = (a0 - b0) >> ROW_SHIFT;
207 static inline void idct(int16_t *block)
210 #if 0 //Alternative, simpler variant
212 #define ROW_IDCT(src0, src4, src1, src5, dst, rounder, shift) \
213 "movq " #src0 ", %%mm0 \n\t" /* R4 R0 r4 r0 */\
214 "movq " #src4 ", %%mm1 \n\t" /* R6 R2 r6 r2 */\
215 "movq " #src1 ", %%mm2 \n\t" /* R3 R1 r3 r1 */\
216 "movq " #src5 ", %%mm3 \n\t" /* R7 R5 r7 r5 */\
217 "movq 16(%2), %%mm4 \n\t" /* C4 C4 C4 C4 */\
218 "pmaddwd %%mm0, %%mm4 \n\t" /* C4R4+C4R0 C4r4+C4r0 */\
219 "movq 24(%2), %%mm5 \n\t" /* -C4 C4 -C4 C4 */\
220 "pmaddwd %%mm5, %%mm0 \n\t" /* -C4R4+C4R0 -C4r4+C4r0 */\
221 "movq 32(%2), %%mm5 \n\t" /* C6 C2 C6 C2 */\
222 "pmaddwd %%mm1, %%mm5 \n\t" /* C6R6+C2R2 C6r6+C2r2 */\
223 "movq 40(%2), %%mm6 \n\t" /* -C2 C6 -C2 C6 */\
224 "pmaddwd %%mm6, %%mm1 \n\t" /* -C2R6+C6R2 -C2r6+C6r2 */\
225 "movq 48(%2), %%mm7 \n\t" /* C3 C1 C3 C1 */\
226 "pmaddwd %%mm2, %%mm7 \n\t" /* C3R3+C1R1 C3r3+C1r1 */\
227 #rounder ", %%mm4 \n\t"\
228 "movq %%mm4, %%mm6 \n\t" /* C4R4+C4R0 C4r4+C4r0 */\
229 "paddd %%mm5, %%mm4 \n\t" /* A0 a0 */\
230 "psubd %%mm5, %%mm6 \n\t" /* A3 a3 */\
231 "movq 56(%2), %%mm5 \n\t" /* C7 C5 C7 C5 */\
232 "pmaddwd %%mm3, %%mm5 \n\t" /* C7R7+C5R5 C7r7+C5r5 */\
233 #rounder ", %%mm0 \n\t"\
234 "paddd %%mm0, %%mm1 \n\t" /* A1 a1 */\
235 "paddd %%mm0, %%mm0 \n\t" \
236 "psubd %%mm1, %%mm0 \n\t" /* A2 a2 */\
237 "pmaddwd 64(%2), %%mm2 \n\t" /* -C7R3+C3R1 -C7r3+C3r1 */\
238 "paddd %%mm5, %%mm7 \n\t" /* B0 b0 */\
239 "movq 72(%2), %%mm5 \n\t" /* -C5 -C1 -C5 -C1 */\
240 "pmaddwd %%mm3, %%mm5 \n\t" /* -C5R7-C1R5 -C5r7-C1r5 */\
241 "paddd %%mm4, %%mm7 \n\t" /* A0+B0 a0+b0 */\
242 "paddd %%mm4, %%mm4 \n\t" /* 2A0 2a0 */\
243 "psubd %%mm7, %%mm4 \n\t" /* A0-B0 a0-b0 */\
244 "paddd %%mm2, %%mm5 \n\t" /* B1 b1 */\
245 "psrad $" #shift ", %%mm7 \n\t"\
246 "psrad $" #shift ", %%mm4 \n\t"\
247 "movq %%mm1, %%mm2 \n\t" /* A1 a1 */\
248 "paddd %%mm5, %%mm1 \n\t" /* A1+B1 a1+b1 */\
249 "psubd %%mm5, %%mm2 \n\t" /* A1-B1 a1-b1 */\
250 "psrad $" #shift ", %%mm1 \n\t"\
251 "psrad $" #shift ", %%mm2 \n\t"\
252 "packssdw %%mm1, %%mm7 \n\t" /* A1+B1 a1+b1 A0+B0 a0+b0 */\
253 "packssdw %%mm4, %%mm2 \n\t" /* A0-B0 a0-b0 A1-B1 a1-b1 */\
254 "movq %%mm7, " #dst " \n\t"\
255 "movq " #src1 ", %%mm1 \n\t" /* R3 R1 r3 r1 */\
256 "movq 80(%2), %%mm4 \n\t" /* -C1 C5 -C1 C5 */\
257 "movq %%mm2, 24+" #dst " \n\t"\
258 "pmaddwd %%mm1, %%mm4 \n\t" /* -C1R3+C5R1 -C1r3+C5r1 */\
259 "movq 88(%2), %%mm7 \n\t" /* C3 C7 C3 C7 */\
260 "pmaddwd 96(%2), %%mm1 \n\t" /* -C5R3+C7R1 -C5r3+C7r1 */\
261 "pmaddwd %%mm3, %%mm7 \n\t" /* C3R7+C7R5 C3r7+C7r5 */\
262 "movq %%mm0, %%mm2 \n\t" /* A2 a2 */\
263 "pmaddwd 104(%2), %%mm3 \n\t" /* -C1R7+C3R5 -C1r7+C3r5 */\
264 "paddd %%mm7, %%mm4 \n\t" /* B2 b2 */\
265 "paddd %%mm4, %%mm2 \n\t" /* A2+B2 a2+b2 */\
266 "psubd %%mm4, %%mm0 \n\t" /* a2-B2 a2-b2 */\
267 "psrad $" #shift ", %%mm2 \n\t"\
268 "psrad $" #shift ", %%mm0 \n\t"\
269 "movq %%mm6, %%mm4 \n\t" /* A3 a3 */\
270 "paddd %%mm1, %%mm3 \n\t" /* B3 b3 */\
271 "paddd %%mm3, %%mm6 \n\t" /* A3+B3 a3+b3 */\
272 "psubd %%mm3, %%mm4 \n\t" /* a3-B3 a3-b3 */\
273 "psrad $" #shift ", %%mm6 \n\t"\
274 "packssdw %%mm6, %%mm2 \n\t" /* A3+B3 a3+b3 A2+B2 a2+b2 */\
275 "movq %%mm2, 8+" #dst " \n\t"\
276 "psrad $" #shift ", %%mm4 \n\t"\
277 "packssdw %%mm0, %%mm4 \n\t" /* A2-B2 a2-b2 A3-B3 a3-b3 */\
278 "movq %%mm4, 16+" #dst " \n\t"\
280 #define COL_IDCT(src0, src4, src1, src5, dst, rounder, shift) \
281 "movq " #src0 ", %%mm0 \n\t" /* R4 R0 r4 r0 */\
282 "movq " #src4 ", %%mm1 \n\t" /* R6 R2 r6 r2 */\
283 "movq " #src1 ", %%mm2 \n\t" /* R3 R1 r3 r1 */\
284 "movq " #src5 ", %%mm3 \n\t" /* R7 R5 r7 r5 */\
285 "movq 16(%2), %%mm4 \n\t" /* C4 C4 C4 C4 */\
286 "pmaddwd %%mm0, %%mm4 \n\t" /* C4R4+C4R0 C4r4+C4r0 */\
287 "movq 24(%2), %%mm5 \n\t" /* -C4 C4 -C4 C4 */\
288 "pmaddwd %%mm5, %%mm0 \n\t" /* -C4R4+C4R0 -C4r4+C4r0 */\
289 "movq 32(%2), %%mm5 \n\t" /* C6 C2 C6 C2 */\
290 "pmaddwd %%mm1, %%mm5 \n\t" /* C6R6+C2R2 C6r6+C2r2 */\
291 "movq 40(%2), %%mm6 \n\t" /* -C2 C6 -C2 C6 */\
292 "pmaddwd %%mm6, %%mm1 \n\t" /* -C2R6+C6R2 -C2r6+C6r2 */\
293 #rounder ", %%mm4 \n\t"\
294 "movq %%mm4, %%mm6 \n\t" /* C4R4+C4R0 C4r4+C4r0 */\
295 "movq 48(%2), %%mm7 \n\t" /* C3 C1 C3 C1 */\
296 #rounder ", %%mm0 \n\t"\
297 "pmaddwd %%mm2, %%mm7 \n\t" /* C3R3+C1R1 C3r3+C1r1 */\
298 "paddd %%mm5, %%mm4 \n\t" /* A0 a0 */\
299 "psubd %%mm5, %%mm6 \n\t" /* A3 a3 */\
300 "movq %%mm0, %%mm5 \n\t" /* -C4R4+C4R0 -C4r4+C4r0 */\
301 "paddd %%mm1, %%mm0 \n\t" /* A1 a1 */\
302 "psubd %%mm1, %%mm5 \n\t" /* A2 a2 */\
303 "movq 56(%2), %%mm1 \n\t" /* C7 C5 C7 C5 */\
304 "pmaddwd %%mm3, %%mm1 \n\t" /* C7R7+C5R5 C7r7+C5r5 */\
305 "pmaddwd 64(%2), %%mm2 \n\t" /* -C7R3+C3R1 -C7r3+C3r1 */\
306 "paddd %%mm1, %%mm7 \n\t" /* B0 b0 */\
307 "movq 72(%2), %%mm1 \n\t" /* -C5 -C1 -C5 -C1 */\
308 "pmaddwd %%mm3, %%mm1 \n\t" /* -C5R7-C1R5 -C5r7-C1r5 */\
309 "paddd %%mm4, %%mm7 \n\t" /* A0+B0 a0+b0 */\
310 "paddd %%mm4, %%mm4 \n\t" /* 2A0 2a0 */\
311 "psubd %%mm7, %%mm4 \n\t" /* A0-B0 a0-b0 */\
312 "paddd %%mm2, %%mm1 \n\t" /* B1 b1 */\
313 "psrad $" #shift ", %%mm7 \n\t"\
314 "psrad $" #shift ", %%mm4 \n\t"\
315 "movq %%mm0, %%mm2 \n\t" /* A1 a1 */\
316 "paddd %%mm1, %%mm0 \n\t" /* A1+B1 a1+b1 */\
317 "psubd %%mm1, %%mm2 \n\t" /* A1-B1 a1-b1 */\
318 "psrad $" #shift ", %%mm0 \n\t"\
319 "psrad $" #shift ", %%mm2 \n\t"\
320 "packssdw %%mm7, %%mm7 \n\t" /* A0+B0 a0+b0 */\
321 "movd %%mm7, " #dst " \n\t"\
322 "packssdw %%mm0, %%mm0 \n\t" /* A1+B1 a1+b1 */\
323 "movd %%mm0, 16+" #dst " \n\t"\
324 "packssdw %%mm2, %%mm2 \n\t" /* A1-B1 a1-b1 */\
325 "movd %%mm2, 96+" #dst " \n\t"\
326 "packssdw %%mm4, %%mm4 \n\t" /* A0-B0 a0-b0 */\
327 "movd %%mm4, 112+" #dst " \n\t"\
328 "movq " #src1 ", %%mm0 \n\t" /* R3 R1 r3 r1 */\
329 "movq 80(%2), %%mm4 \n\t" /* -C1 C5 -C1 C5 */\
330 "pmaddwd %%mm0, %%mm4 \n\t" /* -C1R3+C5R1 -C1r3+C5r1 */\
331 "movq 88(%2), %%mm7 \n\t" /* C3 C7 C3 C7 */\
332 "pmaddwd 96(%2), %%mm0 \n\t" /* -C5R3+C7R1 -C5r3+C7r1 */\
333 "pmaddwd %%mm3, %%mm7 \n\t" /* C3R7+C7R5 C3r7+C7r5 */\
334 "movq %%mm5, %%mm2 \n\t" /* A2 a2 */\
335 "pmaddwd 104(%2), %%mm3 \n\t" /* -C1R7+C3R5 -C1r7+C3r5 */\
336 "paddd %%mm7, %%mm4 \n\t" /* B2 b2 */\
337 "paddd %%mm4, %%mm2 \n\t" /* A2+B2 a2+b2 */\
338 "psubd %%mm4, %%mm5 \n\t" /* a2-B2 a2-b2 */\
339 "psrad $" #shift ", %%mm2 \n\t"\
340 "psrad $" #shift ", %%mm5 \n\t"\
341 "movq %%mm6, %%mm4 \n\t" /* A3 a3 */\
342 "paddd %%mm0, %%mm3 \n\t" /* B3 b3 */\
343 "paddd %%mm3, %%mm6 \n\t" /* A3+B3 a3+b3 */\
344 "psubd %%mm3, %%mm4 \n\t" /* a3-B3 a3-b3 */\
345 "psrad $" #shift ", %%mm6 \n\t"\
346 "psrad $" #shift ", %%mm4 \n\t"\
347 "packssdw %%mm2, %%mm2 \n\t" /* A2+B2 a2+b2 */\
348 "packssdw %%mm6, %%mm6 \n\t" /* A3+B3 a3+b3 */\
349 "movd %%mm2, 32+" #dst " \n\t"\
350 "packssdw %%mm4, %%mm4 \n\t" /* A3-B3 a3-b3 */\
351 "packssdw %%mm5, %%mm5 \n\t" /* A2-B2 a2-b2 */\
352 "movd %%mm6, 48+" #dst " \n\t"\
353 "movd %%mm4, 64+" #dst " \n\t"\
354 "movd %%mm5, 80+" #dst " \n\t"\
357 #define DC_COND_ROW_IDCT(src0, src4, src1, src5, dst, rounder, shift) \
358 "movq " #src0 ", %%mm0 \n\t" /* R4 R0 r4 r0 */\
359 "movq " #src4 ", %%mm1 \n\t" /* R6 R2 r6 r2 */\
360 "movq " #src1 ", %%mm2 \n\t" /* R3 R1 r3 r1 */\
361 "movq " #src5 ", %%mm3 \n\t" /* R7 R5 r7 r5 */\
362 "movq "MANGLE(wm1010)", %%mm4 \n\t"\
363 "pand %%mm0, %%mm4 \n\t"\
364 "por %%mm1, %%mm4 \n\t"\
365 "por %%mm2, %%mm4 \n\t"\
366 "por %%mm3, %%mm4 \n\t"\
367 "packssdw %%mm4,%%mm4 \n\t"\
368 "movd %%mm4, %%eax \n\t"\
369 "orl %%eax, %%eax \n\t"\
371 "movq 16(%2), %%mm4 \n\t" /* C4 C4 C4 C4 */\
372 "pmaddwd %%mm0, %%mm4 \n\t" /* C4R4+C4R0 C4r4+C4r0 */\
373 "movq 24(%2), %%mm5 \n\t" /* -C4 C4 -C4 C4 */\
374 "pmaddwd %%mm5, %%mm0 \n\t" /* -C4R4+C4R0 -C4r4+C4r0 */\
375 "movq 32(%2), %%mm5 \n\t" /* C6 C2 C6 C2 */\
376 "pmaddwd %%mm1, %%mm5 \n\t" /* C6R6+C2R2 C6r6+C2r2 */\
377 "movq 40(%2), %%mm6 \n\t" /* -C2 C6 -C2 C6 */\
378 "pmaddwd %%mm6, %%mm1 \n\t" /* -C2R6+C6R2 -C2r6+C6r2 */\
379 "movq 48(%2), %%mm7 \n\t" /* C3 C1 C3 C1 */\
380 "pmaddwd %%mm2, %%mm7 \n\t" /* C3R3+C1R1 C3r3+C1r1 */\
381 #rounder ", %%mm4 \n\t"\
382 "movq %%mm4, %%mm6 \n\t" /* C4R4+C4R0 C4r4+C4r0 */\
383 "paddd %%mm5, %%mm4 \n\t" /* A0 a0 */\
384 "psubd %%mm5, %%mm6 \n\t" /* A3 a3 */\
385 "movq 56(%2), %%mm5 \n\t" /* C7 C5 C7 C5 */\
386 "pmaddwd %%mm3, %%mm5 \n\t" /* C7R7+C5R5 C7r7+C5r5 */\
387 #rounder ", %%mm0 \n\t"\
388 "paddd %%mm0, %%mm1 \n\t" /* A1 a1 */\
389 "paddd %%mm0, %%mm0 \n\t" \
390 "psubd %%mm1, %%mm0 \n\t" /* A2 a2 */\
391 "pmaddwd 64(%2), %%mm2 \n\t" /* -C7R3+C3R1 -C7r3+C3r1 */\
392 "paddd %%mm5, %%mm7 \n\t" /* B0 b0 */\
393 "movq 72(%2), %%mm5 \n\t" /* -C5 -C1 -C5 -C1 */\
394 "pmaddwd %%mm3, %%mm5 \n\t" /* -C5R7-C1R5 -C5r7-C1r5 */\
395 "paddd %%mm4, %%mm7 \n\t" /* A0+B0 a0+b0 */\
396 "paddd %%mm4, %%mm4 \n\t" /* 2A0 2a0 */\
397 "psubd %%mm7, %%mm4 \n\t" /* A0-B0 a0-b0 */\
398 "paddd %%mm2, %%mm5 \n\t" /* B1 b1 */\
399 "psrad $" #shift ", %%mm7 \n\t"\
400 "psrad $" #shift ", %%mm4 \n\t"\
401 "movq %%mm1, %%mm2 \n\t" /* A1 a1 */\
402 "paddd %%mm5, %%mm1 \n\t" /* A1+B1 a1+b1 */\
403 "psubd %%mm5, %%mm2 \n\t" /* A1-B1 a1-b1 */\
404 "psrad $" #shift ", %%mm1 \n\t"\
405 "psrad $" #shift ", %%mm2 \n\t"\
406 "packssdw %%mm1, %%mm7 \n\t" /* A1+B1 a1+b1 A0+B0 a0+b0 */\
407 "packssdw %%mm4, %%mm2 \n\t" /* A0-B0 a0-b0 A1-B1 a1-b1 */\
408 "movq %%mm7, " #dst " \n\t"\
409 "movq " #src1 ", %%mm1 \n\t" /* R3 R1 r3 r1 */\
410 "movq 80(%2), %%mm4 \n\t" /* -C1 C5 -C1 C5 */\
411 "movq %%mm2, 24+" #dst " \n\t"\
412 "pmaddwd %%mm1, %%mm4 \n\t" /* -C1R3+C5R1 -C1r3+C5r1 */\
413 "movq 88(%2), %%mm7 \n\t" /* C3 C7 C3 C7 */\
414 "pmaddwd 96(%2), %%mm1 \n\t" /* -C5R3+C7R1 -C5r3+C7r1 */\
415 "pmaddwd %%mm3, %%mm7 \n\t" /* C3R7+C7R5 C3r7+C7r5 */\
416 "movq %%mm0, %%mm2 \n\t" /* A2 a2 */\
417 "pmaddwd 104(%2), %%mm3 \n\t" /* -C1R7+C3R5 -C1r7+C3r5 */\
418 "paddd %%mm7, %%mm4 \n\t" /* B2 b2 */\
419 "paddd %%mm4, %%mm2 \n\t" /* A2+B2 a2+b2 */\
420 "psubd %%mm4, %%mm0 \n\t" /* a2-B2 a2-b2 */\
421 "psrad $" #shift ", %%mm2 \n\t"\
422 "psrad $" #shift ", %%mm0 \n\t"\
423 "movq %%mm6, %%mm4 \n\t" /* A3 a3 */\
424 "paddd %%mm1, %%mm3 \n\t" /* B3 b3 */\
425 "paddd %%mm3, %%mm6 \n\t" /* A3+B3 a3+b3 */\
426 "psubd %%mm3, %%mm4 \n\t" /* a3-B3 a3-b3 */\
427 "psrad $" #shift ", %%mm6 \n\t"\
428 "packssdw %%mm6, %%mm2 \n\t" /* A3+B3 a3+b3 A2+B2 a2+b2 */\
429 "movq %%mm2, 8+" #dst " \n\t"\
430 "psrad $" #shift ", %%mm4 \n\t"\
431 "packssdw %%mm0, %%mm4 \n\t" /* A2-B2 a2-b2 A3-B3 a3-b3 */\
432 "movq %%mm4, 16+" #dst " \n\t"\
435 "pslld $16, %%mm0 \n\t"\
436 "#paddd "MANGLE(d40000)", %%mm0 \n\t"\
437 "psrad $13, %%mm0 \n\t"\
438 "packssdw %%mm0, %%mm0 \n\t"\
439 "movq %%mm0, " #dst " \n\t"\
440 "movq %%mm0, 8+" #dst " \n\t"\
441 "movq %%mm0, 16+" #dst " \n\t"\
442 "movq %%mm0, 24+" #dst " \n\t"\
446 //IDCT( src0, src4, src1, src5, dst, rounder, shift)
447 ROW_IDCT( (%0), 8(%0), 16(%0), 24(%0), 0(%1),paddd 8(%2), 11)
448 /*ROW_IDCT( 32(%0), 40(%0), 48(%0), 56(%0), 32(%1), paddd (%2), 11)
449 ROW_IDCT( 64(%0), 72(%0), 80(%0), 88(%0), 64(%1), paddd (%2), 11)
450 ROW_IDCT( 96(%0),104(%0),112(%0),120(%0), 96(%1), paddd (%2), 11)*/
452 DC_COND_ROW_IDCT( 32(%0), 40(%0), 48(%0), 56(%0), 32(%1),paddd (%2), 11)
453 DC_COND_ROW_IDCT( 64(%0), 72(%0), 80(%0), 88(%0), 64(%1),paddd (%2), 11)
454 DC_COND_ROW_IDCT( 96(%0),104(%0),112(%0),120(%0), 96(%1),paddd (%2), 11)
457 //IDCT( src0, src4, src1, src5, dst, rounder, shift)
458 COL_IDCT( (%1), 64(%1), 32(%1), 96(%1), 0(%0),/nop, 20)
459 COL_IDCT( 8(%1), 72(%1), 40(%1), 104(%1), 4(%0),/nop, 20)
460 COL_IDCT( 16(%1), 80(%1), 48(%1), 112(%1), 8(%0),/nop, 20)
461 COL_IDCT( 24(%1), 88(%1), 56(%1), 120(%1), 12(%0),/nop, 20)
465 #define DC_COND_IDCT(src0, src4, src1, src5, dst, rounder, shift) \
466 "movq " #src0 ", %%mm0 \n\t" /* R4 R0 r4 r0 */\
467 "movq " #src4 ", %%mm1 \n\t" /* R6 R2 r6 r2 */\
468 "movq " #src1 ", %%mm2 \n\t" /* R3 R1 r3 r1 */\
469 "movq " #src5 ", %%mm3 \n\t" /* R7 R5 r7 r5 */\
470 "movq "MANGLE(wm1010)", %%mm4 \n\t"\
471 "pand %%mm0, %%mm4 \n\t"\
472 "por %%mm1, %%mm4 \n\t"\
473 "por %%mm2, %%mm4 \n\t"\
474 "por %%mm3, %%mm4 \n\t"\
475 "packssdw %%mm4,%%mm4 \n\t"\
476 "movd %%mm4, %%eax \n\t"\
477 "orl %%eax, %%eax \n\t"\
479 "movq 16(%2), %%mm4 \n\t" /* C4 C4 C4 C4 */\
480 "pmaddwd %%mm0, %%mm4 \n\t" /* C4R4+C4R0 C4r4+C4r0 */\
481 "movq 24(%2), %%mm5 \n\t" /* -C4 C4 -C4 C4 */\
482 "pmaddwd %%mm5, %%mm0 \n\t" /* -C4R4+C4R0 -C4r4+C4r0 */\
483 "movq 32(%2), %%mm5 \n\t" /* C6 C2 C6 C2 */\
484 "pmaddwd %%mm1, %%mm5 \n\t" /* C6R6+C2R2 C6r6+C2r2 */\
485 "movq 40(%2), %%mm6 \n\t" /* -C2 C6 -C2 C6 */\
486 "pmaddwd %%mm6, %%mm1 \n\t" /* -C2R6+C6R2 -C2r6+C6r2 */\
487 "movq 48(%2), %%mm7 \n\t" /* C3 C1 C3 C1 */\
488 "pmaddwd %%mm2, %%mm7 \n\t" /* C3R3+C1R1 C3r3+C1r1 */\
489 #rounder ", %%mm4 \n\t"\
490 "movq %%mm4, %%mm6 \n\t" /* C4R4+C4R0 C4r4+C4r0 */\
491 "paddd %%mm5, %%mm4 \n\t" /* A0 a0 */\
492 "psubd %%mm5, %%mm6 \n\t" /* A3 a3 */\
493 "movq 56(%2), %%mm5 \n\t" /* C7 C5 C7 C5 */\
494 "pmaddwd %%mm3, %%mm5 \n\t" /* C7R7+C5R5 C7r7+C5r5 */\
495 #rounder ", %%mm0 \n\t"\
496 "paddd %%mm0, %%mm1 \n\t" /* A1 a1 */\
497 "paddd %%mm0, %%mm0 \n\t" \
498 "psubd %%mm1, %%mm0 \n\t" /* A2 a2 */\
499 "pmaddwd 64(%2), %%mm2 \n\t" /* -C7R3+C3R1 -C7r3+C3r1 */\
500 "paddd %%mm5, %%mm7 \n\t" /* B0 b0 */\
501 "movq 72(%2), %%mm5 \n\t" /* -C5 -C1 -C5 -C1 */\
502 "pmaddwd %%mm3, %%mm5 \n\t" /* -C5R7-C1R5 -C5r7-C1r5 */\
503 "paddd %%mm4, %%mm7 \n\t" /* A0+B0 a0+b0 */\
504 "paddd %%mm4, %%mm4 \n\t" /* 2A0 2a0 */\
505 "psubd %%mm7, %%mm4 \n\t" /* A0-B0 a0-b0 */\
506 "paddd %%mm2, %%mm5 \n\t" /* B1 b1 */\
507 "psrad $" #shift ", %%mm7 \n\t"\
508 "psrad $" #shift ", %%mm4 \n\t"\
509 "movq %%mm1, %%mm2 \n\t" /* A1 a1 */\
510 "paddd %%mm5, %%mm1 \n\t" /* A1+B1 a1+b1 */\
511 "psubd %%mm5, %%mm2 \n\t" /* A1-B1 a1-b1 */\
512 "psrad $" #shift ", %%mm1 \n\t"\
513 "psrad $" #shift ", %%mm2 \n\t"\
514 "packssdw %%mm1, %%mm7 \n\t" /* A1+B1 a1+b1 A0+B0 a0+b0 */\
515 "packssdw %%mm4, %%mm2 \n\t" /* A0-B0 a0-b0 A1-B1 a1-b1 */\
516 "movq %%mm7, " #dst " \n\t"\
517 "movq " #src1 ", %%mm1 \n\t" /* R3 R1 r3 r1 */\
518 "movq 80(%2), %%mm4 \n\t" /* -C1 C5 -C1 C5 */\
519 "movq %%mm2, 24+" #dst " \n\t"\
520 "pmaddwd %%mm1, %%mm4 \n\t" /* -C1R3+C5R1 -C1r3+C5r1 */\
521 "movq 88(%2), %%mm7 \n\t" /* C3 C7 C3 C7 */\
522 "pmaddwd 96(%2), %%mm1 \n\t" /* -C5R3+C7R1 -C5r3+C7r1 */\
523 "pmaddwd %%mm3, %%mm7 \n\t" /* C3R7+C7R5 C3r7+C7r5 */\
524 "movq %%mm0, %%mm2 \n\t" /* A2 a2 */\
525 "pmaddwd 104(%2), %%mm3 \n\t" /* -C1R7+C3R5 -C1r7+C3r5 */\
526 "paddd %%mm7, %%mm4 \n\t" /* B2 b2 */\
527 "paddd %%mm4, %%mm2 \n\t" /* A2+B2 a2+b2 */\
528 "psubd %%mm4, %%mm0 \n\t" /* a2-B2 a2-b2 */\
529 "psrad $" #shift ", %%mm2 \n\t"\
530 "psrad $" #shift ", %%mm0 \n\t"\
531 "movq %%mm6, %%mm4 \n\t" /* A3 a3 */\
532 "paddd %%mm1, %%mm3 \n\t" /* B3 b3 */\
533 "paddd %%mm3, %%mm6 \n\t" /* A3+B3 a3+b3 */\
534 "psubd %%mm3, %%mm4 \n\t" /* a3-B3 a3-b3 */\
535 "psrad $" #shift ", %%mm6 \n\t"\
536 "packssdw %%mm6, %%mm2 \n\t" /* A3+B3 a3+b3 A2+B2 a2+b2 */\
537 "movq %%mm2, 8+" #dst " \n\t"\
538 "psrad $" #shift ", %%mm4 \n\t"\
539 "packssdw %%mm0, %%mm4 \n\t" /* A2-B2 a2-b2 A3-B3 a3-b3 */\
540 "movq %%mm4, 16+" #dst " \n\t"\
543 "pslld $16, %%mm0 \n\t"\
544 "paddd "MANGLE(d40000)", %%mm0 \n\t"\
545 "psrad $13, %%mm0 \n\t"\
546 "packssdw %%mm0, %%mm0 \n\t"\
547 "movq %%mm0, " #dst " \n\t"\
548 "movq %%mm0, 8+" #dst " \n\t"\
549 "movq %%mm0, 16+" #dst " \n\t"\
550 "movq %%mm0, 24+" #dst " \n\t"\
553 #define Z_COND_IDCT(src0, src4, src1, src5, dst, rounder, shift, bt) \
554 "movq " #src0 ", %%mm0 \n\t" /* R4 R0 r4 r0 */\
555 "movq " #src4 ", %%mm1 \n\t" /* R6 R2 r6 r2 */\
556 "movq " #src1 ", %%mm2 \n\t" /* R3 R1 r3 r1 */\
557 "movq " #src5 ", %%mm3 \n\t" /* R7 R5 r7 r5 */\
558 "movq %%mm0, %%mm4 \n\t"\
559 "por %%mm1, %%mm4 \n\t"\
560 "por %%mm2, %%mm4 \n\t"\
561 "por %%mm3, %%mm4 \n\t"\
562 "packssdw %%mm4,%%mm4 \n\t"\
563 "movd %%mm4, %%eax \n\t"\
564 "orl %%eax, %%eax \n\t"\
566 "movq 16(%2), %%mm4 \n\t" /* C4 C4 C4 C4 */\
567 "pmaddwd %%mm0, %%mm4 \n\t" /* C4R4+C4R0 C4r4+C4r0 */\
568 "movq 24(%2), %%mm5 \n\t" /* -C4 C4 -C4 C4 */\
569 "pmaddwd %%mm5, %%mm0 \n\t" /* -C4R4+C4R0 -C4r4+C4r0 */\
570 "movq 32(%2), %%mm5 \n\t" /* C6 C2 C6 C2 */\
571 "pmaddwd %%mm1, %%mm5 \n\t" /* C6R6+C2R2 C6r6+C2r2 */\
572 "movq 40(%2), %%mm6 \n\t" /* -C2 C6 -C2 C6 */\
573 "pmaddwd %%mm6, %%mm1 \n\t" /* -C2R6+C6R2 -C2r6+C6r2 */\
574 "movq 48(%2), %%mm7 \n\t" /* C3 C1 C3 C1 */\
575 "pmaddwd %%mm2, %%mm7 \n\t" /* C3R3+C1R1 C3r3+C1r1 */\
576 #rounder ", %%mm4 \n\t"\
577 "movq %%mm4, %%mm6 \n\t" /* C4R4+C4R0 C4r4+C4r0 */\
578 "paddd %%mm5, %%mm4 \n\t" /* A0 a0 */\
579 "psubd %%mm5, %%mm6 \n\t" /* A3 a3 */\
580 "movq 56(%2), %%mm5 \n\t" /* C7 C5 C7 C5 */\
581 "pmaddwd %%mm3, %%mm5 \n\t" /* C7R7+C5R5 C7r7+C5r5 */\
582 #rounder ", %%mm0 \n\t"\
583 "paddd %%mm0, %%mm1 \n\t" /* A1 a1 */\
584 "paddd %%mm0, %%mm0 \n\t" \
585 "psubd %%mm1, %%mm0 \n\t" /* A2 a2 */\
586 "pmaddwd 64(%2), %%mm2 \n\t" /* -C7R3+C3R1 -C7r3+C3r1 */\
587 "paddd %%mm5, %%mm7 \n\t" /* B0 b0 */\
588 "movq 72(%2), %%mm5 \n\t" /* -C5 -C1 -C5 -C1 */\
589 "pmaddwd %%mm3, %%mm5 \n\t" /* -C5R7-C1R5 -C5r7-C1r5 */\
590 "paddd %%mm4, %%mm7 \n\t" /* A0+B0 a0+b0 */\
591 "paddd %%mm4, %%mm4 \n\t" /* 2A0 2a0 */\
592 "psubd %%mm7, %%mm4 \n\t" /* A0-B0 a0-b0 */\
593 "paddd %%mm2, %%mm5 \n\t" /* B1 b1 */\
594 "psrad $" #shift ", %%mm7 \n\t"\
595 "psrad $" #shift ", %%mm4 \n\t"\
596 "movq %%mm1, %%mm2 \n\t" /* A1 a1 */\
597 "paddd %%mm5, %%mm1 \n\t" /* A1+B1 a1+b1 */\
598 "psubd %%mm5, %%mm2 \n\t" /* A1-B1 a1-b1 */\
599 "psrad $" #shift ", %%mm1 \n\t"\
600 "psrad $" #shift ", %%mm2 \n\t"\
601 "packssdw %%mm1, %%mm7 \n\t" /* A1+B1 a1+b1 A0+B0 a0+b0 */\
602 "packssdw %%mm4, %%mm2 \n\t" /* A0-B0 a0-b0 A1-B1 a1-b1 */\
603 "movq %%mm7, " #dst " \n\t"\
604 "movq " #src1 ", %%mm1 \n\t" /* R3 R1 r3 r1 */\
605 "movq 80(%2), %%mm4 \n\t" /* -C1 C5 -C1 C5 */\
606 "movq %%mm2, 24+" #dst " \n\t"\
607 "pmaddwd %%mm1, %%mm4 \n\t" /* -C1R3+C5R1 -C1r3+C5r1 */\
608 "movq 88(%2), %%mm7 \n\t" /* C3 C7 C3 C7 */\
609 "pmaddwd 96(%2), %%mm1 \n\t" /* -C5R3+C7R1 -C5r3+C7r1 */\
610 "pmaddwd %%mm3, %%mm7 \n\t" /* C3R7+C7R5 C3r7+C7r5 */\
611 "movq %%mm0, %%mm2 \n\t" /* A2 a2 */\
612 "pmaddwd 104(%2), %%mm3 \n\t" /* -C1R7+C3R5 -C1r7+C3r5 */\
613 "paddd %%mm7, %%mm4 \n\t" /* B2 b2 */\
614 "paddd %%mm4, %%mm2 \n\t" /* A2+B2 a2+b2 */\
615 "psubd %%mm4, %%mm0 \n\t" /* a2-B2 a2-b2 */\
616 "psrad $" #shift ", %%mm2 \n\t"\
617 "psrad $" #shift ", %%mm0 \n\t"\
618 "movq %%mm6, %%mm4 \n\t" /* A3 a3 */\
619 "paddd %%mm1, %%mm3 \n\t" /* B3 b3 */\
620 "paddd %%mm3, %%mm6 \n\t" /* A3+B3 a3+b3 */\
621 "psubd %%mm3, %%mm4 \n\t" /* a3-B3 a3-b3 */\
622 "psrad $" #shift ", %%mm6 \n\t"\
623 "packssdw %%mm6, %%mm2 \n\t" /* A3+B3 a3+b3 A2+B2 a2+b2 */\
624 "movq %%mm2, 8+" #dst " \n\t"\
625 "psrad $" #shift ", %%mm4 \n\t"\
626 "packssdw %%mm0, %%mm4 \n\t" /* A2-B2 a2-b2 A3-B3 a3-b3 */\
627 "movq %%mm4, 16+" #dst " \n\t"\
629 #define ROW_IDCT(src0, src4, src1, src5, dst, rounder, shift) \
630 "movq " #src0 ", %%mm0 \n\t" /* R4 R0 r4 r0 */\
631 "movq " #src4 ", %%mm1 \n\t" /* R6 R2 r6 r2 */\
632 "movq " #src1 ", %%mm2 \n\t" /* R3 R1 r3 r1 */\
633 "movq " #src5 ", %%mm3 \n\t" /* R7 R5 r7 r5 */\
634 "movq 16(%2), %%mm4 \n\t" /* C4 C4 C4 C4 */\
635 "pmaddwd %%mm0, %%mm4 \n\t" /* C4R4+C4R0 C4r4+C4r0 */\
636 "movq 24(%2), %%mm5 \n\t" /* -C4 C4 -C4 C4 */\
637 "pmaddwd %%mm5, %%mm0 \n\t" /* -C4R4+C4R0 -C4r4+C4r0 */\
638 "movq 32(%2), %%mm5 \n\t" /* C6 C2 C6 C2 */\
639 "pmaddwd %%mm1, %%mm5 \n\t" /* C6R6+C2R2 C6r6+C2r2 */\
640 "movq 40(%2), %%mm6 \n\t" /* -C2 C6 -C2 C6 */\
641 "pmaddwd %%mm6, %%mm1 \n\t" /* -C2R6+C6R2 -C2r6+C6r2 */\
642 "movq 48(%2), %%mm7 \n\t" /* C3 C1 C3 C1 */\
643 "pmaddwd %%mm2, %%mm7 \n\t" /* C3R3+C1R1 C3r3+C1r1 */\
644 #rounder ", %%mm4 \n\t"\
645 "movq %%mm4, %%mm6 \n\t" /* C4R4+C4R0 C4r4+C4r0 */\
646 "paddd %%mm5, %%mm4 \n\t" /* A0 a0 */\
647 "psubd %%mm5, %%mm6 \n\t" /* A3 a3 */\
648 "movq 56(%2), %%mm5 \n\t" /* C7 C5 C7 C5 */\
649 "pmaddwd %%mm3, %%mm5 \n\t" /* C7R7+C5R5 C7r7+C5r5 */\
650 #rounder ", %%mm0 \n\t"\
651 "paddd %%mm0, %%mm1 \n\t" /* A1 a1 */\
652 "paddd %%mm0, %%mm0 \n\t" \
653 "psubd %%mm1, %%mm0 \n\t" /* A2 a2 */\
654 "pmaddwd 64(%2), %%mm2 \n\t" /* -C7R3+C3R1 -C7r3+C3r1 */\
655 "paddd %%mm5, %%mm7 \n\t" /* B0 b0 */\
656 "movq 72(%2), %%mm5 \n\t" /* -C5 -C1 -C5 -C1 */\
657 "pmaddwd %%mm3, %%mm5 \n\t" /* -C5R7-C1R5 -C5r7-C1r5 */\
658 "paddd %%mm4, %%mm7 \n\t" /* A0+B0 a0+b0 */\
659 "paddd %%mm4, %%mm4 \n\t" /* 2A0 2a0 */\
660 "psubd %%mm7, %%mm4 \n\t" /* A0-B0 a0-b0 */\
661 "paddd %%mm2, %%mm5 \n\t" /* B1 b1 */\
662 "psrad $" #shift ", %%mm7 \n\t"\
663 "psrad $" #shift ", %%mm4 \n\t"\
664 "movq %%mm1, %%mm2 \n\t" /* A1 a1 */\
665 "paddd %%mm5, %%mm1 \n\t" /* A1+B1 a1+b1 */\
666 "psubd %%mm5, %%mm2 \n\t" /* A1-B1 a1-b1 */\
667 "psrad $" #shift ", %%mm1 \n\t"\
668 "psrad $" #shift ", %%mm2 \n\t"\
669 "packssdw %%mm1, %%mm7 \n\t" /* A1+B1 a1+b1 A0+B0 a0+b0 */\
670 "packssdw %%mm4, %%mm2 \n\t" /* A0-B0 a0-b0 A1-B1 a1-b1 */\
671 "movq %%mm7, " #dst " \n\t"\
672 "movq " #src1 ", %%mm1 \n\t" /* R3 R1 r3 r1 */\
673 "movq 80(%2), %%mm4 \n\t" /* -C1 C5 -C1 C5 */\
674 "movq %%mm2, 24+" #dst " \n\t"\
675 "pmaddwd %%mm1, %%mm4 \n\t" /* -C1R3+C5R1 -C1r3+C5r1 */\
676 "movq 88(%2), %%mm7 \n\t" /* C3 C7 C3 C7 */\
677 "pmaddwd 96(%2), %%mm1 \n\t" /* -C5R3+C7R1 -C5r3+C7r1 */\
678 "pmaddwd %%mm3, %%mm7 \n\t" /* C3R7+C7R5 C3r7+C7r5 */\
679 "movq %%mm0, %%mm2 \n\t" /* A2 a2 */\
680 "pmaddwd 104(%2), %%mm3 \n\t" /* -C1R7+C3R5 -C1r7+C3r5 */\
681 "paddd %%mm7, %%mm4 \n\t" /* B2 b2 */\
682 "paddd %%mm4, %%mm2 \n\t" /* A2+B2 a2+b2 */\
683 "psubd %%mm4, %%mm0 \n\t" /* a2-B2 a2-b2 */\
684 "psrad $" #shift ", %%mm2 \n\t"\
685 "psrad $" #shift ", %%mm0 \n\t"\
686 "movq %%mm6, %%mm4 \n\t" /* A3 a3 */\
687 "paddd %%mm1, %%mm3 \n\t" /* B3 b3 */\
688 "paddd %%mm3, %%mm6 \n\t" /* A3+B3 a3+b3 */\
689 "psubd %%mm3, %%mm4 \n\t" /* a3-B3 a3-b3 */\
690 "psrad $" #shift ", %%mm6 \n\t"\
691 "packssdw %%mm6, %%mm2 \n\t" /* A3+B3 a3+b3 A2+B2 a2+b2 */\
692 "movq %%mm2, 8+" #dst " \n\t"\
693 "psrad $" #shift ", %%mm4 \n\t"\
694 "packssdw %%mm0, %%mm4 \n\t" /* A2-B2 a2-b2 A3-B3 a3-b3 */\
695 "movq %%mm4, 16+" #dst " \n\t"\
697 //IDCT( src0, src4, src1, src5, dst, rounder, shift)
698 DC_COND_IDCT( 0(%0), 8(%0), 16(%0), 24(%0), 0(%1),paddd 8(%2), 11)
699 Z_COND_IDCT( 32(%0), 40(%0), 48(%0), 56(%0), 32(%1),paddd (%2), 11, 4f)
700 Z_COND_IDCT( 64(%0), 72(%0), 80(%0), 88(%0), 64(%1),paddd (%2), 11, 2f)
701 Z_COND_IDCT( 96(%0),104(%0),112(%0),120(%0), 96(%1),paddd (%2), 11, 1f)
704 #define IDCT(src0, src4, src1, src5, dst, rounder, shift) \
705 "movq " #src0 ", %%mm0 \n\t" /* R4 R0 r4 r0 */\
706 "movq " #src4 ", %%mm1 \n\t" /* R6 R2 r6 r2 */\
707 "movq " #src1 ", %%mm2 \n\t" /* R3 R1 r3 r1 */\
708 "movq " #src5 ", %%mm3 \n\t" /* R7 R5 r7 r5 */\
709 "movq 16(%2), %%mm4 \n\t" /* C4 C4 C4 C4 */\
710 "pmaddwd %%mm0, %%mm4 \n\t" /* C4R4+C4R0 C4r4+C4r0 */\
711 "movq 24(%2), %%mm5 \n\t" /* -C4 C4 -C4 C4 */\
712 "pmaddwd %%mm5, %%mm0 \n\t" /* -C4R4+C4R0 -C4r4+C4r0 */\
713 "movq 32(%2), %%mm5 \n\t" /* C6 C2 C6 C2 */\
714 "pmaddwd %%mm1, %%mm5 \n\t" /* C6R6+C2R2 C6r6+C2r2 */\
715 "movq 40(%2), %%mm6 \n\t" /* -C2 C6 -C2 C6 */\
716 "pmaddwd %%mm6, %%mm1 \n\t" /* -C2R6+C6R2 -C2r6+C6r2 */\
717 #rounder ", %%mm4 \n\t"\
718 "movq %%mm4, %%mm6 \n\t" /* C4R4+C4R0 C4r4+C4r0 */\
719 "movq 48(%2), %%mm7 \n\t" /* C3 C1 C3 C1 */\
720 #rounder ", %%mm0 \n\t"\
721 "pmaddwd %%mm2, %%mm7 \n\t" /* C3R3+C1R1 C3r3+C1r1 */\
722 "paddd %%mm5, %%mm4 \n\t" /* A0 a0 */\
723 "psubd %%mm5, %%mm6 \n\t" /* A3 a3 */\
724 "movq %%mm0, %%mm5 \n\t" /* -C4R4+C4R0 -C4r4+C4r0 */\
725 "paddd %%mm1, %%mm0 \n\t" /* A1 a1 */\
726 "psubd %%mm1, %%mm5 \n\t" /* A2 a2 */\
727 "movq 56(%2), %%mm1 \n\t" /* C7 C5 C7 C5 */\
728 "pmaddwd %%mm3, %%mm1 \n\t" /* C7R7+C5R5 C7r7+C5r5 */\
729 "pmaddwd 64(%2), %%mm2 \n\t" /* -C7R3+C3R1 -C7r3+C3r1 */\
730 "paddd %%mm1, %%mm7 \n\t" /* B0 b0 */\
731 "movq 72(%2), %%mm1 \n\t" /* -C5 -C1 -C5 -C1 */\
732 "pmaddwd %%mm3, %%mm1 \n\t" /* -C5R7-C1R5 -C5r7-C1r5 */\
733 "paddd %%mm4, %%mm7 \n\t" /* A0+B0 a0+b0 */\
734 "paddd %%mm4, %%mm4 \n\t" /* 2A0 2a0 */\
735 "psubd %%mm7, %%mm4 \n\t" /* A0-B0 a0-b0 */\
736 "paddd %%mm2, %%mm1 \n\t" /* B1 b1 */\
737 "psrad $" #shift ", %%mm7 \n\t"\
738 "psrad $" #shift ", %%mm4 \n\t"\
739 "movq %%mm0, %%mm2 \n\t" /* A1 a1 */\
740 "paddd %%mm1, %%mm0 \n\t" /* A1+B1 a1+b1 */\
741 "psubd %%mm1, %%mm2 \n\t" /* A1-B1 a1-b1 */\
742 "psrad $" #shift ", %%mm0 \n\t"\
743 "psrad $" #shift ", %%mm2 \n\t"\
744 "packssdw %%mm7, %%mm7 \n\t" /* A0+B0 a0+b0 */\
745 "movd %%mm7, " #dst " \n\t"\
746 "packssdw %%mm0, %%mm0 \n\t" /* A1+B1 a1+b1 */\
747 "movd %%mm0, 16+" #dst " \n\t"\
748 "packssdw %%mm2, %%mm2 \n\t" /* A1-B1 a1-b1 */\
749 "movd %%mm2, 96+" #dst " \n\t"\
750 "packssdw %%mm4, %%mm4 \n\t" /* A0-B0 a0-b0 */\
751 "movd %%mm4, 112+" #dst " \n\t"\
752 "movq " #src1 ", %%mm0 \n\t" /* R3 R1 r3 r1 */\
753 "movq 80(%2), %%mm4 \n\t" /* -C1 C5 -C1 C5 */\
754 "pmaddwd %%mm0, %%mm4 \n\t" /* -C1R3+C5R1 -C1r3+C5r1 */\
755 "movq 88(%2), %%mm7 \n\t" /* C3 C7 C3 C7 */\
756 "pmaddwd 96(%2), %%mm0 \n\t" /* -C5R3+C7R1 -C5r3+C7r1 */\
757 "pmaddwd %%mm3, %%mm7 \n\t" /* C3R7+C7R5 C3r7+C7r5 */\
758 "movq %%mm5, %%mm2 \n\t" /* A2 a2 */\
759 "pmaddwd 104(%2), %%mm3 \n\t" /* -C1R7+C3R5 -C1r7+C3r5 */\
760 "paddd %%mm7, %%mm4 \n\t" /* B2 b2 */\
761 "paddd %%mm4, %%mm2 \n\t" /* A2+B2 a2+b2 */\
762 "psubd %%mm4, %%mm5 \n\t" /* a2-B2 a2-b2 */\
763 "psrad $" #shift ", %%mm2 \n\t"\
764 "psrad $" #shift ", %%mm5 \n\t"\
765 "movq %%mm6, %%mm4 \n\t" /* A3 a3 */\
766 "paddd %%mm0, %%mm3 \n\t" /* B3 b3 */\
767 "paddd %%mm3, %%mm6 \n\t" /* A3+B3 a3+b3 */\
768 "psubd %%mm3, %%mm4 \n\t" /* a3-B3 a3-b3 */\
769 "psrad $" #shift ", %%mm6 \n\t"\
770 "psrad $" #shift ", %%mm4 \n\t"\
771 "packssdw %%mm2, %%mm2 \n\t" /* A2+B2 a2+b2 */\
772 "packssdw %%mm6, %%mm6 \n\t" /* A3+B3 a3+b3 */\
773 "movd %%mm2, 32+" #dst " \n\t"\
774 "packssdw %%mm4, %%mm4 \n\t" /* A3-B3 a3-b3 */\
775 "packssdw %%mm5, %%mm5 \n\t" /* A2-B2 a2-b2 */\
776 "movd %%mm6, 48+" #dst " \n\t"\
777 "movd %%mm4, 64+" #dst " \n\t"\
778 "movd %%mm5, 80+" #dst " \n\t"
781 //IDCT( src0, src4, src1, src5, dst, rounder, shift)
782 IDCT( (%1), 64(%1), 32(%1), 96(%1), 0(%0),/nop, 20)
783 IDCT( 8(%1), 72(%1), 40(%1), 104(%1), 4(%0),/nop, 20)
784 IDCT( 16(%1), 80(%1), 48(%1), 112(%1), 8(%0),/nop, 20)
785 IDCT( 24(%1), 88(%1), 56(%1), 120(%1), 12(%0),/nop, 20)
790 Z_COND_IDCT( 64(%0), 72(%0), 80(%0), 88(%0), 64(%1),paddd (%2), 11, 6f)
791 Z_COND_IDCT( 96(%0),104(%0),112(%0),120(%0), 96(%1),paddd (%2), 11, 5f)
794 #define IDCT(src0, src4, src1, src5, dst, rounder, shift) \
795 "movq " #src0 ", %%mm0 \n\t" /* R4 R0 r4 r0 */\
796 "movq " #src4 ", %%mm1 \n\t" /* R6 R2 r6 r2 */\
797 "movq " #src5 ", %%mm3 \n\t" /* R7 R5 r7 r5 */\
798 "movq 16(%2), %%mm4 \n\t" /* C4 C4 C4 C4 */\
799 "pmaddwd %%mm0, %%mm4 \n\t" /* C4R4+C4R0 C4r4+C4r0 */\
800 "movq 24(%2), %%mm5 \n\t" /* -C4 C4 -C4 C4 */\
801 "pmaddwd %%mm5, %%mm0 \n\t" /* -C4R4+C4R0 -C4r4+C4r0 */\
802 "movq 32(%2), %%mm5 \n\t" /* C6 C2 C6 C2 */\
803 "pmaddwd %%mm1, %%mm5 \n\t" /* C6R6+C2R2 C6r6+C2r2 */\
804 "movq 40(%2), %%mm6 \n\t" /* -C2 C6 -C2 C6 */\
805 "pmaddwd %%mm6, %%mm1 \n\t" /* -C2R6+C6R2 -C2r6+C6r2 */\
806 #rounder ", %%mm4 \n\t"\
807 "movq %%mm4, %%mm6 \n\t" /* C4R4+C4R0 C4r4+C4r0 */\
808 #rounder ", %%mm0 \n\t"\
809 "paddd %%mm5, %%mm4 \n\t" /* A0 a0 */\
810 "psubd %%mm5, %%mm6 \n\t" /* A3 a3 */\
811 "movq %%mm0, %%mm5 \n\t" /* -C4R4+C4R0 -C4r4+C4r0 */\
812 "paddd %%mm1, %%mm0 \n\t" /* A1 a1 */\
813 "psubd %%mm1, %%mm5 \n\t" /* A2 a2 */\
814 "movq 56(%2), %%mm1 \n\t" /* C7 C5 C7 C5 */\
815 "pmaddwd %%mm3, %%mm1 \n\t" /* C7R7+C5R5 C7r7+C5r5 */\
816 "movq 72(%2), %%mm7 \n\t" /* -C5 -C1 -C5 -C1 */\
817 "pmaddwd %%mm3, %%mm7 \n\t" /* -C5R7-C1R5 -C5r7-C1r5 */\
818 "paddd %%mm4, %%mm1 \n\t" /* A0+B0 a0+b0 */\
819 "paddd %%mm4, %%mm4 \n\t" /* 2A0 2a0 */\
820 "psubd %%mm1, %%mm4 \n\t" /* A0-B0 a0-b0 */\
821 "psrad $" #shift ", %%mm1 \n\t"\
822 "psrad $" #shift ", %%mm4 \n\t"\
823 "movq %%mm0, %%mm2 \n\t" /* A1 a1 */\
824 "paddd %%mm7, %%mm0 \n\t" /* A1+B1 a1+b1 */\
825 "psubd %%mm7, %%mm2 \n\t" /* A1-B1 a1-b1 */\
826 "psrad $" #shift ", %%mm0 \n\t"\
827 "psrad $" #shift ", %%mm2 \n\t"\
828 "packssdw %%mm1, %%mm1 \n\t" /* A0+B0 a0+b0 */\
829 "movd %%mm1, " #dst " \n\t"\
830 "packssdw %%mm0, %%mm0 \n\t" /* A1+B1 a1+b1 */\
831 "movd %%mm0, 16+" #dst " \n\t"\
832 "packssdw %%mm2, %%mm2 \n\t" /* A1-B1 a1-b1 */\
833 "movd %%mm2, 96+" #dst " \n\t"\
834 "packssdw %%mm4, %%mm4 \n\t" /* A0-B0 a0-b0 */\
835 "movd %%mm4, 112+" #dst " \n\t"\
836 "movq 88(%2), %%mm1 \n\t" /* C3 C7 C3 C7 */\
837 "pmaddwd %%mm3, %%mm1 \n\t" /* C3R7+C7R5 C3r7+C7r5 */\
838 "movq %%mm5, %%mm2 \n\t" /* A2 a2 */\
839 "pmaddwd 104(%2), %%mm3 \n\t" /* -C1R7+C3R5 -C1r7+C3r5 */\
840 "paddd %%mm1, %%mm2 \n\t" /* A2+B2 a2+b2 */\
841 "psubd %%mm1, %%mm5 \n\t" /* a2-B2 a2-b2 */\
842 "psrad $" #shift ", %%mm2 \n\t"\
843 "psrad $" #shift ", %%mm5 \n\t"\
844 "movq %%mm6, %%mm1 \n\t" /* A3 a3 */\
845 "paddd %%mm3, %%mm6 \n\t" /* A3+B3 a3+b3 */\
846 "psubd %%mm3, %%mm1 \n\t" /* a3-B3 a3-b3 */\
847 "psrad $" #shift ", %%mm6 \n\t"\
848 "psrad $" #shift ", %%mm1 \n\t"\
849 "packssdw %%mm2, %%mm2 \n\t" /* A2+B2 a2+b2 */\
850 "packssdw %%mm6, %%mm6 \n\t" /* A3+B3 a3+b3 */\
851 "movd %%mm2, 32+" #dst " \n\t"\
852 "packssdw %%mm1, %%mm1 \n\t" /* A3-B3 a3-b3 */\
853 "packssdw %%mm5, %%mm5 \n\t" /* A2-B2 a2-b2 */\
854 "movd %%mm6, 48+" #dst " \n\t"\
855 "movd %%mm1, 64+" #dst " \n\t"\
856 "movd %%mm5, 80+" #dst " \n\t"
858 //IDCT( src0, src4, src1, src5, dst, rounder, shift)
859 IDCT( (%1), 64(%1), 32(%1), 96(%1), 0(%0),/nop, 20)
860 IDCT( 8(%1), 72(%1), 40(%1), 104(%1), 4(%0),/nop, 20)
861 IDCT( 16(%1), 80(%1), 48(%1), 112(%1), 8(%0),/nop, 20)
862 IDCT( 24(%1), 88(%1), 56(%1), 120(%1), 12(%0),/nop, 20)
867 Z_COND_IDCT( 96(%0),104(%0),112(%0),120(%0), 96(%1),paddd (%2), 11, 7f)
870 #define IDCT(src0, src4, src1, src5, dst, rounder, shift) \
871 "movq " #src0 ", %%mm0 \n\t" /* R4 R0 r4 r0 */\
872 "movq " #src5 ", %%mm3 \n\t" /* R7 R5 r7 r5 */\
873 "movq 16(%2), %%mm4 \n\t" /* C4 C4 C4 C4 */\
874 "pmaddwd %%mm0, %%mm4 \n\t" /* C4R4+C4R0 C4r4+C4r0 */\
875 "movq 24(%2), %%mm5 \n\t" /* -C4 C4 -C4 C4 */\
876 "pmaddwd %%mm5, %%mm0 \n\t" /* -C4R4+C4R0 -C4r4+C4r0 */\
877 #rounder ", %%mm4 \n\t"\
878 "movq %%mm4, %%mm6 \n\t" /* C4R4+C4R0 C4r4+C4r0 */\
879 #rounder ", %%mm0 \n\t"\
880 "movq %%mm0, %%mm5 \n\t" /* -C4R4+C4R0 -C4r4+C4r0 */\
881 "movq 56(%2), %%mm1 \n\t" /* C7 C5 C7 C5 */\
882 "pmaddwd %%mm3, %%mm1 \n\t" /* C7R7+C5R5 C7r7+C5r5 */\
883 "movq 72(%2), %%mm7 \n\t" /* -C5 -C1 -C5 -C1 */\
884 "pmaddwd %%mm3, %%mm7 \n\t" /* -C5R7-C1R5 -C5r7-C1r5 */\
885 "paddd %%mm4, %%mm1 \n\t" /* A0+B0 a0+b0 */\
886 "paddd %%mm4, %%mm4 \n\t" /* 2A0 2a0 */\
887 "psubd %%mm1, %%mm4 \n\t" /* A0-B0 a0-b0 */\
888 "psrad $" #shift ", %%mm1 \n\t"\
889 "psrad $" #shift ", %%mm4 \n\t"\
890 "movq %%mm0, %%mm2 \n\t" /* A1 a1 */\
891 "paddd %%mm7, %%mm0 \n\t" /* A1+B1 a1+b1 */\
892 "psubd %%mm7, %%mm2 \n\t" /* A1-B1 a1-b1 */\
893 "psrad $" #shift ", %%mm0 \n\t"\
894 "psrad $" #shift ", %%mm2 \n\t"\
895 "packssdw %%mm1, %%mm1 \n\t" /* A0+B0 a0+b0 */\
896 "movd %%mm1, " #dst " \n\t"\
897 "packssdw %%mm0, %%mm0 \n\t" /* A1+B1 a1+b1 */\
898 "movd %%mm0, 16+" #dst " \n\t"\
899 "packssdw %%mm2, %%mm2 \n\t" /* A1-B1 a1-b1 */\
900 "movd %%mm2, 96+" #dst " \n\t"\
901 "packssdw %%mm4, %%mm4 \n\t" /* A0-B0 a0-b0 */\
902 "movd %%mm4, 112+" #dst " \n\t"\
903 "movq 88(%2), %%mm1 \n\t" /* C3 C7 C3 C7 */\
904 "pmaddwd %%mm3, %%mm1 \n\t" /* C3R7+C7R5 C3r7+C7r5 */\
905 "movq %%mm5, %%mm2 \n\t" /* A2 a2 */\
906 "pmaddwd 104(%2), %%mm3 \n\t" /* -C1R7+C3R5 -C1r7+C3r5 */\
907 "paddd %%mm1, %%mm2 \n\t" /* A2+B2 a2+b2 */\
908 "psubd %%mm1, %%mm5 \n\t" /* a2-B2 a2-b2 */\
909 "psrad $" #shift ", %%mm2 \n\t"\
910 "psrad $" #shift ", %%mm5 \n\t"\
911 "movq %%mm6, %%mm1 \n\t" /* A3 a3 */\
912 "paddd %%mm3, %%mm6 \n\t" /* A3+B3 a3+b3 */\
913 "psubd %%mm3, %%mm1 \n\t" /* a3-B3 a3-b3 */\
914 "psrad $" #shift ", %%mm6 \n\t"\
915 "psrad $" #shift ", %%mm1 \n\t"\
916 "packssdw %%mm2, %%mm2 \n\t" /* A2+B2 a2+b2 */\
917 "packssdw %%mm6, %%mm6 \n\t" /* A3+B3 a3+b3 */\
918 "movd %%mm2, 32+" #dst " \n\t"\
919 "packssdw %%mm1, %%mm1 \n\t" /* A3-B3 a3-b3 */\
920 "packssdw %%mm5, %%mm5 \n\t" /* A2-B2 a2-b2 */\
921 "movd %%mm6, 48+" #dst " \n\t"\
922 "movd %%mm1, 64+" #dst " \n\t"\
923 "movd %%mm5, 80+" #dst " \n\t"
926 //IDCT( src0, src4, src1, src5, dst, rounder, shift)
927 IDCT( (%1), 64(%1), 32(%1), 96(%1), 0(%0),/nop, 20)
928 IDCT( 8(%1), 72(%1), 40(%1), 104(%1), 4(%0),/nop, 20)
929 IDCT( 16(%1), 80(%1), 48(%1), 112(%1), 8(%0),/nop, 20)
930 IDCT( 24(%1), 88(%1), 56(%1), 120(%1), 12(%0),/nop, 20)
935 Z_COND_IDCT( 96(%0),104(%0),112(%0),120(%0), 96(%1),paddd (%2), 11, 3f)
938 #define IDCT(src0, src4, src1, src5, dst, rounder, shift) \
939 "movq " #src0 ", %%mm0 \n\t" /* R4 R0 r4 r0 */\
940 "movq " #src1 ", %%mm2 \n\t" /* R3 R1 r3 r1 */\
941 "movq " #src5 ", %%mm3 \n\t" /* R7 R5 r7 r5 */\
942 "movq 16(%2), %%mm4 \n\t" /* C4 C4 C4 C4 */\
943 "pmaddwd %%mm0, %%mm4 \n\t" /* C4R4+C4R0 C4r4+C4r0 */\
944 "movq 24(%2), %%mm5 \n\t" /* -C4 C4 -C4 C4 */\
945 "pmaddwd %%mm5, %%mm0 \n\t" /* -C4R4+C4R0 -C4r4+C4r0 */\
946 #rounder ", %%mm4 \n\t"\
947 "movq %%mm4, %%mm6 \n\t" /* C4R4+C4R0 C4r4+C4r0 */\
948 "movq 48(%2), %%mm7 \n\t" /* C3 C1 C3 C1 */\
949 #rounder ", %%mm0 \n\t"\
950 "pmaddwd %%mm2, %%mm7 \n\t" /* C3R3+C1R1 C3r3+C1r1 */\
951 "movq %%mm0, %%mm5 \n\t" /* -C4R4+C4R0 -C4r4+C4r0 */\
952 "movq 56(%2), %%mm1 \n\t" /* C7 C5 C7 C5 */\
953 "pmaddwd %%mm3, %%mm1 \n\t" /* C7R7+C5R5 C7r7+C5r5 */\
954 "pmaddwd 64(%2), %%mm2 \n\t" /* -C7R3+C3R1 -C7r3+C3r1 */\
955 "paddd %%mm1, %%mm7 \n\t" /* B0 b0 */\
956 "movq 72(%2), %%mm1 \n\t" /* -C5 -C1 -C5 -C1 */\
957 "pmaddwd %%mm3, %%mm1 \n\t" /* -C5R7-C1R5 -C5r7-C1r5 */\
958 "paddd %%mm4, %%mm7 \n\t" /* A0+B0 a0+b0 */\
959 "paddd %%mm4, %%mm4 \n\t" /* 2A0 2a0 */\
960 "psubd %%mm7, %%mm4 \n\t" /* A0-B0 a0-b0 */\
961 "paddd %%mm2, %%mm1 \n\t" /* B1 b1 */\
962 "psrad $" #shift ", %%mm7 \n\t"\
963 "psrad $" #shift ", %%mm4 \n\t"\
964 "movq %%mm0, %%mm2 \n\t" /* A1 a1 */\
965 "paddd %%mm1, %%mm0 \n\t" /* A1+B1 a1+b1 */\
966 "psubd %%mm1, %%mm2 \n\t" /* A1-B1 a1-b1 */\
967 "psrad $" #shift ", %%mm0 \n\t"\
968 "psrad $" #shift ", %%mm2 \n\t"\
969 "packssdw %%mm7, %%mm7 \n\t" /* A0+B0 a0+b0 */\
970 "movd %%mm7, " #dst " \n\t"\
971 "packssdw %%mm0, %%mm0 \n\t" /* A1+B1 a1+b1 */\
972 "movd %%mm0, 16+" #dst " \n\t"\
973 "packssdw %%mm2, %%mm2 \n\t" /* A1-B1 a1-b1 */\
974 "movd %%mm2, 96+" #dst " \n\t"\
975 "packssdw %%mm4, %%mm4 \n\t" /* A0-B0 a0-b0 */\
976 "movd %%mm4, 112+" #dst " \n\t"\
977 "movq " #src1 ", %%mm0 \n\t" /* R3 R1 r3 r1 */\
978 "movq 80(%2), %%mm4 \n\t" /* -C1 C5 -C1 C5 */\
979 "pmaddwd %%mm0, %%mm4 \n\t" /* -C1R3+C5R1 -C1r3+C5r1 */\
980 "movq 88(%2), %%mm7 \n\t" /* C3 C7 C3 C7 */\
981 "pmaddwd 96(%2), %%mm0 \n\t" /* -C5R3+C7R1 -C5r3+C7r1 */\
982 "pmaddwd %%mm3, %%mm7 \n\t" /* C3R7+C7R5 C3r7+C7r5 */\
983 "movq %%mm5, %%mm2 \n\t" /* A2 a2 */\
984 "pmaddwd 104(%2), %%mm3 \n\t" /* -C1R7+C3R5 -C1r7+C3r5 */\
985 "paddd %%mm7, %%mm4 \n\t" /* B2 b2 */\
986 "paddd %%mm4, %%mm2 \n\t" /* A2+B2 a2+b2 */\
987 "psubd %%mm4, %%mm5 \n\t" /* a2-B2 a2-b2 */\
988 "psrad $" #shift ", %%mm2 \n\t"\
989 "psrad $" #shift ", %%mm5 \n\t"\
990 "movq %%mm6, %%mm4 \n\t" /* A3 a3 */\
991 "paddd %%mm0, %%mm3 \n\t" /* B3 b3 */\
992 "paddd %%mm3, %%mm6 \n\t" /* A3+B3 a3+b3 */\
993 "psubd %%mm3, %%mm4 \n\t" /* a3-B3 a3-b3 */\
994 "psrad $" #shift ", %%mm6 \n\t"\
995 "psrad $" #shift ", %%mm4 \n\t"\
996 "packssdw %%mm2, %%mm2 \n\t" /* A2+B2 a2+b2 */\
997 "packssdw %%mm6, %%mm6 \n\t" /* A3+B3 a3+b3 */\
998 "movd %%mm2, 32+" #dst " \n\t"\
999 "packssdw %%mm4, %%mm4 \n\t" /* A3-B3 a3-b3 */\
1000 "packssdw %%mm5, %%mm5 \n\t" /* A2-B2 a2-b2 */\
1001 "movd %%mm6, 48+" #dst " \n\t"\
1002 "movd %%mm4, 64+" #dst " \n\t"\
1003 "movd %%mm5, 80+" #dst " \n\t"
1005 //IDCT( src0, src4, src1, src5, dst, rounder, shift)
1006 IDCT( (%1), 64(%1), 32(%1), 96(%1), 0(%0),/nop, 20)
1007 IDCT( 8(%1), 72(%1), 40(%1), 104(%1), 4(%0),/nop, 20)
1008 IDCT( 16(%1), 80(%1), 48(%1), 112(%1), 8(%0),/nop, 20)
1009 IDCT( 24(%1), 88(%1), 56(%1), 120(%1), 12(%0),/nop, 20)
1015 #define IDCT(src0, src4, src1, src5, dst, rounder, shift) \
1016 "movq " #src0 ", %%mm0 \n\t" /* R4 R0 r4 r0 */\
1017 "movq " #src1 ", %%mm2 \n\t" /* R3 R1 r3 r1 */\
1018 "movq 16(%2), %%mm4 \n\t" /* C4 C4 C4 C4 */\
1019 "pmaddwd %%mm0, %%mm4 \n\t" /* C4R4+C4R0 C4r4+C4r0 */\
1020 "movq 24(%2), %%mm5 \n\t" /* -C4 C4 -C4 C4 */\
1021 "pmaddwd %%mm5, %%mm0 \n\t" /* -C4R4+C4R0 -C4r4+C4r0 */\
1022 #rounder ", %%mm4 \n\t"\
1023 "movq %%mm4, %%mm6 \n\t" /* C4R4+C4R0 C4r4+C4r0 */\
1024 "movq 48(%2), %%mm7 \n\t" /* C3 C1 C3 C1 */\
1025 #rounder ", %%mm0 \n\t"\
1026 "pmaddwd %%mm2, %%mm7 \n\t" /* C3R3+C1R1 C3r3+C1r1 */\
1027 "movq %%mm0, %%mm5 \n\t" /* -C4R4+C4R0 -C4r4+C4r0 */\
1028 "movq 64(%2), %%mm3 \n\t"\
1029 "pmaddwd %%mm2, %%mm3 \n\t" /* -C7R3+C3R1 -C7r3+C3r1 */\
1030 "paddd %%mm4, %%mm7 \n\t" /* A0+B0 a0+b0 */\
1031 "paddd %%mm4, %%mm4 \n\t" /* 2A0 2a0 */\
1032 "psubd %%mm7, %%mm4 \n\t" /* A0-B0 a0-b0 */\
1033 "psrad $" #shift ", %%mm7 \n\t"\
1034 "psrad $" #shift ", %%mm4 \n\t"\
1035 "movq %%mm0, %%mm1 \n\t" /* A1 a1 */\
1036 "paddd %%mm3, %%mm0 \n\t" /* A1+B1 a1+b1 */\
1037 "psubd %%mm3, %%mm1 \n\t" /* A1-B1 a1-b1 */\
1038 "psrad $" #shift ", %%mm0 \n\t"\
1039 "psrad $" #shift ", %%mm1 \n\t"\
1040 "packssdw %%mm7, %%mm7 \n\t" /* A0+B0 a0+b0 */\
1041 "movd %%mm7, " #dst " \n\t"\
1042 "packssdw %%mm0, %%mm0 \n\t" /* A1+B1 a1+b1 */\
1043 "movd %%mm0, 16+" #dst " \n\t"\
1044 "packssdw %%mm1, %%mm1 \n\t" /* A1-B1 a1-b1 */\
1045 "movd %%mm1, 96+" #dst " \n\t"\
1046 "packssdw %%mm4, %%mm4 \n\t" /* A0-B0 a0-b0 */\
1047 "movd %%mm4, 112+" #dst " \n\t"\
1048 "movq 80(%2), %%mm4 \n\t" /* -C1 C5 -C1 C5 */\
1049 "pmaddwd %%mm2, %%mm4 \n\t" /* -C1R3+C5R1 -C1r3+C5r1 */\
1050 "pmaddwd 96(%2), %%mm2 \n\t" /* -C5R3+C7R1 -C5r3+C7r1 */\
1051 "movq %%mm5, %%mm1 \n\t" /* A2 a2 */\
1052 "paddd %%mm4, %%mm1 \n\t" /* A2+B2 a2+b2 */\
1053 "psubd %%mm4, %%mm5 \n\t" /* a2-B2 a2-b2 */\
1054 "psrad $" #shift ", %%mm1 \n\t"\
1055 "psrad $" #shift ", %%mm5 \n\t"\
1056 "movq %%mm6, %%mm4 \n\t" /* A3 a3 */\
1057 "paddd %%mm2, %%mm6 \n\t" /* A3+B3 a3+b3 */\
1058 "psubd %%mm2, %%mm4 \n\t" /* a3-B3 a3-b3 */\
1059 "psrad $" #shift ", %%mm6 \n\t"\
1060 "psrad $" #shift ", %%mm4 \n\t"\
1061 "packssdw %%mm1, %%mm1 \n\t" /* A2+B2 a2+b2 */\
1062 "packssdw %%mm6, %%mm6 \n\t" /* A3+B3 a3+b3 */\
1063 "movd %%mm1, 32+" #dst " \n\t"\
1064 "packssdw %%mm4, %%mm4 \n\t" /* A3-B3 a3-b3 */\
1065 "packssdw %%mm5, %%mm5 \n\t" /* A2-B2 a2-b2 */\
1066 "movd %%mm6, 48+" #dst " \n\t"\
1067 "movd %%mm4, 64+" #dst " \n\t"\
1068 "movd %%mm5, 80+" #dst " \n\t"
1071 //IDCT( src0, src4, src1, src5, dst, rounder, shift)
1072 IDCT( (%1), 64(%1), 32(%1), 96(%1), 0(%0),/nop, 20)
1073 IDCT( 8(%1), 72(%1), 40(%1), 104(%1), 4(%0),/nop, 20)
1074 IDCT( 16(%1), 80(%1), 48(%1), 112(%1), 8(%0),/nop, 20)
1075 IDCT( 24(%1), 88(%1), 56(%1), 120(%1), 12(%0),/nop, 20)
1081 #define IDCT(src0, src4, src1, src5, dst, rounder, shift) \
1082 "movq " #src0 ", %%mm0 \n\t" /* R4 R0 r4 r0 */\
1083 "movq " #src4 ", %%mm1 \n\t" /* R6 R2 r6 r2 */\
1084 "movq 16(%2), %%mm4 \n\t" /* C4 C4 C4 C4 */\
1085 "pmaddwd %%mm0, %%mm4 \n\t" /* C4R4+C4R0 C4r4+C4r0 */\
1086 "movq 24(%2), %%mm5 \n\t" /* -C4 C4 -C4 C4 */\
1087 "pmaddwd %%mm5, %%mm0 \n\t" /* -C4R4+C4R0 -C4r4+C4r0 */\
1088 "movq 32(%2), %%mm5 \n\t" /* C6 C2 C6 C2 */\
1089 "pmaddwd %%mm1, %%mm5 \n\t" /* C6R6+C2R2 C6r6+C2r2 */\
1090 "movq 40(%2), %%mm6 \n\t" /* -C2 C6 -C2 C6 */\
1091 "pmaddwd %%mm6, %%mm1 \n\t" /* -C2R6+C6R2 -C2r6+C6r2 */\
1092 #rounder ", %%mm4 \n\t"\
1093 "movq %%mm4, %%mm6 \n\t" /* C4R4+C4R0 C4r4+C4r0 */\
1094 "paddd %%mm5, %%mm4 \n\t" /* A0 a0 */\
1095 #rounder ", %%mm0 \n\t"\
1096 "psubd %%mm5, %%mm6 \n\t" /* A3 a3 */\
1097 "movq %%mm0, %%mm5 \n\t" /* -C4R4+C4R0 -C4r4+C4r0 */\
1098 "paddd %%mm1, %%mm0 \n\t" /* A1 a1 */\
1099 "psubd %%mm1, %%mm5 \n\t" /* A2 a2 */\
1100 "movq 8+" #src0 ", %%mm2 \n\t" /* R4 R0 r4 r0 */\
1101 "movq 8+" #src4 ", %%mm3 \n\t" /* R6 R2 r6 r2 */\
1102 "movq 16(%2), %%mm1 \n\t" /* C4 C4 C4 C4 */\
1103 "pmaddwd %%mm2, %%mm1 \n\t" /* C4R4+C4R0 C4r4+C4r0 */\
1104 "movq 24(%2), %%mm7 \n\t" /* -C4 C4 -C4 C4 */\
1105 "pmaddwd %%mm7, %%mm2 \n\t" /* -C4R4+C4R0 -C4r4+C4r0 */\
1106 "movq 32(%2), %%mm7 \n\t" /* C6 C2 C6 C2 */\
1107 "pmaddwd %%mm3, %%mm7 \n\t" /* C6R6+C2R2 C6r6+C2r2 */\
1108 "pmaddwd 40(%2), %%mm3 \n\t" /* -C2R6+C6R2 -C2r6+C6r2 */\
1109 #rounder ", %%mm1 \n\t"\
1110 "paddd %%mm1, %%mm7 \n\t" /* A0 a0 */\
1111 "paddd %%mm1, %%mm1 \n\t" /* 2C0 2c0 */\
1112 #rounder ", %%mm2 \n\t"\
1113 "psubd %%mm7, %%mm1 \n\t" /* A3 a3 */\
1114 "paddd %%mm2, %%mm3 \n\t" /* A1 a1 */\
1115 "paddd %%mm2, %%mm2 \n\t" /* 2C1 2c1 */\
1116 "psubd %%mm3, %%mm2 \n\t" /* A2 a2 */\
1117 "psrad $" #shift ", %%mm4 \n\t"\
1118 "psrad $" #shift ", %%mm7 \n\t"\
1119 "psrad $" #shift ", %%mm3 \n\t"\
1120 "packssdw %%mm7, %%mm4 \n\t" /* A0 a0 */\
1121 "movq %%mm4, " #dst " \n\t"\
1122 "psrad $" #shift ", %%mm0 \n\t"\
1123 "packssdw %%mm3, %%mm0 \n\t" /* A1 a1 */\
1124 "movq %%mm0, 16+" #dst " \n\t"\
1125 "movq %%mm0, 96+" #dst " \n\t"\
1126 "movq %%mm4, 112+" #dst " \n\t"\
1127 "psrad $" #shift ", %%mm5 \n\t"\
1128 "psrad $" #shift ", %%mm6 \n\t"\
1129 "psrad $" #shift ", %%mm2 \n\t"\
1130 "packssdw %%mm2, %%mm5 \n\t" /* A2-B2 a2-b2 */\
1131 "movq %%mm5, 32+" #dst " \n\t"\
1132 "psrad $" #shift ", %%mm1 \n\t"\
1133 "packssdw %%mm1, %%mm6 \n\t" /* A3+B3 a3+b3 */\
1134 "movq %%mm6, 48+" #dst " \n\t"\
1135 "movq %%mm6, 64+" #dst " \n\t"\
1136 "movq %%mm5, 80+" #dst " \n\t"
1139 //IDCT( src0, src4, src1, src5, dst, rounder, shift)
1140 IDCT( 0(%1), 64(%1), 32(%1), 96(%1), 0(%0),/nop, 20)
1141 //IDCT( 8(%1), 72(%1), 40(%1), 104(%1), 4(%0),/nop, 20)
1142 IDCT( 16(%1), 80(%1), 48(%1), 112(%1), 8(%0),/nop, 20)
1143 //IDCT( 24(%1), 88(%1), 56(%1), 120(%1), 12(%0),/nop, 20)
1150 #define IDCT(src0, src4, src1, src5, dst, rounder, shift) \
1151 "movq " #src0 ", %%mm0 \n\t" /* R4 R0 r4 r0 */\
1152 "movq " #src4 ", %%mm1 \n\t" /* R6 R2 r6 r2 */\
1153 "movq " #src1 ", %%mm2 \n\t" /* R3 R1 r3 r1 */\
1154 "movq 16(%2), %%mm4 \n\t" /* C4 C4 C4 C4 */\
1155 "pmaddwd %%mm0, %%mm4 \n\t" /* C4R4+C4R0 C4r4+C4r0 */\
1156 "movq 24(%2), %%mm5 \n\t" /* -C4 C4 -C4 C4 */\
1157 "pmaddwd %%mm5, %%mm0 \n\t" /* -C4R4+C4R0 -C4r4+C4r0 */\
1158 "movq 32(%2), %%mm5 \n\t" /* C6 C2 C6 C2 */\
1159 "pmaddwd %%mm1, %%mm5 \n\t" /* C6R6+C2R2 C6r6+C2r2 */\
1160 "movq 40(%2), %%mm6 \n\t" /* -C2 C6 -C2 C6 */\
1161 "pmaddwd %%mm6, %%mm1 \n\t" /* -C2R6+C6R2 -C2r6+C6r2 */\
1162 #rounder ", %%mm4 \n\t"\
1163 "movq %%mm4, %%mm6 \n\t" /* C4R4+C4R0 C4r4+C4r0 */\
1164 "movq 48(%2), %%mm7 \n\t" /* C3 C1 C3 C1 */\
1165 #rounder ", %%mm0 \n\t"\
1166 "pmaddwd %%mm2, %%mm7 \n\t" /* C3R3+C1R1 C3r3+C1r1 */\
1167 "paddd %%mm5, %%mm4 \n\t" /* A0 a0 */\
1168 "psubd %%mm5, %%mm6 \n\t" /* A3 a3 */\
1169 "movq %%mm0, %%mm5 \n\t" /* -C4R4+C4R0 -C4r4+C4r0 */\
1170 "paddd %%mm1, %%mm0 \n\t" /* A1 a1 */\
1171 "psubd %%mm1, %%mm5 \n\t" /* A2 a2 */\
1172 "movq 64(%2), %%mm1 \n\t"\
1173 "pmaddwd %%mm2, %%mm1 \n\t" /* -C7R3+C3R1 -C7r3+C3r1 */\
1174 "paddd %%mm4, %%mm7 \n\t" /* A0+B0 a0+b0 */\
1175 "paddd %%mm4, %%mm4 \n\t" /* 2A0 2a0 */\
1176 "psubd %%mm7, %%mm4 \n\t" /* A0-B0 a0-b0 */\
1177 "psrad $" #shift ", %%mm7 \n\t"\
1178 "psrad $" #shift ", %%mm4 \n\t"\
1179 "movq %%mm0, %%mm3 \n\t" /* A1 a1 */\
1180 "paddd %%mm1, %%mm0 \n\t" /* A1+B1 a1+b1 */\
1181 "psubd %%mm1, %%mm3 \n\t" /* A1-B1 a1-b1 */\
1182 "psrad $" #shift ", %%mm0 \n\t"\
1183 "psrad $" #shift ", %%mm3 \n\t"\
1184 "packssdw %%mm7, %%mm7 \n\t" /* A0+B0 a0+b0 */\
1185 "movd %%mm7, " #dst " \n\t"\
1186 "packssdw %%mm0, %%mm0 \n\t" /* A1+B1 a1+b1 */\
1187 "movd %%mm0, 16+" #dst " \n\t"\
1188 "packssdw %%mm3, %%mm3 \n\t" /* A1-B1 a1-b1 */\
1189 "movd %%mm3, 96+" #dst " \n\t"\
1190 "packssdw %%mm4, %%mm4 \n\t" /* A0-B0 a0-b0 */\
1191 "movd %%mm4, 112+" #dst " \n\t"\
1192 "movq 80(%2), %%mm4 \n\t" /* -C1 C5 -C1 C5 */\
1193 "pmaddwd %%mm2, %%mm4 \n\t" /* -C1R3+C5R1 -C1r3+C5r1 */\
1194 "pmaddwd 96(%2), %%mm2 \n\t" /* -C5R3+C7R1 -C5r3+C7r1 */\
1195 "movq %%mm5, %%mm3 \n\t" /* A2 a2 */\
1196 "paddd %%mm4, %%mm3 \n\t" /* A2+B2 a2+b2 */\
1197 "psubd %%mm4, %%mm5 \n\t" /* a2-B2 a2-b2 */\
1198 "psrad $" #shift ", %%mm3 \n\t"\
1199 "psrad $" #shift ", %%mm5 \n\t"\
1200 "movq %%mm6, %%mm4 \n\t" /* A3 a3 */\
1201 "paddd %%mm2, %%mm6 \n\t" /* A3+B3 a3+b3 */\
1202 "psubd %%mm2, %%mm4 \n\t" /* a3-B3 a3-b3 */\
1203 "psrad $" #shift ", %%mm6 \n\t"\
1204 "packssdw %%mm3, %%mm3 \n\t" /* A2+B2 a2+b2 */\
1205 "movd %%mm3, 32+" #dst " \n\t"\
1206 "psrad $" #shift ", %%mm4 \n\t"\
1207 "packssdw %%mm6, %%mm6 \n\t" /* A3+B3 a3+b3 */\
1208 "movd %%mm6, 48+" #dst " \n\t"\
1209 "packssdw %%mm4, %%mm4 \n\t" /* A3-B3 a3-b3 */\
1210 "packssdw %%mm5, %%mm5 \n\t" /* A2-B2 a2-b2 */\
1211 "movd %%mm4, 64+" #dst " \n\t"\
1212 "movd %%mm5, 80+" #dst " \n\t"
1215 //IDCT( src0, src4, src1, src5, dst, rounder, shift)
1216 IDCT( (%1), 64(%1), 32(%1), 96(%1), 0(%0),/nop, 20)
1217 IDCT( 8(%1), 72(%1), 40(%1), 104(%1), 4(%0),/nop, 20)
1218 IDCT( 16(%1), 80(%1), 48(%1), 112(%1), 8(%0),/nop, 20)
1219 IDCT( 24(%1), 88(%1), 56(%1), 120(%1), 12(%0),/nop, 20)
1226 #define IDCT(src0, src4, src1, src5, dst, rounder, shift) \
1227 "movq " #src0 ", %%mm0 \n\t" /* R4 R0 r4 r0 */\
1228 "movq 16(%2), %%mm4 \n\t" /* C4 C4 C4 C4 */\
1229 "pmaddwd %%mm0, %%mm4 \n\t" /* C4R4+C4R0 C4r4+C4r0 */\
1230 "movq 24(%2), %%mm5 \n\t" /* -C4 C4 -C4 C4 */\
1231 "pmaddwd %%mm5, %%mm0 \n\t" /* -C4R4+C4R0 -C4r4+C4r0 */\
1232 #rounder ", %%mm4 \n\t"\
1233 #rounder ", %%mm0 \n\t"\
1234 "psrad $" #shift ", %%mm4 \n\t"\
1235 "psrad $" #shift ", %%mm0 \n\t"\
1236 "movq 8+" #src0 ", %%mm2 \n\t" /* R4 R0 r4 r0 */\
1237 "movq 16(%2), %%mm1 \n\t" /* C4 C4 C4 C4 */\
1238 "pmaddwd %%mm2, %%mm1 \n\t" /* C4R4+C4R0 C4r4+C4r0 */\
1239 "movq 24(%2), %%mm7 \n\t" /* -C4 C4 -C4 C4 */\
1240 "pmaddwd %%mm7, %%mm2 \n\t" /* -C4R4+C4R0 -C4r4+C4r0 */\
1241 "movq 32(%2), %%mm7 \n\t" /* C6 C2 C6 C2 */\
1242 #rounder ", %%mm1 \n\t"\
1243 #rounder ", %%mm2 \n\t"\
1244 "psrad $" #shift ", %%mm1 \n\t"\
1245 "packssdw %%mm1, %%mm4 \n\t" /* A0 a0 */\
1246 "movq %%mm4, " #dst " \n\t"\
1247 "psrad $" #shift ", %%mm2 \n\t"\
1248 "packssdw %%mm2, %%mm0 \n\t" /* A1 a1 */\
1249 "movq %%mm0, 16+" #dst " \n\t"\
1250 "movq %%mm0, 96+" #dst " \n\t"\
1251 "movq %%mm4, 112+" #dst " \n\t"\
1252 "movq %%mm0, 32+" #dst " \n\t"\
1253 "movq %%mm4, 48+" #dst " \n\t"\
1254 "movq %%mm4, 64+" #dst " \n\t"\
1255 "movq %%mm0, 80+" #dst " \n\t"
1257 //IDCT( src0, src4, src1, src5, dst, rounder, shift)
1258 IDCT( 0(%1), 64(%1), 32(%1), 96(%1), 0(%0),/nop, 20)
1259 //IDCT( 8(%1), 72(%1), 40(%1), 104(%1), 4(%0),/nop, 20)
1260 IDCT( 16(%1), 80(%1), 48(%1), 112(%1), 8(%0),/nop, 20)
1261 //IDCT( 24(%1), 88(%1), 56(%1), 120(%1), 12(%0),/nop, 20)
1268 00 40 04 44 20 60 24 64
1269 10 30 14 34 50 70 54 74
1270 01 41 03 43 21 61 23 63
1271 11 31 13 33 51 71 53 73
1272 02 42 06 46 22 62 26 66
1273 12 32 16 36 52 72 56 76
1274 05 45 07 47 25 65 27 67
1275 15 35 17 37 55 75 57 77
1278 00 04 10 14 20 24 30 34
1279 40 44 50 54 60 64 70 74
1280 01 03 11 13 21 23 31 33
1281 41 43 51 53 61 63 71 73
1282 02 06 12 16 22 26 32 36
1283 42 46 52 56 62 66 72 76
1284 05 07 15 17 25 27 35 37
1285 45 47 55 57 65 67 75 77
1289 :: "r" (block), "r" (temp), "r" (coeffs)
1294 void simple_idct_mmx(int16_t *block)