;; Machine description for IBM RISC System 6000 (POWER) for GNU C compiler
;; Copyright (C) 1990-2015 Free Software Foundation, Inc.
;; Contributed by Richard Kenner (kenner@vlsi1.ultra.nyu.edu)

;; This file is part of GCC.

;; GCC is free software; you can redistribute it and/or modify it
;; under the terms of the GNU General Public License as published
;; by the Free Software Foundation; either version 3, or (at your
;; option) any later version.

;; GCC is distributed in the hope that it will be useful, but WITHOUT
;; ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
;; or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public
;; License for more details.

;; You should have received a copy of the GNU General Public License
;; along with GCC; see the file COPYING3.  If not see
;; <http://www.gnu.org/licenses/>.

;;- See file "rtl.def" for documentation on define_insn, match_*, et. al.

;;
;; REGNOS
;;

(define_constants
  [(FIRST_GPR_REGNO		0)
   (STACK_POINTER_REGNUM	1)
   (TOC_REGNUM			2)
   (STATIC_CHAIN_REGNUM		11)
   (HARD_FRAME_POINTER_REGNUM	31)
   (LAST_GPR_REGNO		31)
   (FIRST_FPR_REGNO		32)
   (LAST_FPR_REGNO		63)
   (LR_REGNO			65)
   (CTR_REGNO			66)
   (ARG_POINTER_REGNUM		67)
   (CR0_REGNO			68)
   (CR1_REGNO			69)
   (CR2_REGNO			70)
   (CR3_REGNO			71)
   (CR4_REGNO			72)
   (CR5_REGNO			73)
   (CR6_REGNO			74)
   (CR7_REGNO			75)
   (MAX_CR_REGNO		75)
   (CA_REGNO			76)
   (FIRST_ALTIVEC_REGNO		77)
   (LAST_ALTIVEC_REGNO		108)
   (VRSAVE_REGNO		109)
   (VSCR_REGNO			110)
   (SPE_ACC_REGNO		111)
   (SPEFSCR_REGNO		112)
   (FRAME_POINTER_REGNUM	113)
   (TFHAR_REGNO			114)
   (TFIAR_REGNO			115)
   (TEXASR_REGNO		116)
   (FIRST_SPE_HIGH_REGNO	117)
   (LAST_SPE_HIGH_REGNO		148)
  ])

;;
;; UNSPEC usage
;;

(define_c_enum "unspec"
  [UNSPEC_FRSP			; frsp for POWER machines
   UNSPEC_PROBE_STACK		; probe stack memory reference
   UNSPEC_TOCPTR		; address of a word pointing to the TOC
   UNSPEC_TOC			; address of the TOC (more-or-less)
   UNSPEC_TOCSLOT		; offset from r1 of toc pointer save slot
   UNSPEC_MOVSI_GOT
   UNSPEC_MV_CR_OV		; move_from_CR_ov_bit
   UNSPEC_FCTIWZ
   UNSPEC_FRIM
   UNSPEC_FRIN
   UNSPEC_FRIP
   UNSPEC_FRIZ
   UNSPEC_LD_MPIC		; load_macho_picbase
   UNSPEC_RELD_MPIC		; re-load_macho_picbase
   UNSPEC_MPIC_CORRECT		; macho_correct_pic
   UNSPEC_TLSGD
   UNSPEC_TLSLD
   UNSPEC_MOVESI_FROM_CR
   UNSPEC_MOVESI_TO_CR
   UNSPEC_TLSDTPREL
   UNSPEC_TLSDTPRELHA
   UNSPEC_TLSDTPRELLO
   UNSPEC_TLSGOTDTPREL
   UNSPEC_TLSTPREL
   UNSPEC_TLSTPRELHA
   UNSPEC_TLSTPRELLO
   UNSPEC_TLSGOTTPREL
   UNSPEC_TLSTLS
   UNSPEC_FIX_TRUNC_TF		; fadd, rounding towards zero
   UNSPEC_MV_CR_GT		; move_from_CR_gt_bit
   UNSPEC_STFIWX
   UNSPEC_POPCNTB
   UNSPEC_FRES
   UNSPEC_SP_SET
   UNSPEC_SP_TEST
   UNSPEC_SYNC
   UNSPEC_LWSYNC
   UNSPEC_SYNC_OP
   UNSPEC_ATOMIC
   UNSPEC_CMPXCHG
   UNSPEC_XCHG
   UNSPEC_AND
   UNSPEC_DLMZB
   UNSPEC_DLMZB_CR
   UNSPEC_DLMZB_STRLEN
   UNSPEC_RSQRT
   UNSPEC_TOCREL
   UNSPEC_MACHOPIC_OFFSET
   UNSPEC_BPERM
   UNSPEC_COPYSIGN
   UNSPEC_PARITY
   UNSPEC_FCTIW
   UNSPEC_FCTID
   UNSPEC_LFIWAX
   UNSPEC_LFIWZX
   UNSPEC_FCTIWUZ
   UNSPEC_GRP_END_NOP
   UNSPEC_P8V_FMRGOW
   UNSPEC_P8V_MTVSRWZ
   UNSPEC_P8V_RELOAD_FROM_GPR
   UNSPEC_P8V_MTVSRD
   UNSPEC_P8V_XXPERMDI
   UNSPEC_P8V_RELOAD_FROM_VSX
   UNSPEC_ADDG6S
   UNSPEC_CDTBCD
   UNSPEC_CBCDTD
   UNSPEC_DIVE
   UNSPEC_DIVEO
   UNSPEC_DIVEU
   UNSPEC_DIVEUO
   UNSPEC_UNPACK_128BIT
   UNSPEC_PACK_128BIT
   UNSPEC_LSQ
   UNSPEC_FUSION_GPR
  ])

;;
;; UNSPEC_VOLATILE usage
;;

(define_c_enum "unspecv"
  [UNSPECV_BLOCK
   UNSPECV_LL			; load-locked
   UNSPECV_SC			; store-conditional
   UNSPECV_PROBE_STACK_RANGE	; probe range of stack addresses
   UNSPECV_EH_RR		; eh_reg_restore
   UNSPECV_ISYNC		; isync instruction
   UNSPECV_MFTB			; move from time base
   UNSPECV_NLGR			; non-local goto receiver
   UNSPECV_MFFS			; Move from FPSCR
   UNSPECV_MTFSF		; Move to FPSCR Fields
  ])


;; Define an insn type attribute.  This is used in function unit delay
;; computations.
(define_attr "type"
  "integer,two,three,
   add,logical,shift,insert,
   mul,halfmul,div,
   exts,cntlz,popcnt,isel,
   load,store,fpload,fpstore,vecload,vecstore,
   cmp,
   branch,jmpreg,mfjmpr,mtjmpr,trap,isync,sync,load_l,store_c,
   cr_logical,delayed_cr,mfcr,mfcrf,mtcr,
   fpcompare,fp,fpsimple,dmul,sdiv,ddiv,ssqrt,dsqrt,
   brinc,
   vecsimple,veccomplex,vecdiv,veccmp,veccmpsimple,vecperm,
   vecfloat,vecfdiv,vecdouble,mffgpr,mftgpr,crypto,
   htm"
  (const_string "integer"))

;; What data size does this instruction work on?
;; This is used for insert, mul.
(define_attr "size" "8,16,32,64" (const_string "32"))

;; Is this instruction record form ("dot", signed compare to 0, writing CR0)?
;; This is used for add, logical, shift, exts, mul.
(define_attr "dot" "no,yes" (const_string "no"))

;; Does this instruction sign-extend its result?
;; This is used for load insns.
(define_attr "sign_extend" "no,yes" (const_string "no"))

;; Does this instruction use indexed (that is, reg+reg) addressing?
;; This is used for load and store insns.  If operand 0 or 1 is a MEM
;; it is automatically set based on that.  If a load or store instruction
;; has fewer than two operands it needs to set this attribute manually
;; or the compiler will crash.
(define_attr "indexed" "no,yes"
  (if_then_else (ior (match_operand 0 "indexed_address_mem")
		     (match_operand 1 "indexed_address_mem"))
		(const_string "yes")
		(const_string "no")))

;; Does this instruction use update addressing?
;; This is used for load and store insns.  See the comments for "indexed".
(define_attr "update" "no,yes"
  (if_then_else (ior (match_operand 0 "update_address_mem")
		     (match_operand 1 "update_address_mem"))
		(const_string "yes")
		(const_string "no")))

;; Is this instruction using operands[2] as shift amount, and can that be a
;; register?
;; This is used for shift insns.
(define_attr "maybe_var_shift" "no,yes" (const_string "no"))

;; Is this instruction using a shift amount from a register?
;; This is used for shift insns.
(define_attr "var_shift" "no,yes"
  (if_then_else (and (eq_attr "type" "shift")
		     (eq_attr "maybe_var_shift" "yes"))
		(if_then_else (match_operand 2 "gpc_reg_operand")
			      (const_string "yes")
			      (const_string "no"))
		(const_string "no")))

;; Define floating point instruction sub-types for use with Xfpu.md
(define_attr "fp_type" "fp_default,fp_addsub_s,fp_addsub_d,fp_mul_s,fp_mul_d,fp_div_s,fp_div_d,fp_maddsub_s,fp_maddsub_d,fp_sqrt_s,fp_sqrt_d" (const_string "fp_default"))

;; Length (in bytes).
; '(pc)' in the following doesn't include the instruction itself; it is
; calculated as if the instruction had zero size.
(define_attr "length" ""
  (if_then_else (eq_attr "type" "branch")
		(if_then_else (and (ge (minus (match_dup 0) (pc))
				       (const_int -32768))
				   (lt (minus (match_dup 0) (pc))
				       (const_int 32764)))
			      (const_int 4)
			      (const_int 8))
		(const_int 4)))

;; Processor type -- this attribute must exactly match the processor_type
;; enumeration in rs6000-opts.h.
(define_attr "cpu"
  "ppc601,ppc603,ppc604,ppc604e,ppc620,ppc630,
   ppc750,ppc7400,ppc7450,
   ppc403,ppc405,ppc440,ppc476,
   ppc8540,ppc8548,ppce300c2,ppce300c3,ppce500mc,ppce500mc64,ppce5500,ppce6500,
   power4,power5,power6,power7,power8,
   rs64a,mpccore,cell,ppca2,titan"
  (const (symbol_ref "rs6000_cpu_attr")))


;; If this instruction is microcoded on the CELL processor
; The default for load extended, the recorded instructions and rotate/shifts by a variable is always microcoded
(define_attr "cell_micro" "not,conditional,always"
  (if_then_else (ior (and (eq_attr "type" "shift,exts,mul")
			  (eq_attr "dot" "yes"))
		     (and (eq_attr "type" "load")
			  (eq_attr "sign_extend" "yes"))
		     (and (eq_attr "type" "shift")
			  (eq_attr "var_shift" "yes")))
		(const_string "always")
		(const_string "not")))

(automata_option "ndfa")

(include "rs64.md")
(include "mpc.md")
(include "40x.md")
(include "440.md")
(include "476.md")
(include "601.md")
(include "603.md")
(include "6xx.md")
(include "7xx.md")
(include "7450.md")
(include "8540.md")
(include "e300c2c3.md")
(include "e500mc.md")
(include "e500mc64.md")
(include "e5500.md")
(include "e6500.md")
(include "power4.md")
(include "power5.md")
(include "power6.md")
(include "power7.md")
(include "power8.md")
(include "cell.md")
(include "xfpu.md")
(include "a2.md")
(include "titan.md")

(include "predicates.md")
(include "constraints.md")

(include "darwin.md")


;; Mode iterators

; This mode iterator allows :GPR to be used to indicate the allowable size
; of whole values in GPRs.
(define_mode_iterator GPR [SI (DI "TARGET_POWERPC64")])

; Any supported integer mode.
(define_mode_iterator INT [QI HI SI DI TI PTI])

; Any supported integer mode that fits in one register.
(define_mode_iterator INT1 [QI HI SI (DI "TARGET_POWERPC64")])

; Everything we can extend QImode to.
(define_mode_iterator EXTQI [HI SI (DI "TARGET_POWERPC64")])

; Everything we can extend HImode to.
(define_mode_iterator EXTHI [SI (DI "TARGET_POWERPC64")])

; Everything we can extend SImode to.
(define_mode_iterator EXTSI [(DI "TARGET_POWERPC64")])

; QImode or HImode for small atomic ops
(define_mode_iterator QHI [QI HI])

; HImode or SImode for sign extended fusion ops
(define_mode_iterator HSI [HI SI])

; SImode or DImode, even if DImode doesn't fit in GPRs.
(define_mode_iterator SDI [SI DI])

; The size of a pointer.  Also, the size of the value that a record-condition
; (one with a '.') will compare; and the size used for arithmetic carries.
(define_mode_iterator P [(SI "TARGET_32BIT") (DI "TARGET_64BIT")])

; Iterator to add PTImode along with TImode (TImode can go in VSX registers,
; PTImode is GPR only)
(define_mode_iterator TI2 [TI PTI])

; Any hardware-supported floating-point mode
(define_mode_iterator FP [
  (SF "TARGET_HARD_FLOAT 
   && ((TARGET_FPRS && TARGET_SINGLE_FLOAT) || TARGET_E500_SINGLE)")
  (DF "TARGET_HARD_FLOAT 
   && ((TARGET_FPRS && TARGET_DOUBLE_FLOAT) || TARGET_E500_DOUBLE)")
  (TF "!TARGET_IEEEQUAD
   && TARGET_HARD_FLOAT
   && (TARGET_FPRS || TARGET_E500_DOUBLE)
   && TARGET_LONG_DOUBLE_128")
  (DD "TARGET_DFP")
  (TD "TARGET_DFP")])

; Any fma capable floating-point mode.
(define_mode_iterator FMA_F [
  (SF "TARGET_HARD_FLOAT && TARGET_FPRS && TARGET_SINGLE_FLOAT")
  (DF "(TARGET_HARD_FLOAT && TARGET_FPRS && TARGET_DOUBLE_FLOAT)
       || VECTOR_UNIT_VSX_P (DFmode)")
  (V2SF "TARGET_PAIRED_FLOAT")
  (V4SF "VECTOR_UNIT_ALTIVEC_OR_VSX_P (V4SFmode)")
  (V2DF "VECTOR_UNIT_ALTIVEC_OR_VSX_P (V2DFmode)")
  ])

; Floating point move iterators to combine binary and decimal moves
(define_mode_iterator FMOVE32 [SF SD])
(define_mode_iterator FMOVE64 [DF DD])
(define_mode_iterator FMOVE64X [DI DF DD])
(define_mode_iterator FMOVE128 [(TF "!TARGET_IEEEQUAD && TARGET_LONG_DOUBLE_128")
				(TD "TARGET_HARD_FLOAT && TARGET_FPRS")])

; Iterators for 128 bit types for direct move
(define_mode_iterator FMOVE128_GPR [(TI    "TARGET_VSX_TIMODE")
				    (V16QI "")
				    (V8HI  "")
				    (V4SI  "")
				    (V4SF  "")
				    (V2DI  "")
				    (V2DF  "")
				    (V1TI  "")])

; Whether a floating point move is ok, don't allow SD without hardware FP
(define_mode_attr fmove_ok [(SF "")
			    (DF "")
			    (SD "TARGET_HARD_FLOAT && TARGET_FPRS")
			    (DD "")])

; Convert REAL_VALUE to the appropriate bits
(define_mode_attr real_value_to_target [(SF "REAL_VALUE_TO_TARGET_SINGLE")
					(DF "REAL_VALUE_TO_TARGET_DOUBLE")
					(SD "REAL_VALUE_TO_TARGET_DECIMAL32")
					(DD "REAL_VALUE_TO_TARGET_DECIMAL64")])

; Definitions for load to 32-bit fpr register
(define_mode_attr f32_lr [(SF "f")		 (SD "wz")])
(define_mode_attr f32_lm [(SF "m")		 (SD "Z")])
(define_mode_attr f32_li [(SF "lfs%U1%X1 %0,%1") (SD "lfiwzx %0,%y1")])
(define_mode_attr f32_lv [(SF "lxsspx %x0,%y1")	 (SD "lxsiwzx %x0,%y1")])

; Definitions for store from 32-bit fpr register
(define_mode_attr f32_sr [(SF "f")		  (SD "wx")])
(define_mode_attr f32_sm [(SF "m")		  (SD "Z")])
(define_mode_attr f32_si [(SF "stfs%U0%X0 %1,%0") (SD "stfiwx %1,%y0")])
(define_mode_attr f32_sv [(SF "stxsspx %x1,%y0")  (SD "stxsiwzx %x1,%y0")])

; Definitions for 32-bit fpr direct move
; At present, the decimal modes are not allowed in the traditional altivec
; registers, so restrict the constraints to just the traditional FPRs.
(define_mode_attr f32_dm [(SF "wn") (SD "wh")])

; Definitions for 32-bit VSX
(define_mode_attr f32_vsx [(SF "ww") (SD "wn")])

; Definitions for 32-bit use of altivec registers
(define_mode_attr f32_av  [(SF "wu") (SD "wn")])

; Definitions for 64-bit VSX
(define_mode_attr f64_vsx [(DF "ws") (DD "wn")])

; Definitions for 64-bit direct move
(define_mode_attr f64_dm  [(DF "wk") (DD "wh")])

; Definitions for 64-bit use of altivec registers
(define_mode_attr f64_av  [(DF "wv") (DD "wn")])

; These modes do not fit in integer registers in 32-bit mode.
; but on e500v2, the gpr are 64 bit registers
(define_mode_iterator DIFD [DI (DF "!TARGET_E500_DOUBLE") DD])

; Iterator for reciprocal estimate instructions
(define_mode_iterator RECIPF [SF DF V4SF V2DF])

; Iterator for just SF/DF
(define_mode_iterator SFDF [SF DF])

; SF/DF suffix for traditional floating instructions
(define_mode_attr Ftrad		[(SF "s") (DF "")])

; SF/DF suffix for VSX instructions
(define_mode_attr Fvsx		[(SF "sp") (DF	"dp")])

; SF/DF constraint for arithmetic on traditional floating point registers
(define_mode_attr Ff		[(SF "f") (DF "d")])

; SF/DF constraint for arithmetic on VSX registers
(define_mode_attr Fv		[(SF "wy") (DF "ws")])

; SF/DF constraint for arithmetic on altivec registers
(define_mode_attr Fa		[(SF "wu") (DF "wv")])

; s/d suffix for things like fp_addsub_s/fp_addsub_d
(define_mode_attr Fs		[(SF "s")  (DF "d")])

; FRE/FRES support
(define_mode_attr Ffre		[(SF "fres") (DF "fre")])
(define_mode_attr FFRE		[(SF "FRES") (DF "FRE")])

; Conditional returns.
(define_code_iterator any_return [return simple_return])
(define_code_attr return_pred [(return "direct_return ()")
			       (simple_return "1")])
(define_code_attr return_str [(return "") (simple_return "simple_")])

; Logical operators.
(define_code_iterator iorxor [ior xor])

; Signed/unsigned variants of ops.
(define_code_iterator any_extend [sign_extend zero_extend])
(define_code_attr u [(sign_extend "") (zero_extend "u")])
(define_code_attr su [(sign_extend "s") (zero_extend "u")])

; Various instructions that come in SI and DI forms.
; A generic w/d attribute, for things like cmpw/cmpd.
(define_mode_attr wd [(QI    "b")
		      (HI    "h")
		      (SI    "w")
		      (DI    "d")
		      (V16QI "b")
		      (V8HI  "h")
		      (V4SI  "w")
		      (V2DI  "d")])

;; How many bits in this mode?
(define_mode_attr bits [(QI "8") (HI "16") (SI "32") (DI "64")])

; DImode bits
(define_mode_attr dbits [(QI "56") (HI "48") (SI "32")])

;; ISEL/ISEL64 target selection
(define_mode_attr sel [(SI "") (DI "64")])

;; Bitmask for shift instructions
(define_mode_attr hH [(SI "h") (DI "H")])

;; A mode twice the size of the given mode
(define_mode_attr dmode [(SI "di") (DI "ti")])
(define_mode_attr DMODE [(SI "DI") (DI "TI")])

;; Suffix for reload patterns
(define_mode_attr ptrsize [(SI "32bit")
			   (DI "64bit")])

(define_mode_attr tptrsize [(SI "TARGET_32BIT")
			    (DI "TARGET_64BIT")])

(define_mode_attr mptrsize [(SI "si")
			    (DI "di")])

(define_mode_attr ptrload [(SI "lwz")
			   (DI "ld")])

(define_mode_attr ptrm [(SI "m")
			(DI "Y")])

(define_mode_attr rreg [(SF   "f")
			(DF   "ws")
			(TF   "f")
			(TD   "f")
			(V4SF "wf")
			(V2DF "wd")])

(define_mode_attr rreg2 [(SF   "f")
			 (DF   "d")])

(define_mode_attr SI_CONVERT_FP [(SF "TARGET_FCFIDS")
				 (DF "TARGET_FCFID")])

(define_mode_attr E500_CONVERT [(SF "!TARGET_FPRS")
				(DF "TARGET_E500_DOUBLE")])

(define_mode_attr TARGET_FLOAT [(SF "TARGET_SINGLE_FLOAT")
				(DF "TARGET_DOUBLE_FLOAT")])

;; Mode iterator for logical operations on 128-bit types
(define_mode_iterator BOOL_128		[TI
					 PTI
					 (V16QI	"TARGET_ALTIVEC")
					 (V8HI	"TARGET_ALTIVEC")
					 (V4SI	"TARGET_ALTIVEC")
					 (V4SF	"TARGET_ALTIVEC")
					 (V2DI	"TARGET_ALTIVEC")
					 (V2DF	"TARGET_ALTIVEC")
					 (V1TI  "TARGET_ALTIVEC")])

;; For the GPRs we use 3 constraints for register outputs, two that are the
;; same as the output register, and a third where the output register is an
;; early clobber, so we don't have to deal with register overlaps.  For the
;; vector types, we prefer to use the vector registers.  For TI mode, allow
;; either.

;; Mode attribute for boolean operation register constraints for output
(define_mode_attr BOOL_REGS_OUTPUT	[(TI	"&r,r,r,wt,v")
					 (PTI	"&r,r,r")
					 (V16QI	"wa,v,&?r,?r,?r")
					 (V8HI	"wa,v,&?r,?r,?r")
					 (V4SI	"wa,v,&?r,?r,?r")
					 (V4SF	"wa,v,&?r,?r,?r")
					 (V2DI	"wa,v,&?r,?r,?r")
					 (V2DF	"wa,v,&?r,?r,?r")
					 (V1TI	"wa,v,&?r,?r,?r")])

;; Mode attribute for boolean operation register constraints for operand1
(define_mode_attr BOOL_REGS_OP1		[(TI	"r,0,r,wt,v")
					 (PTI	"r,0,r")
					 (V16QI	"wa,v,r,0,r")
					 (V8HI	"wa,v,r,0,r")
					 (V4SI	"wa,v,r,0,r")
					 (V4SF	"wa,v,r,0,r")
					 (V2DI	"wa,v,r,0,r")
					 (V2DF	"wa,v,r,0,r")
					 (V1TI	"wa,v,r,0,r")])

;; Mode attribute for boolean operation register constraints for operand2
(define_mode_attr BOOL_REGS_OP2		[(TI	"r,r,0,wt,v")
					 (PTI	"r,r,0")
					 (V16QI	"wa,v,r,r,0")
					 (V8HI	"wa,v,r,r,0")
					 (V4SI	"wa,v,r,r,0")
					 (V4SF	"wa,v,r,r,0")
					 (V2DI	"wa,v,r,r,0")
					 (V2DF	"wa,v,r,r,0")
					 (V1TI	"wa,v,r,r,0")])

;; Mode attribute for boolean operation register constraints for operand1
;; for one_cmpl.  To simplify things, we repeat the constraint where 0
;; is used for operand1 or operand2
(define_mode_attr BOOL_REGS_UNARY	[(TI	"r,0,0,wt,v")
					 (PTI	"r,0,0")
					 (V16QI	"wa,v,r,0,0")
					 (V8HI	"wa,v,r,0,0")
					 (V4SI	"wa,v,r,0,0")
					 (V4SF	"wa,v,r,0,0")
					 (V2DI	"wa,v,r,0,0")
					 (V2DF	"wa,v,r,0,0")
					 (V1TI	"wa,v,r,0,0")])

;; Reload iterator for creating the function to allocate a base register to
;; supplement addressing modes.
(define_mode_iterator RELOAD [V16QI V8HI V4SI V2DI V4SF V2DF V1TI
			      SF SD SI DF DD DI TI PTI])


;; Start with fixed-point load and store insns.  Here we put only the more
;; complex forms.  Basic data transfer is done later.

(define_insn "zero_extendqi<mode>2"
  [(set (match_operand:EXTQI 0 "gpc_reg_operand" "=r,r")
	(zero_extend:EXTQI (match_operand:QI 1 "reg_or_mem_operand" "m,r")))]
  ""
  "@
   lbz%U1%X1 %0,%1
   rlwinm %0,%1,0,0xff"
  [(set_attr "type" "load,shift")])

(define_insn_and_split "*zero_extendqi<mode>2_dot"
  [(set (match_operand:CC 2 "cc_reg_operand" "=x,?y")
	(compare:CC (zero_extend:EXTQI (match_operand:QI 1 "gpc_reg_operand" "r,r"))
		    (const_int 0)))
   (clobber (match_scratch:EXTQI 0 "=r,r"))]
  "rs6000_gen_cell_microcode"
  "@
   andi. %0,%1,0xff
   #"
  "&& reload_completed && cc_reg_not_cr0_operand (operands[2], CCmode)"
  [(set (match_dup 0)
	(zero_extend:EXTQI (match_dup 1)))
   (set (match_dup 2)
	(compare:CC (match_dup 0)
		    (const_int 0)))]
  ""
  [(set_attr "type" "logical")
   (set_attr "dot" "yes")
   (set_attr "length" "4,8")])

(define_insn_and_split "*zero_extendqi<mode>2_dot2"
  [(set (match_operand:CC 2 "cc_reg_operand" "=x,?y")
	(compare:CC (zero_extend:EXTQI (match_operand:QI 1 "gpc_reg_operand" "r,r"))
		    (const_int 0)))
   (set (match_operand:EXTQI 0 "gpc_reg_operand" "=r,r")
	(zero_extend:EXTQI (match_dup 1)))]
  "rs6000_gen_cell_microcode"
  "@
   andi. %0,%1,0xff
   #"
  "&& reload_completed && cc_reg_not_cr0_operand (operands[2], CCmode)"
  [(set (match_dup 0)
	(zero_extend:EXTQI (match_dup 1)))
   (set (match_dup 2)
	(compare:CC (match_dup 0)
		    (const_int 0)))]
  ""
  [(set_attr "type" "logical")
   (set_attr "dot" "yes")
   (set_attr "length" "4,8")])


(define_insn "zero_extendhi<mode>2"
  [(set (match_operand:EXTHI 0 "gpc_reg_operand" "=r,r")
	(zero_extend:EXTHI (match_operand:HI 1 "reg_or_mem_operand" "m,r")))]
  ""
  "@
   lhz%U1%X1 %0,%1
   rlwinm %0,%1,0,0xffff"
  [(set_attr "type" "load,shift")])

(define_insn_and_split "*zero_extendhi<mode>2_dot"
  [(set (match_operand:CC 2 "cc_reg_operand" "=x,?y")
	(compare:CC (zero_extend:EXTHI (match_operand:HI 1 "gpc_reg_operand" "r,r"))
		    (const_int 0)))
   (clobber (match_scratch:EXTHI 0 "=r,r"))]
  "rs6000_gen_cell_microcode"
  "@
   andi. %0,%1,0xffff
   #"
  "&& reload_completed && cc_reg_not_cr0_operand (operands[2], CCmode)"
  [(set (match_dup 0)
	(zero_extend:EXTHI (match_dup 1)))
   (set (match_dup 2)
	(compare:CC (match_dup 0)
		    (const_int 0)))]
  ""
  [(set_attr "type" "logical")
   (set_attr "dot" "yes")
   (set_attr "length" "4,8")])

(define_insn_and_split "*zero_extendhi<mode>2_dot2"
  [(set (match_operand:CC 2 "cc_reg_operand" "=x,?y")
	(compare:CC (zero_extend:EXTHI (match_operand:HI 1 "gpc_reg_operand" "r,r"))
		    (const_int 0)))
   (set (match_operand:EXTHI 0 "gpc_reg_operand" "=r,r")
	(zero_extend:EXTHI (match_dup 1)))]
  "rs6000_gen_cell_microcode"
  "@
   andi. %0,%1,0xffff
   #"
  "&& reload_completed && cc_reg_not_cr0_operand (operands[2], CCmode)"
  [(set (match_dup 0)
	(zero_extend:EXTHI (match_dup 1)))
   (set (match_dup 2)
	(compare:CC (match_dup 0)
		    (const_int 0)))]
  ""
  [(set_attr "type" "logical")
   (set_attr "dot" "yes")
   (set_attr "length" "4,8")])


(define_insn "zero_extendsi<mode>2"
  [(set (match_operand:EXTSI 0 "gpc_reg_operand" "=r,r,??wj,!wz,!wu")
	(zero_extend:EXTSI (match_operand:SI 1 "reg_or_mem_operand" "m,r,r,Z,Z")))]
  ""
  "@
   lwz%U1%X1 %0,%1
   rldicl %0,%1,0,32
   mtvsrwz %x0,%1
   lfiwzx %0,%y1
   lxsiwzx %x0,%y1"
  [(set_attr "type" "load,shift,mffgpr,fpload,fpload")])

(define_insn_and_split "*zero_extendsi<mode>2_dot"
  [(set (match_operand:CC 2 "cc_reg_operand" "=x,?y")
	(compare:CC (zero_extend:EXTSI (match_operand:SI 1 "gpc_reg_operand" "r,r"))
		    (const_int 0)))
   (clobber (match_scratch:EXTSI 0 "=r,r"))]
  "rs6000_gen_cell_microcode"
  "@
   rldicl. %0,%1,0,32
   #"
  "&& reload_completed && cc_reg_not_cr0_operand (operands[2], CCmode)"
  [(set (match_dup 0)
	(zero_extend:DI (match_dup 1)))
   (set (match_dup 2)
	(compare:CC (match_dup 0)
		    (const_int 0)))]
  ""
  [(set_attr "type" "shift")
   (set_attr "dot" "yes")
   (set_attr "length" "4,8")])

(define_insn_and_split "*zero_extendsi<mode>2_dot2"
  [(set (match_operand:CC 2 "cc_reg_operand" "=x,?y")
	(compare:CC (zero_extend:EXTSI (match_operand:SI 1 "gpc_reg_operand" "r,r"))
		    (const_int 0)))
   (set (match_operand:EXTSI 0 "gpc_reg_operand" "=r,r")
	(zero_extend:EXTSI (match_dup 1)))]
  "rs6000_gen_cell_microcode"
  "@
   rldicl. %0,%1,0,32
   #"
  "&& reload_completed && cc_reg_not_cr0_operand (operands[2], CCmode)"
  [(set (match_dup 0)
	(zero_extend:EXTSI (match_dup 1)))
   (set (match_dup 2)
	(compare:CC (match_dup 0)
		    (const_int 0)))]
  ""
  [(set_attr "type" "shift")
   (set_attr "dot" "yes")
   (set_attr "length" "4,8")])


(define_insn "extendqi<mode>2"
  [(set (match_operand:EXTQI 0 "gpc_reg_operand" "=r")
	(sign_extend:EXTQI (match_operand:QI 1 "gpc_reg_operand" "r")))]
  ""
  "extsb %0,%1"
  [(set_attr "type" "exts")])

(define_insn_and_split "*extendqi<mode>2_dot"
  [(set (match_operand:CC 2 "cc_reg_operand" "=x,?y")
	(compare:CC (sign_extend:EXTQI (match_operand:QI 1 "gpc_reg_operand" "r,r"))
		    (const_int 0)))
   (clobber (match_scratch:EXTQI 0 "=r,r"))]
  "rs6000_gen_cell_microcode"
  "@
   extsb. %0,%1
   #"
  "&& reload_completed && cc_reg_not_cr0_operand (operands[2], CCmode)"
  [(set (match_dup 0)
	(sign_extend:EXTQI (match_dup 1)))
   (set (match_dup 2)
	(compare:CC (match_dup 0)
		    (const_int 0)))]
  ""
  [(set_attr "type" "exts")
   (set_attr "dot" "yes")
   (set_attr "length" "4,8")])

(define_insn_and_split "*extendqi<mode>2_dot2"
  [(set (match_operand:CC 2 "cc_reg_operand" "=x,?y")
	(compare:CC (sign_extend:EXTQI (match_operand:QI 1 "gpc_reg_operand" "r,r"))
		    (const_int 0)))
   (set (match_operand:EXTQI 0 "gpc_reg_operand" "=r,r")
	(sign_extend:EXTQI (match_dup 1)))]
  "rs6000_gen_cell_microcode"
  "@
   extsb. %0,%1
   #"
  "&& reload_completed && cc_reg_not_cr0_operand (operands[2], CCmode)"
  [(set (match_dup 0)
	(sign_extend:EXTQI (match_dup 1)))
   (set (match_dup 2)
	(compare:CC (match_dup 0)
		    (const_int 0)))]
  ""
  [(set_attr "type" "exts")
   (set_attr "dot" "yes")
   (set_attr "length" "4,8")])


(define_expand "extendhi<mode>2"
  [(set (match_operand:EXTHI 0 "gpc_reg_operand" "")
	(sign_extend:EXTHI (match_operand:HI 1 "gpc_reg_operand" "")))]
  ""
  "")

(define_insn "*extendhi<mode>2"
  [(set (match_operand:EXTHI 0 "gpc_reg_operand" "=r,r")
	(sign_extend:EXTHI (match_operand:HI 1 "reg_or_mem_operand" "m,r")))]
  "rs6000_gen_cell_microcode"
  "@
   lha%U1%X1 %0,%1
   extsh %0,%1"
  [(set_attr "type" "load,exts")
   (set_attr "sign_extend" "yes")])

(define_insn "*extendhi<mode>2_noload"
  [(set (match_operand:EXTHI 0 "gpc_reg_operand" "=r")
        (sign_extend:EXTHI (match_operand:HI 1 "gpc_reg_operand" "r")))]
  "!rs6000_gen_cell_microcode"
  "extsh %0,%1"
  [(set_attr "type" "exts")])

(define_insn_and_split "*extendhi<mode>2_dot"
  [(set (match_operand:CC 2 "cc_reg_operand" "=x,?y")
	(compare:CC (sign_extend:EXTHI (match_operand:HI 1 "gpc_reg_operand" "r,r"))
		    (const_int 0)))
   (clobber (match_scratch:EXTHI 0 "=r,r"))]
  "rs6000_gen_cell_microcode"
  "@
   extsh. %0,%1
   #"
  "&& reload_completed && cc_reg_not_cr0_operand (operands[2], CCmode)"
  [(set (match_dup 0)
	(sign_extend:EXTHI (match_dup 1)))
   (set (match_dup 2)
	(compare:CC (match_dup 0)
		    (const_int 0)))]
  ""
  [(set_attr "type" "exts")
   (set_attr "dot" "yes")
   (set_attr "length" "4,8")])

(define_insn_and_split "*extendhi<mode>2_dot2"
  [(set (match_operand:CC 2 "cc_reg_operand" "=x,?y")
	(compare:CC (sign_extend:EXTHI (match_operand:HI 1 "gpc_reg_operand" "r,r"))
		    (const_int 0)))
   (set (match_operand:EXTHI 0 "gpc_reg_operand" "=r,r")
	(sign_extend:EXTHI (match_dup 1)))]
  "rs6000_gen_cell_microcode"
  "@
   extsh. %0,%1
   #"
  "&& reload_completed && cc_reg_not_cr0_operand (operands[2], CCmode)"
  [(set (match_dup 0)
	(sign_extend:EXTHI (match_dup 1)))
   (set (match_dup 2)
	(compare:CC (match_dup 0)
		    (const_int 0)))]
  ""
  [(set_attr "type" "exts")
   (set_attr "dot" "yes")
   (set_attr "length" "4,8")])


(define_insn "extendsi<mode>2"
  [(set (match_operand:EXTSI 0 "gpc_reg_operand" "=r,r,??wj,!wl,!wu")
	(sign_extend:EXTSI (match_operand:SI 1 "lwa_operand" "Y,r,r,Z,Z")))]
  ""
  "@
   lwa%U1%X1 %0,%1
   extsw %0,%1
   mtvsrwa %x0,%1
   lfiwax %0,%y1
   lxsiwax %x0,%y1"
  [(set_attr "type" "load,exts,mffgpr,fpload,fpload")
   (set_attr "sign_extend" "yes")])

(define_insn_and_split "*extendsi<mode>2_dot"
  [(set (match_operand:CC 2 "cc_reg_operand" "=x,?y")
	(compare:CC (sign_extend:EXTSI (match_operand:SI 1 "gpc_reg_operand" "r,r"))
		    (const_int 0)))
   (clobber (match_scratch:EXTSI 0 "=r,r"))]
  "rs6000_gen_cell_microcode"
  "@
   extsw. %0,%1
   #"
  "&& reload_completed && cc_reg_not_cr0_operand (operands[2], CCmode)"
  [(set (match_dup 0)
	(sign_extend:EXTSI (match_dup 1)))
   (set (match_dup 2)
	(compare:CC (match_dup 0)
		    (const_int 0)))]
  ""
  [(set_attr "type" "exts")
   (set_attr "dot" "yes")
   (set_attr "length" "4,8")])

(define_insn_and_split "*extendsi<mode>2_dot2"
  [(set (match_operand:CC 2 "cc_reg_operand" "=x,?y")
	(compare:CC (sign_extend:EXTSI (match_operand:SI 1 "gpc_reg_operand" "r,r"))
		    (const_int 0)))
   (set (match_operand:EXTSI 0 "gpc_reg_operand" "=r,r")
	(sign_extend:EXTSI (match_dup 1)))]
  "rs6000_gen_cell_microcode"
  "@
   extsw. %0,%1
   #"
  "&& reload_completed && cc_reg_not_cr0_operand (operands[2], CCmode)"
  [(set (match_dup 0)
	(sign_extend:EXTSI (match_dup 1)))
   (set (match_dup 2)
	(compare:CC (match_dup 0)
		    (const_int 0)))]
  ""
  [(set_attr "type" "exts")
   (set_attr "dot" "yes")
   (set_attr "length" "4,8")])

;; IBM 405, 440, 464 and 476 half-word multiplication operations.

(define_insn "*macchwc"
  [(set (match_operand:CC 3 "cc_reg_operand" "=x")
        (compare:CC (plus:SI (mult:SI (ashiftrt:SI
                                       (match_operand:SI 2 "gpc_reg_operand" "r")
                                       (const_int 16))
                                      (sign_extend:SI
                                       (match_operand:HI 1 "gpc_reg_operand" "r")))
                             (match_operand:SI 4 "gpc_reg_operand" "0"))
                    (const_int 0)))
   (set (match_operand:SI 0 "gpc_reg_operand" "=r")
        (plus:SI (mult:SI (ashiftrt:SI
                           (match_dup 2)
                           (const_int 16))
                          (sign_extend:SI
                           (match_dup 1)))
                 (match_dup 4)))]
  "TARGET_MULHW"
  "macchw. %0,%1,%2"
  [(set_attr "type" "halfmul")])

(define_insn "*macchw"
  [(set (match_operand:SI 0 "gpc_reg_operand" "=r")
        (plus:SI (mult:SI (ashiftrt:SI
                           (match_operand:SI 2 "gpc_reg_operand" "r")
                           (const_int 16))
                          (sign_extend:SI
                           (match_operand:HI 1 "gpc_reg_operand" "r")))
                 (match_operand:SI 3 "gpc_reg_operand" "0")))]
  "TARGET_MULHW"
  "macchw %0,%1,%2"
  [(set_attr "type" "halfmul")])

(define_insn "*macchwuc"
  [(set (match_operand:CC 3 "cc_reg_operand" "=x")
        (compare:CC (plus:SI (mult:SI (lshiftrt:SI
                                       (match_operand:SI 2 "gpc_reg_operand" "r")
                                       (const_int 16))
                                      (zero_extend:SI
                                       (match_operand:HI 1 "gpc_reg_operand" "r")))
                             (match_operand:SI 4 "gpc_reg_operand" "0"))
                    (const_int 0)))
   (set (match_operand:SI 0 "gpc_reg_operand" "=r")
        (plus:SI (mult:SI (lshiftrt:SI
                           (match_dup 2)
                           (const_int 16))
                          (zero_extend:SI
                           (match_dup 1)))
                 (match_dup 4)))]
  "TARGET_MULHW"
  "macchwu. %0,%1,%2"
  [(set_attr "type" "halfmul")])

(define_insn "*macchwu"
  [(set (match_operand:SI 0 "gpc_reg_operand" "=r")
        (plus:SI (mult:SI (lshiftrt:SI
                           (match_operand:SI 2 "gpc_reg_operand" "r")
                           (const_int 16))
                          (zero_extend:SI
                           (match_operand:HI 1 "gpc_reg_operand" "r")))
                 (match_operand:SI 3 "gpc_reg_operand" "0")))]
  "TARGET_MULHW"
  "macchwu %0,%1,%2"
  [(set_attr "type" "halfmul")])

(define_insn "*machhwc"
  [(set (match_operand:CC 3 "cc_reg_operand" "=x")
        (compare:CC (plus:SI (mult:SI (ashiftrt:SI
                                       (match_operand:SI 1 "gpc_reg_operand" "%r")
                                       (const_int 16))
                                      (ashiftrt:SI
                                       (match_operand:SI 2 "gpc_reg_operand" "r")
                                       (const_int 16)))
                             (match_operand:SI 4 "gpc_reg_operand" "0"))
                    (const_int 0)))
   (set (match_operand:SI 0 "gpc_reg_operand" "=r")
        (plus:SI (mult:SI (ashiftrt:SI
                           (match_dup 1)
                           (const_int 16))
                          (ashiftrt:SI
                           (match_dup 2)
                           (const_int 16)))
                 (match_dup 4)))]
  "TARGET_MULHW"
  "machhw. %0,%1,%2"
  [(set_attr "type" "halfmul")])

(define_insn "*machhw"
  [(set (match_operand:SI 0 "gpc_reg_operand" "=r")
        (plus:SI (mult:SI (ashiftrt:SI
                           (match_operand:SI 1 "gpc_reg_operand" "%r")
                           (const_int 16))
                          (ashiftrt:SI
                           (match_operand:SI 2 "gpc_reg_operand" "r")
                           (const_int 16)))
                 (match_operand:SI 3 "gpc_reg_operand" "0")))]
  "TARGET_MULHW"
  "machhw %0,%1,%2"
  [(set_attr "type" "halfmul")])

(define_insn "*machhwuc"
  [(set (match_operand:CC 3 "cc_reg_operand" "=x")
        (compare:CC (plus:SI (mult:SI (lshiftrt:SI
                                       (match_operand:SI 1 "gpc_reg_operand" "%r")
                                       (const_int 16))
                                      (lshiftrt:SI
                                       (match_operand:SI 2 "gpc_reg_operand" "r")
                                       (const_int 16)))
                             (match_operand:SI 4 "gpc_reg_operand" "0"))
                    (const_int 0)))
   (set (match_operand:SI 0 "gpc_reg_operand" "=r")
        (plus:SI (mult:SI (lshiftrt:SI
                           (match_dup 1)
                           (const_int 16))
                          (lshiftrt:SI
                           (match_dup 2)
                           (const_int 16)))
                 (match_dup 4)))]
  "TARGET_MULHW"
  "machhwu. %0,%1,%2"
  [(set_attr "type" "halfmul")])

(define_insn "*machhwu"
  [(set (match_operand:SI 0 "gpc_reg_operand" "=r")
        (plus:SI (mult:SI (lshiftrt:SI
                           (match_operand:SI 1 "gpc_reg_operand" "%r")
                           (const_int 16))
                          (lshiftrt:SI
                           (match_operand:SI 2 "gpc_reg_operand" "r")
                           (const_int 16)))
                 (match_operand:SI 3 "gpc_reg_operand" "0")))]
  "TARGET_MULHW"
  "machhwu %0,%1,%2"
  [(set_attr "type" "halfmul")])

(define_insn "*maclhwc"
  [(set (match_operand:CC 3 "cc_reg_operand" "=x")
        (compare:CC (plus:SI (mult:SI (sign_extend:SI
                                       (match_operand:HI 1 "gpc_reg_operand" "%r"))
                                      (sign_extend:SI
                                       (match_operand:HI 2 "gpc_reg_operand" "r")))
                             (match_operand:SI 4 "gpc_reg_operand" "0"))
                    (const_int 0)))
   (set (match_operand:SI 0 "gpc_reg_operand" "=r")
        (plus:SI (mult:SI (sign_extend:SI
                           (match_dup 1))
                          (sign_extend:SI
                           (match_dup 2)))
                 (match_dup 4)))]
  "TARGET_MULHW"
  "maclhw. %0,%1,%2"
  [(set_attr "type" "halfmul")])

(define_insn "*maclhw"
  [(set (match_operand:SI 0 "gpc_reg_operand" "=r")
        (plus:SI (mult:SI (sign_extend:SI
                           (match_operand:HI 1 "gpc_reg_operand" "%r"))
                          (sign_extend:SI
                           (match_operand:HI 2 "gpc_reg_operand" "r")))
                 (match_operand:SI 3 "gpc_reg_operand" "0")))]
  "TARGET_MULHW"
  "maclhw %0,%1,%2"
  [(set_attr "type" "halfmul")])

(define_insn "*maclhwuc"
  [(set (match_operand:CC 3 "cc_reg_operand" "=x")
        (compare:CC (plus:SI (mult:SI (zero_extend:SI
                                       (match_operand:HI 1 "gpc_reg_operand" "%r"))
                                      (zero_extend:SI
                                       (match_operand:HI 2 "gpc_reg_operand" "r")))
                             (match_operand:SI 4 "gpc_reg_operand" "0"))
                    (const_int 0)))
   (set (match_operand:SI 0 "gpc_reg_operand" "=r")
        (plus:SI (mult:SI (zero_extend:SI
                           (match_dup 1))
                          (zero_extend:SI
                           (match_dup 2)))
                 (match_dup 4)))]
  "TARGET_MULHW"
  "maclhwu. %0,%1,%2"
  [(set_attr "type" "halfmul")])

(define_insn "*maclhwu"
  [(set (match_operand:SI 0 "gpc_reg_operand" "=r")
        (plus:SI (mult:SI (zero_extend:SI
                           (match_operand:HI 1 "gpc_reg_operand" "%r"))
                          (zero_extend:SI
                           (match_operand:HI 2 "gpc_reg_operand" "r")))
                 (match_operand:SI 3 "gpc_reg_operand" "0")))]
  "TARGET_MULHW"
  "maclhwu %0,%1,%2"
  [(set_attr "type" "halfmul")])

(define_insn "*nmacchwc"
  [(set (match_operand:CC 3 "cc_reg_operand" "=x")
        (compare:CC (minus:SI (match_operand:SI 4 "gpc_reg_operand" "0")
                              (mult:SI (ashiftrt:SI
                                        (match_operand:SI 2 "gpc_reg_operand" "r")
                                        (const_int 16))
                                       (sign_extend:SI
                                        (match_operand:HI 1 "gpc_reg_operand" "r"))))
                    (const_int 0)))
   (set (match_operand:SI 0 "gpc_reg_operand" "=r")
        (minus:SI (match_dup 4)
                  (mult:SI (ashiftrt:SI
                            (match_dup 2)
                            (const_int 16))
                           (sign_extend:SI
                            (match_dup 1)))))]
  "TARGET_MULHW"
  "nmacchw. %0,%1,%2"
  [(set_attr "type" "halfmul")])

(define_insn "*nmacchw"
  [(set (match_operand:SI 0 "gpc_reg_operand" "=r")
        (minus:SI (match_operand:SI 3 "gpc_reg_operand" "0")
                  (mult:SI (ashiftrt:SI
                            (match_operand:SI 2 "gpc_reg_operand" "r")
                            (const_int 16))
                           (sign_extend:SI
                            (match_operand:HI 1 "gpc_reg_operand" "r")))))]
  "TARGET_MULHW"
  "nmacchw %0,%1,%2"
  [(set_attr "type" "halfmul")])

(define_insn "*nmachhwc"
  [(set (match_operand:CC 3 "cc_reg_operand" "=x")
        (compare:CC (minus:SI (match_operand:SI 4 "gpc_reg_operand" "0")
                              (mult:SI (ashiftrt:SI
                                        (match_operand:SI 1 "gpc_reg_operand" "%r")
                                        (const_int 16))
                                       (ashiftrt:SI
                                        (match_operand:SI 2 "gpc_reg_operand" "r")
                                        (const_int 16))))
                    (const_int 0)))
   (set (match_operand:SI 0 "gpc_reg_operand" "=r")
        (minus:SI (match_dup 4)
                  (mult:SI (ashiftrt:SI
                            (match_dup 1)
                            (const_int 16))
                           (ashiftrt:SI
                            (match_dup 2)
                            (const_int 16)))))]
  "TARGET_MULHW"
  "nmachhw. %0,%1,%2"
  [(set_attr "type" "halfmul")])

(define_insn "*nmachhw"
  [(set (match_operand:SI 0 "gpc_reg_operand" "=r")
        (minus:SI (match_operand:SI 3 "gpc_reg_operand" "0")
                  (mult:SI (ashiftrt:SI
                            (match_operand:SI 1 "gpc_reg_operand" "%r")
                            (const_int 16))
                           (ashiftrt:SI
                            (match_operand:SI 2 "gpc_reg_operand" "r")
                            (const_int 16)))))]
  "TARGET_MULHW"
  "nmachhw %0,%1,%2"
  [(set_attr "type" "halfmul")])

(define_insn "*nmaclhwc"
  [(set (match_operand:CC 3 "cc_reg_operand" "=x")
        (compare:CC (minus:SI (match_operand:SI 4 "gpc_reg_operand" "0")
                              (mult:SI (sign_extend:SI
                                        (match_operand:HI 1 "gpc_reg_operand" "%r"))
                                       (sign_extend:SI
                                        (match_operand:HI 2 "gpc_reg_operand" "r"))))
                    (const_int 0)))
   (set (match_operand:SI 0 "gpc_reg_operand" "=r")
        (minus:SI (match_dup 4)
                  (mult:SI (sign_extend:SI
                            (match_dup 1))
                           (sign_extend:SI
                            (match_dup 2)))))]
  "TARGET_MULHW"
  "nmaclhw. %0,%1,%2"
  [(set_attr "type" "halfmul")])

(define_insn "*nmaclhw"
  [(set (match_operand:SI 0 "gpc_reg_operand" "=r")
        (minus:SI (match_operand:SI 3 "gpc_reg_operand" "0")
                  (mult:SI (sign_extend:SI
                            (match_operand:HI 1 "gpc_reg_operand" "%r"))
                           (sign_extend:SI
                            (match_operand:HI 2 "gpc_reg_operand" "r")))))]
  "TARGET_MULHW"
  "nmaclhw %0,%1,%2"
  [(set_attr "type" "halfmul")])

(define_insn "*mulchwc"
  [(set (match_operand:CC 3 "cc_reg_operand" "=x")
        (compare:CC (mult:SI (ashiftrt:SI
                              (match_operand:SI 2 "gpc_reg_operand" "r")
                              (const_int 16))
                             (sign_extend:SI
                              (match_operand:HI 1 "gpc_reg_operand" "r")))
                    (const_int 0)))
   (set (match_operand:SI 0 "gpc_reg_operand" "=r")
        (mult:SI (ashiftrt:SI
                  (match_dup 2)
                  (const_int 16))
                 (sign_extend:SI
                  (match_dup 1))))]
  "TARGET_MULHW"
  "mulchw. %0,%1,%2"
  [(set_attr "type" "halfmul")])

(define_insn "*mulchw"
  [(set (match_operand:SI 0 "gpc_reg_operand" "=r")
        (mult:SI (ashiftrt:SI
                  (match_operand:SI 2 "gpc_reg_operand" "r")
                  (const_int 16))
                 (sign_extend:SI
                  (match_operand:HI 1 "gpc_reg_operand" "r"))))]
  "TARGET_MULHW"
  "mulchw %0,%1,%2"
  [(set_attr "type" "halfmul")])

(define_insn "*mulchwuc"
  [(set (match_operand:CC 3 "cc_reg_operand" "=x")
        (compare:CC (mult:SI (lshiftrt:SI
                              (match_operand:SI 2 "gpc_reg_operand" "r")
                              (const_int 16))
                             (zero_extend:SI
                              (match_operand:HI 1 "gpc_reg_operand" "r")))
                    (const_int 0)))
   (set (match_operand:SI 0 "gpc_reg_operand" "=r")
        (mult:SI (lshiftrt:SI
                  (match_dup 2)
                  (const_int 16))
                 (zero_extend:SI
                  (match_dup 1))))]
  "TARGET_MULHW"
  "mulchwu. %0,%1,%2"
  [(set_attr "type" "halfmul")])

(define_insn "*mulchwu"
  [(set (match_operand:SI 0 "gpc_reg_operand" "=r")
        (mult:SI (lshiftrt:SI
                  (match_operand:SI 2 "gpc_reg_operand" "r")
                  (const_int 16))
                 (zero_extend:SI
                  (match_operand:HI 1 "gpc_reg_operand" "r"))))]
  "TARGET_MULHW"
  "mulchwu %0,%1,%2"
  [(set_attr "type" "halfmul")])

(define_insn "*mulhhwc"
  [(set (match_operand:CC 3 "cc_reg_operand" "=x")
        (compare:CC (mult:SI (ashiftrt:SI
                              (match_operand:SI 1 "gpc_reg_operand" "%r")
                              (const_int 16))
                             (ashiftrt:SI
                              (match_operand:SI 2 "gpc_reg_operand" "r")
                              (const_int 16)))
                    (const_int 0)))
   (set (match_operand:SI 0 "gpc_reg_operand" "=r")
        (mult:SI (ashiftrt:SI
                  (match_dup 1)
                  (const_int 16))
                 (ashiftrt:SI
                  (match_dup 2)
                  (const_int 16))))]
  "TARGET_MULHW"
  "mulhhw. %0,%1,%2"
  [(set_attr "type" "halfmul")])

(define_insn "*mulhhw"
  [(set (match_operand:SI 0 "gpc_reg_operand" "=r")
        (mult:SI (ashiftrt:SI
                  (match_operand:SI 1 "gpc_reg_operand" "%r")
                  (const_int 16))
                 (ashiftrt:SI
                  (match_operand:SI 2 "gpc_reg_operand" "r")
                  (const_int 16))))]
  "TARGET_MULHW"
  "mulhhw %0,%1,%2"
  [(set_attr "type" "halfmul")])

(define_insn "*mulhhwuc"
  [(set (match_operand:CC 3 "cc_reg_operand" "=x")
        (compare:CC (mult:SI (lshiftrt:SI
                              (match_operand:SI 1 "gpc_reg_operand" "%r")
                              (const_int 16))
                             (lshiftrt:SI
                              (match_operand:SI 2 "gpc_reg_operand" "r")
                              (const_int 16)))
                    (const_int 0)))
   (set (match_operand:SI 0 "gpc_reg_operand" "=r")
        (mult:SI (lshiftrt:SI
                  (match_dup 1)
                  (const_int 16))
                 (lshiftrt:SI
                  (match_dup 2)
                  (const_int 16))))]
  "TARGET_MULHW"
  "mulhhwu. %0,%1,%2"
  [(set_attr "type" "halfmul")])

(define_insn "*mulhhwu"
  [(set (match_operand:SI 0 "gpc_reg_operand" "=r")
        (mult:SI (lshiftrt:SI
                  (match_operand:SI 1 "gpc_reg_operand" "%r")
                  (const_int 16))
                 (lshiftrt:SI
                  (match_operand:SI 2 "gpc_reg_operand" "r")
                  (const_int 16))))]
  "TARGET_MULHW"
  "mulhhwu %0,%1,%2"
  [(set_attr "type" "halfmul")])

(define_insn "*mullhwc"
  [(set (match_operand:CC 3 "cc_reg_operand" "=x")
        (compare:CC (mult:SI (sign_extend:SI
                              (match_operand:HI 1 "gpc_reg_operand" "%r"))
                             (sign_extend:SI
                              (match_operand:HI 2 "gpc_reg_operand" "r")))
                    (const_int 0)))
   (set (match_operand:SI 0 "gpc_reg_operand" "=r")
        (mult:SI (sign_extend:SI
                  (match_dup 1))
                 (sign_extend:SI
                  (match_dup 2))))]
  "TARGET_MULHW"
  "mullhw. %0,%1,%2"
  [(set_attr "type" "halfmul")])

(define_insn "*mullhw"
  [(set (match_operand:SI 0 "gpc_reg_operand" "=r")
        (mult:SI (sign_extend:SI
                  (match_operand:HI 1 "gpc_reg_operand" "%r"))
                 (sign_extend:SI
                  (match_operand:HI 2 "gpc_reg_operand" "r"))))]
  "TARGET_MULHW"
  "mullhw %0,%1,%2"
  [(set_attr "type" "halfmul")])

(define_insn "*mullhwuc"
  [(set (match_operand:CC 3 "cc_reg_operand" "=x")
        (compare:CC (mult:SI (zero_extend:SI
                              (match_operand:HI 1 "gpc_reg_operand" "%r"))
                             (zero_extend:SI
                              (match_operand:HI 2 "gpc_reg_operand" "r")))
                    (const_int 0)))
   (set (match_operand:SI 0 "gpc_reg_operand" "=r")
        (mult:SI (zero_extend:SI
                  (match_dup 1))
                 (zero_extend:SI
                  (match_dup 2))))]
  "TARGET_MULHW"
  "mullhwu. %0,%1,%2"
  [(set_attr "type" "halfmul")])

(define_insn "*mullhwu"
  [(set (match_operand:SI 0 "gpc_reg_operand" "=r")
        (mult:SI (zero_extend:SI
                  (match_operand:HI 1 "gpc_reg_operand" "%r"))
                 (zero_extend:SI
                  (match_operand:HI 2 "gpc_reg_operand" "r"))))]
  "TARGET_MULHW"
  "mullhwu %0,%1,%2"
  [(set_attr "type" "halfmul")])

;; IBM 405, 440, 464 and 476 string-search dlmzb instruction support.
(define_insn "dlmzb"
  [(set (match_operand:CC 3 "cc_reg_operand" "=x")
        (unspec:CC [(match_operand:SI 1 "gpc_reg_operand" "r")
                    (match_operand:SI 2 "gpc_reg_operand" "r")]
                   UNSPEC_DLMZB_CR))
   (set (match_operand:SI 0 "gpc_reg_operand" "=r")
        (unspec:SI [(match_dup 1)
                    (match_dup 2)]
                   UNSPEC_DLMZB))]
  "TARGET_DLMZB"
  "dlmzb. %0,%1,%2")

(define_expand "strlensi"
  [(set (match_operand:SI 0 "gpc_reg_operand" "")
        (unspec:SI [(match_operand:BLK 1 "general_operand" "")
                    (match_operand:QI 2 "const_int_operand" "")
                    (match_operand 3 "const_int_operand" "")]
                   UNSPEC_DLMZB_STRLEN))
   (clobber (match_scratch:CC 4 "=x"))]
  "TARGET_DLMZB && WORDS_BIG_ENDIAN && !optimize_size"
{
  rtx result = operands[0];
  rtx src = operands[1];
  rtx search_char = operands[2];
  rtx align = operands[3];
  rtx addr, scratch_string, word1, word2, scratch_dlmzb;
  rtx loop_label, end_label, mem, cr0, cond;
  if (search_char != const0_rtx
      || GET_CODE (align) != CONST_INT
      || INTVAL (align) < 8)
        FAIL;
  word1 = gen_reg_rtx (SImode);
  word2 = gen_reg_rtx (SImode);
  scratch_dlmzb = gen_reg_rtx (SImode);
  scratch_string = gen_reg_rtx (Pmode);
  loop_label = gen_label_rtx ();
  end_label = gen_label_rtx ();
  addr = force_reg (Pmode, XEXP (src, 0));
  emit_move_insn (scratch_string, addr);
  emit_label (loop_label);
  mem = change_address (src, SImode, scratch_string);
  emit_move_insn (word1, mem);
  emit_move_insn (word2, adjust_address (mem, SImode, 4));
  cr0 = gen_rtx_REG (CCmode, CR0_REGNO);
  emit_insn (gen_dlmzb (scratch_dlmzb, word1, word2, cr0));
  cond = gen_rtx_NE (VOIDmode, cr0, const0_rtx);
  emit_jump_insn (gen_rtx_SET (VOIDmode,
                               pc_rtx,
                               gen_rtx_IF_THEN_ELSE (VOIDmode,
                                                     cond,
                                                     gen_rtx_LABEL_REF
                                                       (VOIDmode,
                                                        end_label),
                                                     pc_rtx)));
  emit_insn (gen_addsi3 (scratch_string, scratch_string, GEN_INT (8)));
  emit_jump_insn (gen_rtx_SET (VOIDmode,
                               pc_rtx,
                               gen_rtx_LABEL_REF (VOIDmode, loop_label)));
  emit_barrier ();
  emit_label (end_label);
  emit_insn (gen_addsi3 (scratch_string, scratch_string, scratch_dlmzb));
  emit_insn (gen_subsi3 (result, scratch_string, addr));
  emit_insn (gen_addsi3 (result, result, constm1_rtx));
  DONE;
})

;; Fixed-point arithmetic insns.

(define_expand "add<mode>3"
  [(set (match_operand:SDI 0 "gpc_reg_operand" "")
	(plus:SDI (match_operand:SDI 1 "gpc_reg_operand" "")
		  (match_operand:SDI 2 "reg_or_add_cint_operand" "")))]
  ""
{
  if (<MODE>mode == DImode && !TARGET_POWERPC64)
    {
      rtx lo0 = gen_lowpart (SImode, operands[0]);
      rtx lo1 = gen_lowpart (SImode, operands[1]);
      rtx lo2 = gen_lowpart (SImode, operands[2]);
      rtx hi0 = gen_highpart (SImode, operands[0]);
      rtx hi1 = gen_highpart (SImode, operands[1]);
      rtx hi2 = gen_highpart_mode (SImode, DImode, operands[2]);

      if (!reg_or_short_operand (lo2, SImode))
	lo2 = force_reg (SImode, lo2);
      if (!adde_operand (hi2, SImode))
	hi2 = force_reg (SImode, hi2);

      emit_insn (gen_addsi3_carry (lo0, lo1, lo2));
      emit_insn (gen_addsi3_carry_in (hi0, hi1, hi2));
      DONE;
    }

  if (CONST_INT_P (operands[2]) && !add_operand (operands[2], <MODE>mode))
    {
      rtx tmp = ((!can_create_pseudo_p ()
		  || rtx_equal_p (operands[0], operands[1]))
		 ? operands[0] : gen_reg_rtx (<MODE>mode));

      HOST_WIDE_INT val = INTVAL (operands[2]);
      HOST_WIDE_INT low = ((val & 0xffff) ^ 0x8000) - 0x8000;
      HOST_WIDE_INT rest = trunc_int_for_mode (val - low, <MODE>mode);

      if (<MODE>mode == DImode && !satisfies_constraint_L (GEN_INT (rest)))
	FAIL;

      /* The ordering here is important for the prolog expander.
	 When space is allocated from the stack, adding 'low' first may
	 produce a temporary deallocation (which would be bad).  */
      emit_insn (gen_add<mode>3 (tmp, operands[1], GEN_INT (rest)));
      emit_insn (gen_add<mode>3 (operands[0], tmp, GEN_INT (low)));
      DONE;
    }
})

(define_insn "*add<mode>3"
  [(set (match_operand:GPR 0 "gpc_reg_operand" "=r,r,r")
	(plus:GPR (match_operand:GPR 1 "gpc_reg_operand" "%r,b,b")
		  (match_operand:GPR 2 "add_operand" "r,I,L")))]
  ""
  "@
   add %0,%1,%2
   addi %0,%1,%2
   addis %0,%1,%v2"
  [(set_attr "type" "add")])

(define_insn "addsi3_high"
  [(set (match_operand:SI 0 "gpc_reg_operand" "=b")
        (plus:SI (match_operand:SI 1 "gpc_reg_operand" "b")
                 (high:SI (match_operand 2 "" ""))))]
  "TARGET_MACHO && !TARGET_64BIT"
  "addis %0,%1,ha16(%2)"
  [(set_attr "type" "add")])

(define_insn_and_split "*add<mode>3_dot"
  [(set (match_operand:CC 3 "cc_reg_operand" "=x,?y")
	(compare:CC (plus:GPR (match_operand:GPR 1 "gpc_reg_operand" "r,r")
			      (match_operand:GPR 2 "gpc_reg_operand" "r,r"))
		    (const_int 0)))
   (clobber (match_scratch:GPR 0 "=r,r"))]
  "<MODE>mode == Pmode"
  "@
   add. %0,%1,%2
   #"
  "&& reload_completed && cc_reg_not_cr0_operand (operands[3], CCmode)"
  [(set (match_dup 0)
	(plus:GPR (match_dup 1)
		 (match_dup 2)))
   (set (match_dup 3)
	(compare:CC (match_dup 0)
		    (const_int 0)))]
  ""
  [(set_attr "type" "add")
   (set_attr "dot" "yes")
   (set_attr "length" "4,8")])

(define_insn_and_split "*add<mode>3_dot2"
  [(set (match_operand:CC 3 "cc_reg_operand" "=x,?y")
	(compare:CC (plus:GPR (match_operand:GPR 1 "gpc_reg_operand" "r,r")
			      (match_operand:GPR 2 "gpc_reg_operand" "r,r"))
		    (const_int 0)))
   (set (match_operand:GPR 0 "gpc_reg_operand" "=r,r")
	(plus:GPR (match_dup 1)
		  (match_dup 2)))]
  "<MODE>mode == Pmode"
  "@
   add. %0,%1,%2
   #"
  "&& reload_completed && cc_reg_not_cr0_operand (operands[3], CCmode)"
  [(set (match_dup 0)
	(plus:GPR (match_dup 1)
		  (match_dup 2)))
   (set (match_dup 3)
	(compare:CC (match_dup 0)
		    (const_int 0)))]
  ""
  [(set_attr "type" "add")
   (set_attr "dot" "yes")
   (set_attr "length" "4,8")])

(define_insn_and_split "*add<mode>3_imm_dot"
  [(set (match_operand:CC 3 "cc_reg_operand" "=x,?y")
	(compare:CC (plus:GPR (match_operand:GPR 1 "gpc_reg_operand" "%r,b")
			      (match_operand:GPR 2 "short_cint_operand" "I,I"))
		    (const_int 0)))
   (clobber (match_scratch:GPR 0 "=r,r"))
   (clobber (reg:GPR CA_REGNO))]
  "<MODE>mode == Pmode"
  "@
   addic. %0,%1,%2
   #"
  "&& reload_completed && cc_reg_not_cr0_operand (operands[3], CCmode)"
  [(set (match_dup 0)
	(plus:GPR (match_dup 1)
		  (match_dup 2)))
   (set (match_dup 3)
	(compare:CC (match_dup 0)
		    (const_int 0)))]
  ""
  [(set_attr "type" "add")
   (set_attr "dot" "yes")
   (set_attr "length" "4,8")])

(define_insn_and_split "*add<mode>3_imm_dot2"
  [(set (match_operand:CC 3 "cc_reg_operand" "=x,?y")
	(compare:CC (plus:GPR (match_operand:GPR 1 "gpc_reg_operand" "%r,b")
			      (match_operand:GPR 2 "short_cint_operand" "I,I"))
		    (const_int 0)))
   (set (match_operand:GPR 0 "gpc_reg_operand" "=r,r")
	(plus:GPR (match_dup 1)
		  (match_dup 2)))
   (clobber (reg:GPR CA_REGNO))]
  "<MODE>mode == Pmode"
  "@
   addic. %0,%1,%2
   #"
  "&& reload_completed && cc_reg_not_cr0_operand (operands[3], CCmode)"
  [(set (match_dup 0)
	(plus:GPR (match_dup 1)
		  (match_dup 2)))
   (set (match_dup 3)
	(compare:CC (match_dup 0)
		    (const_int 0)))]
  ""
  [(set_attr "type" "add")
   (set_attr "dot" "yes")
   (set_attr "length" "4,8")])

;; Split an add that we can't do in one insn into two insns, each of which
;; does one 16-bit part.  This is used by combine.  Note that the low-order
;; add should be last in case the result gets used in an address.

(define_split
  [(set (match_operand:GPR 0 "gpc_reg_operand" "")
	(plus:GPR (match_operand:GPR 1 "gpc_reg_operand" "")
		  (match_operand:GPR 2 "non_add_cint_operand" "")))]
  ""
  [(set (match_dup 0) (plus:GPR (match_dup 1) (match_dup 3)))
   (set (match_dup 0) (plus:GPR (match_dup 0) (match_dup 4)))]
{
  HOST_WIDE_INT val = INTVAL (operands[2]);
  HOST_WIDE_INT low = ((val & 0xffff) ^ 0x8000) - 0x8000;
  HOST_WIDE_INT rest = trunc_int_for_mode (val - low, <MODE>mode);

  operands[4] = GEN_INT (low);
  if (<MODE>mode == SImode || satisfies_constraint_L (GEN_INT (rest)))
    operands[3] = GEN_INT (rest);
  else if (can_create_pseudo_p ())
    {
      operands[3] = gen_reg_rtx (DImode);
      emit_move_insn (operands[3], operands[2]);
      emit_insn (gen_adddi3 (operands[0], operands[1], operands[3]));
      DONE;
    }
  else
    FAIL;
})


(define_insn "add<mode>3_carry"
  [(set (match_operand:P 0 "gpc_reg_operand" "=r")
	(plus:P (match_operand:P 1 "gpc_reg_operand" "r")
		(match_operand:P 2 "reg_or_short_operand" "rI")))
   (set (reg:P CA_REGNO)
	(ltu:P (plus:P (match_dup 1)
		       (match_dup 2))
	       (match_dup 1)))]
  ""
  "add%I2c %0,%1,%2"
  [(set_attr "type" "add")])

(define_insn "*add<mode>3_imm_carry_pos"
  [(set (match_operand:P 0 "gpc_reg_operand" "=r")
	(plus:P (match_operand:P 1 "gpc_reg_operand" "r")
		(match_operand:P 2 "short_cint_operand" "n")))
   (set (reg:P CA_REGNO)
	(geu:P (match_dup 1)
	       (match_operand:P 3 "const_int_operand" "n")))]
  "INTVAL (operands[2]) > 0
   && INTVAL (operands[2]) + INTVAL (operands[3]) == 0"
  "addic %0,%1,%2"
  [(set_attr "type" "add")])

(define_insn "*add<mode>3_imm_carry_0"
  [(set (match_operand:P 0 "gpc_reg_operand" "=r")
	(match_operand:P 1 "gpc_reg_operand" "r"))
   (set (reg:P CA_REGNO)
	(const_int 0))]
  ""
  "addic %0,%1,0"
  [(set_attr "type" "add")])

(define_insn "*add<mode>3_imm_carry_m1"
  [(set (match_operand:P 0 "gpc_reg_operand" "=r")
	(plus:P (match_operand:P 1 "gpc_reg_operand" "r")
		(const_int -1)))
   (set (reg:P CA_REGNO)
	(ne:P (match_dup 1)
	      (const_int 0)))]
  ""
  "addic %0,%1,-1"
  [(set_attr "type" "add")])

(define_insn "*add<mode>3_imm_carry_neg"
  [(set (match_operand:P 0 "gpc_reg_operand" "=r")
	(plus:P (match_operand:P 1 "gpc_reg_operand" "r")
		(match_operand:P 2 "short_cint_operand" "n")))
   (set (reg:P CA_REGNO)
	(gtu:P (match_dup 1)
	       (match_operand:P 3 "const_int_operand" "n")))]
  "INTVAL (operands[2]) < 0
   && INTVAL (operands[2]) + INTVAL (operands[3]) == -1"
  "addic %0,%1,%2"
  [(set_attr "type" "add")])


(define_expand "add<mode>3_carry_in"
  [(parallel [
     (set (match_operand:GPR 0 "gpc_reg_operand")
	  (plus:GPR (plus:GPR (match_operand:GPR 1 "gpc_reg_operand")
			      (match_operand:GPR 2 "adde_operand"))
		    (reg:GPR CA_REGNO)))
     (clobber (reg:GPR CA_REGNO))])]
  ""
{
  if (operands[2] == const0_rtx)
    {
      emit_insn (gen_add<mode>3_carry_in_0 (operands[0], operands[1]));
      DONE;
    }
  if (operands[2] == constm1_rtx)
    {
      emit_insn (gen_add<mode>3_carry_in_m1 (operands[0], operands[1]));
      DONE;
    }
})

(define_insn "*add<mode>3_carry_in_internal"
  [(set (match_operand:GPR 0 "gpc_reg_operand" "=r")
	(plus:GPR (plus:GPR (match_operand:GPR 1 "gpc_reg_operand" "r")
			    (match_operand:GPR 2 "gpc_reg_operand" "r"))
		  (reg:GPR CA_REGNO)))
   (clobber (reg:GPR CA_REGNO))]
  ""
  "adde %0,%1,%2"
  [(set_attr "type" "add")])

(define_insn "add<mode>3_carry_in_0"
  [(set (match_operand:GPR 0 "gpc_reg_operand" "=r")
	(plus:GPR (match_operand:GPR 1 "gpc_reg_operand" "r")
		  (reg:GPR CA_REGNO)))
   (clobber (reg:GPR CA_REGNO))]
  ""
  "addze %0,%1"
  [(set_attr "type" "add")])

(define_insn "add<mode>3_carry_in_m1"
  [(set (match_operand:GPR 0 "gpc_reg_operand" "=r")
	(plus:GPR (plus:GPR (match_operand:GPR 1 "gpc_reg_operand" "r")
			    (reg:GPR CA_REGNO))
		  (const_int -1)))
   (clobber (reg:GPR CA_REGNO))]
  ""
  "addme %0,%1"
  [(set_attr "type" "add")])


(define_expand "one_cmpl<mode>2"
  [(set (match_operand:SDI 0 "gpc_reg_operand" "")
	(not:SDI (match_operand:SDI 1 "gpc_reg_operand" "")))]
  ""
{
  if (<MODE>mode == DImode && !TARGET_POWERPC64)
    {
      rs6000_split_logical (operands, NOT, false, false, false);
      DONE;
    }
})

(define_insn "*one_cmpl<mode>2"
  [(set (match_operand:GPR 0 "gpc_reg_operand" "=r")
	(not:GPR (match_operand:GPR 1 "gpc_reg_operand" "r")))]
  ""
  "not %0,%1")

(define_insn_and_split "*one_cmpl<mode>2_dot"
  [(set (match_operand:CC 2 "cc_reg_operand" "=x,?y")
	(compare:CC (not:GPR (match_operand:GPR 1 "gpc_reg_operand" "r,r"))
		    (const_int 0)))
   (clobber (match_scratch:GPR 0 "=r,r"))]
  "<MODE>mode == Pmode && rs6000_gen_cell_microcode"
  "@
   not. %0,%1
   #"
  "&& reload_completed && cc_reg_not_cr0_operand (operands[2], CCmode)"
  [(set (match_dup 0)
	(not:GPR (match_dup 1)))
   (set (match_dup 2)
	(compare:CC (match_dup 0)
		    (const_int 0)))]
  ""
  [(set_attr "type" "logical")
   (set_attr "dot" "yes")
   (set_attr "length" "4,8")])

(define_insn_and_split "*one_cmpl<mode>2_dot2"
  [(set (match_operand:CC 2 "cc_reg_operand" "=x,?y")
	(compare:CC (not:GPR (match_operand:GPR 1 "gpc_reg_operand" "r,r"))
		    (const_int 0)))
   (set (match_operand:GPR 0 "gpc_reg_operand" "=r,r")
	(not:GPR (match_dup 1)))]
  "<MODE>mode == Pmode && rs6000_gen_cell_microcode"
  "@
   not. %0,%1
   #"
  "&& reload_completed && cc_reg_not_cr0_operand (operands[2], CCmode)"
  [(set (match_dup 0)
	(not:GPR (match_dup 1)))
   (set (match_dup 2)
	(compare:CC (match_dup 0)
		    (const_int 0)))]
  ""
  [(set_attr "type" "logical")
   (set_attr "dot" "yes")
   (set_attr "length" "4,8")])


(define_expand "sub<mode>3"
  [(set (match_operand:SDI 0 "gpc_reg_operand" "")
	(minus:SDI (match_operand:SDI 1 "reg_or_short_operand" "")
		   (match_operand:SDI 2 "gpc_reg_operand" "")))]
  ""
{
  if (<MODE>mode == DImode && !TARGET_POWERPC64)
    {
      rtx lo0 = gen_lowpart (SImode, operands[0]);
      rtx lo1 = gen_lowpart (SImode, operands[1]);
      rtx lo2 = gen_lowpart (SImode, operands[2]);
      rtx hi0 = gen_highpart (SImode, operands[0]);
      rtx hi1 = gen_highpart_mode (SImode, DImode, operands[1]);
      rtx hi2 = gen_highpart (SImode, operands[2]);

      if (!reg_or_short_operand (lo1, SImode))
	lo1 = force_reg (SImode, lo1);
      if (!adde_operand (hi1, SImode))
	hi1 = force_reg (SImode, hi1);

      emit_insn (gen_subfsi3_carry (lo0, lo2, lo1));
      emit_insn (gen_subfsi3_carry_in (hi0, hi2, hi1));
      DONE;
    }

  if (short_cint_operand (operands[1], <MODE>mode))
    {
      emit_insn (gen_subf<mode>3_imm (operands[0], operands[2], operands[1]));
      DONE;
    }
})

(define_insn "*subf<mode>3"
  [(set (match_operand:GPR 0 "gpc_reg_operand" "=r")
	(minus:GPR (match_operand:GPR 2 "gpc_reg_operand" "r")
		   (match_operand:GPR 1 "gpc_reg_operand" "r")))]
  ""
  "subf %0,%1,%2"
  [(set_attr "type" "add")])

(define_insn_and_split "*subf<mode>3_dot"
  [(set (match_operand:CC 3 "cc_reg_operand" "=x,?y")
	(compare:CC (minus:GPR (match_operand:GPR 2 "gpc_reg_operand" "r,r")
			       (match_operand:GPR 1 "gpc_reg_operand" "r,r"))
		    (const_int 0)))
   (clobber (match_scratch:GPR 0 "=r,r"))]
  "<MODE>mode == Pmode"
  "@
   subf. %0,%1,%2
   #"
  "&& reload_completed && cc_reg_not_cr0_operand (operands[3], CCmode)"
  [(set (match_dup 0)
	(minus:GPR (match_dup 2)
		   (match_dup 1)))
   (set (match_dup 3)
	(compare:CC (match_dup 0)
		    (const_int 0)))]
  ""
  [(set_attr "type" "add")
   (set_attr "dot" "yes")
   (set_attr "length" "4,8")])

(define_insn_and_split "*subf<mode>3_dot2"
  [(set (match_operand:CC 3 "cc_reg_operand" "=x,?y")
	(compare:CC (minus:GPR (match_operand:GPR 2 "gpc_reg_operand" "r,r")
			       (match_operand:GPR 1 "gpc_reg_operand" "r,r"))
		    (const_int 0)))
   (set (match_operand:GPR 0 "gpc_reg_operand" "=r,r")
	(minus:GPR (match_dup 2)
		   (match_dup 1)))]
  "<MODE>mode == Pmode"
  "@
   subf. %0,%1,%2
   #"
  "&& reload_completed && cc_reg_not_cr0_operand (operands[3], CCmode)"
  [(set (match_dup 0)
	(minus:GPR (match_dup 2)
		   (match_dup 1)))
   (set (match_dup 3)
	(compare:CC (match_dup 0)
		    (const_int 0)))]
  ""
  [(set_attr "type" "add")
   (set_attr "dot" "yes")
   (set_attr "length" "4,8")])

(define_insn "subf<mode>3_imm"
  [(set (match_operand:GPR 0 "gpc_reg_operand" "=r")
	(minus:GPR (match_operand:GPR 2 "short_cint_operand" "I")
		   (match_operand:GPR 1 "gpc_reg_operand" "r")))
   (clobber (reg:GPR CA_REGNO))]
  ""
  "subfic %0,%1,%2"
  [(set_attr "type" "add")])


(define_insn "subf<mode>3_carry"
  [(set (match_operand:P 0 "gpc_reg_operand" "=r")
	(minus:P (match_operand:P 2 "reg_or_short_operand" "rI")
		 (match_operand:P 1 "gpc_reg_operand" "r")))
   (set (reg:P CA_REGNO)
	(leu:P (match_dup 1)
	       (match_dup 2)))]
  ""
  "subf%I2c %0,%1,%2"
  [(set_attr "type" "add")])

(define_insn "*subf<mode>3_imm_carry_0"
  [(set (match_operand:P 0 "gpc_reg_operand" "=r")
	(neg:P (match_operand:P 1 "gpc_reg_operand" "r")))
   (set (reg:P CA_REGNO)
	(eq:P (match_dup 1)
	      (const_int 0)))]
  ""
  "subfic %0,%1,0"
  [(set_attr "type" "add")])

(define_insn "*subf<mode>3_imm_carry_m1"
  [(set (match_operand:P 0 "gpc_reg_operand" "=r")
	(not:P (match_operand:P 1 "gpc_reg_operand" "r")))
   (set (reg:P CA_REGNO)
	(const_int 1))]
  ""
  "subfic %0,%1,-1"
  [(set_attr "type" "add")])


(define_expand "subf<mode>3_carry_in"
  [(parallel [
     (set (match_operand:GPR 0 "gpc_reg_operand")
	  (plus:GPR (plus:GPR (not:GPR (match_operand:GPR 1 "gpc_reg_operand"))
			      (reg:GPR CA_REGNO))
		    (match_operand:GPR 2 "adde_operand")))
     (clobber (reg:GPR CA_REGNO))])]
  ""
{
  if (operands[2] == const0_rtx)
    {
      emit_insn (gen_subf<mode>3_carry_in_0 (operands[0], operands[1]));
      DONE;
    }
  if (operands[2] == constm1_rtx)
    {
      emit_insn (gen_subf<mode>3_carry_in_m1 (operands[0], operands[1]));
      DONE;
    }
})

(define_insn "*subf<mode>3_carry_in_internal"
  [(set (match_operand:GPR 0 "gpc_reg_operand" "=r")
	(plus:GPR (plus:GPR (not:GPR (match_operand:GPR 1 "gpc_reg_operand" "r"))
			    (reg:GPR CA_REGNO))
		  (match_operand:GPR 2 "gpc_reg_operand" "r")))
   (clobber (reg:GPR CA_REGNO))]
  ""
  "subfe %0,%1,%2"
  [(set_attr "type" "add")])

(define_insn "subf<mode>3_carry_in_0"
  [(set (match_operand:GPR 0 "gpc_reg_operand" "=r")
	(plus:GPR (not:GPR (match_operand:GPR 1 "gpc_reg_operand" "r"))
		  (reg:GPR CA_REGNO)))
   (clobber (reg:GPR CA_REGNO))]
  ""
  "subfze %0,%1"
  [(set_attr "type" "add")])

(define_insn "subf<mode>3_carry_in_m1"
  [(set (match_operand:GPR 0 "gpc_reg_operand" "=r")
	(plus:GPR (minus:GPR (reg:GPR CA_REGNO)
			     (match_operand:GPR 1 "gpc_reg_operand" "r"))
		  (const_int -2)))
   (clobber (reg:GPR CA_REGNO))]
  ""
  "subfme %0,%1"
  [(set_attr "type" "add")])

(define_insn "subf<mode>3_carry_in_xx"
  [(set (match_operand:GPR 0 "gpc_reg_operand" "=r")
	(plus:GPR (reg:GPR CA_REGNO)
		  (const_int -1)))
   (clobber (reg:GPR CA_REGNO))]
  ""
  "subfe %0,%0,%0"
  [(set_attr "type" "add")])


(define_insn "neg<mode>2"
  [(set (match_operand:GPR 0 "gpc_reg_operand" "=r")
	(neg:GPR (match_operand:GPR 1 "gpc_reg_operand" "r")))]
  ""
  "neg %0,%1"
  [(set_attr "type" "add")])

(define_insn_and_split "*neg<mode>2_dot"
  [(set (match_operand:CC 2 "cc_reg_operand" "=x,?y")
	(compare:CC (neg:GPR (match_operand:GPR 1 "gpc_reg_operand" "r,r"))
		    (const_int 0)))
   (clobber (match_scratch:GPR 0 "=r,r"))]
  "<MODE>mode == Pmode"
  "@
   neg. %0,%1
   #"
  "&& reload_completed && cc_reg_not_cr0_operand (operands[2], CCmode)"
  [(set (match_dup 0)
	(neg:GPR (match_dup 1)))
   (set (match_dup 2)
	(compare:CC (match_dup 0)
		    (const_int 0)))]
  ""
  [(set_attr "type" "add")
   (set_attr "dot" "yes")
   (set_attr "length" "4,8")])

(define_insn_and_split "*neg<mode>2_dot2"
  [(set (match_operand:CC 2 "cc_reg_operand" "=x,?y")
	(compare:CC (neg:GPR (match_operand:GPR 1 "gpc_reg_operand" "r,r"))
		    (const_int 0)))
   (set (match_operand:GPR 0 "gpc_reg_operand" "=r,r")
	(neg:GPR (match_dup 1)))]
  "<MODE>mode == Pmode"
  "@
   neg. %0,%1
   #"
  "&& reload_completed && cc_reg_not_cr0_operand (operands[2], CCmode)"
  [(set (match_dup 0)
	(neg:GPR (match_dup 1)))
   (set (match_dup 2)
	(compare:CC (match_dup 0)
		    (const_int 0)))]
  ""
  [(set_attr "type" "add")
   (set_attr "dot" "yes")
   (set_attr "length" "4,8")])


(define_insn "clz<mode>2"
  [(set (match_operand:GPR 0 "gpc_reg_operand" "=r")
	(clz:GPR (match_operand:GPR 1 "gpc_reg_operand" "r")))]
  ""
  "cntlz<wd> %0,%1"
  [(set_attr "type" "cntlz")])

(define_expand "ctz<mode>2"
  [(set (match_dup 2)
	(neg:GPR (match_operand:GPR 1 "gpc_reg_operand" "")))
   (set (match_dup 3)
	(and:GPR (match_dup 1)
		 (match_dup 2)))
   (set (match_dup 4)
	(clz:GPR (match_dup 3)))
   (parallel [(set (match_operand:GPR 0 "gpc_reg_operand" "")
		   (minus:GPR (match_dup 5)
			      (match_dup 4)))
	      (clobber (reg:GPR CA_REGNO))])]
  ""
{
  operands[2] = gen_reg_rtx (<MODE>mode);
  operands[3] = gen_reg_rtx (<MODE>mode);
  operands[4] = gen_reg_rtx (<MODE>mode);
  operands[5] = GEN_INT (GET_MODE_BITSIZE (<MODE>mode) - 1);
})

(define_expand "ffs<mode>2"
  [(set (match_dup 2)
	(neg:GPR (match_operand:GPR 1 "gpc_reg_operand" "")))
   (set (match_dup 3)
	(and:GPR (match_dup 1)
		 (match_dup 2)))
   (set (match_dup 4)
	(clz:GPR (match_dup 3)))
   (parallel [(set (match_operand:GPR 0 "gpc_reg_operand" "")
		   (minus:GPR (match_dup 5)
			      (match_dup 4)))
	      (clobber (reg:GPR CA_REGNO))])]
  ""
{
  operands[2] = gen_reg_rtx (<MODE>mode);
  operands[3] = gen_reg_rtx (<MODE>mode);
  operands[4] = gen_reg_rtx (<MODE>mode);
  operands[5] = GEN_INT (GET_MODE_BITSIZE (<MODE>mode));
})


(define_expand "popcount<mode>2"
  [(set (match_operand:GPR 0 "gpc_reg_operand" "")
	(popcount:GPR (match_operand:GPR 1 "gpc_reg_operand" "")))]
  "TARGET_POPCNTB || TARGET_POPCNTD"
{
  rs6000_emit_popcount (operands[0], operands[1]);
  DONE;
})

(define_insn "popcntb<mode>2"
  [(set (match_operand:GPR 0 "gpc_reg_operand" "=r")
	(unspec:GPR [(match_operand:GPR 1 "gpc_reg_operand" "r")]
		    UNSPEC_POPCNTB))]
  "TARGET_POPCNTB"
  "popcntb %0,%1"
  [(set_attr "type" "popcnt")])

(define_insn "popcntd<mode>2"
  [(set (match_operand:GPR 0 "gpc_reg_operand" "=r")
	(popcount:GPR (match_operand:GPR 1 "gpc_reg_operand" "r")))]
  "TARGET_POPCNTD"
  "popcnt<wd> %0,%1"
  [(set_attr "type" "popcnt")])


(define_expand "parity<mode>2"
  [(set (match_operand:GPR 0 "gpc_reg_operand" "")
	(parity:GPR (match_operand:GPR 1 "gpc_reg_operand" "")))]
  "TARGET_POPCNTB"
{
  rs6000_emit_parity (operands[0], operands[1]);
  DONE;
})

(define_insn "parity<mode>2_cmpb"
  [(set (match_operand:GPR 0 "gpc_reg_operand" "=r")
	(unspec:GPR [(match_operand:GPR 1 "gpc_reg_operand" "r")] UNSPEC_PARITY))]
  "TARGET_CMPB && TARGET_POPCNTB"
  "prty<wd> %0,%1"
  [(set_attr "type" "popcnt")])


;; Since the hardware zeros the upper part of the register, save generating the
;; AND immediate if we are converting to unsigned
(define_insn "*bswaphi2_extenddi"
  [(set (match_operand:DI 0 "gpc_reg_operand" "=r")
	(zero_extend:DI
	 (bswap:HI (match_operand:HI 1 "memory_operand" "Z"))))]
  "TARGET_POWERPC64"
  "lhbrx %0,%y1"
  [(set_attr "length" "4")
   (set_attr "type" "load")])

(define_insn "*bswaphi2_extendsi"
  [(set (match_operand:SI 0 "gpc_reg_operand" "=r")
	(zero_extend:SI
	 (bswap:HI (match_operand:HI 1 "memory_operand" "Z"))))]
  ""
  "lhbrx %0,%y1"
  [(set_attr "length" "4")
   (set_attr "type" "load")])

(define_expand "bswaphi2"
  [(parallel [(set (match_operand:HI 0 "reg_or_mem_operand" "")
		   (bswap:HI
		    (match_operand:HI 1 "reg_or_mem_operand" "")))
	      (clobber (match_scratch:SI 2 ""))])]
  ""
{
  if (!REG_P (operands[0]) && !REG_P (operands[1]))
    operands[1] = force_reg (HImode, operands[1]);
})

(define_insn "bswaphi2_internal"
  [(set (match_operand:HI 0 "reg_or_mem_operand" "=r,Z,&r")
	(bswap:HI
	 (match_operand:HI 1 "reg_or_mem_operand" "Z,r,r")))
   (clobber (match_scratch:SI 2 "=X,X,&r"))]
  ""
  "@
   lhbrx %0,%y1
   sthbrx %1,%y0
   #"
  [(set_attr "length" "4,4,12")
   (set_attr "type" "load,store,*")])

;; We are always BITS_BIG_ENDIAN, so the (const_int 16) below is
;; correct for -mlittle as well as -mbig.
(define_split
  [(set (match_operand:HI 0 "gpc_reg_operand" "")
	(bswap:HI (match_operand:HI 1 "gpc_reg_operand" "")))
   (clobber (match_operand:SI 2 "gpc_reg_operand" ""))]
  "reload_completed"
  [(set (match_dup 3)
	(zero_extract:SI (match_dup 4)
			 (const_int 8)
			 (const_int 16)))
   (set (match_dup 2)
	(and:SI (ashift:SI (match_dup 4)
			   (const_int 8))
		(const_int 65280)))		;; 0xff00
   (set (match_dup 3)
	(ior:SI (match_dup 3)
		(match_dup 2)))]
  "
{
  operands[3] = simplify_gen_subreg (SImode, operands[0], HImode, 0);
  operands[4] = simplify_gen_subreg (SImode, operands[1], HImode, 0);
}")

(define_insn "*bswapsi2_extenddi"
  [(set (match_operand:DI 0 "gpc_reg_operand" "=r")
	(zero_extend:DI
	 (bswap:SI (match_operand:SI 1 "memory_operand" "Z"))))]
  "TARGET_POWERPC64"
  "lwbrx %0,%y1"
  [(set_attr "length" "4")
   (set_attr "type" "load")])

(define_expand "bswapsi2"
  [(set (match_operand:SI 0 "reg_or_mem_operand" "")
	(bswap:SI
	 (match_operand:SI 1 "reg_or_mem_operand" "")))]
  ""
{
  if (!REG_P (operands[0]) && !REG_P (operands[1]))
    operands[1] = force_reg (SImode, operands[1]);
})

(define_insn "*bswapsi2_internal"
  [(set (match_operand:SI 0 "reg_or_mem_operand" "=r,Z,&r")
	(bswap:SI
	 (match_operand:SI 1 "reg_or_mem_operand" "Z,r,r")))]
  ""
  "@
   lwbrx %0,%y1
   stwbrx %1,%y0
   #"
  [(set_attr "length" "4,4,12")
   (set_attr "type" "load,store,*")])

;; We are always BITS_BIG_ENDIAN, so the bit positions below in
;; zero_extract insns do not change for -mlittle.
(define_split
  [(set (match_operand:SI 0 "gpc_reg_operand" "")
	(bswap:SI (match_operand:SI 1 "gpc_reg_operand" "")))]
  "reload_completed"
  [(set (match_dup 0)
	(rotate:SI (match_dup 1) (const_int 8)))
   (set (zero_extract:SI (match_dup 0)
			 (const_int 8)
			 (const_int 0))
	(match_dup 1))
   (set (zero_extract:SI (match_dup 0)
			 (const_int 8)
			 (const_int 16))
	(rotate:SI (match_dup 1)
		   (const_int 16)))]
  "")

(define_expand "bswapdi2"
  [(parallel [(set (match_operand:DI 0 "reg_or_mem_operand" "")
		   (bswap:DI
		    (match_operand:DI 1 "reg_or_mem_operand" "")))
	      (clobber (match_scratch:DI 2 ""))
	      (clobber (match_scratch:DI 3 ""))])]
  ""
{
  if (!REG_P (operands[0]) && !REG_P (operands[1]))
    operands[1] = force_reg (DImode, operands[1]);

  if (!TARGET_POWERPC64)
    {
      /* 32-bit mode needs fewer scratch registers, but 32-bit addressing mode
	 that uses 64-bit registers needs the same scratch registers as 64-bit
	 mode.  */
      emit_insn (gen_bswapdi2_32bit (operands[0], operands[1]));
      DONE;
    }
})

;; Power7/cell has ldbrx/stdbrx, so use it directly
(define_insn "*bswapdi2_ldbrx"
  [(set (match_operand:DI 0 "reg_or_mem_operand" "=r,Z,&r")
	(bswap:DI (match_operand:DI 1 "reg_or_mem_operand" "Z,r,r")))
   (clobber (match_scratch:DI 2 "=X,X,&r"))
   (clobber (match_scratch:DI 3 "=X,X,&r"))]
  "TARGET_POWERPC64 && TARGET_LDBRX
   && (REG_P (operands[0]) || REG_P (operands[1]))"
  "@
   ldbrx %0,%y1
   stdbrx %1,%y0
   #"
  [(set_attr "length" "4,4,36")
   (set_attr "type" "load,store,*")])

;; Non-power7/cell, fall back to use lwbrx/stwbrx
(define_insn "*bswapdi2_64bit"
  [(set (match_operand:DI 0 "reg_or_mem_operand" "=r,Z,&r")
	(bswap:DI (match_operand:DI 1 "reg_or_mem_operand" "Z,r,r")))
   (clobber (match_scratch:DI 2 "=&b,&b,&r"))
   (clobber (match_scratch:DI 3 "=&r,&r,&r"))]
  "TARGET_POWERPC64 && !TARGET_LDBRX
   && (REG_P (operands[0]) || REG_P (operands[1]))
   && !(MEM_P (operands[0]) && MEM_VOLATILE_P (operands[0]))
   && !(MEM_P (operands[1]) && MEM_VOLATILE_P (operands[1]))"
  "#"
  [(set_attr "length" "16,12,36")])

(define_split
  [(set (match_operand:DI 0 "gpc_reg_operand" "")
	(bswap:DI (match_operand:DI 1 "indexed_or_indirect_operand" "")))
   (clobber (match_operand:DI 2 "gpc_reg_operand" ""))
   (clobber (match_operand:DI 3 "gpc_reg_operand" ""))]
  "TARGET_POWERPC64 && !TARGET_LDBRX && reload_completed"
  [(const_int 0)]
  "
{
  rtx dest   = operands[0];
  rtx src    = operands[1];
  rtx op2    = operands[2];
  rtx op3    = operands[3];
  rtx op3_32 = simplify_gen_subreg (SImode, op3, DImode,
				    BYTES_BIG_ENDIAN ? 4 : 0);
  rtx dest_32 = simplify_gen_subreg (SImode, dest, DImode,
				     BYTES_BIG_ENDIAN ? 4 : 0);
  rtx addr1;
  rtx addr2;
  rtx word1;
  rtx word2;

  addr1 = XEXP (src, 0);
  if (GET_CODE (addr1) == PLUS)
    {
      emit_insn (gen_add3_insn (op2, XEXP (addr1, 0), GEN_INT (4)));
      if (TARGET_AVOID_XFORM)
	{
	  emit_insn (gen_add3_insn (op2, XEXP (addr1, 1), op2));
	  addr2 = op2;
	}
      else
	addr2 = gen_rtx_PLUS (Pmode, op2, XEXP (addr1, 1));
    }
  else if (TARGET_AVOID_XFORM)
    {
      emit_insn (gen_add3_insn (op2, addr1, GEN_INT (4)));
      addr2 = op2;
    }
  else
    {
      emit_move_insn (op2, GEN_INT (4));
      addr2 = gen_rtx_PLUS (Pmode, op2, addr1);
    }

  word1 = change_address (src, SImode, addr1);
  word2 = change_address (src, SImode, addr2);

  if (BYTES_BIG_ENDIAN)
    {
      emit_insn (gen_bswapsi2 (op3_32, word2));
      emit_insn (gen_bswapsi2 (dest_32, word1));
    }
  else
    {
      emit_insn (gen_bswapsi2 (op3_32, word1));
      emit_insn (gen_bswapsi2 (dest_32, word2));
    }

  emit_insn (gen_ashldi3 (op3, op3, GEN_INT (32)));
  emit_insn (gen_iordi3 (dest, dest, op3));
  DONE;
}")

(define_split
  [(set (match_operand:DI 0 "indexed_or_indirect_operand" "")
	(bswap:DI (match_operand:DI 1 "gpc_reg_operand" "")))
   (clobber (match_operand:DI 2 "gpc_reg_operand" ""))
   (clobber (match_operand:DI 3 "gpc_reg_operand" ""))]
  "TARGET_POWERPC64 && !TARGET_LDBRX && reload_completed"
  [(const_int 0)]
  "
{
  rtx dest   = operands[0];
  rtx src    = operands[1];
  rtx op2    = operands[2];
  rtx op3    = operands[3];
  rtx src_si = simplify_gen_subreg (SImode, src, DImode,
				    BYTES_BIG_ENDIAN ? 4 : 0);
  rtx op3_si = simplify_gen_subreg (SImode, op3, DImode,
				    BYTES_BIG_ENDIAN ? 4 : 0);
  rtx addr1;
  rtx addr2;
  rtx word1;
  rtx word2;

  addr1 = XEXP (dest, 0);
  if (GET_CODE (addr1) == PLUS)
    {
      emit_insn (gen_add3_insn (op2, XEXP (addr1, 0), GEN_INT (4)));
      if (TARGET_AVOID_XFORM)
	{
	  emit_insn (gen_add3_insn (op2, XEXP (addr1, 1), op2));
	  addr2 = op2;
	}
      else
	addr2 = gen_rtx_PLUS (Pmode, op2, XEXP (addr1, 1));
    }
  else if (TARGET_AVOID_XFORM)
    {
      emit_insn (gen_add3_insn (op2, addr1, GEN_INT (4)));
      addr2 = op2;
    }
  else
    {
      emit_move_insn (op2, GEN_INT (4));
      addr2 = gen_rtx_PLUS (Pmode, op2, addr1);
    }

  word1 = change_address (dest, SImode, addr1);
  word2 = change_address (dest, SImode, addr2);

  emit_insn (gen_lshrdi3 (op3, src, GEN_INT (32)));

  if (BYTES_BIG_ENDIAN)
    {
      emit_insn (gen_bswapsi2 (word1, src_si));
      emit_insn (gen_bswapsi2 (word2, op3_si));
    }
  else
    {
      emit_insn (gen_bswapsi2 (word2, src_si));
      emit_insn (gen_bswapsi2 (word1, op3_si));
    }
  DONE;
}")

(define_split
  [(set (match_operand:DI 0 "gpc_reg_operand" "")
	(bswap:DI (match_operand:DI 1 "gpc_reg_operand" "")))
   (clobber (match_operand:DI 2 "gpc_reg_operand" ""))
   (clobber (match_operand:DI 3 "gpc_reg_operand" ""))]
  "TARGET_POWERPC64 && reload_completed"
  [(const_int 0)]
  "
{
  rtx dest    = operands[0];
  rtx src     = operands[1];
  rtx op2     = operands[2];
  rtx op3     = operands[3];
  int lo_off  = BYTES_BIG_ENDIAN ? 4 : 0;
  rtx dest_si = simplify_gen_subreg (SImode, dest, DImode, lo_off);
  rtx src_si  = simplify_gen_subreg (SImode, src, DImode, lo_off);
  rtx op2_si  = simplify_gen_subreg (SImode, op2, DImode, lo_off);
  rtx op3_si  = simplify_gen_subreg (SImode, op3, DImode, lo_off);

  emit_insn (gen_lshrdi3 (op2, src, GEN_INT (32)));
  emit_insn (gen_bswapsi2 (dest_si, src_si));
  emit_insn (gen_bswapsi2 (op3_si, op2_si));
  emit_insn (gen_ashldi3 (dest, dest, GEN_INT (32)));
  emit_insn (gen_iordi3 (dest, dest, op3));
  DONE;
}")

(define_insn "bswapdi2_32bit"
  [(set (match_operand:DI 0 "reg_or_mem_operand" "=r,Z,?&r")
	(bswap:DI (match_operand:DI 1 "reg_or_mem_operand" "Z,r,r")))
   (clobber (match_scratch:SI 2 "=&b,&b,X"))]
  "!TARGET_POWERPC64 && (REG_P (operands[0]) || REG_P (operands[1]))"
  "#"
  [(set_attr "length" "16,12,36")])

(define_split
  [(set (match_operand:DI 0 "gpc_reg_operand" "")
	(bswap:DI (match_operand:DI 1 "indexed_or_indirect_operand" "")))
   (clobber (match_operand:SI 2 "gpc_reg_operand" ""))]
  "!TARGET_POWERPC64 && reload_completed"
  [(const_int 0)]
  "
{
  rtx dest  = operands[0];
  rtx src   = operands[1];
  rtx op2   = operands[2];
  rtx dest1 = simplify_gen_subreg (SImode, dest, DImode, 0);
  rtx dest2 = simplify_gen_subreg (SImode, dest, DImode, 4);
  rtx addr1;
  rtx addr2;
  rtx word1;
  rtx word2;

  addr1 = XEXP (src, 0);
  if (GET_CODE (addr1) == PLUS)
    {
      emit_insn (gen_add3_insn (op2, XEXP (addr1, 0), GEN_INT (4)));
      if (TARGET_AVOID_XFORM
	  || REGNO (XEXP (addr1, 1)) == REGNO (dest2))
	{
	  emit_insn (gen_add3_insn (op2, XEXP (addr1, 1), op2));
	  addr2 = op2;
	}
      else
	addr2 = gen_rtx_PLUS (SImode, op2, XEXP (addr1, 1));
    }
  else if (TARGET_AVOID_XFORM
	   || REGNO (addr1) == REGNO (dest2))
    {
      emit_insn (gen_add3_insn (op2, addr1, GEN_INT (4)));
      addr2 = op2;
    }
  else
    {
      emit_move_insn (op2, GEN_INT (4));
      addr2 = gen_rtx_PLUS (SImode, op2, addr1);
    }

  word1 = change_address (src, SImode, addr1);
  word2 = change_address (src, SImode, addr2);

  emit_insn (gen_bswapsi2 (dest2, word1));
  /* The REGNO (dest2) tests above ensure that addr2 has not been trashed,
     thus allowing us to omit an early clobber on the output.  */
  emit_insn (gen_bswapsi2 (dest1, word2));
  DONE;
}")

(define_split
  [(set (match_operand:DI 0 "indexed_or_indirect_operand" "")
	(bswap:DI (match_operand:DI 1 "gpc_reg_operand" "")))
   (clobber (match_operand:SI 2 "gpc_reg_operand" ""))]
  "!TARGET_POWERPC64 && reload_completed"
  [(const_int 0)]
  "
{
  rtx dest = operands[0];
  rtx src  = operands[1];
  rtx op2  = operands[2];
  rtx src1 = simplify_gen_subreg (SImode, src, DImode, 0);
  rtx src2 = simplify_gen_subreg (SImode, src, DImode, 4);
  rtx addr1;
  rtx addr2;
  rtx word1;
  rtx word2;

  addr1 = XEXP (dest, 0);
  if (GET_CODE (addr1) == PLUS)
    {
      emit_insn (gen_add3_insn (op2, XEXP (addr1, 0), GEN_INT (4)));
      if (TARGET_AVOID_XFORM)
	{
	  emit_insn (gen_add3_insn (op2, XEXP (addr1, 1), op2));
	  addr2 = op2;
	}
      else
	addr2 = gen_rtx_PLUS (SImode, op2, XEXP (addr1, 1));
    }
  else if (TARGET_AVOID_XFORM)
    {
      emit_insn (gen_add3_insn (op2, addr1, GEN_INT (4)));
      addr2 = op2;
    }
  else
    {
      emit_move_insn (op2, GEN_INT (4));
      addr2 = gen_rtx_PLUS (SImode, op2, addr1);
    }

  word1 = change_address (dest, SImode, addr1);
  word2 = change_address (dest, SImode, addr2);

  emit_insn (gen_bswapsi2 (word2, src1));
  emit_insn (gen_bswapsi2 (word1, src2));
  DONE;
}")

(define_split
  [(set (match_operand:DI 0 "gpc_reg_operand" "")
	(bswap:DI (match_operand:DI 1 "gpc_reg_operand" "")))
   (clobber (match_operand:SI 2 "" ""))]
  "!TARGET_POWERPC64 && reload_completed"
  [(const_int 0)]
  "
{
  rtx dest  = operands[0];
  rtx src   = operands[1];
  rtx src1  = simplify_gen_subreg (SImode, src, DImode, 0);
  rtx src2  = simplify_gen_subreg (SImode, src, DImode, 4);
  rtx dest1 = simplify_gen_subreg (SImode, dest, DImode, 0);
  rtx dest2 = simplify_gen_subreg (SImode, dest, DImode, 4);

  emit_insn (gen_bswapsi2 (dest1, src2));
  emit_insn (gen_bswapsi2 (dest2, src1));
  DONE;
}")


(define_insn "mul<mode>3"
  [(set (match_operand:GPR 0 "gpc_reg_operand" "=r,r")
	(mult:GPR (match_operand:GPR 1 "gpc_reg_operand" "%r,r")
		  (match_operand:GPR 2 "reg_or_short_operand" "r,I")))]
  ""
  "@
   mull<wd> %0,%1,%2
   mulli %0,%1,%2"
   [(set_attr "type" "mul")
    (set (attr "size")
      (cond [(match_operand:GPR 2 "s8bit_cint_operand" "")
		(const_string "8")
             (match_operand:GPR 2 "short_cint_operand" "")
		(const_string "16")]
	(const_string "<bits>")))])

(define_insn_and_split "*mul<mode>3_dot"
  [(set (match_operand:CC 3 "cc_reg_operand" "=x,?y")
	(compare:CC (mult:GPR (match_operand:GPR 1 "gpc_reg_operand" "r,r")
			      (match_operand:GPR 2 "gpc_reg_operand" "r,r"))
		    (const_int 0)))
   (clobber (match_scratch:GPR 0 "=r,r"))]
  "<MODE>mode == Pmode && rs6000_gen_cell_microcode"
  "@
   mull<wd>. %0,%1,%2
   #"
  "&& reload_completed && cc_reg_not_cr0_operand (operands[3], CCmode)"
  [(set (match_dup 0)
	(mult:GPR (match_dup 1)
		  (match_dup 2)))
   (set (match_dup 3)
	(compare:CC (match_dup 0)
		    (const_int 0)))]
  ""
  [(set_attr "type" "mul")
   (set_attr "size" "<bits>")
   (set_attr "dot" "yes")
   (set_attr "length" "4,8")])

(define_insn_and_split "*mul<mode>3_dot2"
  [(set (match_operand:CC 3 "cc_reg_operand" "=x,?y")
	(compare:CC (mult:GPR (match_operand:GPR 1 "gpc_reg_operand" "r,r")
			      (match_operand:GPR 2 "gpc_reg_operand" "r,r"))
		    (const_int 0)))
   (set (match_operand:GPR 0 "gpc_reg_operand" "=r,r")
	(mult:GPR (match_dup 1)
		  (match_dup 2)))]
  "<MODE>mode == Pmode && rs6000_gen_cell_microcode"
  "@
   mull<wd>. %0,%1,%2
   #"
  "&& reload_completed && cc_reg_not_cr0_operand (operands[3], CCmode)"
  [(set (match_dup 0)
	(mult:GPR (match_dup 1)
		  (match_dup 2)))
   (set (match_dup 3)
	(compare:CC (match_dup 0)
		    (const_int 0)))]
  ""
  [(set_attr "type" "mul")
   (set_attr "size" "<bits>")
   (set_attr "dot" "yes")
   (set_attr "length" "4,8")])


(define_expand "<su>mul<mode>3_highpart"
  [(set (match_operand:GPR 0 "gpc_reg_operand")
	(subreg:GPR
	  (mult:<DMODE> (any_extend:<DMODE>
			  (match_operand:GPR 1 "gpc_reg_operand"))
			(any_extend:<DMODE>
			  (match_operand:GPR 2 "gpc_reg_operand")))
	 0))]
  ""
{
  if (<MODE>mode == SImode && TARGET_POWERPC64)
    {
      emit_insn (gen_<su>mulsi3_highpart_64 (operands[0], operands[1],
					     operands[2]));
      DONE;
    }

  if (!WORDS_BIG_ENDIAN)
    {
      emit_insn (gen_<su>mul<mode>3_highpart_le (operands[0], operands[1],
						 operands[2]));
      DONE;
    }
})

(define_insn "*<su>mul<mode>3_highpart"
  [(set (match_operand:GPR 0 "gpc_reg_operand" "=r")
	(subreg:GPR
	  (mult:<DMODE> (any_extend:<DMODE>
			  (match_operand:GPR 1 "gpc_reg_operand" "r"))
			(any_extend:<DMODE>
			  (match_operand:GPR 2 "gpc_reg_operand" "r")))
	 0))]
  "WORDS_BIG_ENDIAN && !(<MODE>mode == SImode && TARGET_POWERPC64)"
  "mulh<wd><u> %0,%1,%2"
  [(set_attr "type" "mul")
   (set_attr "size" "<bits>")])

(define_insn "<su>mulsi3_highpart_le"
  [(set (match_operand:SI 0 "gpc_reg_operand" "=r")
	(subreg:SI
	  (mult:DI (any_extend:DI
		     (match_operand:SI 1 "gpc_reg_operand" "r"))
		   (any_extend:DI
		     (match_operand:SI 2 "gpc_reg_operand" "r")))
	 4))]
  "!WORDS_BIG_ENDIAN && !TARGET_POWERPC64"
  "mulhw<u> %0,%1,%2"
  [(set_attr "type" "mul")])

(define_insn "<su>muldi3_highpart_le"
  [(set (match_operand:DI 0 "gpc_reg_operand" "=r")
	(subreg:DI
	  (mult:TI (any_extend:TI
		     (match_operand:DI 1 "gpc_reg_operand" "r"))
		   (any_extend:TI
		     (match_operand:DI 2 "gpc_reg_operand" "r")))
	 8))]
  "!WORDS_BIG_ENDIAN && TARGET_POWERPC64"
  "mulhd<u> %0,%1,%2"
  [(set_attr "type" "mul")
   (set_attr "size" "64")])

(define_insn "<su>mulsi3_highpart_64"
  [(set (match_operand:SI 0 "gpc_reg_operand" "=r")
	(truncate:SI
	  (lshiftrt:DI
	    (mult:DI (any_extend:DI
		       (match_operand:SI 1 "gpc_reg_operand" "r"))
		     (any_extend:DI
		       (match_operand:SI 2 "gpc_reg_operand" "r")))
	    (const_int 32))))]
  "TARGET_POWERPC64"
  "mulhw<u> %0,%1,%2"
  [(set_attr "type" "mul")])

(define_expand "<u>mul<mode><dmode>3"
  [(set (match_operand:<DMODE> 0 "gpc_reg_operand")
	(mult:<DMODE> (any_extend:<DMODE>
			(match_operand:GPR 1 "gpc_reg_operand"))
		      (any_extend:<DMODE>
			(match_operand:GPR 2 "gpc_reg_operand"))))]
  "!(<MODE>mode == SImode && TARGET_POWERPC64)"
{
  rtx l = gen_reg_rtx (<MODE>mode);
  rtx h = gen_reg_rtx (<MODE>mode);
  emit_insn (gen_mul<mode>3 (l, operands[1], operands[2]));
  emit_insn (gen_<su>mul<mode>3_highpart (h, operands[1], operands[2]));
  emit_move_insn (gen_lowpart (<MODE>mode, operands[0]), l);
  emit_move_insn (gen_highpart (<MODE>mode, operands[0]), h);
  DONE;
})


(define_insn "udiv<mode>3"
  [(set (match_operand:GPR 0 "gpc_reg_operand" "=r")
        (udiv:GPR (match_operand:GPR 1 "gpc_reg_operand" "r")
		  (match_operand:GPR 2 "gpc_reg_operand" "r")))]
  ""
  "div<wd>u %0,%1,%2"
  [(set_attr "type" "div")
   (set_attr "size" "<bits>")])


;; For powers of two we can do sra[wd]i/addze for divide and then adjust for
;; modulus.  If it isn't a power of two, force operands into register and do
;; a normal divide.
(define_expand "div<mode>3"
  [(set (match_operand:GPR 0 "gpc_reg_operand" "")
	(div:GPR (match_operand:GPR 1 "gpc_reg_operand" "")
		 (match_operand:GPR 2 "reg_or_cint_operand" "")))]
  ""
{
  if (CONST_INT_P (operands[2])
      && INTVAL (operands[2]) > 0
      && exact_log2 (INTVAL (operands[2])) >= 0)
    {
      emit_insn (gen_div<mode>3_sra (operands[0], operands[1], operands[2]));
      DONE;
    }

  operands[2] = force_reg (<MODE>mode, operands[2]);
})

(define_insn "*div<mode>3"
  [(set (match_operand:GPR 0 "gpc_reg_operand" "=r")
        (div:GPR (match_operand:GPR 1 "gpc_reg_operand" "r")
		 (match_operand:GPR 2 "gpc_reg_operand" "r")))]
  ""
  "div<wd> %0,%1,%2"
  [(set_attr "type" "div")
   (set_attr "size" "<bits>")])

(define_insn "div<mode>3_sra"
  [(set (match_operand:GPR 0 "gpc_reg_operand" "=r")
	(div:GPR (match_operand:GPR 1 "gpc_reg_operand" "r")
		 (match_operand:GPR 2 "exact_log2_cint_operand" "N")))
   (clobber (reg:GPR CA_REGNO))]
  ""
  "sra<wd>i %0,%1,%p2\;addze %0,%0"
  [(set_attr "type" "two")
   (set_attr "length" "8")])

(define_insn_and_split "*div<mode>3_sra_dot"
  [(set (match_operand:CC 3 "cc_reg_operand" "=x,?y")
	(compare:CC (div:GPR (match_operand:GPR 1 "gpc_reg_operand" "r,r")
			     (match_operand:GPR 2 "exact_log2_cint_operand" "N,N"))
		    (const_int 0)))
   (clobber (match_scratch:GPR 0 "=r,r"))
   (clobber (reg:GPR CA_REGNO))]
  "<MODE>mode == Pmode"
  "@
   sra<wd>i %0,%1,%p2\;addze. %0,%0
   #"
  "&& reload_completed && cc_reg_not_cr0_operand (operands[3], CCmode)"
  [(parallel [(set (match_dup 0)
		   (div:GPR (match_dup 1)
			    (match_dup 2)))
	      (clobber (reg:GPR CA_REGNO))])
   (set (match_dup 3)
	(compare:CC (match_dup 0)
		    (const_int 0)))]
  ""
  [(set_attr "type" "two")
   (set_attr "length" "8,12")
   (set_attr "cell_micro" "not")])

(define_insn_and_split "*div<mode>3_sra_dot2"
  [(set (match_operand:CC 3 "cc_reg_operand" "=x,?y")
	(compare:CC (div:GPR (match_operand:GPR 1 "gpc_reg_operand" "r,r")
			     (match_operand:GPR 2 "exact_log2_cint_operand" "N,N"))
		    (const_int 0)))
   (set (match_operand:GPR 0 "gpc_reg_operand" "=r,r")
	(div:GPR (match_dup 1)
		 (match_dup 2)))
   (clobber (reg:GPR CA_REGNO))]
  "<MODE>mode == Pmode"
  "@
   sra<wd>i %0,%1,%p2\;addze. %0,%0
   #"
  "&& reload_completed && cc_reg_not_cr0_operand (operands[3], CCmode)"
  [(parallel [(set (match_dup 0)
		   (div:GPR (match_dup 1)
			    (match_dup 2)))
	      (clobber (reg:GPR CA_REGNO))])
   (set (match_dup 3)
	(compare:CC (match_dup 0)
		    (const_int 0)))]
  ""
  [(set_attr "type" "two")
   (set_attr "length" "8,12")
   (set_attr "cell_micro" "not")])

(define_expand "mod<mode>3"
  [(use (match_operand:GPR 0 "gpc_reg_operand" ""))
   (use (match_operand:GPR 1 "gpc_reg_operand" ""))
   (use (match_operand:GPR 2 "reg_or_cint_operand" ""))]
  ""
{
  int i;
  rtx temp1;
  rtx temp2;

  if (GET_CODE (operands[2]) != CONST_INT
      || INTVAL (operands[2]) <= 0
      || (i = exact_log2 (INTVAL (operands[2]))) < 0)
    FAIL;

  temp1 = gen_reg_rtx (<MODE>mode);
  temp2 = gen_reg_rtx (<MODE>mode);

  emit_insn (gen_div<mode>3 (temp1, operands[1], operands[2]));
  emit_insn (gen_ashl<mode>3 (temp2, temp1, GEN_INT (i)));
  emit_insn (gen_sub<mode>3 (operands[0], operands[1], temp2));
  DONE;
})

;; Logical instructions
;; The logical instructions are mostly combined by using match_operator,
;; but the plain AND insns are somewhat different because there is no
;; plain 'andi' (only 'andi.'), no plain 'andis', and there are all
;; those rotate-and-mask operations.  Thus, the AND insns come first.

(define_expand "and<mode>3"
  [(set (match_operand:SDI 0 "gpc_reg_operand" "")
	(and:SDI (match_operand:SDI 1 "gpc_reg_operand" "")
		 (match_operand:SDI 2 "reg_or_cint_operand" "")))]
  ""
{
  if (<MODE>mode == DImode && !TARGET_POWERPC64)
    {
      rs6000_split_logical (operands, AND, false, false, false);
      DONE;
    }

  if (logical_const_operand (operands[2], <MODE>mode)
      && !any_mask_operand (operands[2], <MODE>mode))
    {
      if (rs6000_gen_cell_microcode)
	{
	  emit_insn (gen_and<mode>3_imm (operands[0], operands[1], operands[2]));
	  DONE;
	}
      else
	operands[2] = force_reg (<MODE>mode, operands[2]);
    }

  if ((<MODE>mode == DImode && !and64_2_operand (operands[2], <MODE>mode))
      || (<MODE>mode != DImode && !and_operand (operands[2], <MODE>mode)))
    operands[2] = force_reg (<MODE>mode, operands[2]);
})


(define_insn "and<mode>3_imm"
  [(set (match_operand:GPR 0 "gpc_reg_operand" "=r")
	(and:GPR (match_operand:GPR 1 "gpc_reg_operand" "%r")
		 (match_operand:GPR 2 "logical_const_operand" "n")))
   (clobber (match_scratch:CC 3 "=x"))]
  "rs6000_gen_cell_microcode
   && !any_mask_operand (operands[2], <MODE>mode)"
  "andi%e2. %0,%1,%u2"
  [(set_attr "type" "logical")
   (set_attr "dot" "yes")])

(define_insn_and_split "*and<mode>3_imm_dot"
  [(set (match_operand:CC 3 "cc_reg_operand" "=x,??y")
	(compare:CC (and:GPR (match_operand:GPR 1 "gpc_reg_operand" "%r,r")
			     (match_operand:GPR 2 "logical_const_operand" "n,n"))
		    (const_int 0)))
   (clobber (match_scratch:GPR 0 "=r,r"))
   (clobber (match_scratch:CC 4 "=X,x"))]
  "(<MODE>mode == Pmode || UINTVAL (operands[2]) <= 0x7fffffff)
   && rs6000_gen_cell_microcode
   && !any_mask_operand (operands[2], <MODE>mode)"
  "@
   andi%e2. %0,%1,%u2
   #"
  "&& reload_completed && cc_reg_not_cr0_operand (operands[3], CCmode)"
  [(parallel [(set (match_dup 0)
		   (and:GPR (match_dup 1)
			    (match_dup 2)))
	      (clobber (match_dup 4))])
   (set (match_dup 3)
	(compare:CC (match_dup 0)
		    (const_int 0)))]
  ""
  [(set_attr "type" "logical")
   (set_attr "dot" "yes")
   (set_attr "length" "4,8")])

(define_insn_and_split "*and<mode>3_imm_dot2"
  [(set (match_operand:CC 3 "cc_reg_operand" "=x,??y")
	(compare:CC (and:GPR (match_operand:GPR 1 "gpc_reg_operand" "%r,r")
			     (match_operand:GPR 2 "logical_const_operand" "n,n"))
		    (const_int 0)))
   (set (match_operand:GPR 0 "gpc_reg_operand" "=r,r")
	(and:GPR (match_dup 1)
		 (match_dup 2)))
   (clobber (match_scratch:CC 4 "=X,x"))]
  "(<MODE>mode == Pmode || UINTVAL (operands[2]) <= 0x7fffffff)
   && rs6000_gen_cell_microcode
   && !any_mask_operand (operands[2], <MODE>mode)"
  "@
   andi%e2. %0,%1,%u2
   #"
  "&& reload_completed && cc_reg_not_cr0_operand (operands[3], CCmode)"
  [(parallel [(set (match_dup 0)
		   (and:GPR (match_dup 1)
			    (match_dup 2)))
	      (clobber (match_dup 4))])
   (set (match_dup 3)
	(compare:CC (match_dup 0)
		    (const_int 0)))]
  ""
  [(set_attr "type" "logical")
   (set_attr "dot" "yes")
   (set_attr "length" "4,8")])

(define_insn_and_split "*and<mode>3_imm_mask_dot"
  [(set (match_operand:CC 3 "cc_reg_operand" "=x,??y")
	(compare:CC (and:GPR (match_operand:GPR 1 "gpc_reg_operand" "%r,r")
			     (match_operand:GPR 2 "logical_const_operand" "n,n"))
		    (const_int 0)))
   (clobber (match_scratch:GPR 0 "=r,r"))]
  "(<MODE>mode == Pmode || UINTVAL (operands[2]) <= 0x7fffffff)
   && rs6000_gen_cell_microcode
   && any_mask_operand (operands[2], <MODE>mode)"
  "@
   andi%e2. %0,%1,%u2
   #"
  "&& reload_completed && cc_reg_not_cr0_operand (operands[3], CCmode)"
  [(set (match_dup 0)
	(and:GPR (match_dup 1)
		 (match_dup 2)))
   (set (match_dup 3)
	(compare:CC (match_dup 0)
		    (const_int 0)))]
  ""
  [(set_attr "type" "logical")
   (set_attr "dot" "yes")
   (set_attr "length" "4,8")])

(define_insn_and_split "*and<mode>3_imm_mask_dot2"
  [(set (match_operand:CC 3 "cc_reg_operand" "=x,??y")
	(compare:CC (and:GPR (match_operand:GPR 1 "gpc_reg_operand" "%r,r")
			     (match_operand:GPR 2 "logical_const_operand" "n,n"))
		    (const_int 0)))
   (set (match_operand:GPR 0 "gpc_reg_operand" "=r,r")
	(and:GPR (match_dup 1)
		 (match_dup 2)))]
  "(<MODE>mode == Pmode || UINTVAL (operands[2]) <= 0x7fffffff)
   && rs6000_gen_cell_microcode
   && any_mask_operand (operands[2], <MODE>mode)"
  "@
   andi%e2. %0,%1,%u2
   #"
  "&& reload_completed && cc_reg_not_cr0_operand (operands[3], CCmode)"
  [(set (match_dup 0)
	(and:GPR (match_dup 1)
		 (match_dup 2)))
   (set (match_dup 3)
	(compare:CC (match_dup 0)
		    (const_int 0)))]
  ""
  [(set_attr "type" "logical")
   (set_attr "dot" "yes")
   (set_attr "length" "4,8")])


(define_insn "*and<mode>3_mask"
  [(set (match_operand:GPR 0 "gpc_reg_operand" "=r,r")
	(and:GPR (match_operand:GPR 1 "gpc_reg_operand" "%r,r")
		 (match_operand:GPR 2 "any_mask_operand" "S,T")))]
  ""
  "@
   rldic%B2 %0,%1,0,%S2
   rlwinm %0,%1,0,%m2,%M2"
  [(set_attr "type" "shift")])

(define_insn_and_split "*and<mode>3_mask_dot"
  [(set (match_operand:CC 3 "cc_reg_operand" "=x,x,?y,?y")
	(compare:CC (and:GPR (match_operand:GPR 1 "gpc_reg_operand" "%r,r,r,r")
			     (match_operand:GPR 2 "any_mask_operand" "S,T,S,T"))
		    (const_int 0)))
   (clobber (match_scratch:GPR 0 "=r,r,r,r"))]
  "(<MODE>mode == Pmode || UINTVAL (operands[2]) <= 0x7fffffff)
   && rs6000_gen_cell_microcode
   && !logical_const_operand (operands[2], <MODE>mode)"
  "@
   rldic%B2. %0,%1,0,%S2
   rlwinm. %0,%1,0,%m2,%M2
   #
   #"
  "&& reload_completed && cc_reg_not_cr0_operand (operands[3], CCmode)"
  [(set (match_dup 0)
	(and:GPR (match_dup 1)
		 (match_dup 2)))
   (set (match_dup 3)
	(compare:CC (match_dup 0)
		    (const_int 0)))]
  ""
  [(set_attr "type" "shift")
   (set_attr "dot" "yes")
   (set_attr "length" "4,4,8,8")])

(define_insn_and_split "*and<mode>3_mask_dot2"
  [(set (match_operand:CC 3 "cc_reg_operand" "=x,x,?y,?y")
	(compare:CC (and:GPR (match_operand:GPR 1 "gpc_reg_operand" "%r,r,r,r")
			     (match_operand:GPR 2 "any_mask_operand" "S,T,S,T"))
		    (const_int 0)))
   (set (match_operand:GPR 0 "gpc_reg_operand" "=r,r,r,r")
	(and:GPR (match_dup 1)
		 (match_dup 2)))]
  "(<MODE>mode == Pmode || UINTVAL (operands[2]) <= 0x7fffffff)
   && rs6000_gen_cell_microcode
   && !logical_const_operand (operands[2], <MODE>mode)"
  "@
   rldic%B2. %0,%1,0,%S2
   rlwinm. %0,%1,0,%m2,%M2
   #
   #"
  "&& reload_completed && cc_reg_not_cr0_operand (operands[3], CCmode)"
  [(set (match_dup 0)
	(and:GPR (match_dup 1)
		 (match_dup 2)))
   (set (match_dup 3)
	(compare:CC (match_dup 0)
		    (const_int 0)))]
  ""
  [(set_attr "type" "shift")
   (set_attr "dot" "yes")
   (set_attr "length" "4,4,8,8")])



(define_insn "andsi3_internal0_nomc"
  [(set (match_operand:SI 0 "gpc_reg_operand" "=r,r")
        (and:SI (match_operand:SI 1 "gpc_reg_operand" "%r,r")
                (match_operand:SI 2 "and_operand" "?r,T")))]
  "!rs6000_gen_cell_microcode"
  "@
   and %0,%1,%2
   rlwinm %0,%1,0,%m2,%M2"
  [(set_attr "type" "logical,shift")])


;; Handle the PowerPC64 rlwinm corner case

(define_insn_and_split "*andsi3_internal6"
  [(set (match_operand:SI 0 "gpc_reg_operand" "=r")
	(and:SI (match_operand:SI 1 "gpc_reg_operand" "r")
		(match_operand:SI 2 "mask_operand_wrap" "i")))]
  "TARGET_POWERPC64"
  "#"
  "TARGET_POWERPC64"
  [(set (match_dup 0)
	(and:SI (rotate:SI (match_dup 1) (match_dup 3))
		(match_dup 4)))
   (set (match_dup 0)
	(rotate:SI (match_dup 0) (match_dup 5)))]
  "
{
  int mb = extract_MB (operands[2]);
  int me = extract_ME (operands[2]);
  operands[3] = GEN_INT (me + 1);
  operands[5] = GEN_INT (32 - (me + 1));
  operands[4] = GEN_INT (~((HOST_WIDE_INT) -1 << (33 + me - mb)));
}"
  [(set_attr "length" "8")])


(define_expand "<code><mode>3"
  [(set (match_operand:SDI 0 "gpc_reg_operand" "")
	(iorxor:SDI (match_operand:SDI 1 "gpc_reg_operand" "")
		    (match_operand:SDI 2 "reg_or_cint_operand" "")))]
  ""
{
  if (<MODE>mode == DImode && !TARGET_POWERPC64)
    {
      rs6000_split_logical (operands, <CODE>, false, false, false);
      DONE;
    }

  if (non_logical_cint_operand (operands[2], <MODE>mode))
    {
      rtx tmp = ((!can_create_pseudo_p ()
		  || rtx_equal_p (operands[0], operands[1]))
		 ? operands[0] : gen_reg_rtx (<MODE>mode));

      HOST_WIDE_INT value = INTVAL (operands[2]);
      HOST_WIDE_INT lo = value & 0xffff;
      HOST_WIDE_INT hi = value - lo;

      emit_insn (gen_<code><mode>3 (tmp, operands[1], GEN_INT (hi)));
      emit_insn (gen_<code><mode>3 (operands[0], tmp, GEN_INT (lo)));
      DONE;
    }

  if (!reg_or_logical_cint_operand (operands[2], <MODE>mode))
    operands[2] = force_reg (<MODE>mode, operands[2]);
})

(define_split
  [(set (match_operand:GPR 0 "gpc_reg_operand" "")
	(iorxor:GPR (match_operand:GPR 1 "gpc_reg_operand" "")
		    (match_operand:GPR 2 "non_logical_cint_operand" "")))]
  ""
  [(set (match_dup 3)
	(iorxor:GPR (match_dup 1)
		    (match_dup 4)))
   (set (match_dup 0)
	(iorxor:GPR (match_dup 3)
		    (match_dup 5)))]
{
  operands[3] = ((!can_create_pseudo_p ()
		  || rtx_equal_p (operands[0], operands[1]))
		 ? operands[0] : gen_reg_rtx (<MODE>mode));

  HOST_WIDE_INT value = INTVAL (operands[2]);
  HOST_WIDE_INT lo = value & 0xffff;
  HOST_WIDE_INT hi = value - lo;

  operands[4] = GEN_INT (hi);
  operands[5] = GEN_INT (lo);
})

(define_insn "*bool<mode>3_imm"
  [(set (match_operand:GPR 0 "gpc_reg_operand" "=r")
	(match_operator:GPR 3 "boolean_or_operator"
	 [(match_operand:GPR 1 "gpc_reg_operand" "%r")
	  (match_operand:GPR 2 "logical_const_operand" "n")]))]
  ""
  "%q3i%e2 %0,%1,%u2"
  [(set_attr "type" "logical")])

(define_insn "*bool<mode>3"
  [(set (match_operand:GPR 0 "gpc_reg_operand" "=r")
	(match_operator:GPR 3 "boolean_operator"
	 [(match_operand:GPR 1 "gpc_reg_operand" "r")
	  (match_operand:GPR 2 "gpc_reg_operand" "r")]))]
  ""
  "%q3 %0,%1,%2"
  [(set_attr "type" "logical")])

(define_insn_and_split "*bool<mode>3_dot"
  [(set (match_operand:CC 4 "cc_reg_operand" "=x,?y")
	(compare:CC (match_operator:GPR 3 "boolean_operator"
	 [(match_operand:GPR 1 "gpc_reg_operand" "r,r")
	  (match_operand:GPR 2 "gpc_reg_operand" "r,r")])
	 (const_int 0)))
   (clobber (match_scratch:GPR 0 "=r,r"))]
  "<MODE>mode == Pmode && rs6000_gen_cell_microcode"
  "@
   %q3. %0,%1,%2
   #"
  "&& reload_completed && cc_reg_not_cr0_operand (operands[4], CCmode)"
  [(set (match_dup 0)
	(match_dup 3))
   (set (match_dup 4)
	(compare:CC (match_dup 0)
		    (const_int 0)))]
  ""
  [(set_attr "type" "logical")
   (set_attr "dot" "yes")
   (set_attr "length" "4,8")])

(define_insn_and_split "*bool<mode>3_dot2"
  [(set (match_operand:CC 4 "cc_reg_operand" "=x,?y")
	(compare:CC (match_operator:GPR 3 "boolean_operator"
	 [(match_operand:GPR 1 "gpc_reg_operand" "r,r")
	  (match_operand:GPR 2 "gpc_reg_operand" "r,r")])
	 (const_int 0)))
   (set (match_operand:GPR 0 "gpc_reg_operand" "=r,r")
	(match_dup 3))]
  "<MODE>mode == Pmode && rs6000_gen_cell_microcode"
  "@
   %q3. %0,%1,%2
   #"
  "&& reload_completed && cc_reg_not_cr0_operand (operands[4], CCmode)"
  [(set (match_dup 0)
	(match_dup 3))
   (set (match_dup 4)
	(compare:CC (match_dup 0)
		    (const_int 0)))]
  ""
  [(set_attr "type" "logical")
   (set_attr "dot" "yes")
   (set_attr "length" "4,8")])


(define_insn "*boolc<mode>3"
  [(set (match_operand:GPR 0 "gpc_reg_operand" "=r")
	(match_operator:GPR 3 "boolean_operator"
	 [(not:GPR (match_operand:GPR 2 "gpc_reg_operand" "r"))
	  (match_operand:GPR 1 "gpc_reg_operand" "r")]))]
  ""
  "%q3 %0,%1,%2"
  [(set_attr "type" "logical")])

(define_insn_and_split "*boolc<mode>3_dot"
  [(set (match_operand:CC 4 "cc_reg_operand" "=x,?y")
	(compare:CC (match_operator:GPR 3 "boolean_operator"
	 [(not:GPR (match_operand:GPR 2 "gpc_reg_operand" "r,r"))
	  (match_operand:GPR 1 "gpc_reg_operand" "r,r")])
	 (const_int 0)))
   (clobber (match_scratch:GPR 0 "=r,r"))]
  "<MODE>mode == Pmode && rs6000_gen_cell_microcode"
  "@
   %q3. %0,%1,%2
   #"
  "&& reload_completed && cc_reg_not_cr0_operand (operands[4], CCmode)"
  [(set (match_dup 0)
	(match_dup 3))
   (set (match_dup 4)
	(compare:CC (match_dup 0)
		    (const_int 0)))]
  ""
  [(set_attr "type" "logical")
   (set_attr "dot" "yes")
   (set_attr "length" "4,8")])

(define_insn_and_split "*boolc<mode>3_dot2"
  [(set (match_operand:CC 4 "cc_reg_operand" "=x,?y")
	(compare:CC (match_operator:GPR 3 "boolean_operator"
	 [(not:GPR (match_operand:GPR 2 "gpc_reg_operand" "r,r"))
	  (match_operand:GPR 1 "gpc_reg_operand" "r,r")])
	 (const_int 0)))
   (set (match_operand:GPR 0 "gpc_reg_operand" "=r,r")
	(match_dup 3))]
  "<MODE>mode == Pmode && rs6000_gen_cell_microcode"
  "@
   %q3. %0,%1,%2
   #"
  "&& reload_completed && cc_reg_not_cr0_operand (operands[4], CCmode)"
  [(set (match_dup 0)
	(match_dup 3))
   (set (match_dup 4)
	(compare:CC (match_dup 0)
		    (const_int 0)))]
  ""
  [(set_attr "type" "logical")
   (set_attr "dot" "yes")
   (set_attr "length" "4,8")])


(define_insn "*boolcc<mode>3"
  [(set (match_operand:GPR 0 "gpc_reg_operand" "=r")
	(match_operator:GPR 3 "boolean_operator"
	 [(not:GPR (match_operand:GPR 1 "gpc_reg_operand" "r"))
	  (not:GPR (match_operand:GPR 2 "gpc_reg_operand" "r"))]))]
  ""
  "%q3 %0,%1,%2"
  [(set_attr "type" "logical")])

(define_insn_and_split "*boolcc<mode>3_dot"
  [(set (match_operand:CC 4 "cc_reg_operand" "=x,?y")
	(compare:CC (match_operator:GPR 3 "boolean_operator"
	 [(not:GPR (match_operand:GPR 1 "gpc_reg_operand" "r,r"))
	  (not:GPR (match_operand:GPR 2 "gpc_reg_operand" "r,r"))])
	 (const_int 0)))
   (clobber (match_scratch:GPR 0 "=r,r"))]
  "<MODE>mode == Pmode && rs6000_gen_cell_microcode"
  "@
   %q3. %0,%1,%2
   #"
  "&& reload_completed && cc_reg_not_cr0_operand (operands[4], CCmode)"
  [(set (match_dup 0)
	(match_dup 3))
   (set (match_dup 4)
	(compare:CC (match_dup 0)
		    (const_int 0)))]
  ""
  [(set_attr "type" "logical")
   (set_attr "dot" "yes")
   (set_attr "length" "4,8")])

(define_insn_and_split "*boolcc<mode>3_dot2"
  [(set (match_operand:CC 4 "cc_reg_operand" "=x,?y")
	(compare:CC (match_operator:GPR 3 "boolean_operator"
	 [(not:GPR (match_operand:GPR 1 "gpc_reg_operand" "r,r"))
	  (not:GPR (match_operand:GPR 2 "gpc_reg_operand" "r,r"))])
	 (const_int 0)))
   (set (match_operand:GPR 0 "gpc_reg_operand" "=r,r")
	(match_dup 3))]
  "<MODE>mode == Pmode && rs6000_gen_cell_microcode"
  "@
   %q3. %0,%1,%2
   #"
  "&& reload_completed && cc_reg_not_cr0_operand (operands[4], CCmode)"
  [(set (match_dup 0)
	(match_dup 3))
   (set (match_dup 4)
	(compare:CC (match_dup 0)
		    (const_int 0)))]
  ""
  [(set_attr "type" "logical")
   (set_attr "dot" "yes")
   (set_attr "length" "4,8")])


;; TODO: Should have dots of this as well.
(define_insn "*eqv<mode>3"
  [(set (match_operand:GPR 0 "gpc_reg_operand" "=r")
	(not:GPR (xor:GPR (match_operand:GPR 1 "gpc_reg_operand" "r")
			  (match_operand:GPR 2 "gpc_reg_operand" "r"))))]
  ""
  "eqv %0,%1,%2"
  [(set_attr "type" "logical")])

;; Rotate and shift insns, in all their variants.  These support shifts,
;; field inserts and extracts, and various combinations thereof.
(define_expand "insv"
  [(set (zero_extract (match_operand 0 "gpc_reg_operand" "")
		       (match_operand:SI 1 "const_int_operand" "")
		       (match_operand:SI 2 "const_int_operand" ""))
	(match_operand 3 "gpc_reg_operand" ""))]
  ""
  "
{
  /* Do not handle 16/8 bit structures that fit in HI/QI modes directly, since
     the (SUBREG:SI (REG:HI xxx)) that is otherwise generated can confuse the
     compiler if the address of the structure is taken later.  Likewise, do
     not handle invalid E500 subregs.  */
  if (GET_CODE (operands[0]) == SUBREG
      && (GET_MODE_SIZE (GET_MODE (SUBREG_REG (operands[0]))) < UNITS_PER_WORD
	  || ((TARGET_E500_DOUBLE || TARGET_SPE)
	      && invalid_e500_subreg (operands[0], GET_MODE (operands[0])))))
    FAIL;

  if (TARGET_POWERPC64 && GET_MODE (operands[0]) == DImode)
    emit_insn (gen_insvdi_internal (operands[0], operands[1], operands[2],
				    operands[3]));
  else
    emit_insn (gen_insvsi_internal (operands[0], operands[1], operands[2],
				    operands[3]));
  DONE;
}")

(define_insn "insvsi_internal"
  [(set (zero_extract:SI (match_operand:SI 0 "gpc_reg_operand" "+r")
			 (match_operand:SI 1 "const_int_operand" "i")
			 (match_operand:SI 2 "const_int_operand" "i"))
	(match_operand:SI 3 "gpc_reg_operand" "r"))]
  ""
  "*
{
  int start = INTVAL (operands[2]) & 31;
  int size = INTVAL (operands[1]) & 31;

  operands[4] = GEN_INT (32 - start - size);
  operands[1] = GEN_INT (start + size - 1);
  return \"rlwimi %0,%3,%h4,%h2,%h1\";
}"
  [(set_attr "type" "insert")])

(define_insn "*insvsi_internal1"
  [(set (zero_extract:SI (match_operand:SI 0 "gpc_reg_operand" "+r")
			 (match_operand:SI 1 "const_int_operand" "i")
			 (match_operand:SI 2 "const_int_operand" "i"))
	(rotate:SI (match_operand:SI 3 "gpc_reg_operand" "r")
		   (match_operand:SI 4 "const_int_operand" "i")))]
  "(32 - (INTVAL (operands[4]) & 31)) >= INTVAL (operands[1])"
  "*
{
  int shift = INTVAL (operands[4]) & 31;
  int start = INTVAL (operands[2]) & 31;
  int size = INTVAL (operands[1]) & 31;

  operands[4] = GEN_INT (shift - start - size);
  operands[1] = GEN_INT (start + size - 1);
  return \"rlwimi %0,%3,%h4,%h2,%h1\";
}"
  [(set_attr "type" "insert")])

(define_insn "*insvsi_internal2"
  [(set (zero_extract:SI (match_operand:SI 0 "gpc_reg_operand" "+r")
			 (match_operand:SI 1 "const_int_operand" "i")
			 (match_operand:SI 2 "const_int_operand" "i"))
	(ashiftrt:SI (match_operand:SI 3 "gpc_reg_operand" "r")
		     (match_operand:SI 4 "const_int_operand" "i")))]
  "(32 - (INTVAL (operands[4]) & 31)) >= INTVAL (operands[1])"
  "*
{
  int shift = INTVAL (operands[4]) & 31;
  int start = INTVAL (operands[2]) & 31;
  int size = INTVAL (operands[1]) & 31;

  operands[4] = GEN_INT (32 - shift - start - size);
  operands[1] = GEN_INT (start + size - 1);
  return \"rlwimi %0,%3,%h4,%h2,%h1\";
}"
  [(set_attr "type" "insert")])

(define_insn "*insvsi_internal3"
  [(set (zero_extract:SI (match_operand:SI 0 "gpc_reg_operand" "+r")
			 (match_operand:SI 1 "const_int_operand" "i")
			 (match_operand:SI 2 "const_int_operand" "i"))
	(lshiftrt:SI (match_operand:SI 3 "gpc_reg_operand" "r")
		     (match_operand:SI 4 "const_int_operand" "i")))]
  "(32 - (INTVAL (operands[4]) & 31)) >= INTVAL (operands[1])"
  "*
{
  int shift = INTVAL (operands[4]) & 31;
  int start = INTVAL (operands[2]) & 31;
  int size = INTVAL (operands[1]) & 31;

  operands[4] = GEN_INT (32 - shift - start - size);
  operands[1] = GEN_INT (start + size - 1);
  return \"rlwimi %0,%3,%h4,%h2,%h1\";
}"
  [(set_attr "type" "insert")])

(define_insn "*insvsi_internal4"
  [(set (zero_extract:SI (match_operand:SI 0 "gpc_reg_operand" "+r")
			 (match_operand:SI 1 "const_int_operand" "i")
			 (match_operand:SI 2 "const_int_operand" "i"))
	(zero_extract:SI (match_operand:SI 3 "gpc_reg_operand" "r")
			 (match_operand:SI 4 "const_int_operand" "i")
			 (match_operand:SI 5 "const_int_operand" "i")))]
  "INTVAL (operands[4]) >= INTVAL (operands[1])"
  "*
{
  int extract_start = INTVAL (operands[5]) & 31;
  int extract_size = INTVAL (operands[4]) & 31;
  int insert_start = INTVAL (operands[2]) & 31;
  int insert_size = INTVAL (operands[1]) & 31;

/* Align extract field with insert field */
  operands[5] = GEN_INT (extract_start + extract_size - insert_start - insert_size);
  operands[1] = GEN_INT (insert_start + insert_size - 1);
  return \"rlwimi %0,%3,%h5,%h2,%h1\";
}"
  [(set_attr "type" "insert")])

;; combine patterns for rlwimi
(define_insn "*insvsi_internal5"
  [(set (match_operand:SI 0 "gpc_reg_operand" "=r")
        (ior:SI (and:SI (match_operand:SI 4 "gpc_reg_operand" "0")
                        (match_operand:SI 1 "mask_operand" "i"))
                (and:SI (lshiftrt:SI (match_operand:SI 3 "gpc_reg_operand" "r")
                                     (match_operand:SI 2 "const_int_operand" "i"))
                        (match_operand:SI 5 "mask_operand" "i"))))]
  "INTVAL(operands[1]) == ~INTVAL(operands[5])"
  "*
{
 int me = extract_ME(operands[5]);
 int mb = extract_MB(operands[5]);
 operands[4] = GEN_INT(32 - INTVAL(operands[2]));
 operands[2] = GEN_INT(mb);
 operands[1] = GEN_INT(me);
 return \"rlwimi %0,%3,%h4,%h2,%h1\";
}"
  [(set_attr "type" "insert")])

(define_insn "*insvsi_internal6"
  [(set (match_operand:SI 0 "gpc_reg_operand" "=r")
        (ior:SI (and:SI (lshiftrt:SI (match_operand:SI 3 "gpc_reg_operand" "r")
                                     (match_operand:SI 2 "const_int_operand" "i"))
                        (match_operand:SI 5 "mask_operand" "i"))
                (and:SI (match_operand:SI 4 "gpc_reg_operand" "0")
                        (match_operand:SI 1 "mask_operand" "i"))))]
  "INTVAL(operands[1]) == ~INTVAL(operands[5])"
  "*
{
 int me = extract_ME(operands[5]);
 int mb = extract_MB(operands[5]);
 operands[4] = GEN_INT(32 - INTVAL(operands[2]));
 operands[2] = GEN_INT(mb);
 operands[1] = GEN_INT(me);
 return \"rlwimi %0,%3,%h4,%h2,%h1\";
}"
  [(set_attr "type" "insert")])

(define_insn "insvdi_internal"
  [(set (zero_extract:DI (match_operand:DI 0 "gpc_reg_operand" "+r")
			 (match_operand:SI 1 "const_int_operand" "i")
			 (match_operand:SI 2 "const_int_operand" "i"))
	(match_operand:DI 3 "gpc_reg_operand" "r"))]
  "TARGET_POWERPC64"
  "*
{
  int start = INTVAL (operands[2]) & 63;
  int size = INTVAL (operands[1]) & 63;

  operands[1] = GEN_INT (64 - start - size);
  return \"rldimi %0,%3,%H1,%H2\";
}"
  [(set_attr "type" "insert")
   (set_attr "size" "64")])

(define_insn "*insvdi_internal2"
  [(set (zero_extract:DI (match_operand:DI 0 "gpc_reg_operand" "+r")
			 (match_operand:SI 1 "const_int_operand" "i")
			 (match_operand:SI 2 "const_int_operand" "i"))
	(ashiftrt:DI (match_operand:DI 3 "gpc_reg_operand" "r")
		     (match_operand:SI 4 "const_int_operand" "i")))]
  "TARGET_POWERPC64
   && insvdi_rshift_rlwimi_p (operands[1], operands[2], operands[4])"
  "*
{
  int shift = INTVAL (operands[4]) & 63;
  int start = (INTVAL (operands[2]) & 63) - 32;
  int size = INTVAL (operands[1]) & 63;

  operands[4] = GEN_INT (64 - shift - start - size);
  operands[2] = GEN_INT (start);
  operands[1] = GEN_INT (start + size - 1);
  return \"rlwimi %0,%3,%h4,%h2,%h1\";
}")

(define_insn "*insvdi_internal3"
  [(set (zero_extract:DI (match_operand:DI 0 "gpc_reg_operand" "+r")
			 (match_operand:SI 1 "const_int_operand" "i")
			 (match_operand:SI 2 "const_int_operand" "i"))
	(lshiftrt:DI (match_operand:DI 3 "gpc_reg_operand" "r")
		     (match_operand:SI 4 "const_int_operand" "i")))]
  "TARGET_POWERPC64
   && insvdi_rshift_rlwimi_p (operands[1], operands[2], operands[4])"
  "*
{
  int shift = INTVAL (operands[4]) & 63;
  int start = (INTVAL (operands[2]) & 63) - 32;
  int size = INTVAL (operands[1]) & 63;

  operands[4] = GEN_INT (64 - shift - start - size);
  operands[2] = GEN_INT (start);
  operands[1] = GEN_INT (start + size - 1);
  return \"rlwimi %0,%3,%h4,%h2,%h1\";
}")

(define_expand "extzv"
  [(set (match_operand 0 "gpc_reg_operand" "")
	(zero_extract (match_operand 1 "gpc_reg_operand" "")
		       (match_operand:SI 2 "const_int_operand" "")
		       (match_operand:SI 3 "const_int_operand" "")))]
  ""
  "
{
  /* Do not handle 16/8 bit structures that fit in HI/QI modes directly, since
     the (SUBREG:SI (REG:HI xxx)) that is otherwise generated can confuse the
     compiler if the address of the structure is taken later.  */
  if (GET_CODE (operands[0]) == SUBREG
      && (GET_MODE_SIZE (GET_MODE (SUBREG_REG (operands[0]))) < UNITS_PER_WORD))
    FAIL;

  if (TARGET_POWERPC64 && GET_MODE (operands[1]) == DImode)
    emit_insn (gen_extzvdi_internal (operands[0], operands[1], operands[2],
				     operands[3]));
  else
    emit_insn (gen_extzvsi_internal (operands[0], operands[1], operands[2],
				     operands[3]));
  DONE;
}")

(define_insn "extzvsi_internal"
  [(set (match_operand:SI 0 "gpc_reg_operand" "=r")
	(zero_extract:SI (match_operand:SI 1 "gpc_reg_operand" "r")
			 (match_operand:SI 2 "const_int_operand" "i")
			 (match_operand:SI 3 "const_int_operand" "i")))]
  ""
  "*
{
  int start = INTVAL (operands[3]) & 31;
  int size = INTVAL (operands[2]) & 31;

  if (start + size >= 32)
    operands[3] = const0_rtx;
  else
    operands[3] = GEN_INT (start + size);
  return \"rlwinm %0,%1,%3,%s2,31\";
}"
  [(set_attr "type" "shift")])

(define_insn "*extzvsi_internal1"
  [(set (match_operand:CC 0 "cc_reg_operand" "=x,?y")
	(compare:CC (zero_extract:SI (match_operand:SI 1 "gpc_reg_operand" "r,r")
			 (match_operand:SI 2 "const_int_operand" "i,i")
			 (match_operand:SI 3 "const_int_operand" "i,i"))
		    (const_int 0)))
   (clobber (match_scratch:SI 4 "=r,r"))]
  ""
  "*
{
  int start = INTVAL (operands[3]) & 31;
  int size = INTVAL (operands[2]) & 31;

  /* Force split for non-cc0 compare.  */
  if (which_alternative == 1)
     return \"#\";

  /* If the bit-field being tested fits in the upper or lower half of a
     word, it is possible to use andiu. or andil. to test it.  This is
     useful because the condition register set-use delay is smaller for
     andi[ul]. than for rlinm.  This doesn't work when the starting bit
     position is 0 because the LT and GT bits may be set wrong.  */

  if ((start > 0 && start + size <= 16) || start >= 16)
    {
      operands[3] = GEN_INT (((1 << (16 - (start & 15)))
			      - (1 << (16 - (start & 15) - size))));
      if (start < 16)
	return \"andis. %4,%1,%3\";
      else
	return \"andi. %4,%1,%3\";
    }

  if (start + size >= 32)
    operands[3] = const0_rtx;
  else
    operands[3] = GEN_INT (start + size);
  return \"rlwinm. %4,%1,%3,%s2,31\";
}"
  [(set_attr "type" "shift")
   (set_attr "dot" "yes")
   (set_attr "length" "4,8")])

(define_split
  [(set (match_operand:CC 0 "cc_reg_not_micro_cr0_operand" "")
	(compare:CC (zero_extract:SI (match_operand:SI 1 "gpc_reg_operand" "")
			 (match_operand:SI 2 "const_int_operand" "")
			 (match_operand:SI 3 "const_int_operand" ""))
		    (const_int 0)))
   (clobber (match_scratch:SI 4 ""))]
  "reload_completed"
  [(set (match_dup 4)
	(zero_extract:SI (match_dup 1) (match_dup 2)
			 (match_dup 3)))
   (set (match_dup 0)
	(compare:CC (match_dup 4)
		    (const_int 0)))]
  "")

(define_insn "*extzvsi_internal2"
  [(set (match_operand:CC 4 "cc_reg_operand" "=x,?y")
	(compare:CC (zero_extract:SI (match_operand:SI 1 "gpc_reg_operand" "r,r")
			 (match_operand:SI 2 "const_int_operand" "i,i")
			 (match_operand:SI 3 "const_int_operand" "i,i"))
		    (const_int 0)))
   (set (match_operand:SI 0 "gpc_reg_operand" "=r,r")
	(zero_extract:SI (match_dup 1) (match_dup 2) (match_dup 3)))]
  ""
  "*
{
  int start = INTVAL (operands[3]) & 31;
  int size = INTVAL (operands[2]) & 31;

  /* Force split for non-cc0 compare.  */
  if (which_alternative == 1)
     return \"#\";

  /* Since we are using the output value, we can't ignore any need for
     a shift.  The bit-field must end at the LSB.  */
  if (start >= 16 && start + size == 32)
    {
      operands[3] = GEN_INT ((1 << size) - 1);
      return \"andi. %0,%1,%3\";
    }

  if (start + size >= 32)
    operands[3] = const0_rtx;
  else
    operands[3] = GEN_INT (start + size);
  return \"rlwinm. %0,%1,%3,%s2,31\";
}"
  [(set_attr "type" "shift")
   (set_attr "dot" "yes")
   (set_attr "length" "4,8")])

(define_split
  [(set (match_operand:CC 4 "cc_reg_not_micro_cr0_operand" "")
	(compare:CC (zero_extract:SI (match_operand:SI 1 "gpc_reg_operand" "")
			 (match_operand:SI 2 "const_int_operand" "")
			 (match_operand:SI 3 "const_int_operand" ""))
		    (const_int 0)))
   (set (match_operand:SI 0 "gpc_reg_operand" "")
	(zero_extract:SI (match_dup 1) (match_dup 2) (match_dup 3)))]
  "reload_completed"
  [(set (match_dup 0)
	(zero_extract:SI (match_dup 1) (match_dup 2) (match_dup 3)))
   (set (match_dup 4)
	(compare:CC (match_dup 0)
		    (const_int 0)))]
  "")

(define_insn "extzvdi_internal"
  [(set (match_operand:DI 0 "gpc_reg_operand" "=r")
	(zero_extract:DI (match_operand:DI 1 "gpc_reg_operand" "r")
			 (match_operand:SI 2 "const_int_operand" "i")
			 (match_operand:SI 3 "const_int_operand" "i")))]
  "TARGET_POWERPC64"
  "*
{
  int start = INTVAL (operands[3]) & 63;
  int size = INTVAL (operands[2]) & 63;

  if (start + size >= 64)
    operands[3] = const0_rtx;
  else
    operands[3] = GEN_INT (start + size);
  operands[2] = GEN_INT (64 - size);
  return \"rldicl %0,%1,%3,%2\";
}"
  [(set_attr "type" "shift")])

(define_insn "*extzvdi_internal1"
  [(set (match_operand:CC 0 "gpc_reg_operand" "=x")
	(compare:CC (zero_extract:DI (match_operand:DI 1 "gpc_reg_operand" "r")
			 (match_operand:SI 2 "const_int_operand" "i")
			 (match_operand:SI 3 "const_int_operand" "i"))
		    (const_int 0)))
   (clobber (match_scratch:DI 4 "=r"))]
  "TARGET_64BIT && rs6000_gen_cell_microcode"
  "*
{
  int start = INTVAL (operands[3]) & 63;
  int size = INTVAL (operands[2]) & 63;

  if (start + size >= 64)
    operands[3] = const0_rtx;
  else
    operands[3] = GEN_INT (start + size);
  operands[2] = GEN_INT (64 - size);
  return \"rldicl. %4,%1,%3,%2\";
}"
  [(set_attr "type" "shift")
   (set_attr "dot" "yes")])

(define_insn "*extzvdi_internal2"
  [(set (match_operand:CC 4 "gpc_reg_operand" "=x")
	(compare:CC (zero_extract:DI (match_operand:DI 1 "gpc_reg_operand" "r")
			 (match_operand:SI 2 "const_int_operand" "i")
			 (match_operand:SI 3 "const_int_operand" "i"))
		    (const_int 0)))
   (set (match_operand:DI 0 "gpc_reg_operand" "=r")
	(zero_extract:DI (match_dup 1) (match_dup 2) (match_dup 3)))]
  "TARGET_64BIT && rs6000_gen_cell_microcode"
  "*
{
  int start = INTVAL (operands[3]) & 63;
  int size = INTVAL (operands[2]) & 63;

  if (start + size >= 64)
    operands[3] = const0_rtx;
  else
    operands[3] = GEN_INT (start + size);
  operands[2] = GEN_INT (64 - size);
  return \"rldicl. %0,%1,%3,%2\";
}"
  [(set_attr "type" "shift")
   (set_attr "dot" "yes")])


(define_insn "rotl<mode>3"
  [(set (match_operand:GPR 0 "gpc_reg_operand" "=r")
	(rotate:GPR (match_operand:GPR 1 "gpc_reg_operand" "r")
		    (match_operand:SI 2 "reg_or_cint_operand" "rn")))]
  ""
  "rotl<wd>%I2 %0,%1,%<hH>2"
  [(set_attr "type" "shift")
   (set_attr "maybe_var_shift" "yes")])

(define_insn "*rotlsi3_64"
  [(set (match_operand:DI 0 "gpc_reg_operand" "=r")
	(zero_extend:DI
	    (rotate:SI (match_operand:SI 1 "gpc_reg_operand" "r")
		       (match_operand:SI 2 "reg_or_cint_operand" "rn"))))]
  "TARGET_POWERPC64"
  "rotlw%I2 %0,%1,%h2"
  [(set_attr "type" "shift")
   (set_attr "maybe_var_shift" "yes")])

(define_insn_and_split "*rotl<mode>3_dot"
  [(set (match_operand:CC 3 "cc_reg_operand" "=x,?y")
	(compare:CC (rotate:GPR (match_operand:GPR 1 "gpc_reg_operand" "r,r")
				(match_operand:SI 2 "reg_or_cint_operand" "rn,rn"))
		    (const_int 0)))
   (clobber (match_scratch:GPR 0 "=r,r"))]
  "<MODE>mode == Pmode && rs6000_gen_cell_microcode"
  "@
   rotl<wd>%I2. %0,%1,%<hH>2
   #"
  "&& reload_completed && cc_reg_not_cr0_operand (operands[3], CCmode)"
  [(set (match_dup 0)
	(rotate:GPR (match_dup 1)
		    (match_dup 2)))
   (set (match_dup 3)
	(compare:CC (match_dup 0)
		    (const_int 0)))]
  ""
  [(set_attr "type" "shift")
   (set_attr "maybe_var_shift" "yes")
   (set_attr "dot" "yes")
   (set_attr "length" "4,8")])

(define_insn_and_split "*rotl<mode>3_dot2"
  [(set (match_operand:CC 3 "cc_reg_operand" "=x,?y")
	(compare:CC (rotate:GPR (match_operand:GPR 1 "gpc_reg_operand" "r,r")
				(match_operand:SI 2 "reg_or_cint_operand" "rn,rn"))
		    (const_int 0)))
   (set (match_operand:GPR 0 "gpc_reg_operand" "=r,r")
	(rotate:GPR (match_dup 1)
		    (match_dup 2)))]
  "<MODE>mode == Pmode && rs6000_gen_cell_microcode"
  "@
   rotl<wd>%I2. %0,%1,%<hH>2
   #"
  "&& reload_completed && cc_reg_not_cr0_operand (operands[3], CCmode)"
  [(set (match_dup 0)
	(rotate:GPR (match_dup 1)
		    (match_dup 2)))
   (set (match_dup 3)
	(compare:CC (match_dup 0)
		    (const_int 0)))]
  ""
  [(set_attr "type" "shift")
   (set_attr "maybe_var_shift" "yes")
   (set_attr "dot" "yes")
   (set_attr "length" "4,8")])


(define_insn "*rotlsi3_internal4"
  [(set (match_operand:SI 0 "gpc_reg_operand" "=r")
	(and:SI (rotate:SI (match_operand:SI 1 "gpc_reg_operand" "r")
			   (match_operand:SI 2 "reg_or_cint_operand" "rn"))
		(match_operand:SI 3 "mask_operand" "n")))]
  ""
  "rlw%I2nm %0,%1,%h2,%m3,%M3"
  [(set_attr "type" "shift")
   (set_attr "maybe_var_shift" "yes")])

(define_insn "*rotlsi3_internal5"
  [(set (match_operand:CC 0 "cc_reg_operand" "=x,?y")
	(compare:CC (and:SI
		     (rotate:SI (match_operand:SI 1 "gpc_reg_operand" "r,r")
				(match_operand:SI 2 "reg_or_cint_operand" "rn,rn"))
		     (match_operand:SI 3 "mask_operand" "n,n"))
		    (const_int 0)))
   (clobber (match_scratch:SI 4 "=r,r"))]
  ""
  "@
   rlw%I2nm. %4,%1,%h2,%m3,%M3
   #"
  [(set_attr "type" "shift")
   (set_attr "maybe_var_shift" "yes")
   (set_attr "dot" "yes")
   (set_attr "length" "4,8")])

(define_split
  [(set (match_operand:CC 0 "cc_reg_not_micro_cr0_operand" "")
	(compare:CC (and:SI
		     (rotate:SI (match_operand:SI 1 "gpc_reg_operand" "")
				(match_operand:SI 2 "reg_or_cint_operand" ""))
		     (match_operand:SI 3 "mask_operand" ""))
		    (const_int 0)))
   (clobber (match_scratch:SI 4 ""))]
  "reload_completed"
  [(set (match_dup 4)
	(and:SI (rotate:SI (match_dup 1)
				(match_dup 2))
		     (match_dup 3)))
   (set (match_dup 0)
	(compare:CC (match_dup 4)
		    (const_int 0)))]
  "")

(define_insn "*rotlsi3_internal6"
  [(set (match_operand:CC 4 "cc_reg_operand" "=x,?y")
	(compare:CC (and:SI
		     (rotate:SI (match_operand:SI 1 "gpc_reg_operand" "r,r")
				(match_operand:SI 2 "reg_or_cint_operand" "rn,rn"))
		     (match_operand:SI 3 "mask_operand" "n,n"))
		    (const_int 0)))
   (set (match_operand:SI 0 "gpc_reg_operand" "=r,r")
	(and:SI (rotate:SI (match_dup 1) (match_dup 2)) (match_dup 3)))]
  ""
  "@
   rlw%I2nm. %0,%1,%h2,%m3,%M3
   #"
  [(set_attr "type" "shift")
   (set_attr "maybe_var_shift" "yes")
   (set_attr "dot" "yes")
   (set_attr "length" "4,8")])

(define_split
  [(set (match_operand:CC 4 "cc_reg_not_micro_cr0_operand" "")
	(compare:CC (and:SI
		     (rotate:SI (match_operand:SI 1 "gpc_reg_operand" "")
				(match_operand:SI 2 "reg_or_cint_operand" ""))
		     (match_operand:SI 3 "mask_operand" ""))
		    (const_int 0)))
   (set (match_operand:SI 0 "gpc_reg_operand" "")
	(and:SI (rotate:SI (match_dup 1) (match_dup 2)) (match_dup 3)))]
  "reload_completed"
  [(set (match_dup 0)
	(and:SI (rotate:SI (match_dup 1) (match_dup 2)) (match_dup 3)))
   (set (match_dup 4)
	(compare:CC (match_dup 0)
		    (const_int 0)))]
  "")

(define_insn "*rotlsi3_internal7le"
  [(set (match_operand:SI 0 "gpc_reg_operand" "=r")
	(zero_extend:SI
	 (subreg:QI
	  (rotate:SI (match_operand:SI 1 "gpc_reg_operand" "r")
		     (match_operand:SI 2 "reg_or_cint_operand" "ri")) 0)))]
  "!BYTES_BIG_ENDIAN"
  "rlw%I2nm %0,%1,%h2,0xff"
  [(set (attr "cell_micro")
     (if_then_else (match_operand:SI 2 "const_int_operand" "")
	(const_string "not")
	(const_string "always")))
   (set_attr "type" "shift")])

(define_insn "*rotlsi3_internal7be"
  [(set (match_operand:SI 0 "gpc_reg_operand" "=r")
	(zero_extend:SI
	 (subreg:QI
	  (rotate:SI (match_operand:SI 1 "gpc_reg_operand" "r")
		     (match_operand:SI 2 "reg_or_cint_operand" "ri")) 3)))]
  "BYTES_BIG_ENDIAN"
  "rlw%I2nm %0,%1,%h2,0xff"
  [(set (attr "cell_micro")
     (if_then_else (match_operand:SI 2 "const_int_operand" "")
	(const_string "not")
	(const_string "always")))
   (set_attr "type" "shift")])

(define_insn "*rotlsi3_internal8le"
  [(set (match_operand:CC 0 "cc_reg_operand" "=x,?y")
	(compare:CC (zero_extend:SI
		     (subreg:QI
		      (rotate:SI (match_operand:SI 1 "gpc_reg_operand" "r,r")
				 (match_operand:SI 2 "reg_or_cint_operand" "rn,rn")) 0))
		    (const_int 0)))
   (clobber (match_scratch:SI 3 "=r,r"))]
  "!BYTES_BIG_ENDIAN"
  "@
   rlw%I2nm. %3,%1,%h2,0xff
   #"
  [(set_attr "type" "shift")
   (set_attr "maybe_var_shift" "yes")
   (set_attr "dot" "yes")
   (set_attr "length" "4,8")])

(define_insn "*rotlsi3_internal8be"
  [(set (match_operand:CC 0 "cc_reg_operand" "=x,?y")
	(compare:CC (zero_extend:SI
		     (subreg:QI
		      (rotate:SI (match_operand:SI 1 "gpc_reg_operand" "r,r")
				 (match_operand:SI 2 "reg_or_cint_operand" "rn,rn")) 3))
		    (const_int 0)))
   (clobber (match_scratch:SI 3 "=r,r"))]
  "BYTES_BIG_ENDIAN"
  "@
   rlw%I2nm. %3,%1,%h2,0xff
   #"
  [(set_attr "type" "shift")
   (set_attr "maybe_var_shift" "yes")
   (set_attr "dot" "yes")
   (set_attr "length" "4,8")])

(define_split
  [(set (match_operand:CC 0 "cc_reg_not_micro_cr0_operand" "")
	(compare:CC (zero_extend:SI
		     (subreg:QI
		      (rotate:SI (match_operand:SI 1 "gpc_reg_operand" "")
				 (match_operand:SI 2 "reg_or_cint_operand" "")) 0))
		    (const_int 0)))
   (clobber (match_scratch:SI 3 ""))]
  "!BYTES_BIG_ENDIAN && reload_completed"
  [(set (match_dup 3)
	(zero_extend:SI (subreg:QI
		      (rotate:SI (match_dup 1)
				 (match_dup 2)) 0)))
   (set (match_dup 0)
	(compare:CC (match_dup 3)
		    (const_int 0)))]
  "")

(define_split
  [(set (match_operand:CC 0 "cc_reg_not_micro_cr0_operand" "")
	(compare:CC (zero_extend:SI
		     (subreg:QI
		      (rotate:SI (match_operand:SI 1 "gpc_reg_operand" "")
				 (match_operand:SI 2 "reg_or_cint_operand" "")) 3))
		    (const_int 0)))
   (clobber (match_scratch:SI 3 ""))]
  "BYTES_BIG_ENDIAN && reload_completed"
  [(set (match_dup 3)
	(zero_extend:SI (subreg:QI
		      (rotate:SI (match_dup 1)
				 (match_dup 2)) 3)))
   (set (match_dup 0)
	(compare:CC (match_dup 3)
		    (const_int 0)))]
  "")

(define_insn "*rotlsi3_internal9le"
  [(set (match_operand:CC 3 "cc_reg_operand" "=x,?y")
	(compare:CC (zero_extend:SI
		     (subreg:QI
		      (rotate:SI (match_operand:SI 1 "gpc_reg_operand" "r,r")
				 (match_operand:SI 2 "reg_or_cint_operand" "rn,rn")) 0))
		    (const_int 0)))
   (set (match_operand:SI 0 "gpc_reg_operand" "=r,r")
	(zero_extend:SI (subreg:QI (rotate:SI (match_dup 1) (match_dup 2)) 0)))]
  "!BYTES_BIG_ENDIAN"
  "@
   rlw%I2nm. %0,%1,%h2,0xff
   #"
  [(set_attr "type" "shift")
   (set_attr "maybe_var_shift" "yes")
   (set_attr "dot" "yes")
   (set_attr "length" "4,8")])

(define_insn "*rotlsi3_internal9be"
  [(set (match_operand:CC 3 "cc_reg_operand" "=x,?y")
	(compare:CC (zero_extend:SI
		     (subreg:QI
		      (rotate:SI (match_operand:SI 1 "gpc_reg_operand" "r,r")
				 (match_operand:SI 2 "reg_or_cint_operand" "rn,rn")) 3))
		    (const_int 0)))
   (set (match_operand:SI 0 "gpc_reg_operand" "=r,r")
	(zero_extend:SI (subreg:QI (rotate:SI (match_dup 1) (match_dup 2)) 3)))]
  "BYTES_BIG_ENDIAN"
  "@
   rlw%I2nm. %0,%1,%h2,0xff
   #"
  [(set_attr "type" "shift")
   (set_attr "maybe_var_shift" "yes")
   (set_attr "dot" "yes")
   (set_attr "length" "4,8")])

(define_split
  [(set (match_operand:CC 3 "cc_reg_not_micro_cr0_operand" "")
	(compare:CC (zero_extend:SI
		     (subreg:QI
		      (rotate:SI (match_operand:SI 1 "gpc_reg_operand" "")
				 (match_operand:SI 2 "reg_or_cint_operand" "")) 0))
		    (const_int 0)))
   (set (match_operand:SI 0 "gpc_reg_operand" "")
	(zero_extend:SI (subreg:QI (rotate:SI (match_dup 1) (match_dup 2)) 0)))]
  "!BYTES_BIG_ENDIAN && reload_completed"
  [(set (match_dup 0)
	(zero_extend:SI (subreg:QI (rotate:SI (match_dup 1) (match_dup 2)) 0)))
   (set (match_dup 3)
	(compare:CC (match_dup 0)
		    (const_int 0)))]
  "")

(define_split
  [(set (match_operand:CC 3 "cc_reg_not_micro_cr0_operand" "")
	(compare:CC (zero_extend:SI
		     (subreg:QI
		      (rotate:SI (match_operand:SI 1 "gpc_reg_operand" "")
				 (match_operand:SI 2 "reg_or_cint_operand" "")) 3))
		    (const_int 0)))
   (set (match_operand:SI 0 "gpc_reg_operand" "")
	(zero_extend:SI (subreg:QI (rotate:SI (match_dup 1) (match_dup 2)) 3)))]
  "BYTES_BIG_ENDIAN && reload_completed"
  [(set (match_dup 0)
	(zero_extend:SI (subreg:QI (rotate:SI (match_dup 1) (match_dup 2)) 3)))
   (set (match_dup 3)
	(compare:CC (match_dup 0)
		    (const_int 0)))]
  "")

(define_insn "*rotlsi3_internal10le"
  [(set (match_operand:SI 0 "gpc_reg_operand" "=r")
	(zero_extend:SI
	 (subreg:HI
	  (rotate:SI (match_operand:SI 1 "gpc_reg_operand" "r")
		     (match_operand:SI 2 "reg_or_cint_operand" "rn")) 0)))]
  "!BYTES_BIG_ENDIAN"
  "rlw%I2nm %0,%1,%h2,0xffff"
  [(set_attr "type" "shift")
   (set_attr "maybe_var_shift" "yes")])

(define_insn "*rotlsi3_internal10be"
  [(set (match_operand:SI 0 "gpc_reg_operand" "=r")
	(zero_extend:SI
	 (subreg:HI
	  (rotate:SI (match_operand:SI 1 "gpc_reg_operand" "r")
		     (match_operand:SI 2 "reg_or_cint_operand" "rn")) 2)))]
  "BYTES_BIG_ENDIAN"
  "rlw%I2nm %0,%1,%h2,0xffff"
  [(set_attr "type" "shift")
   (set_attr "maybe_var_shift" "yes")])

(define_insn "*rotlsi3_internal11le"
  [(set (match_operand:CC 0 "cc_reg_operand" "=x,?y")
	(compare:CC (zero_extend:SI
		     (subreg:HI
		      (rotate:SI (match_operand:SI 1 "gpc_reg_operand" "r,r")
				 (match_operand:SI 2 "reg_or_cint_operand" "rn,rn")) 0))
		    (const_int 0)))
   (clobber (match_scratch:SI 3 "=r,r"))]
  "!BYTES_BIG_ENDIAN"
  "@
   rlw%I2nm. %3,%1,%h2,0xffff
   #"
  [(set_attr "type" "shift")
   (set_attr "maybe_var_shift" "yes")
   (set_attr "dot" "yes")
   (set_attr "length" "4,8")])

(define_insn "*rotlsi3_internal11be"
  [(set (match_operand:CC 0 "cc_reg_operand" "=x,?y")
	(compare:CC (zero_extend:SI
		     (subreg:HI
		      (rotate:SI (match_operand:SI 1 "gpc_reg_operand" "r,r")
				 (match_operand:SI 2 "reg_or_cint_operand" "rn,rn")) 2))
		    (const_int 0)))
   (clobber (match_scratch:SI 3 "=r,r"))]
  "BYTES_BIG_ENDIAN"
  "@
   rlw%I2nm. %3,%1,%h2,0xffff
   #"
  [(set_attr "type" "shift")
   (set_attr "maybe_var_shift" "yes")
   (set_attr "dot" "yes")
   (set_attr "length" "4,8")])

(define_split
  [(set (match_operand:CC 0 "cc_reg_not_micro_cr0_operand" "")
	(compare:CC (zero_extend:SI
		     (subreg:HI
		      (rotate:SI (match_operand:SI 1 "gpc_reg_operand" "")
				 (match_operand:SI 2 "reg_or_cint_operand" "")) 0))
		    (const_int 0)))
   (clobber (match_scratch:SI 3 ""))]
  "!BYTES_BIG_ENDIAN && reload_completed"
  [(set (match_dup 3)
	(zero_extend:SI (subreg:HI
		      (rotate:SI (match_dup 1)
				 (match_dup 2)) 0)))
   (set (match_dup 0)
	(compare:CC (match_dup 3)
		    (const_int 0)))]
  "")

(define_split
  [(set (match_operand:CC 0 "cc_reg_not_micro_cr0_operand" "")
	(compare:CC (zero_extend:SI
		     (subreg:HI
		      (rotate:SI (match_operand:SI 1 "gpc_reg_operand" "")
				 (match_operand:SI 2 "reg_or_cint_operand" "")) 2))
		    (const_int 0)))
   (clobber (match_scratch:SI 3 ""))]
  "BYTES_BIG_ENDIAN && reload_completed"
  [(set (match_dup 3)
	(zero_extend:SI (subreg:HI
		      (rotate:SI (match_dup 1)
				 (match_dup 2)) 2)))
   (set (match_dup 0)
	(compare:CC (match_dup 3)
		    (const_int 0)))]
  "")

(define_insn "*rotlsi3_internal12le"
  [(set (match_operand:CC 3 "cc_reg_operand" "=x,?y")
	(compare:CC (zero_extend:SI
		     (subreg:HI
		      (rotate:SI (match_operand:SI 1 "gpc_reg_operand" "r,r")
				 (match_operand:SI 2 "reg_or_cint_operand" "rn,rn")) 0))
		    (const_int 0)))
   (set (match_operand:SI 0 "gpc_reg_operand" "=r,r")
	(zero_extend:SI (subreg:HI (rotate:SI (match_dup 1) (match_dup 2)) 0)))]
  "!BYTES_BIG_ENDIAN"
  "@
   rlw%I2nm. %0,%1,%h2,0xffff
   #"
  [(set_attr "type" "shift")
   (set_attr "maybe_var_shift" "yes")
   (set_attr "dot" "yes")
   (set_attr "length" "4,8")])

(define_insn "*rotlsi3_internal12be"
  [(set (match_operand:CC 3 "cc_reg_operand" "=x,?y")
	(compare:CC (zero_extend:SI
		     (subreg:HI
		      (rotate:SI (match_operand:SI 1 "gpc_reg_operand" "r,r")
				 (match_operand:SI 2 "reg_or_cint_operand" "rn,rn")) 2))
		    (const_int 0)))
   (set (match_operand:SI 0 "gpc_reg_operand" "=r,r")
	(zero_extend:SI (subreg:HI (rotate:SI (match_dup 1) (match_dup 2)) 2)))]
  "BYTES_BIG_ENDIAN"
  "@
   rlw%I2nm. %0,%1,%h2,0xffff
   #"
  [(set_attr "type" "shift")
   (set_attr "maybe_var_shift" "yes")
   (set_attr "dot" "yes")
   (set_attr "length" "4,8")])

(define_split
  [(set (match_operand:CC 3 "cc_reg_not_micro_cr0_operand" "")
	(compare:CC (zero_extend:SI
		     (subreg:HI
		      (rotate:SI (match_operand:SI 1 "gpc_reg_operand" "")
				 (match_operand:SI 2 "reg_or_cint_operand" "")) 0))
		    (const_int 0)))
   (set (match_operand:SI 0 "gpc_reg_operand" "")
	(zero_extend:SI (subreg:HI (rotate:SI (match_dup 1) (match_dup 2)) 0)))]
  "!BYTES_BIG_ENDIAN && reload_completed"
  [(set (match_dup 0)
	(zero_extend:SI (subreg:HI (rotate:SI (match_dup 1) (match_dup 2)) 0)))
   (set (match_dup 3)
	(compare:CC (match_dup 0)
		    (const_int 0)))]
  "")

(define_split
  [(set (match_operand:CC 3 "cc_reg_not_micro_cr0_operand" "")
	(compare:CC (zero_extend:SI
		     (subreg:HI
		      (rotate:SI (match_operand:SI 1 "gpc_reg_operand" "")
				 (match_operand:SI 2 "reg_or_cint_operand" "")) 2))
		    (const_int 0)))
   (set (match_operand:SI 0 "gpc_reg_operand" "")
	(zero_extend:SI (subreg:HI (rotate:SI (match_dup 1) (match_dup 2)) 2)))]
  "BYTES_BIG_ENDIAN && reload_completed"
  [(set (match_dup 0)
	(zero_extend:SI (subreg:HI (rotate:SI (match_dup 1) (match_dup 2)) 2)))
   (set (match_dup 3)
	(compare:CC (match_dup 0)
		    (const_int 0)))]
  "")


(define_insn "ashl<mode>3"
  [(set (match_operand:GPR 0 "gpc_reg_operand" "=r")
	(ashift:GPR (match_operand:GPR 1 "gpc_reg_operand" "r")
		    (match_operand:SI 2 "reg_or_cint_operand" "rn")))]
  ""
  "sl<wd>%I2 %0,%1,%<hH>2"
  [(set_attr "type" "shift")
   (set_attr "maybe_var_shift" "yes")])

(define_insn "*ashlsi3_64"
  [(set (match_operand:DI 0 "gpc_reg_operand" "=r")
	(zero_extend:DI
	    (ashift:SI (match_operand:SI 1 "gpc_reg_operand" "r")
		       (match_operand:SI 2 "reg_or_cint_operand" "rn"))))]
  "TARGET_POWERPC64"
  "slw%I2 %0,%1,%h2"
  [(set_attr "type" "shift")
   (set_attr "maybe_var_shift" "yes")])

(define_insn_and_split "*ashl<mode>3_dot"
  [(set (match_operand:CC 3 "cc_reg_operand" "=x,?y")
	(compare:CC (ashift:GPR (match_operand:GPR 1 "gpc_reg_operand" "r,r")
				(match_operand:SI 2 "reg_or_cint_operand" "rn,rn"))
		    (const_int 0)))
   (clobber (match_scratch:GPR 0 "=r,r"))]
  "<MODE>mode == Pmode && rs6000_gen_cell_microcode"
  "@
   sl<wd>%I2. %0,%1,%<hH>2
   #"
  "&& reload_completed && cc_reg_not_cr0_operand (operands[3], CCmode)"
  [(set (match_dup 0)
	(ashift:GPR (match_dup 1)
		    (match_dup 2)))
   (set (match_dup 3)
	(compare:CC (match_dup 0)
		    (const_int 0)))]
  ""
  [(set_attr "type" "shift")
   (set_attr "maybe_var_shift" "yes")
   (set_attr "dot" "yes")
   (set_attr "length" "4,8")])

(define_insn_and_split "*ashl<mode>3_dot2"
  [(set (match_operand:CC 3 "cc_reg_operand" "=x,?y")
	(compare:CC (ashift:GPR (match_operand:GPR 1 "gpc_reg_operand" "r,r")
				(match_operand:SI 2 "reg_or_cint_operand" "rn,rn"))
		    (const_int 0)))
   (set (match_operand:GPR 0 "gpc_reg_operand" "=r,r")
	(ashift:GPR (match_dup 1)
		    (match_dup 2)))]
  "<MODE>mode == Pmode && rs6000_gen_cell_microcode"
  "@
   sl<wd>%I2. %0,%1,%<hH>2
   #"
  "&& reload_completed && cc_reg_not_cr0_operand (operands[3], CCmode)"
  [(set (match_dup 0)
	(ashift:GPR (match_dup 1)
		    (match_dup 2)))
   (set (match_dup 3)
	(compare:CC (match_dup 0)
		    (const_int 0)))]
  ""
  [(set_attr "type" "shift")
   (set_attr "maybe_var_shift" "yes")
   (set_attr "dot" "yes")
   (set_attr "length" "4,8")])


(define_insn "rlwinm"
  [(set (match_operand:SI 0 "gpc_reg_operand" "=r")
	(and:SI (ashift:SI (match_operand:SI 1 "gpc_reg_operand" "r")
			   (match_operand:SI 2 "const_int_operand" "i"))
		(match_operand:SI 3 "mask_operand" "n")))]
  "includes_lshift_p (operands[2], operands[3])"
  "rlwinm %0,%1,%h2,%m3,%M3"
  [(set_attr "type" "shift")])

(define_insn ""
  [(set (match_operand:CC 0 "cc_reg_operand" "=x,?y")
	(compare:CC
	 (and:SI (ashift:SI (match_operand:SI 1 "gpc_reg_operand" "r,r")
			    (match_operand:SI 2 "const_int_operand" "i,i"))
		 (match_operand:SI 3 "mask_operand" "n,n"))
	 (const_int 0)))
   (clobber (match_scratch:SI 4 "=r,r"))]
  "includes_lshift_p (operands[2], operands[3])"
  "@
   rlwinm. %4,%1,%h2,%m3,%M3
   #"
  [(set_attr "type" "shift")
   (set_attr "dot" "yes")
   (set_attr "length" "4,8")])

(define_split
  [(set (match_operand:CC 0 "cc_reg_not_micro_cr0_operand" "")
	(compare:CC
	 (and:SI (ashift:SI (match_operand:SI 1 "gpc_reg_operand" "")
			    (match_operand:SI 2 "const_int_operand" ""))
		 (match_operand:SI 3 "mask_operand" ""))
	 (const_int 0)))
   (clobber (match_scratch:SI 4 ""))]
  "includes_lshift_p (operands[2], operands[3]) && reload_completed"
  [(set (match_dup 4)
	(and:SI (ashift:SI (match_dup 1) (match_dup 2))
		 (match_dup 3)))
   (set (match_dup 0)
	(compare:CC (match_dup 4)
		    (const_int 0)))]
  "")

(define_insn ""
  [(set (match_operand:CC 4 "cc_reg_operand" "=x,?y")
	(compare:CC
	 (and:SI (ashift:SI (match_operand:SI 1 "gpc_reg_operand" "r,r")
			    (match_operand:SI 2 "const_int_operand" "i,i"))
		 (match_operand:SI 3 "mask_operand" "n,n"))
	 (const_int 0)))
   (set (match_operand:SI 0 "gpc_reg_operand" "=r,r")
	(and:SI (ashift:SI (match_dup 1) (match_dup 2)) (match_dup 3)))]
  "includes_lshift_p (operands[2], operands[3])"
  "@
   rlwinm. %0,%1,%h2,%m3,%M3
   #"
  [(set_attr "type" "shift")
   (set_attr "dot" "yes")
   (set_attr "length" "4,8")])

(define_split
  [(set (match_operand:CC 4 "cc_reg_not_micro_cr0_operand" "")
	(compare:CC
	 (and:SI (ashift:SI (match_operand:SI 1 "gpc_reg_operand" "")
			    (match_operand:SI 2 "const_int_operand" ""))
		 (match_operand:SI 3 "mask_operand" ""))
	 (const_int 0)))
   (set (match_operand:SI 0 "gpc_reg_operand" "")
	(and:SI (ashift:SI (match_dup 1) (match_dup 2)) (match_dup 3)))]
  "includes_lshift_p (operands[2], operands[3]) && reload_completed"
  [(set (match_dup 0)
	(and:SI (ashift:SI (match_dup 1) (match_dup 2)) (match_dup 3)))
   (set (match_dup 4)
	(compare:CC (match_dup 0)
		    (const_int 0)))]
  "")


(define_insn "lshr<mode>3"
  [(set (match_operand:GPR 0 "gpc_reg_operand" "=r")
	(lshiftrt:GPR (match_operand:GPR 1 "gpc_reg_operand" "r")
		      (match_operand:SI 2 "reg_or_cint_operand" "rn")))]
  ""
  "sr<wd>%I2 %0,%1,%<hH>2"
  [(set_attr "type" "shift")
   (set_attr "maybe_var_shift" "yes")])

(define_insn "*lshrsi3_64"
  [(set (match_operand:DI 0 "gpc_reg_operand" "=r")
	(zero_extend:DI
	    (lshiftrt:SI (match_operand:SI 1 "gpc_reg_operand" "r")
			 (match_operand:SI 2 "reg_or_cint_operand" "rn"))))]
  "TARGET_POWERPC64"
  "srw%I2 %0,%1,%h2"
  [(set_attr "type" "shift")
   (set_attr "maybe_var_shift" "yes")])

(define_insn_and_split "*lshr<mode>3_dot"
  [(set (match_operand:CC 3 "cc_reg_operand" "=x,?y")
	(compare:CC (lshiftrt:GPR (match_operand:GPR 1 "gpc_reg_operand" "r,r")
				  (match_operand:SI 2 "reg_or_cint_operand" "rn,rn"))
		    (const_int 0)))
   (clobber (match_scratch:GPR 0 "=r,r"))]
  "<MODE>mode == Pmode && rs6000_gen_cell_microcode"
  "@
   sr<wd>%I2. %0,%1,%<hH>2
   #"
  "&& reload_completed && cc_reg_not_cr0_operand (operands[3], CCmode)"
  [(set (match_dup 0)
	(lshiftrt:GPR (match_dup 1)
		      (match_dup 2)))
   (set (match_dup 3)
	(compare:CC (match_dup 0)
		    (const_int 0)))]
  ""
  [(set_attr "type" "shift")
   (set_attr "maybe_var_shift" "yes")
   (set_attr "dot" "yes")
   (set_attr "length" "4,8")])

(define_insn_and_split "*lshr<mode>3_dot2"
  [(set (match_operand:CC 3 "cc_reg_operand" "=x,?y")
	(compare:CC (lshiftrt:GPR (match_operand:GPR 1 "gpc_reg_operand" "r,r")
				  (match_operand:SI 2 "reg_or_cint_operand" "rn,rn"))
		    (const_int 0)))
   (set (match_operand:GPR 0 "gpc_reg_operand" "=r,r")
	(lshiftrt:GPR (match_dup 1)
		      (match_dup 2)))]
  "<MODE>mode == Pmode && rs6000_gen_cell_microcode"
  "@
   sr<wd>%I2. %0,%1,%<hH>2
   #"
  "&& reload_completed && cc_reg_not_cr0_operand (operands[3], CCmode)"
  [(set (match_dup 0)
	(lshiftrt:GPR (match_dup 1)
		      (match_dup 2)))
   (set (match_dup 3)
	(compare:CC (match_dup 0)
		    (const_int 0)))]
  ""
  [(set_attr "type" "shift")
   (set_attr "maybe_var_shift" "yes")
   (set_attr "dot" "yes")
   (set_attr "length" "4,8")])


(define_insn ""
  [(set (match_operand:SI 0 "gpc_reg_operand" "=r")
	(and:SI (lshiftrt:SI (match_operand:SI 1 "gpc_reg_operand" "r")
			     (match_operand:SI 2 "const_int_operand" "i"))
		(match_operand:SI 3 "mask_operand" "n")))]
  "includes_rshift_p (operands[2], operands[3])"
  "rlwinm %0,%1,%s2,%m3,%M3"
  [(set_attr "type" "shift")])

(define_insn ""
  [(set (match_operand:CC 0 "cc_reg_operand" "=x,?y")
	(compare:CC
	 (and:SI (lshiftrt:SI (match_operand:SI 1 "gpc_reg_operand" "r,r")
			      (match_operand:SI 2 "const_int_operand" "i,i"))
		 (match_operand:SI 3 "mask_operand" "n,n"))
	 (const_int 0)))
   (clobber (match_scratch:SI 4 "=r,r"))]
  "includes_rshift_p (operands[2], operands[3])"
  "@
   rlwinm. %4,%1,%s2,%m3,%M3
   #"
  [(set_attr "type" "shift")
   (set_attr "dot" "yes")
   (set_attr "length" "4,8")])

(define_split
  [(set (match_operand:CC 0 "cc_reg_not_micro_cr0_operand" "")
	(compare:CC
	 (and:SI (lshiftrt:SI (match_operand:SI 1 "gpc_reg_operand" "")
			      (match_operand:SI 2 "const_int_operand" ""))
		 (match_operand:SI 3 "mask_operand" ""))
	 (const_int 0)))
   (clobber (match_scratch:SI 4 ""))]
  "includes_rshift_p (operands[2], operands[3]) && reload_completed"
  [(set (match_dup 4)
	(and:SI (lshiftrt:SI (match_dup 1) (match_dup 2))
		 (match_dup 3)))
   (set (match_dup 0)
	(compare:CC (match_dup 4)
		    (const_int 0)))]
  "")

(define_insn ""
  [(set (match_operand:CC 4 "cc_reg_operand" "=x,?y")
	(compare:CC
	 (and:SI (lshiftrt:SI (match_operand:SI 1 "gpc_reg_operand" "r,r")
			      (match_operand:SI 2 "const_int_operand" "i,i"))
		 (match_operand:SI 3 "mask_operand" "n,n"))
	 (const_int 0)))
   (set (match_operand:SI 0 "gpc_reg_operand" "=r,r")
	(and:SI (lshiftrt:SI (match_dup 1) (match_dup 2)) (match_dup 3)))]
  "includes_rshift_p (operands[2], operands[3])"
  "@
   rlwinm. %0,%1,%s2,%m3,%M3
   #"
  [(set_attr "type" "shift")
   (set_attr "dot" "yes")
   (set_attr "length" "4,8")])

(define_split
  [(set (match_operand:CC 4 "cc_reg_not_micro_cr0_operand" "")
	(compare:CC
	 (and:SI (lshiftrt:SI (match_operand:SI 1 "gpc_reg_operand" "")
			      (match_operand:SI 2 "const_int_operand" ""))
		 (match_operand:SI 3 "mask_operand" ""))
	 (const_int 0)))
   (set (match_operand:SI 0 "gpc_reg_operand" "")
	(and:SI (lshiftrt:SI (match_dup 1) (match_dup 2)) (match_dup 3)))]
  "includes_rshift_p (operands[2], operands[3]) && reload_completed"
  [(set (match_dup 0)
	(and:SI (lshiftrt:SI (match_dup 1) (match_dup 2)) (match_dup 3)))
   (set (match_dup 4)
	(compare:CC (match_dup 0)
		    (const_int 0)))]
  "")

(define_insn "*lshiftrt_internal1le"
  [(set (match_operand:SI 0 "gpc_reg_operand" "=r")
	(zero_extend:SI
	 (subreg:QI
	  (lshiftrt:SI (match_operand:SI 1 "gpc_reg_operand" "r")
		       (match_operand:SI 2 "const_int_operand" "i")) 0)))]
  "!BYTES_BIG_ENDIAN && includes_rshift_p (operands[2], GEN_INT (255))"
  "rlwinm %0,%1,%s2,0xff"
  [(set_attr "type" "shift")])

(define_insn "*lshiftrt_internal1be"
  [(set (match_operand:SI 0 "gpc_reg_operand" "=r")
	(zero_extend:SI
	 (subreg:QI
	  (lshiftrt:SI (match_operand:SI 1 "gpc_reg_operand" "r")
		       (match_operand:SI 2 "const_int_operand" "i")) 3)))]
  "BYTES_BIG_ENDIAN && includes_rshift_p (operands[2], GEN_INT (255))"
  "rlwinm %0,%1,%s2,0xff"
  [(set_attr "type" "shift")])

(define_insn "*lshiftrt_internal2le"
  [(set (match_operand:CC 0 "cc_reg_operand" "=x,?y")
	(compare:CC
	 (zero_extend:SI
	  (subreg:QI
	   (lshiftrt:SI (match_operand:SI 1 "gpc_reg_operand" "r,r")
			(match_operand:SI 2 "const_int_operand" "i,i")) 0))
	 (const_int 0)))
   (clobber (match_scratch:SI 3 "=r,r"))]
  "!BYTES_BIG_ENDIAN && includes_rshift_p (operands[2], GEN_INT (255))"
  "@
   rlwinm. %3,%1,%s2,0xff
   #"
  [(set_attr "type" "shift")
   (set_attr "dot" "yes")
   (set_attr "length" "4,8")])

(define_insn "*lshiftrt_internal2be"
  [(set (match_operand:CC 0 "cc_reg_operand" "=x,?y")
	(compare:CC
	 (zero_extend:SI
	  (subreg:QI
	   (lshiftrt:SI (match_operand:SI 1 "gpc_reg_operand" "r,r")
			(match_operand:SI 2 "const_int_operand" "i,i")) 3))
	 (const_int 0)))
   (clobber (match_scratch:SI 3 "=r,r"))]
  "BYTES_BIG_ENDIAN && includes_rshift_p (operands[2], GEN_INT (255))"
  "@
   rlwinm. %3,%1,%s2,0xff
   #"
  [(set_attr "type" "shift")
   (set_attr "dot" "yes")
   (set_attr "length" "4,8")])

(define_split
  [(set (match_operand:CC 0 "cc_reg_not_micro_cr0_operand" "")
	(compare:CC
	 (zero_extend:SI
	  (subreg:QI
	   (lshiftrt:SI (match_operand:SI 1 "gpc_reg_operand" "")
			(match_operand:SI 2 "const_int_operand" "")) 0))
	 (const_int 0)))
   (clobber (match_scratch:SI 3 ""))]
  "!BYTES_BIG_ENDIAN && includes_rshift_p (operands[2], GEN_INT (255)) && reload_completed"
  [(set (match_dup 3)
	(zero_extend:SI (subreg:QI
	   (lshiftrt:SI (match_dup 1)
			(match_dup 2)) 0)))
   (set (match_dup 0)
	(compare:CC (match_dup 3)
		    (const_int 0)))]
  "")

(define_split
  [(set (match_operand:CC 0 "cc_reg_not_micro_cr0_operand" "")
	(compare:CC
	 (zero_extend:SI
	  (subreg:QI
	   (lshiftrt:SI (match_operand:SI 1 "gpc_reg_operand" "")
			(match_operand:SI 2 "const_int_operand" "")) 3))
	 (const_int 0)))
   (clobber (match_scratch:SI 3 ""))]
  "BYTES_BIG_ENDIAN && includes_rshift_p (operands[2], GEN_INT (255)) && reload_completed"
  [(set (match_dup 3)
	(zero_extend:SI (subreg:QI
	   (lshiftrt:SI (match_dup 1)
			(match_dup 2)) 3)))
   (set (match_dup 0)
	(compare:CC (match_dup 3)
		    (const_int 0)))]
  "")

(define_insn "*lshiftrt_internal3le"
  [(set (match_operand:CC 3 "cc_reg_operand" "=x,?y")
	(compare:CC
	 (zero_extend:SI
	  (subreg:QI
	   (lshiftrt:SI (match_operand:SI 1 "gpc_reg_operand" "r,r")
			(match_operand:SI 2 "const_int_operand" "i,i")) 0))
	 (const_int 0)))
   (set (match_operand:SI 0 "gpc_reg_operand" "=r,r")
	(zero_extend:SI (subreg:QI (lshiftrt:SI (match_dup 1) (match_dup 2)) 0)))]
  "!BYTES_BIG_ENDIAN && includes_rshift_p (operands[2], GEN_INT (255))"
  "@
   rlwinm. %0,%1,%s2,0xff
   #"
  [(set_attr "type" "shift")
   (set_attr "dot" "yes")
   (set_attr "length" "4,8")])

(define_insn "*lshiftrt_internal3be"
  [(set (match_operand:CC 3 "cc_reg_operand" "=x,?y")
	(compare:CC
	 (zero_extend:SI
	  (subreg:QI
	   (lshiftrt:SI (match_operand:SI 1 "gpc_reg_operand" "r,r")
			(match_operand:SI 2 "const_int_operand" "i,i")) 3))
	 (const_int 0)))
   (set (match_operand:SI 0 "gpc_reg_operand" "=r,r")
	(zero_extend:SI (subreg:QI (lshiftrt:SI (match_dup 1) (match_dup 2)) 3)))]
  "BYTES_BIG_ENDIAN && includes_rshift_p (operands[2], GEN_INT (255))"
  "@
   rlwinm. %0,%1,%s2,0xff
   #"
  [(set_attr "type" "shift")
   (set_attr "dot" "yes")
   (set_attr "length" "4,8")])

(define_split
  [(set (match_operand:CC 3 "cc_reg_not_micro_cr0_operand" "")
	(compare:CC
	 (zero_extend:SI
	  (subreg:QI
	   (lshiftrt:SI (match_operand:SI 1 "gpc_reg_operand" "")
			(match_operand:SI 2 "const_int_operand" "")) 0))
	 (const_int 0)))
   (set (match_operand:SI 0 "gpc_reg_operand" "")
	(zero_extend:SI (subreg:QI (lshiftrt:SI (match_dup 1) (match_dup 2)) 0)))]
  "!BYTES_BIG_ENDIAN && includes_rshift_p (operands[2], GEN_INT (255)) && reload_completed"
  [(set (match_dup 0)
	(zero_extend:SI (subreg:QI (lshiftrt:SI (match_dup 1) (match_dup 2)) 0)))
   (set (match_dup 3)
	(compare:CC (match_dup 0)
		    (const_int 0)))]
  "")

(define_split
  [(set (match_operand:CC 3 "cc_reg_not_micro_cr0_operand" "")
	(compare:CC
	 (zero_extend:SI
	  (subreg:QI
	   (lshiftrt:SI (match_operand:SI 1 "gpc_reg_operand" "")
			(match_operand:SI 2 "const_int_operand" "")) 3))
	 (const_int 0)))
   (set (match_operand:SI 0 "gpc_reg_operand" "")
	(zero_extend:SI (subreg:QI (lshiftrt:SI (match_dup 1) (match_dup 2)) 3)))]
  "BYTES_BIG_ENDIAN && includes_rshift_p (operands[2], GEN_INT (255)) && reload_completed"
  [(set (match_dup 0)
	(zero_extend:SI (subreg:QI (lshiftrt:SI (match_dup 1) (match_dup 2)) 3)))
   (set (match_dup 3)
	(compare:CC (match_dup 0)
		    (const_int 0)))]
  "")

(define_insn "*lshiftrt_internal4le"
  [(set (match_operand:SI 0 "gpc_reg_operand" "=r")
	(zero_extend:SI
	 (subreg:HI
	  (lshiftrt:SI (match_operand:SI 1 "gpc_reg_operand" "r")
		       (match_operand:SI 2 "const_int_operand" "i")) 0)))]
  "!BYTES_BIG_ENDIAN && includes_rshift_p (operands[2], GEN_INT (65535))"
  "rlwinm %0,%1,%s2,0xffff"
  [(set_attr "type" "shift")])

(define_insn "*lshiftrt_internal4be"
  [(set (match_operand:SI 0 "gpc_reg_operand" "=r")
	(zero_extend:SI
	 (subreg:HI
	  (lshiftrt:SI (match_operand:SI 1 "gpc_reg_operand" "r")
		       (match_operand:SI 2 "const_int_operand" "i")) 2)))]
  "BYTES_BIG_ENDIAN && includes_rshift_p (operands[2], GEN_INT (65535))"
  "rlwinm %0,%1,%s2,0xffff"
  [(set_attr "type" "shift")])

(define_insn "*lshiftrt_internal5le"
  [(set (match_operand:CC 0 "cc_reg_operand" "=x,?y")
	(compare:CC
	 (zero_extend:SI
	  (subreg:HI
	   (lshiftrt:SI (match_operand:SI 1 "gpc_reg_operand" "r,r")
			(match_operand:SI 2 "const_int_operand" "i,i")) 0))
	 (const_int 0)))
   (clobber (match_scratch:SI 3 "=r,r"))]
  "!BYTES_BIG_ENDIAN && includes_rshift_p (operands[2], GEN_INT (65535))"
  "@
   rlwinm. %3,%1,%s2,0xffff
   #"
  [(set_attr "type" "shift")
   (set_attr "dot" "yes")
   (set_attr "length" "4,8")])

(define_insn "*lshiftrt_internal5be"
  [(set (match_operand:CC 0 "cc_reg_operand" "=x,?y")
	(compare:CC
	 (zero_extend:SI
	  (subreg:HI
	   (lshiftrt:SI (match_operand:SI 1 "gpc_reg_operand" "r,r")
			(match_operand:SI 2 "const_int_operand" "i,i")) 2))
	 (const_int 0)))
   (clobber (match_scratch:SI 3 "=r,r"))]
  "BYTES_BIG_ENDIAN && includes_rshift_p (operands[2], GEN_INT (65535))"
  "@
   rlwinm. %3,%1,%s2,0xffff
   #"
  [(set_attr "type" "shift")
   (set_attr "dot" "yes")
   (set_attr "length" "4,8")])

(define_split
  [(set (match_operand:CC 0 "cc_reg_not_micro_cr0_operand" "")
	(compare:CC
	 (zero_extend:SI
	  (subreg:HI
	   (lshiftrt:SI (match_operand:SI 1 "gpc_reg_operand" "")
			(match_operand:SI 2 "const_int_operand" "")) 0))
	 (const_int 0)))
   (clobber (match_scratch:SI 3 ""))]
  "!BYTES_BIG_ENDIAN && includes_rshift_p (operands[2], GEN_INT (65535)) && reload_completed"
  [(set (match_dup 3)
	(zero_extend:SI (subreg:HI
	   (lshiftrt:SI (match_dup 1)
			(match_dup 2)) 0)))
   (set (match_dup 0)
	(compare:CC (match_dup 3)
		    (const_int 0)))]
  "")

(define_split
  [(set (match_operand:CC 0 "cc_reg_not_micro_cr0_operand" "")
	(compare:CC
	 (zero_extend:SI
	  (subreg:HI
	   (lshiftrt:SI (match_operand:SI 1 "gpc_reg_operand" "")
			(match_operand:SI 2 "const_int_operand" "")) 2))
	 (const_int 0)))
   (clobber (match_scratch:SI 3 ""))]
  "BYTES_BIG_ENDIAN && includes_rshift_p (operands[2], GEN_INT (65535)) && reload_completed"
  [(set (match_dup 3)
	(zero_extend:SI (subreg:HI
	   (lshiftrt:SI (match_dup 1)
			(match_dup 2)) 2)))
   (set (match_dup 0)
	(compare:CC (match_dup 3)
		    (const_int 0)))]
  "")

(define_insn "*lshiftrt_internal5le"
  [(set (match_operand:CC 3 "cc_reg_operand" "=x,?y")
	(compare:CC
	 (zero_extend:SI
	  (subreg:HI
	   (lshiftrt:SI (match_operand:SI 1 "gpc_reg_operand" "r,r")
			(match_operand:SI 2 "const_int_operand" "i,i")) 0))
	 (const_int 0)))
   (set (match_operand:SI 0 "gpc_reg_operand" "=r,r")
	(zero_extend:SI (subreg:HI (lshiftrt:SI (match_dup 1) (match_dup 2)) 0)))]
  "!BYTES_BIG_ENDIAN && includes_rshift_p (operands[2], GEN_INT (65535))"
  "@
   rlwinm. %0,%1,%s2,0xffff
   #"
  [(set_attr "type" "shift")
   (set_attr "dot" "yes")
   (set_attr "length" "4,8")])

(define_insn "*lshiftrt_internal5be"
  [(set (match_operand:CC 3 "cc_reg_operand" "=x,?y")
	(compare:CC
	 (zero_extend:SI
	  (subreg:HI
	   (lshiftrt:SI (match_operand:SI 1 "gpc_reg_operand" "r,r")
			(match_operand:SI 2 "const_int_operand" "i,i")) 2))
	 (const_int 0)))
   (set (match_operand:SI 0 "gpc_reg_operand" "=r,r")
	(zero_extend:SI (subreg:HI (lshiftrt:SI (match_dup 1) (match_dup 2)) 2)))]
  "BYTES_BIG_ENDIAN && includes_rshift_p (operands[2], GEN_INT (65535))"
  "@
   rlwinm. %0,%1,%s2,0xffff
   #"
  [(set_attr "type" "shift")
   (set_attr "dot" "yes")
   (set_attr "length" "4,8")])

(define_split
  [(set (match_operand:CC 3 "cc_reg_not_micro_cr0_operand" "")
	(compare:CC
	 (zero_extend:SI
	  (subreg:HI
	   (lshiftrt:SI (match_operand:SI 1 "gpc_reg_operand" "")
			(match_operand:SI 2 "const_int_operand" "")) 0))
	 (const_int 0)))
   (set (match_operand:SI 0 "gpc_reg_operand" "")
	(zero_extend:SI (subreg:HI (lshiftrt:SI (match_dup 1) (match_dup 2)) 0)))]
  "!BYTES_BIG_ENDIAN && includes_rshift_p (operands[2], GEN_INT (65535)) && reload_completed"
  [(set (match_dup 0)
	(zero_extend:SI (subreg:HI (lshiftrt:SI (match_dup 1) (match_dup 2)) 0)))
   (set (match_dup 3)
	(compare:CC (match_dup 0)
		    (const_int 0)))]
  "")

(define_split
  [(set (match_operand:CC 3 "cc_reg_not_micro_cr0_operand" "")
	(compare:CC
	 (zero_extend:SI
	  (subreg:HI
	   (lshiftrt:SI (match_operand:SI 1 "gpc_reg_operand" "")
			(match_operand:SI 2 "const_int_operand" "")) 2))
	 (const_int 0)))
   (set (match_operand:SI 0 "gpc_reg_operand" "")
	(zero_extend:SI (subreg:HI (lshiftrt:SI (match_dup 1) (match_dup 2)) 2)))]
  "BYTES_BIG_ENDIAN && includes_rshift_p (operands[2], GEN_INT (65535)) && reload_completed"
  [(set (match_dup 0)
	(zero_extend:SI (subreg:HI (lshiftrt:SI (match_dup 1) (match_dup 2)) 2)))
   (set (match_dup 3)
	(compare:CC (match_dup 0)
		    (const_int 0)))]
  "")


(define_expand "ashr<mode>3"
  [(parallel [(set (match_operand:GPR 0 "gpc_reg_operand" "")
		   (ashiftrt:GPR (match_operand:GPR 1 "gpc_reg_operand" "")
				 (match_operand:SI 2 "reg_or_cint_operand" "")))
	      (clobber (reg:GPR CA_REGNO))])]
  ""
{
  /* The generic code does not generate optimal code for the low word
     (it should be a rlwimi and a rot).  Until we have target code to
     solve this generically, keep this expander.  */

  if (<MODE>mode == DImode && !TARGET_POWERPC64)
    {
      if (CONST_INT_P (operands[2]))
	{
	  emit_insn (gen_ashrdi3_no_power (operands[0], operands[1], operands[2]));
	  DONE;
	}
      else
	FAIL;
    }
})

(define_insn "*ashr<mode>3"
  [(set (match_operand:GPR 0 "gpc_reg_operand" "=r")
	(ashiftrt:GPR (match_operand:GPR 1 "gpc_reg_operand" "r")
		      (match_operand:SI 2 "reg_or_cint_operand" "rn")))
   (clobber (reg:GPR CA_REGNO))]
  ""
  "sra<wd>%I2 %0,%1,%<hH>2"
  [(set_attr "type" "shift")
   (set_attr "maybe_var_shift" "yes")])

(define_insn "*ashrsi3_64"
  [(set (match_operand:DI 0 "gpc_reg_operand" "=r")
	(sign_extend:DI
	    (ashiftrt:SI (match_operand:SI 1 "gpc_reg_operand" "r")
			 (match_operand:SI 2 "reg_or_cint_operand" "rn"))))
   (clobber (reg:SI CA_REGNO))]
  "TARGET_POWERPC64"
  "sraw%I2 %0,%1,%h2"
  [(set_attr "type" "shift")
   (set_attr "maybe_var_shift" "yes")])

(define_insn_and_split "*ashr<mode>3_dot"
  [(set (match_operand:CC 3 "cc_reg_operand" "=x,?y")
	(compare:CC (ashiftrt:GPR (match_operand:GPR 1 "gpc_reg_operand" "r,r")
				  (match_operand:SI 2 "reg_or_cint_operand" "rn,rn"))
		    (const_int 0)))
   (clobber (match_scratch:GPR 0 "=r,r"))
   (clobber (reg:GPR CA_REGNO))]
  "<MODE>mode == Pmode && rs6000_gen_cell_microcode"
  "@
   sra<wd>%I2. %0,%1,%<hH>2
   #"
  "&& reload_completed && cc_reg_not_cr0_operand (operands[3], CCmode)"
  [(parallel [(set (match_dup 0)
		   (ashiftrt:GPR (match_dup 1)
				 (match_dup 2)))
	      (clobber (reg:GPR CA_REGNO))])
   (set (match_dup 3)
	(compare:CC (match_dup 0)
		    (const_int 0)))]
  ""
  [(set_attr "type" "shift")
   (set_attr "maybe_var_shift" "yes")
   (set_attr "dot" "yes")
   (set_attr "length" "4,8")])

(define_insn_and_split "*ashr<mode>3_dot2"
  [(set (match_operand:CC 3 "cc_reg_operand" "=x,?y")
	(compare:CC (ashiftrt:GPR (match_operand:GPR 1 "gpc_reg_operand" "r,r")
				  (match_operand:SI 2 "reg_or_cint_operand" "rn,rn"))
		    (const_int 0)))
   (set (match_operand:GPR 0 "gpc_reg_operand" "=r,r")
	(ashiftrt:GPR (match_dup 1)
		      (match_dup 2)))
   (clobber (reg:GPR CA_REGNO))]
  "<MODE>mode == Pmode && rs6000_gen_cell_microcode"
  "@
   sra<wd>%I2. %0,%1,%<hH>2
   #"
  "&& reload_completed && cc_reg_not_cr0_operand (operands[3], CCmode)"
  [(parallel [(set (match_dup 0)
		   (ashiftrt:GPR (match_dup 1)
				 (match_dup 2)))
	      (clobber (reg:GPR CA_REGNO))])
   (set (match_dup 3)
	(compare:CC (match_dup 0)
		    (const_int 0)))]
  ""
  [(set_attr "type" "shift")
   (set_attr "maybe_var_shift" "yes")
   (set_attr "dot" "yes")
   (set_attr "length" "4,8")])

;; Builtins to replace a division to generate FRE reciprocal estimate
;; instructions and the necessary fixup instructions
(define_expand "recip<mode>3"
  [(match_operand:RECIPF 0 "gpc_reg_operand" "")
   (match_operand:RECIPF 1 "gpc_reg_operand" "")
   (match_operand:RECIPF 2 "gpc_reg_operand" "")]
  "RS6000_RECIP_HAVE_RE_P (<MODE>mode)"
{
   rs6000_emit_swdiv (operands[0], operands[1], operands[2], false);
   DONE;
})

;; Split to create division from FRE/FRES/etc. and fixup instead of the normal
;; hardware division.  This is only done before register allocation and with
;; -ffast-math.  This must appear before the divsf3/divdf3 insns.
(define_split
  [(set (match_operand:RECIPF 0 "gpc_reg_operand" "")
	(div:RECIPF (match_operand 1 "gpc_reg_operand" "")
		    (match_operand 2 "gpc_reg_operand" "")))]
  "RS6000_RECIP_AUTO_RE_P (<MODE>mode)
   && can_create_pseudo_p () && optimize_insn_for_speed_p ()
   && flag_finite_math_only && !flag_trapping_math && flag_reciprocal_math"
  [(const_int 0)]
{
  rs6000_emit_swdiv (operands[0], operands[1], operands[2], true);
  DONE;
})

;; Builtins to replace 1/sqrt(x) with instructions using RSQRTE and the
;; appropriate fixup.
(define_expand "rsqrt<mode>2"
  [(match_operand:RECIPF 0 "gpc_reg_operand" "")
   (match_operand:RECIPF 1 "gpc_reg_operand" "")]
  "RS6000_RECIP_HAVE_RSQRTE_P (<MODE>mode)"
{
  rs6000_emit_swrsqrt (operands[0], operands[1]);
  DONE;
})

;; Floating-point insns, excluding normal data motion.  We combine the SF/DF
;; modes here, and also add in conditional vsx/power8-vector support to access
;; values in the traditional Altivec registers if the appropriate
;; -mupper-regs-{df,sf} option is enabled.

(define_expand "abs<mode>2"
  [(set (match_operand:SFDF 0 "gpc_reg_operand" "")
	(abs:SFDF (match_operand:SFDF 1 "gpc_reg_operand" "")))]
  "TARGET_<MODE>_INSN"
  "")

(define_insn "*abs<mode>2_fpr"
  [(set (match_operand:SFDF 0 "gpc_reg_operand" "=<Ff>,<Fv>")
	(abs:SFDF (match_operand:SFDF 1 "gpc_reg_operand" "<Ff>,<Fv>")))]
  "TARGET_<MODE>_FPR"
  "@
   fabs %0,%1
   xsabsdp %x0,%x1"
  [(set_attr "type" "fp")
   (set_attr "fp_type" "fp_addsub_<Fs>")])

(define_insn "*nabs<mode>2_fpr"
  [(set (match_operand:SFDF 0 "gpc_reg_operand" "=<Ff>,<Fv>")
	(neg:SFDF
	 (abs:SFDF
	  (match_operand:SFDF 1 "gpc_reg_operand" "<Ff>,<Fv>"))))]
  "TARGET_<MODE>_FPR"
  "@
   fnabs %0,%1
   xsnabsdp %x0,%x1"
  [(set_attr "type" "fp")
   (set_attr "fp_type" "fp_addsub_<Fs>")])

(define_expand "neg<mode>2"
  [(set (match_operand:SFDF 0 "gpc_reg_operand" "")
	(neg:SFDF (match_operand:SFDF 1 "gpc_reg_operand" "")))]
  "TARGET_<MODE>_INSN"
  "")

(define_insn "*neg<mode>2_fpr"
  [(set (match_operand:SFDF 0 "gpc_reg_operand" "=<Ff>,<Fv>")
	(neg:SFDF (match_operand:SFDF 1 "gpc_reg_operand" "<Ff>,<Fv>")))]
  "TARGET_<MODE>_FPR"
  "@
   fneg %0,%1
   xsnegdp %x0,%x1"
  [(set_attr "type" "fp")
   (set_attr "fp_type" "fp_addsub_<Fs>")])

(define_expand "add<mode>3"
  [(set (match_operand:SFDF 0 "gpc_reg_operand" "")
	(plus:SFDF (match_operand:SFDF 1 "gpc_reg_operand" "")
		   (match_operand:SFDF 2 "gpc_reg_operand" "")))]
  "TARGET_<MODE>_INSN"
  "")

(define_insn "*add<mode>3_fpr"
  [(set (match_operand:SFDF 0 "gpc_reg_operand" "=<Ff>,<Fv>")
	(plus:SFDF (match_operand:SFDF 1 "gpc_reg_operand" "%<Ff>,<Fv>")
		   (match_operand:SFDF 2 "gpc_reg_operand" "<Ff>,<Fv>")))]
  "TARGET_<MODE>_FPR"
  "@
   fadd<Ftrad> %0,%1,%2
   xsadd<Fvsx> %x0,%x1,%x2"
  [(set_attr "type" "fp")
   (set_attr "fp_type" "fp_addsub_<Fs>")])

(define_expand "sub<mode>3"
  [(set (match_operand:SFDF 0 "gpc_reg_operand" "")
	(minus:SFDF (match_operand:SFDF 1 "gpc_reg_operand" "")
		    (match_operand:SFDF 2 "gpc_reg_operand" "")))]
  "TARGET_<MODE>_INSN"
  "")

(define_insn "*sub<mode>3_fpr"
  [(set (match_operand:SFDF 0 "gpc_reg_operand" "=<Ff>,<Fv>")
	(minus:SFDF (match_operand:SFDF 1 "gpc_reg_operand" "<Ff>,<Fv>")
		    (match_operand:SFDF 2 "gpc_reg_operand" "<Ff>,<Fv>")))]
  "TARGET_<MODE>_FPR"
  "@
   fsub<Ftrad> %0,%1,%2
   xssub<Fvsx> %x0,%x1,%x2"
  [(set_attr "type" "fp")
   (set_attr "fp_type" "fp_addsub_<Fs>")])

(define_expand "mul<mode>3"
  [(set (match_operand:SFDF 0 "gpc_reg_operand" "")
	(mult:SFDF (match_operand:SFDF 1 "gpc_reg_operand" "")
		   (match_operand:SFDF 2 "gpc_reg_operand" "")))]
  "TARGET_<MODE>_INSN"
  "")

(define_insn "*mul<mode>3_fpr"
  [(set (match_operand:SFDF 0 "gpc_reg_operand" "=<Ff>,<Fv>")
	(mult:SFDF (match_operand:SFDF 1 "gpc_reg_operand" "%<Ff>,<Fv>")
		   (match_operand:SFDF 2 "gpc_reg_operand" "<Ff>,<Fv>")))]
  "TARGET_<MODE>_FPR"
  "@
   fmul<Ftrad> %0,%1,%2
   xsmul<Fvsx> %x0,%x1,%x2"
  [(set_attr "type" "dmul")
   (set_attr "fp_type" "fp_mul_<Fs>")])

(define_expand "div<mode>3"
  [(set (match_operand:SFDF 0 "gpc_reg_operand" "")
	(div:SFDF (match_operand:SFDF 1 "gpc_reg_operand" "")
		  (match_operand:SFDF 2 "gpc_reg_operand" "")))]
  "TARGET_<MODE>_INSN && !TARGET_SIMPLE_FPU"
  "")

(define_insn "*div<mode>3_fpr"
  [(set (match_operand:SFDF 0 "gpc_reg_operand" "=<Ff>,<Fv>")
	(div:SFDF (match_operand:SFDF 1 "gpc_reg_operand" "<Ff>,<Fv>")
		  (match_operand:SFDF 2 "gpc_reg_operand" "<Ff>,<Fv>")))]
  "TARGET_<MODE>_FPR && !TARGET_SIMPLE_FPU"
  "@
   fdiv<Ftrad> %0,%1,%2
   xsdiv<Fvsx> %x0,%x1,%x2"
  [(set_attr "type" "<Fs>div")
   (set_attr "fp_type" "fp_div_<Fs>")])

(define_insn "sqrt<mode>2"
  [(set (match_operand:SFDF 0 "gpc_reg_operand" "=<Ff>,<Fv>")
	(sqrt:SFDF (match_operand:SFDF 1 "gpc_reg_operand" "<Ff>,<Fv>")))]
  "TARGET_<MODE>_FPR && !TARGET_SIMPLE_FPU
   && (TARGET_PPC_GPOPT || (<MODE>mode == SFmode && TARGET_XILINX_FPU))"
  "@
   fsqrt<Ftrad> %0,%1
   xssqrt<Fvsx> %x0,%x1"
  [(set_attr "type" "<Fs>sqrt")
   (set_attr "fp_type" "fp_sqrt_<Fs>")])

;; Floating point reciprocal approximation
(define_insn "fre<Fs>"
  [(set (match_operand:SFDF 0 "gpc_reg_operand" "=<Ff>,<Fv>")
	(unspec:SFDF [(match_operand:SFDF 1 "gpc_reg_operand" "<Ff>,<Fv>")]
		     UNSPEC_FRES))]
  "TARGET_<FFRE>"
  "@
   fre<Ftrad> %0,%1
   xsre<Fvsx> %x0,%x1"
  [(set_attr "type" "fp")])

(define_insn "*rsqrt<mode>2"
  [(set (match_operand:SFDF 0 "gpc_reg_operand" "=<Ff>,<Fv>")
	(unspec:SFDF [(match_operand:SFDF 1 "gpc_reg_operand" "<Ff>,<Fv>")]
		     UNSPEC_RSQRT))]
  "RS6000_RECIP_HAVE_RSQRTE_P (<MODE>mode)"
  "@
   frsqrte<Ftrad> %0,%1
   xsrsqrte<Fvsx> %x0,%x1"
  [(set_attr "type" "fp")])

;; Floating point comparisons
(define_insn "*cmp<mode>_fpr"
  [(set (match_operand:CCFP 0 "cc_reg_operand" "=y,y")
	(compare:CCFP (match_operand:SFDF 1 "gpc_reg_operand" "<Ff>,<Fv>")
		      (match_operand:SFDF 2 "gpc_reg_operand" "<Ff>,<Fv>")))]
  "TARGET_<MODE>_FPR"
  "@
   fcmpu %0,%1,%2
   xscmpudp %0,%x1,%x2"
  [(set_attr "type" "fpcompare")])

;; Floating point conversions
(define_expand "extendsfdf2"
  [(set (match_operand:DF 0 "gpc_reg_operand" "")
	(float_extend:DF (match_operand:SF 1 "reg_or_none500mem_operand" "")))]
  "TARGET_HARD_FLOAT && ((TARGET_FPRS && TARGET_DOUBLE_FLOAT) || TARGET_E500_DOUBLE)"
  "")

(define_insn_and_split "*extendsfdf2_fpr"
  [(set (match_operand:DF 0 "gpc_reg_operand" "=d,?d,d,ws,?ws,wu")
	(float_extend:DF (match_operand:SF 1 "reg_or_mem_operand" "0,f,m,0,wy,Z")))]
  "TARGET_HARD_FLOAT && TARGET_FPRS && TARGET_DOUBLE_FLOAT"
  "@
   #
   fmr %0,%1
   lfs%U1%X1 %0,%1
   #
   xscpsgndp %x0,%x1,%x1
   lxsspx %x0,%y1"
  "&& reload_completed && REG_P (operands[1]) && REGNO (operands[0]) == REGNO (operands[1])"
  [(const_int 0)]
{
  emit_note (NOTE_INSN_DELETED);
  DONE;
}
  [(set_attr "type" "fp,fp,fpload,fp,fp,fpload")])

(define_expand "truncdfsf2"
  [(set (match_operand:SF 0 "gpc_reg_operand" "")
	(float_truncate:SF (match_operand:DF 1 "gpc_reg_operand" "")))]
  "TARGET_HARD_FLOAT && ((TARGET_FPRS && TARGET_DOUBLE_FLOAT) || TARGET_E500_DOUBLE)"
  "")

(define_insn "*truncdfsf2_fpr"
  [(set (match_operand:SF 0 "gpc_reg_operand" "=f,wy")
	(float_truncate:SF (match_operand:DF 1 "gpc_reg_operand" "d,ws")))]
  "TARGET_HARD_FLOAT && TARGET_FPRS && TARGET_DOUBLE_FLOAT"
  "@
   frsp %0,%1
   xsrsp %x0,%x1"
  [(set_attr "type" "fp")])

;; This expander is here to avoid FLOAT_WORDS_BIGENDIAN tests in
;; builtins.c and optabs.c that are not correct for IBM long double
;; when little-endian.
(define_expand "signbittf2"
  [(set (match_dup 2)
	(float_truncate:DF (match_operand:TF 1 "gpc_reg_operand" "")))
   (set (match_dup 3)
   	(subreg:DI (match_dup 2) 0))
   (set (match_dup 4)
   	(match_dup 5))
   (set (match_operand:SI 0 "gpc_reg_operand" "")
  	(match_dup 6))]
  "!TARGET_IEEEQUAD
   && TARGET_HARD_FLOAT
   && (TARGET_FPRS || TARGET_E500_DOUBLE)
   && TARGET_LONG_DOUBLE_128"
{
  operands[2] = gen_reg_rtx (DFmode);
  operands[3] = gen_reg_rtx (DImode);
  if (TARGET_POWERPC64)
    {
      operands[4] = gen_reg_rtx (DImode);
      operands[5] = gen_rtx_LSHIFTRT (DImode, operands[3], GEN_INT (63));
      operands[6] = gen_rtx_SUBREG (SImode, operands[4],
				    WORDS_BIG_ENDIAN ? 4 : 0);
    }
  else
    {
      operands[4] = gen_reg_rtx (SImode);
      operands[5] = gen_rtx_SUBREG (SImode, operands[3],
				    WORDS_BIG_ENDIAN ? 0 : 4);
      operands[6] = gen_rtx_LSHIFTRT (SImode, operands[4], GEN_INT (31));
    }
})

(define_expand "copysign<mode>3"
  [(set (match_dup 3)
        (abs:SFDF (match_operand:SFDF 1 "gpc_reg_operand" "")))
   (set (match_dup 4)
	(neg:SFDF (abs:SFDF (match_dup 1))))
   (set (match_operand:SFDF 0 "gpc_reg_operand" "")
        (if_then_else:SFDF (ge (match_operand:SFDF 2 "gpc_reg_operand" "")
			       (match_dup 5))
			 (match_dup 3)
			 (match_dup 4)))]
  "TARGET_HARD_FLOAT && TARGET_FPRS && <TARGET_FLOAT>
   && ((TARGET_PPC_GFXOPT
        && !HONOR_NANS (<MODE>mode)
        && !HONOR_SIGNED_ZEROS (<MODE>mode))
       || TARGET_CMPB
       || VECTOR_UNIT_VSX_P (<MODE>mode))"
{
  if (TARGET_CMPB || VECTOR_UNIT_VSX_P (<MODE>mode))
    {
      emit_insn (gen_copysign<mode>3_fcpsgn (operands[0], operands[1],
					     operands[2]));
      DONE;
    }

   operands[3] = gen_reg_rtx (<MODE>mode);
   operands[4] = gen_reg_rtx (<MODE>mode);
   operands[5] = CONST0_RTX (<MODE>mode);
  })

;; Use an unspec rather providing an if-then-else in RTL, to prevent the
;; compiler from optimizing -0.0
(define_insn "copysign<mode>3_fcpsgn"
  [(set (match_operand:SFDF 0 "gpc_reg_operand" "=<Ff>,<Fv>")
	(unspec:SFDF [(match_operand:SFDF 1 "gpc_reg_operand" "<Ff>,<Fv>")
		      (match_operand:SFDF 2 "gpc_reg_operand" "<Ff>,<Fv>")]
		     UNSPEC_COPYSIGN))]
  "TARGET_<MODE>_FPR && TARGET_CMPB"
  "@
   fcpsgn %0,%2,%1
   xscpsgndp %x0,%x2,%x1"
  [(set_attr "type" "fp")])

;; For MIN, MAX, and conditional move, we use DEFINE_EXPAND's that involve a
;; fsel instruction and some auxiliary computations.  Then we just have a
;; single DEFINE_INSN for fsel and the define_splits to make them if made by
;; combine.
;; For MIN, MAX on non-VSX machines, and conditional move all of the time, we
;; use DEFINE_EXPAND's that involve a fsel instruction and some auxiliary
;; computations.  Then we just have a single DEFINE_INSN for fsel and the
;; define_splits to make them if made by combine.  On VSX machines we have the
;; min/max instructions.
;;
;; On VSX, we only check for TARGET_VSX instead of checking for a vsx/p8 vector
;; to allow either DF/SF to use only traditional registers.

(define_expand "smax<mode>3"
  [(set (match_operand:SFDF 0 "gpc_reg_operand" "")
	(if_then_else:SFDF (ge (match_operand:SFDF 1 "gpc_reg_operand" "")
			       (match_operand:SFDF 2 "gpc_reg_operand" ""))
			   (match_dup 1)
			   (match_dup 2)))]
  "TARGET_<MODE>_FPR && TARGET_PPC_GFXOPT && !flag_trapping_math"
{
  rs6000_emit_minmax (operands[0], SMAX, operands[1], operands[2]);
  DONE;
})

(define_insn "*smax<mode>3_vsx"
  [(set (match_operand:SFDF 0 "gpc_reg_operand" "=<Ff>,<Fv>")
	(smax:SFDF (match_operand:SFDF 1 "gpc_reg_operand" "%<Ff>,<Fv>")
		   (match_operand:SFDF 2 "gpc_reg_operand" "<Ff>,<Fv>")))]
  "TARGET_<MODE>_FPR && TARGET_VSX"
  "xsmaxdp %x0,%x1,%x2"
  [(set_attr "type" "fp")])

(define_expand "smin<mode>3"
  [(set (match_operand:SFDF 0 "gpc_reg_operand" "")
	(if_then_else:SFDF (ge (match_operand:SFDF 1 "gpc_reg_operand" "")
			       (match_operand:SFDF 2 "gpc_reg_operand" ""))
			   (match_dup 2)
			   (match_dup 1)))]
  "TARGET_<MODE>_FPR && TARGET_PPC_GFXOPT && !flag_trapping_math"
{
  rs6000_emit_minmax (operands[0], SMIN, operands[1], operands[2]);
  DONE;
})

(define_insn "*smin<mode>3_vsx"
  [(set (match_operand:SFDF 0 "gpc_reg_operand" "=<Ff>,<Fv>")
	(smin:SFDF (match_operand:SFDF 1 "gpc_reg_operand" "%<Ff>,<Fv>")
		   (match_operand:SFDF 2 "gpc_reg_operand" "<Ff>,<Fv>")))]
  "TARGET_<MODE>_FPR && TARGET_VSX"
  "xsmindp %x0,%x1,%x2"
  [(set_attr "type" "fp")])

(define_split
  [(set (match_operand:SFDF 0 "gpc_reg_operand" "")
	(match_operator:SFDF 3 "min_max_operator"
	 [(match_operand:SFDF 1 "gpc_reg_operand" "")
	  (match_operand:SFDF 2 "gpc_reg_operand" "")]))]
  "TARGET_<MODE>_FPR && TARGET_PPC_GFXOPT && !flag_trapping_math
   && !TARGET_VSX"
  [(const_int 0)]
{
  rs6000_emit_minmax (operands[0], GET_CODE (operands[3]), operands[1],
		      operands[2]);
  DONE;
})

(define_split
  [(set (match_operand:SF 0 "gpc_reg_operand" "")
	(match_operator:SF 3 "min_max_operator"
	 [(match_operand:SF 1 "gpc_reg_operand" "")
	  (match_operand:SF 2 "gpc_reg_operand" "")]))]
  "TARGET_PPC_GFXOPT && TARGET_HARD_FLOAT && TARGET_FPRS 
   && TARGET_SINGLE_FLOAT && !flag_trapping_math"
  [(const_int 0)]
  "
{ rs6000_emit_minmax (operands[0], GET_CODE (operands[3]),
		      operands[1], operands[2]);
  DONE;
}")

(define_expand "mov<mode>cc"
   [(set (match_operand:GPR 0 "gpc_reg_operand" "")
	 (if_then_else:GPR (match_operand 1 "comparison_operator" "")
			   (match_operand:GPR 2 "gpc_reg_operand" "")
			   (match_operand:GPR 3 "gpc_reg_operand" "")))]
  "TARGET_ISEL<sel>"
  "
{
  if (rs6000_emit_cmove (operands[0], operands[1], operands[2], operands[3]))
    DONE;
  else
    FAIL;
}")

;; We use the BASE_REGS for the isel input operands because, if rA is
;; 0, the value of 0 is placed in rD upon truth.  Similarly for rB
;; because we may switch the operands and rB may end up being rA.
;;
;; We need 2 patterns: an unsigned and a signed pattern.  We could
;; leave out the mode in operand 4 and use one pattern, but reload can
;; change the mode underneath our feet and then gets confused trying
;; to reload the value.
(define_insn "isel_signed_<mode>"
  [(set (match_operand:GPR 0 "gpc_reg_operand" "=r,r")
	(if_then_else:GPR
	 (match_operator 1 "scc_comparison_operator"
			 [(match_operand:CC 4 "cc_reg_operand" "y,y")
			  (const_int 0)])
	 (match_operand:GPR 2 "reg_or_cint_operand" "O,b")
	 (match_operand:GPR 3 "gpc_reg_operand" "r,r")))]
  "TARGET_ISEL<sel>"
  "*
{ return output_isel (operands); }"
  [(set_attr "type" "isel")
   (set_attr "length" "4")])

(define_insn "isel_unsigned_<mode>"
  [(set (match_operand:GPR 0 "gpc_reg_operand" "=r,r")
	(if_then_else:GPR
	 (match_operator 1 "scc_comparison_operator"
			 [(match_operand:CCUNS 4 "cc_reg_operand" "y,y")
			  (const_int 0)])
	 (match_operand:GPR 2 "reg_or_cint_operand" "O,b")
	 (match_operand:GPR 3 "gpc_reg_operand" "r,r")))]
  "TARGET_ISEL<sel>"
  "*
{ return output_isel (operands); }"
  [(set_attr "type" "isel")
   (set_attr "length" "4")])

;; These patterns can be useful for combine; they let combine know that
;; isel can handle reversed comparisons so long as the operands are
;; registers.

(define_insn "*isel_reversed_signed_<mode>"
  [(set (match_operand:GPR 0 "gpc_reg_operand" "=r")
	(if_then_else:GPR
	 (match_operator 1 "scc_rev_comparison_operator"
			 [(match_operand:CC 4 "cc_reg_operand" "y")
			  (const_int 0)])
	 (match_operand:GPR 2 "gpc_reg_operand" "b")
	 (match_operand:GPR 3 "gpc_reg_operand" "b")))]
  "TARGET_ISEL<sel>"
  "*
{ return output_isel (operands); }"
  [(set_attr "type" "isel")
   (set_attr "length" "4")])

(define_insn "*isel_reversed_unsigned_<mode>"
  [(set (match_operand:GPR 0 "gpc_reg_operand" "=r")
	(if_then_else:GPR
	 (match_operator 1 "scc_rev_comparison_operator"
			 [(match_operand:CCUNS 4 "cc_reg_operand" "y")
			  (const_int 0)])
	 (match_operand:GPR 2 "gpc_reg_operand" "b")
	 (match_operand:GPR 3 "gpc_reg_operand" "b")))]
  "TARGET_ISEL<sel>"
  "*
{ return output_isel (operands); }"
  [(set_attr "type" "isel")
   (set_attr "length" "4")])

(define_expand "movsfcc"
   [(set (match_operand:SF 0 "gpc_reg_operand" "")
	 (if_then_else:SF (match_operand 1 "comparison_operator" "")
			  (match_operand:SF 2 "gpc_reg_operand" "")
			  (match_operand:SF 3 "gpc_reg_operand" "")))]
  "TARGET_PPC_GFXOPT && TARGET_HARD_FLOAT && TARGET_FPRS && TARGET_SINGLE_FLOAT"
  "
{
  if (rs6000_emit_cmove (operands[0], operands[1], operands[2], operands[3]))
    DONE;
  else
    FAIL;
}")

(define_insn "*fselsfsf4"
  [(set (match_operand:SF 0 "gpc_reg_operand" "=f")
	(if_then_else:SF (ge (match_operand:SF 1 "gpc_reg_operand" "f")
			     (match_operand:SF 4 "zero_fp_constant" "F"))
			 (match_operand:SF 2 "gpc_reg_operand" "f")
			 (match_operand:SF 3 "gpc_reg_operand" "f")))]
  "TARGET_PPC_GFXOPT && TARGET_HARD_FLOAT && TARGET_FPRS && TARGET_SINGLE_FLOAT"
  "fsel %0,%1,%2,%3"
  [(set_attr "type" "fp")])

(define_insn "*fseldfsf4"
  [(set (match_operand:SF 0 "gpc_reg_operand" "=f")
	(if_then_else:SF (ge (match_operand:DF 1 "gpc_reg_operand" "d")
			     (match_operand:DF 4 "zero_fp_constant" "F"))
			 (match_operand:SF 2 "gpc_reg_operand" "f")
			 (match_operand:SF 3 "gpc_reg_operand" "f")))]
  "TARGET_PPC_GFXOPT && TARGET_HARD_FLOAT && TARGET_FPRS && TARGET_DOUBLE_FLOAT && TARGET_SINGLE_FLOAT"
  "fsel %0,%1,%2,%3"
  [(set_attr "type" "fp")])

;; The conditional move instructions allow us to perform max and min
;; operations even when

(define_split
  [(set (match_operand:DF 0 "gpc_reg_operand" "")
	(match_operator:DF 3 "min_max_operator"
	 [(match_operand:DF 1 "gpc_reg_operand" "")
	  (match_operand:DF 2 "gpc_reg_operand" "")]))]
  "TARGET_PPC_GFXOPT && TARGET_HARD_FLOAT && TARGET_FPRS && TARGET_DOUBLE_FLOAT 
   && !flag_trapping_math"
  [(const_int 0)]
  "
{ rs6000_emit_minmax (operands[0], GET_CODE (operands[3]),
		      operands[1], operands[2]);
  DONE;
}")

(define_expand "movdfcc"
   [(set (match_operand:DF 0 "gpc_reg_operand" "")
	 (if_then_else:DF (match_operand 1 "comparison_operator" "")
			  (match_operand:DF 2 "gpc_reg_operand" "")
			  (match_operand:DF 3 "gpc_reg_operand" "")))]
  "TARGET_PPC_GFXOPT && TARGET_HARD_FLOAT && TARGET_FPRS && TARGET_DOUBLE_FLOAT"
  "
{
  if (rs6000_emit_cmove (operands[0], operands[1], operands[2], operands[3]))
    DONE;
  else
    FAIL;
}")

(define_insn "*fseldfdf4"
  [(set (match_operand:DF 0 "gpc_reg_operand" "=d")
	(if_then_else:DF (ge (match_operand:DF 1 "gpc_reg_operand" "d")
			     (match_operand:DF 4 "zero_fp_constant" "F"))
			 (match_operand:DF 2 "gpc_reg_operand" "d")
			 (match_operand:DF 3 "gpc_reg_operand" "d")))]
  "TARGET_PPC_GFXOPT && TARGET_HARD_FLOAT && TARGET_FPRS && TARGET_DOUBLE_FLOAT"
  "fsel %0,%1,%2,%3"
  [(set_attr "type" "fp")])

(define_insn "*fselsfdf4"
  [(set (match_operand:DF 0 "gpc_reg_operand" "=d")
	(if_then_else:DF (ge (match_operand:SF 1 "gpc_reg_operand" "f")
			     (match_operand:SF 4 "zero_fp_constant" "F"))
			 (match_operand:DF 2 "gpc_reg_operand" "d")
			 (match_operand:DF 3 "gpc_reg_operand" "d")))]
  "TARGET_PPC_GFXOPT && TARGET_HARD_FLOAT && TARGET_FPRS && TARGET_DOUBLE_FLOAT && TARGET_SINGLE_FLOAT"
  "fsel %0,%1,%2,%3"
  [(set_attr "type" "fp")])

;; Conversions to and from floating-point.

; We don't define lfiwax/lfiwzx with the normal definition, because we
; don't want to support putting SImode in FPR registers.
(define_insn "lfiwax"
  [(set (match_operand:DI 0 "gpc_reg_operand" "=d,wj,!wj")
	(unspec:DI [(match_operand:SI 1 "reg_or_indexed_operand" "Z,Z,r")]
		   UNSPEC_LFIWAX))]
  "TARGET_HARD_FLOAT && TARGET_FPRS && TARGET_DOUBLE_FLOAT && TARGET_LFIWAX"
  "@
   lfiwax %0,%y1
   lxsiwax %x0,%y1
   mtvsrwa %x0,%1"
  [(set_attr "type" "fpload,fpload,mffgpr")])

; This split must be run before register allocation because it allocates the
; memory slot that is needed to move values to/from the FPR.  We don't allocate
; it earlier to allow for the combiner to merge insns together where it might
; not be needed and also in case the insns are deleted as dead code.

(define_insn_and_split "floatsi<mode>2_lfiwax"
  [(set (match_operand:SFDF 0 "gpc_reg_operand" "=<Fv>")
	(float:SFDF (match_operand:SI 1 "nonimmediate_operand" "r")))
   (clobber (match_scratch:DI 2 "=wj"))]
  "TARGET_HARD_FLOAT && TARGET_FPRS && TARGET_DOUBLE_FLOAT && TARGET_LFIWAX
   && <SI_CONVERT_FP> && can_create_pseudo_p ()"
  "#"
  ""
  [(pc)]
  "
{
  rtx dest = operands[0];
  rtx src = operands[1];
  rtx tmp;

  if (!MEM_P (src) && TARGET_POWERPC64
      && (TARGET_MFPGPR || TARGET_DIRECT_MOVE))
    tmp = convert_to_mode (DImode, src, false);
  else
    {
      tmp = operands[2];
      if (GET_CODE (tmp) == SCRATCH)
	tmp = gen_reg_rtx (DImode);
      if (MEM_P (src))
	{
	  src = rs6000_address_for_fpconvert (src);
	  emit_insn (gen_lfiwax (tmp, src));
	}
      else
	{
	  rtx stack = rs6000_allocate_stack_temp (SImode, false, true);
	  emit_move_insn (stack, src);
	  emit_insn (gen_lfiwax (tmp, stack));
	}
    }
  emit_insn (gen_floatdi<mode>2 (dest, tmp));
  DONE;
}"
  [(set_attr "length" "12")
   (set_attr "type" "fpload")])

(define_insn_and_split "floatsi<mode>2_lfiwax_mem"
  [(set (match_operand:SFDF 0 "gpc_reg_operand" "=<Ff>,<Fa>")
	(float:SFDF
	 (sign_extend:DI
	  (match_operand:SI 1 "indexed_or_indirect_operand" "Z,Z"))))
   (clobber (match_scratch:DI 2 "=0,d"))]
  "TARGET_HARD_FLOAT && TARGET_FPRS && TARGET_DOUBLE_FLOAT && TARGET_LFIWAX
   && <SI_CONVERT_FP>"
  "#"
  ""
  [(pc)]
  "
{
  operands[1] = rs6000_address_for_fpconvert (operands[1]);
  if (GET_CODE (operands[2]) == SCRATCH)
    operands[2] = gen_reg_rtx (DImode);
  emit_insn (gen_lfiwax (operands[2], operands[1]));
  emit_insn (gen_floatdi<mode>2 (operands[0], operands[2]));
  DONE;
}"
  [(set_attr "length" "8")
   (set_attr "type" "fpload")])

(define_insn "lfiwzx"
  [(set (match_operand:DI 0 "gpc_reg_operand" "=d,wj,!wj")
	(unspec:DI [(match_operand:SI 1 "reg_or_indexed_operand" "Z,Z,r")]
		   UNSPEC_LFIWZX))]
  "TARGET_HARD_FLOAT && TARGET_FPRS && TARGET_DOUBLE_FLOAT && TARGET_LFIWZX"
  "@
   lfiwzx %0,%y1
   lxsiwzx %x0,%y1
   mtvsrwz %x0,%1"
  [(set_attr "type" "fpload,fpload,mftgpr")])

(define_insn_and_split "floatunssi<mode>2_lfiwzx"
  [(set (match_operand:SFDF 0 "gpc_reg_operand" "=<Fv>")
	(unsigned_float:SFDF (match_operand:SI 1 "nonimmediate_operand" "r")))
   (clobber (match_scratch:DI 2 "=wj"))]
  "TARGET_HARD_FLOAT && TARGET_FPRS && TARGET_DOUBLE_FLOAT && TARGET_LFIWZX
   && <SI_CONVERT_FP>"
  "#"
  ""
  [(pc)]
  "
{
  rtx dest = operands[0];
  rtx src = operands[1];
  rtx tmp;

  if (!MEM_P (src) && TARGET_POWERPC64
      && (TARGET_MFPGPR || TARGET_DIRECT_MOVE))
    tmp = convert_to_mode (DImode, src, true);
  else
    {
      tmp = operands[2];
      if (GET_CODE (tmp) == SCRATCH)
	tmp = gen_reg_rtx (DImode);
      if (MEM_P (src))
	{
	  src = rs6000_address_for_fpconvert (src);
	  emit_insn (gen_lfiwzx (tmp, src));
	}
      else
	{
	  rtx stack = rs6000_allocate_stack_temp (SImode, false, true);
	  emit_move_insn (stack, src);
	  emit_insn (gen_lfiwzx (tmp, stack));
	}
    }
  emit_insn (gen_floatdi<mode>2 (dest, tmp));
  DONE;
}"
  [(set_attr "length" "12")
   (set_attr "type" "fpload")])

(define_insn_and_split "floatunssi<mode>2_lfiwzx_mem"
  [(set (match_operand:SFDF 0 "gpc_reg_operand" "=<Ff>,<Fa>")
	(unsigned_float:SFDF
	 (zero_extend:DI
	  (match_operand:SI 1 "indexed_or_indirect_operand" "Z,Z"))))
   (clobber (match_scratch:DI 2 "=0,d"))]
  "TARGET_HARD_FLOAT && TARGET_FPRS && TARGET_DOUBLE_FLOAT && TARGET_LFIWZX
   && <SI_CONVERT_FP>"
  "#"
  ""
  [(pc)]
  "
{
  operands[1] = rs6000_address_for_fpconvert (operands[1]);
  if (GET_CODE (operands[2]) == SCRATCH)
    operands[2] = gen_reg_rtx (DImode);
  emit_insn (gen_lfiwzx (operands[2], operands[1]));
  emit_insn (gen_floatdi<mode>2 (operands[0], operands[2]));
  DONE;
}"
  [(set_attr "length" "8")
   (set_attr "type" "fpload")])

; For each of these conversions, there is a define_expand, a define_insn
; with a '#' template, and a define_split (with C code).  The idea is
; to allow constant folding with the template of the define_insn,
; then to have the insns split later (between sched1 and final).

(define_expand "floatsidf2"
  [(parallel [(set (match_operand:DF 0 "gpc_reg_operand" "")
		   (float:DF (match_operand:SI 1 "nonimmediate_operand" "")))
	      (use (match_dup 2))
	      (use (match_dup 3))
	      (clobber (match_dup 4))
	      (clobber (match_dup 5))
	      (clobber (match_dup 6))])]
  "TARGET_HARD_FLOAT 
   && ((TARGET_FPRS && TARGET_DOUBLE_FLOAT) || TARGET_E500_DOUBLE)"
  "
{
  if (TARGET_E500_DOUBLE)
    {
      if (!REG_P (operands[1]))
	operands[1] = force_reg (SImode, operands[1]);
      emit_insn (gen_spe_floatsidf2 (operands[0], operands[1]));
      DONE;
    }
  else if (TARGET_LFIWAX && TARGET_FCFID)
    {
      emit_insn (gen_floatsidf2_lfiwax (operands[0], operands[1]));
      DONE;
    }
  else if (TARGET_FCFID)
    {
      rtx dreg = operands[1];
      if (!REG_P (dreg))
	dreg = force_reg (SImode, dreg);
      dreg = convert_to_mode (DImode, dreg, false);
      emit_insn (gen_floatdidf2 (operands[0], dreg));
      DONE;
    }

  if (!REG_P (operands[1]))
    operands[1] = force_reg (SImode, operands[1]);
  operands[2] = force_reg (SImode, GEN_INT (0x43300000));
  operands[3] = force_reg (DFmode, CONST_DOUBLE_ATOF (\"4503601774854144\", DFmode));
  operands[4] = rs6000_allocate_stack_temp (DFmode, true, false);
  operands[5] = gen_reg_rtx (DFmode);
  operands[6] = gen_reg_rtx (SImode);
}")

(define_insn_and_split "*floatsidf2_internal"
  [(set (match_operand:DF 0 "gpc_reg_operand" "=&d")
	(float:DF (match_operand:SI 1 "gpc_reg_operand" "r")))
   (use (match_operand:SI 2 "gpc_reg_operand" "r"))
   (use (match_operand:DF 3 "gpc_reg_operand" "d"))
   (clobber (match_operand:DF 4 "offsettable_mem_operand" "=o"))
   (clobber (match_operand:DF 5 "gpc_reg_operand" "=&d"))
   (clobber (match_operand:SI 6 "gpc_reg_operand" "=&r"))]
  "! TARGET_FCFID && TARGET_HARD_FLOAT && TARGET_FPRS && TARGET_DOUBLE_FLOAT"
  "#"
  ""
  [(pc)]
  "
{
  rtx lowword, highword;
  gcc_assert (MEM_P (operands[4]));
  highword = adjust_address (operands[4], SImode, 0);
  lowword = adjust_address (operands[4], SImode, 4);
  if (! WORDS_BIG_ENDIAN)
    std::swap (lowword, highword);

  emit_insn (gen_xorsi3 (operands[6], operands[1],
			 GEN_INT (~ (HOST_WIDE_INT) 0x7fffffff)));
  emit_move_insn (lowword, operands[6]);
  emit_move_insn (highword, operands[2]);
  emit_move_insn (operands[5], operands[4]);
  emit_insn (gen_subdf3 (operands[0], operands[5], operands[3]));
  DONE;
}"
  [(set_attr "length" "24")
   (set_attr "type" "fp")])

;; If we don't have a direct conversion to single precision, don't enable this
;; conversion for 32-bit without fast math, because we don't have the insn to
;; generate the fixup swizzle to avoid double rounding problems.
(define_expand "floatunssisf2"
  [(set (match_operand:SF 0 "gpc_reg_operand" "")
        (unsigned_float:SF (match_operand:SI 1 "nonimmediate_operand" "")))]
  "TARGET_HARD_FLOAT && TARGET_SINGLE_FLOAT
   && (!TARGET_FPRS
       || (TARGET_FPRS
	   && ((TARGET_FCFIDUS && TARGET_LFIWZX)
	       || (TARGET_DOUBLE_FLOAT && TARGET_FCFID
		   && (TARGET_POWERPC64 || flag_unsafe_math_optimizations)))))"
  "
{
  if (!TARGET_FPRS)
    {
      if (!REG_P (operands[1]))
	operands[1] = force_reg (SImode, operands[1]);
    }
  else if (TARGET_LFIWZX && TARGET_FCFIDUS)
    {
      emit_insn (gen_floatunssisf2_lfiwzx (operands[0], operands[1]));
      DONE;
    }
  else
    {
      rtx dreg = operands[1];
      if (!REG_P (dreg))
	dreg = force_reg (SImode, dreg);
      dreg = convert_to_mode (DImode, dreg, true);
      emit_insn (gen_floatdisf2 (operands[0], dreg));
      DONE;
    }
}")

(define_expand "floatunssidf2"
  [(parallel [(set (match_operand:DF 0 "gpc_reg_operand" "")
		   (unsigned_float:DF (match_operand:SI 1 "nonimmediate_operand" "")))
	      (use (match_dup 2))
	      (use (match_dup 3))
	      (clobber (match_dup 4))
	      (clobber (match_dup 5))])]
  "TARGET_HARD_FLOAT
   && ((TARGET_FPRS && TARGET_DOUBLE_FLOAT) || TARGET_E500_DOUBLE)"
  "
{
  if (TARGET_E500_DOUBLE)
    {
      if (!REG_P (operands[1]))
	operands[1] = force_reg (SImode, operands[1]);
      emit_insn (gen_spe_floatunssidf2 (operands[0], operands[1]));
      DONE;
    }
  else if (TARGET_LFIWZX && TARGET_FCFID)
    {
      emit_insn (gen_floatunssidf2_lfiwzx (operands[0], operands[1]));
      DONE;
    }
  else if (TARGET_FCFID)
    {
      rtx dreg = operands[1];
      if (!REG_P (dreg))
	dreg = force_reg (SImode, dreg);
      dreg = convert_to_mode (DImode, dreg, true);
      emit_insn (gen_floatdidf2 (operands[0], dreg));
      DONE;
    }

  if (!REG_P (operands[1]))
    operands[1] = force_reg (SImode, operands[1]);
  operands[2] = force_reg (SImode, GEN_INT (0x43300000));
  operands[3] = force_reg (DFmode, CONST_DOUBLE_ATOF (\"4503599627370496\", DFmode));
  operands[4] = rs6000_allocate_stack_temp (DFmode, true, false);
  operands[5] = gen_reg_rtx (DFmode);
}")

(define_insn_and_split "*floatunssidf2_internal"
  [(set (match_operand:DF 0 "gpc_reg_operand" "=&d")
	(unsigned_float:DF (match_operand:SI 1 "gpc_reg_operand" "r")))
   (use (match_operand:SI 2 "gpc_reg_operand" "r"))
   (use (match_operand:DF 3 "gpc_reg_operand" "d"))
   (clobber (match_operand:DF 4 "offsettable_mem_operand" "=o"))
   (clobber (match_operand:DF 5 "gpc_reg_operand" "=&d"))]
  "! TARGET_FCFIDU && TARGET_HARD_FLOAT && TARGET_FPRS && TARGET_DOUBLE_FLOAT
   && !(TARGET_FCFID && TARGET_POWERPC64)"
  "#"
  ""
  [(pc)]
  "
{
  rtx lowword, highword;
  gcc_assert (MEM_P (operands[4]));
  highword = adjust_address (operands[4], SImode, 0);
  lowword = adjust_address (operands[4], SImode, 4);
  if (! WORDS_BIG_ENDIAN)
    std::swap (lowword, highword);

  emit_move_insn (lowword, operands[1]);
  emit_move_insn (highword, operands[2]);
  emit_move_insn (operands[5], operands[4]);
  emit_insn (gen_subdf3 (operands[0], operands[5], operands[3]));
  DONE;
}"
  [(set_attr "length" "20")
   (set_attr "type" "fp")])

(define_expand "fix_trunc<mode>si2"
  [(set (match_operand:SI 0 "gpc_reg_operand" "")
	(fix:SI (match_operand:SFDF 1 "gpc_reg_operand" "")))]
  "TARGET_HARD_FLOAT && ((TARGET_FPRS && <TARGET_FLOAT>) || <E500_CONVERT>)"
  "
{
  if (!<E500_CONVERT>)
    {
      rtx tmp, stack;

      if (TARGET_STFIWX)
	emit_insn (gen_fix_trunc<mode>si2_stfiwx (operands[0], operands[1]));
      else
	{
	  tmp = gen_reg_rtx (DImode);
	  stack = rs6000_allocate_stack_temp (DImode, true, false);
	  emit_insn (gen_fix_trunc<mode>si2_internal (operands[0], operands[1],
						      tmp, stack));
	}
      DONE;
    }
}")

; Like the convert to float patterns, this insn must be split before
; register allocation so that it can allocate the memory slot if it
; needed
(define_insn_and_split "fix_trunc<mode>si2_stfiwx"
  [(set (match_operand:SI 0 "nonimmediate_operand" "=rm")
	(fix:SI (match_operand:SFDF 1 "gpc_reg_operand" "d")))
   (clobber (match_scratch:DI 2 "=d"))]
  "TARGET_HARD_FLOAT && TARGET_FPRS && TARGET_DOUBLE_FLOAT
   && (<MODE>mode != SFmode || TARGET_SINGLE_FLOAT)
   && TARGET_STFIWX && can_create_pseudo_p ()"
  "#"
  ""
  [(pc)]
{
  rtx dest = operands[0];
  rtx src = operands[1];
  rtx tmp = operands[2];

  if (GET_CODE (tmp) == SCRATCH)
    tmp = gen_reg_rtx (DImode);

  emit_insn (gen_fctiwz_<mode> (tmp, src));
  if (MEM_P (dest))
    {
      dest = rs6000_address_for_fpconvert (dest);
      emit_insn (gen_stfiwx (dest, tmp));
      DONE;
    }
  else if (TARGET_POWERPC64 && (TARGET_MFPGPR || TARGET_DIRECT_MOVE))
    {
      dest = gen_lowpart (DImode, dest);
      emit_move_insn (dest, tmp);
      DONE;
    }
  else
    {
      rtx stack = rs6000_allocate_stack_temp (SImode, false, true);
      emit_insn (gen_stfiwx (stack, tmp));
      emit_move_insn (dest, stack);
      DONE;
    }
}
  [(set_attr "length" "12")
   (set_attr "type" "fp")])

(define_insn_and_split "fix_trunc<mode>si2_internal"
  [(set (match_operand:SI 0 "gpc_reg_operand" "=r,?r")
	(fix:SI (match_operand:SFDF 1 "gpc_reg_operand" "d,<rreg>")))
   (clobber (match_operand:DI 2 "gpc_reg_operand" "=1,d"))
   (clobber (match_operand:DI 3 "offsettable_mem_operand" "=o,o"))]
  "TARGET_HARD_FLOAT && TARGET_FPRS && TARGET_DOUBLE_FLOAT"
  "#"
  ""
  [(pc)]
  "
{
  rtx lowword;
  gcc_assert (MEM_P (operands[3]));
  lowword = adjust_address (operands[3], SImode, WORDS_BIG_ENDIAN ? 4 : 0);

  emit_insn (gen_fctiwz_<mode> (operands[2], operands[1]));
  emit_move_insn (operands[3], operands[2]);
  emit_move_insn (operands[0], lowword);
  DONE;
}"
  [(set_attr "length" "16")
   (set_attr "type" "fp")])

(define_expand "fix_trunc<mode>di2"
  [(set (match_operand:DI 0 "gpc_reg_operand" "")
	(fix:DI (match_operand:SFDF 1 "gpc_reg_operand" "")))]
  "TARGET_HARD_FLOAT && TARGET_DOUBLE_FLOAT && TARGET_FPRS
   && TARGET_FCFID"
  "")

(define_insn "*fix_trunc<mode>di2_fctidz"
  [(set (match_operand:DI 0 "gpc_reg_operand" "=d,wi")
	(fix:DI (match_operand:SFDF 1 "gpc_reg_operand" "<Ff>,<Fa>")))]
  "TARGET_HARD_FLOAT && TARGET_DOUBLE_FLOAT && TARGET_FPRS
    && TARGET_FCFID"
  "@
   fctidz %0,%1
   xscvdpsxds %x0,%x1"
  [(set_attr "type" "fp")])

(define_expand "fixuns_trunc<mode>si2"
  [(set (match_operand:SI 0 "gpc_reg_operand" "")
	(unsigned_fix:SI (match_operand:SFDF 1 "gpc_reg_operand" "")))]
  "TARGET_HARD_FLOAT
   && ((TARGET_FPRS && <TARGET_FLOAT> && TARGET_FCTIWUZ && TARGET_STFIWX)
       || <E500_CONVERT>)"
  "
{
  if (!<E500_CONVERT>)
    {
      emit_insn (gen_fixuns_trunc<mode>si2_stfiwx (operands[0], operands[1]));
      DONE;
    }
}")

(define_insn_and_split "fixuns_trunc<mode>si2_stfiwx"
  [(set (match_operand:SI 0 "nonimmediate_operand" "=rm")
	(unsigned_fix:SI (match_operand:SFDF 1 "gpc_reg_operand" "d")))
   (clobber (match_scratch:DI 2 "=d"))]
  "TARGET_HARD_FLOAT && TARGET_FPRS && <TARGET_FLOAT> && TARGET_FCTIWUZ
   && TARGET_STFIWX && can_create_pseudo_p ()"
  "#"
  ""
  [(pc)]
{
  rtx dest = operands[0];
  rtx src = operands[1];
  rtx tmp = operands[2];

  if (GET_CODE (tmp) == SCRATCH)
    tmp = gen_reg_rtx (DImode);

  emit_insn (gen_fctiwuz_<mode> (tmp, src));
  if (MEM_P (dest))
    {
      dest = rs6000_address_for_fpconvert (dest);
      emit_insn (gen_stfiwx (dest, tmp));
      DONE;
    }
  else if (TARGET_POWERPC64 && (TARGET_MFPGPR || TARGET_DIRECT_MOVE))
    {
      dest = gen_lowpart (DImode, dest);
      emit_move_insn (dest, tmp);
      DONE;
    }
  else
    {
      rtx stack = rs6000_allocate_stack_temp (SImode, false, true);
      emit_insn (gen_stfiwx (stack, tmp));
      emit_move_insn (dest, stack);
      DONE;
    }
}
  [(set_attr "length" "12")
   (set_attr "type" "fp")])

(define_expand "fixuns_trunc<mode>di2"
  [(set (match_operand:DI 0 "register_operand" "")
	(unsigned_fix:DI (match_operand:SFDF 1 "register_operand" "")))]
  "TARGET_HARD_FLOAT && (TARGET_FCTIDUZ || VECTOR_UNIT_VSX_P (<MODE>mode))"
  "")

(define_insn "*fixuns_trunc<mode>di2_fctiduz"
  [(set (match_operand:DI 0 "gpc_reg_operand" "=d,wi")
	(unsigned_fix:DI (match_operand:SFDF 1 "gpc_reg_operand" "<Ff>,<Fa>")))]
  "TARGET_HARD_FLOAT && TARGET_DOUBLE_FLOAT && TARGET_FPRS
    && TARGET_FCTIDUZ"
  "@
   fctiduz %0,%1
   xscvdpuxds %x0,%x1"
  [(set_attr "type" "fp")])

; Here, we use (set (reg) (unspec:DI [(fix:SI ...)] UNSPEC_FCTIWZ))
; rather than (set (subreg:SI (reg)) (fix:SI ...))
; because the first makes it clear that operand 0 is not live
; before the instruction.
(define_insn "fctiwz_<mode>"
  [(set (match_operand:DI 0 "gpc_reg_operand" "=d,wi")
	(unspec:DI [(fix:SI (match_operand:SFDF 1 "gpc_reg_operand" "<Ff>,<Fv>"))]
		   UNSPEC_FCTIWZ))]
  "TARGET_HARD_FLOAT && TARGET_FPRS && TARGET_DOUBLE_FLOAT"
  "@
   fctiwz %0,%1
   xscvdpsxws %x0,%x1"
  [(set_attr "type" "fp")])

(define_insn "fctiwuz_<mode>"
  [(set (match_operand:DI 0 "gpc_reg_operand" "=d,wi")
	(unspec:DI [(unsigned_fix:SI
		     (match_operand:SFDF 1 "gpc_reg_operand" "<Ff>,<Fv>"))]
		   UNSPEC_FCTIWUZ))]
  "TARGET_HARD_FLOAT && TARGET_FPRS && <TARGET_FLOAT> && TARGET_FCTIWUZ"
  "@
   fctiwuz %0,%1
   xscvdpuxws %x0,%x1"
  [(set_attr "type" "fp")])

;; Only optimize (float (fix x)) -> frz if we are in fast-math mode, since
;; since the friz instruction does not truncate the value if the floating
;; point value is < LONG_MIN or > LONG_MAX.
(define_insn "*friz"
  [(set (match_operand:DF 0 "gpc_reg_operand" "=d,ws")
	(float:DF (fix:DI (match_operand:DF 1 "gpc_reg_operand" "d,ws"))))]
  "TARGET_HARD_FLOAT && TARGET_FPRS && TARGET_DOUBLE_FLOAT && TARGET_FPRND
   && flag_unsafe_math_optimizations && !flag_trapping_math && TARGET_FRIZ"
  "@
   friz %0,%1
   xsrdpiz %x0,%x1"
  [(set_attr "type" "fp")])

;; Since FCTIWZ doesn't sign extend the upper bits, we have to do a store and a
;; load to properly sign extend the value, but at least doing a store, load
;; into a GPR to sign extend, a store from the GPR and a load back into the FPR
;; if we have 32-bit memory ops
(define_insn_and_split "*round32<mode>2_fprs"
  [(set (match_operand:SFDF 0 "gpc_reg_operand" "=d")
	(float:SFDF
	 (fix:SI (match_operand:SFDF 1 "gpc_reg_operand" "d"))))
   (clobber (match_scratch:DI 2 "=d"))
   (clobber (match_scratch:DI 3 "=d"))]
  "TARGET_HARD_FLOAT && TARGET_FPRS && TARGET_DOUBLE_FLOAT
   && <SI_CONVERT_FP> && TARGET_LFIWAX && TARGET_STFIWX && TARGET_FCFID
   && can_create_pseudo_p ()"
  "#"
  ""
  [(pc)]
{
  rtx dest = operands[0];
  rtx src = operands[1];
  rtx tmp1 = operands[2];
  rtx tmp2 = operands[3];
  rtx stack = rs6000_allocate_stack_temp (SImode, false, true);

  if (GET_CODE (tmp1) == SCRATCH)
    tmp1 = gen_reg_rtx (DImode);
  if (GET_CODE (tmp2) == SCRATCH)
    tmp2 = gen_reg_rtx (DImode);

  emit_insn (gen_fctiwz_<mode> (tmp1, src));
  emit_insn (gen_stfiwx (stack, tmp1));
  emit_insn (gen_lfiwax (tmp2, stack));
  emit_insn (gen_floatdi<mode>2 (dest, tmp2));
  DONE;
}
  [(set_attr "type" "fpload")
   (set_attr "length" "16")])

(define_insn_and_split "*roundu32<mode>2_fprs"
  [(set (match_operand:SFDF 0 "gpc_reg_operand" "=d")
	(unsigned_float:SFDF
	 (unsigned_fix:SI (match_operand:SFDF 1 "gpc_reg_operand" "d"))))
   (clobber (match_scratch:DI 2 "=d"))
   (clobber (match_scratch:DI 3 "=d"))]
  "TARGET_HARD_FLOAT && TARGET_FPRS && TARGET_DOUBLE_FLOAT
   && TARGET_LFIWZX && TARGET_STFIWX && TARGET_FCFIDU
   && can_create_pseudo_p ()"
  "#"
  ""
  [(pc)]
{
  rtx dest = operands[0];
  rtx src = operands[1];
  rtx tmp1 = operands[2];
  rtx tmp2 = operands[3];
  rtx stack = rs6000_allocate_stack_temp (SImode, false, true);

  if (GET_CODE (tmp1) == SCRATCH)
    tmp1 = gen_reg_rtx (DImode);
  if (GET_CODE (tmp2) == SCRATCH)
    tmp2 = gen_reg_rtx (DImode);

  emit_insn (gen_fctiwuz_<mode> (tmp1, src));
  emit_insn (gen_stfiwx (stack, tmp1));
  emit_insn (gen_lfiwzx (tmp2, stack));
  emit_insn (gen_floatdi<mode>2 (dest, tmp2));
  DONE;
}
  [(set_attr "type" "fpload")
   (set_attr "length" "16")])

;; No VSX equivalent to fctid
(define_insn "lrint<mode>di2"
  [(set (match_operand:DI 0 "gpc_reg_operand" "=d")
	(unspec:DI [(match_operand:SFDF 1 "gpc_reg_operand" "<rreg2>")]
		   UNSPEC_FCTID))]
  "TARGET_<MODE>_FPR && TARGET_FPRND"
  "fctid %0,%1"
  [(set_attr "type" "fp")])

(define_insn "btrunc<mode>2"
  [(set (match_operand:SFDF 0 "gpc_reg_operand" "=<Ff>,<Fv>")
	(unspec:SFDF [(match_operand:SFDF 1 "gpc_reg_operand" "<Ff>,<Fv>")]
		     UNSPEC_FRIZ))]
  "TARGET_<MODE>_FPR && TARGET_FPRND"
  "@
   friz %0,%1
   xsrdpiz %x0,%x1"
  [(set_attr "type" "fp")
   (set_attr "fp_type" "fp_addsub_<Fs>")])

(define_insn "ceil<mode>2"
  [(set (match_operand:SFDF 0 "gpc_reg_operand" "=<Ff>,<Fv>")
	(unspec:SFDF [(match_operand:SFDF 1 "gpc_reg_operand" "<Ff>,<Fv>")]
		     UNSPEC_FRIP))]
  "TARGET_<MODE>_FPR && TARGET_FPRND"
  "@
   frip %0,%1
   xsrdpip %x0,%x1"
  [(set_attr "type" "fp")
   (set_attr "fp_type" "fp_addsub_<Fs>")])

(define_insn "floor<mode>2"
  [(set (match_operand:SFDF 0 "gpc_reg_operand" "=<Ff>,<Fv>")
	(unspec:SFDF [(match_operand:SFDF 1 "gpc_reg_operand" "<Ff>,<Fv>")]
		     UNSPEC_FRIM))]
  "TARGET_<MODE>_FPR && TARGET_FPRND"
  "@
   frim %0,%1
   xsrdpim %x0,%x1"
  [(set_attr "type" "fp")
   (set_attr "fp_type" "fp_addsub_<Fs>")])

;; No VSX equivalent to frin
(define_insn "round<mode>2"
  [(set (match_operand:SFDF 0 "gpc_reg_operand" "=<rreg2>")
	(unspec:SFDF [(match_operand:SFDF 1 "gpc_reg_operand" "<rreg2>")]
		     UNSPEC_FRIN))]
  "TARGET_<MODE>_FPR && TARGET_FPRND"
  "frin %0,%1"
  [(set_attr "type" "fp")
   (set_attr "fp_type" "fp_addsub_<Fs>")])

; An UNSPEC is used so we don't have to support SImode in FP registers.
(define_insn "stfiwx"
  [(set (match_operand:SI 0 "memory_operand" "=Z")
	(unspec:SI [(match_operand:DI 1 "gpc_reg_operand" "d")]
		   UNSPEC_STFIWX))]
  "TARGET_PPC_GFXOPT"
  "stfiwx %1,%y0"
  [(set_attr "type" "fpstore")])

;; If we don't have a direct conversion to single precision, don't enable this
;; conversion for 32-bit without fast math, because we don't have the insn to
;; generate the fixup swizzle to avoid double rounding problems.
(define_expand "floatsisf2"
  [(set (match_operand:SF 0 "gpc_reg_operand" "")
        (float:SF (match_operand:SI 1 "nonimmediate_operand" "")))]
  "TARGET_HARD_FLOAT && TARGET_SINGLE_FLOAT
   && (!TARGET_FPRS
       || (TARGET_FPRS
	   && ((TARGET_FCFIDS && TARGET_LFIWAX)
	       || (TARGET_DOUBLE_FLOAT && TARGET_FCFID
		   && (TARGET_POWERPC64 || flag_unsafe_math_optimizations)))))"
  "
{
  if (!TARGET_FPRS)
    {
      if (!REG_P (operands[1]))
	operands[1] = force_reg (SImode, operands[1]);
    }
  else if (TARGET_FCFIDS && TARGET_LFIWAX)
    {
      emit_insn (gen_floatsisf2_lfiwax (operands[0], operands[1]));
      DONE;
    }
  else if (TARGET_FCFID && TARGET_LFIWAX)
    {
      rtx dfreg = gen_reg_rtx (DFmode);
      emit_insn (gen_floatsidf2_lfiwax (dfreg, operands[1]));
      emit_insn (gen_truncdfsf2 (operands[0], dfreg));
      DONE;
    }
  else
    {
      rtx dreg = operands[1];
      if (!REG_P (dreg))
	dreg = force_reg (SImode, dreg);
      dreg = convert_to_mode (DImode, dreg, false);
      emit_insn (gen_floatdisf2 (operands[0], dreg));
      DONE;
    }
}")

(define_expand "floatdidf2"
  [(set (match_operand:DF 0 "gpc_reg_operand" "")
	(float:DF (match_operand:DI 1 "gpc_reg_operand" "")))]
  "TARGET_FCFID && TARGET_HARD_FLOAT && TARGET_DOUBLE_FLOAT && TARGET_FPRS"
  "")

(define_insn "*floatdidf2_fpr"
  [(set (match_operand:DF 0 "gpc_reg_operand" "=d,ws")
	(float:DF (match_operand:DI 1 "gpc_reg_operand" "d,wi")))]
  "TARGET_FCFID && TARGET_HARD_FLOAT && TARGET_DOUBLE_FLOAT && TARGET_FPRS"
  "@
   fcfid %0,%1
   xscvsxddp %x0,%x1"
  [(set_attr "type" "fp")])

; Allow the combiner to merge source memory operands to the conversion so that
; the optimizer/register allocator doesn't try to load the value too early in a
; GPR and then use store/load to move it to a FPR and suffer from a store-load
; hit.  We will split after reload to avoid the trip through the GPRs

(define_insn_and_split "*floatdidf2_mem"
  [(set (match_operand:DF 0 "gpc_reg_operand" "=d,ws")
	(float:DF (match_operand:DI 1 "memory_operand" "m,Z")))
   (clobber (match_scratch:DI 2 "=d,wi"))]
  "TARGET_HARD_FLOAT && TARGET_DOUBLE_FLOAT && TARGET_FPRS && TARGET_FCFID"
  "#"
  "&& reload_completed"
  [(set (match_dup 2) (match_dup 1))
   (set (match_dup 0) (float:DF (match_dup 2)))]
  ""
  [(set_attr "length" "8")
   (set_attr "type" "fpload")])

(define_expand "floatunsdidf2"
  [(set (match_operand:DF 0 "gpc_reg_operand" "")
	(unsigned_float:DF
	 (match_operand:DI 1 "gpc_reg_operand" "")))]
  "TARGET_HARD_FLOAT && TARGET_FCFIDU"
  "")

(define_insn "*floatunsdidf2_fcfidu"
  [(set (match_operand:DF 0 "gpc_reg_operand" "=d,ws")
	(unsigned_float:DF (match_operand:DI 1 "gpc_reg_operand" "d,wi")))]
  "TARGET_HARD_FLOAT && TARGET_FCFIDU"
  "@
   fcfidu %0,%1
   xscvuxddp %x0,%x1"
  [(set_attr "type" "fp")
   (set_attr "length" "4")])

(define_insn_and_split "*floatunsdidf2_mem"
  [(set (match_operand:DF 0 "gpc_reg_operand" "=d,ws")
	(unsigned_float:DF (match_operand:DI 1 "memory_operand" "m,Z")))
   (clobber (match_scratch:DI 2 "=d,wi"))]
  "TARGET_HARD_FLOAT && (TARGET_FCFIDU || VECTOR_UNIT_VSX_P (DFmode))"
  "#"
  "&& reload_completed"
  [(set (match_dup 2) (match_dup 1))
   (set (match_dup 0) (unsigned_float:DF (match_dup 2)))]
  ""
  [(set_attr "length" "8")
   (set_attr "type" "fpload")])

(define_expand "floatdisf2"
  [(set (match_operand:SF 0 "gpc_reg_operand" "")
        (float:SF (match_operand:DI 1 "gpc_reg_operand" "")))]
  "TARGET_FCFID && TARGET_HARD_FLOAT && TARGET_FPRS && TARGET_SINGLE_FLOAT
   && (TARGET_FCFIDS || TARGET_POWERPC64 || flag_unsafe_math_optimizations)"
  "
{
  if (!TARGET_FCFIDS)
    {
      rtx val = operands[1];
      if (!flag_unsafe_math_optimizations)
	{
	  rtx label = gen_label_rtx ();
	  val = gen_reg_rtx (DImode);
	  emit_insn (gen_floatdisf2_internal2 (val, operands[1], label));
	  emit_label (label);
	}
      emit_insn (gen_floatdisf2_internal1 (operands[0], val));
      DONE;
    }
}")

(define_insn "floatdisf2_fcfids"
  [(set (match_operand:SF 0 "gpc_reg_operand" "=f,wy")
	(float:SF (match_operand:DI 1 "gpc_reg_operand" "d,wi")))]
  "TARGET_HARD_FLOAT && TARGET_FPRS && TARGET_SINGLE_FLOAT
   && TARGET_DOUBLE_FLOAT && TARGET_FCFIDS"
  "@
   fcfids %0,%1
   xscvsxdsp %x0,%x1"
  [(set_attr "type" "fp")])

(define_insn_and_split "*floatdisf2_mem"
  [(set (match_operand:SF 0 "gpc_reg_operand" "=f,wy,wy")
	(float:SF (match_operand:DI 1 "memory_operand" "m,m,Z")))
   (clobber (match_scratch:DI 2 "=d,d,wi"))]
  "TARGET_HARD_FLOAT && TARGET_FPRS && TARGET_SINGLE_FLOAT
   && TARGET_DOUBLE_FLOAT && TARGET_FCFIDS"
  "#"
  "&& reload_completed"
  [(pc)]
  "
{
  emit_move_insn (operands[2], operands[1]);
  emit_insn (gen_floatdisf2_fcfids (operands[0], operands[2]));
  DONE;
}"
  [(set_attr "length" "8")])

;; This is not IEEE compliant if rounding mode is "round to nearest".
;; If the DI->DF conversion is inexact, then it's possible to suffer
;; from double rounding.
;; Instead of creating a new cpu type for two FP operations, just use fp
(define_insn_and_split "floatdisf2_internal1"
  [(set (match_operand:SF 0 "gpc_reg_operand" "=f")
        (float:SF (match_operand:DI 1 "gpc_reg_operand" "d")))
   (clobber (match_scratch:DF 2 "=d"))]
  "TARGET_FCFID && TARGET_HARD_FLOAT && TARGET_FPRS && TARGET_SINGLE_FLOAT
   && !TARGET_FCFIDS"
  "#"
  "&& reload_completed"
  [(set (match_dup 2)
        (float:DF (match_dup 1)))
   (set (match_dup 0)
        (float_truncate:SF (match_dup 2)))]
  ""
  [(set_attr "length" "8")
   (set_attr "type" "fp")])

;; Twiddles bits to avoid double rounding.
;; Bits that might be truncated when converting to DFmode are replaced
;; by a bit that won't be lost at that stage, but is below the SFmode
;; rounding position.
(define_expand "floatdisf2_internal2"
  [(parallel [(set (match_dup 3) (ashiftrt:DI (match_operand:DI 1 "" "")
					      (const_int 53)))
	      (clobber (reg:DI CA_REGNO))])
   (set (match_operand:DI 0 "" "") (and:DI (match_dup 1)
					   (const_int 2047)))
   (set (match_dup 3) (plus:DI (match_dup 3)
			       (const_int 1)))
   (set (match_dup 0) (plus:DI (match_dup 0)
			       (const_int 2047)))
   (set (match_dup 4) (compare:CCUNS (match_dup 3)
				     (const_int 2)))
   (set (match_dup 0) (ior:DI (match_dup 0)
			      (match_dup 1)))
   (set (match_dup 0) (and:DI (match_dup 0)
			      (const_int -2048)))
   (set (pc) (if_then_else (geu (match_dup 4) (const_int 0))
			   (label_ref (match_operand:DI 2 "" ""))
			   (pc)))
   (set (match_dup 0) (match_dup 1))]
  "TARGET_POWERPC64 && TARGET_HARD_FLOAT && TARGET_FPRS && TARGET_SINGLE_FLOAT
   && !TARGET_FCFIDS"
  "
{
  operands[3] = gen_reg_rtx (DImode);
  operands[4] = gen_reg_rtx (CCUNSmode);
}")

(define_expand "floatunsdisf2"
  [(set (match_operand:SF 0 "gpc_reg_operand" "")
        (unsigned_float:SF (match_operand:DI 1 "gpc_reg_operand" "")))]
  "TARGET_HARD_FLOAT && TARGET_FPRS && TARGET_SINGLE_FLOAT
   && TARGET_DOUBLE_FLOAT && TARGET_FCFIDUS"
  "")

(define_insn "floatunsdisf2_fcfidus"
  [(set (match_operand:SF 0 "gpc_reg_operand" "=f,wu")
        (unsigned_float:SF (match_operand:DI 1 "gpc_reg_operand" "d,wi")))]
  "TARGET_HARD_FLOAT && TARGET_FPRS && TARGET_SINGLE_FLOAT
   && TARGET_DOUBLE_FLOAT && TARGET_FCFIDUS"
  "@
   fcfidus %0,%1
   xscvuxdsp %x0,%x1"
  [(set_attr "type" "fp")])

(define_insn_and_split "*floatunsdisf2_mem"
  [(set (match_operand:SF 0 "gpc_reg_operand" "=f,wy,wy")
	(unsigned_float:SF (match_operand:DI 1 "memory_operand" "m,m,Z")))
   (clobber (match_scratch:DI 2 "=d,d,wi"))]
  "TARGET_HARD_FLOAT && TARGET_FPRS && TARGET_SINGLE_FLOAT
   && TARGET_DOUBLE_FLOAT && TARGET_FCFIDUS"
  "#"
  "&& reload_completed"
  [(pc)]
  "
{
  emit_move_insn (operands[2], operands[1]);
  emit_insn (gen_floatunsdisf2_fcfidus (operands[0], operands[2]));
  DONE;
}"
  [(set_attr "length" "8")
   (set_attr "type" "fpload")])

;; Define the TImode operations that can be done in a small number
;; of instructions.  The & constraints are to prevent the register
;; allocator from allocating registers that overlap with the inputs
;; (for example, having an input in 7,8 and an output in 6,7).  We
;; also allow for the output being the same as one of the inputs.

(define_expand "addti3"
  [(set (match_operand:TI 0 "gpc_reg_operand" "")
	(plus:TI (match_operand:TI 1 "gpc_reg_operand" "")
		 (match_operand:TI 2 "reg_or_short_operand" "")))]
  "TARGET_64BIT"
{
  rtx lo0 = gen_lowpart (DImode, operands[0]);
  rtx lo1 = gen_lowpart (DImode, operands[1]);
  rtx lo2 = gen_lowpart (DImode, operands[2]);
  rtx hi0 = gen_highpart (DImode, operands[0]);
  rtx hi1 = gen_highpart (DImode, operands[1]);
  rtx hi2 = gen_highpart_mode (DImode, TImode, operands[2]);

  if (!reg_or_short_operand (lo2, DImode))
    lo2 = force_reg (DImode, lo2);
  if (!adde_operand (hi2, DImode))
    hi2 = force_reg (DImode, hi2);

  emit_insn (gen_adddi3_carry (lo0, lo1, lo2));
  emit_insn (gen_adddi3_carry_in (hi0, hi1, hi2));
  DONE;
})

(define_expand "subti3"
  [(set (match_operand:TI 0 "gpc_reg_operand" "")
	(minus:TI (match_operand:TI 1 "reg_or_short_operand" "")
		  (match_operand:TI 2 "gpc_reg_operand" "")))]
  "TARGET_64BIT"
{
  rtx lo0 = gen_lowpart (DImode, operands[0]);
  rtx lo1 = gen_lowpart (DImode, operands[1]);
  rtx lo2 = gen_lowpart (DImode, operands[2]);
  rtx hi0 = gen_highpart (DImode, operands[0]);
  rtx hi1 = gen_highpart_mode (DImode, TImode, operands[1]);
  rtx hi2 = gen_highpart (DImode, operands[2]);

  if (!reg_or_short_operand (lo1, DImode))
    lo1 = force_reg (DImode, lo1);
  if (!adde_operand (hi1, DImode))
    hi1 = force_reg (DImode, hi1);

  emit_insn (gen_subfdi3_carry (lo0, lo2, lo1));
  emit_insn (gen_subfdi3_carry_in (hi0, hi2, hi1));
  DONE;
})


;; Shift by a variable amount is too complex to be worth open-coding.  We
;; just handle shifts by constants.
(define_insn "ashrdi3_no_power"
  [(set (match_operand:DI 0 "gpc_reg_operand" "=&r,&r")
	(ashiftrt:DI (match_operand:DI 1 "gpc_reg_operand" "r,r")
		     (match_operand:SI 2 "const_int_operand" "M,i")))
   (clobber (reg:SI CA_REGNO))]
  "!TARGET_POWERPC64"
{
  switch (which_alternative)
    {
    default:
      gcc_unreachable ();
    case 0:
      if (WORDS_BIG_ENDIAN)
        return \"srawi %0,%1,31\;srawi %L0,%1,%h2\";
      else
        return \"srawi %L0,%L1,31\;srawi %0,%L1,%h2\";
    case 1:
      if (WORDS_BIG_ENDIAN)
	return \"srwi %L0,%L1,%h2\;insrwi %L0,%1,%h2,0\;srawi %0,%1,%h2\";
      else
	return \"srwi %0,%1,%h2\;insrwi %0,%L1,%h2,0\;srawi %L0,%L1,%h2\";
    }
}
  [(set_attr "type" "two,three")
   (set_attr "length" "8,12")])

(define_insn "*ashrdisi3_noppc64be"
  [(set (match_operand:SI 0 "gpc_reg_operand" "=r")
        (subreg:SI (ashiftrt:DI (match_operand:DI 1 "gpc_reg_operand" "r")
                                (const_int 32)) 4))]
  "TARGET_32BIT && !TARGET_POWERPC64 && WORDS_BIG_ENDIAN"
  "*
{
  if (REGNO (operands[0]) == REGNO (operands[1]))
    return \"\";
  else
    return \"mr %0,%1\";
}"
   [(set_attr "length" "4")])


;; PowerPC64 DImode operations.

(define_insn "*rotldi3_internal4"
  [(set (match_operand:DI 0 "gpc_reg_operand" "=r")
	(and:DI (rotate:DI (match_operand:DI 1 "gpc_reg_operand" "r")
			   (match_operand:DI 2 "reg_or_cint_operand" "rn"))
		(match_operand:DI 3 "mask64_operand" "n")))]
  "TARGET_POWERPC64"
  "rld%I2c%B3 %0,%1,%H2,%S3"
  [(set_attr "type" "shift")
   (set_attr "maybe_var_shift" "yes")])

(define_insn "*rotldi3_internal5"
  [(set (match_operand:CC 0 "cc_reg_operand" "=x,?y")
	(compare:CC (and:DI
		     (rotate:DI (match_operand:DI 1 "gpc_reg_operand" "r,r")
				(match_operand:DI 2 "reg_or_cint_operand" "rn,rn"))
		     (match_operand:DI 3 "mask64_operand" "n,n"))
		    (const_int 0)))
   (clobber (match_scratch:DI 4 "=r,r"))]
  "TARGET_64BIT"
  "@
   rld%I2c%B3. %4,%1,%H2,%S3
   #"
  [(set_attr "type" "shift")
   (set_attr "maybe_var_shift" "yes")
   (set_attr "dot" "yes")
   (set_attr "length" "4,8")])

(define_split
  [(set (match_operand:CC 0 "cc_reg_not_micro_cr0_operand" "")
	(compare:CC (and:DI
		     (rotate:DI (match_operand:DI 1 "gpc_reg_operand" "")
				(match_operand:DI 2 "reg_or_cint_operand" ""))
		     (match_operand:DI 3 "mask64_operand" ""))
		    (const_int 0)))
   (clobber (match_scratch:DI 4 ""))]
  "TARGET_POWERPC64 && reload_completed"
  [(set (match_dup 4)
	(and:DI (rotate:DI (match_dup 1)
				(match_dup 2))
		     (match_dup 3)))
   (set (match_dup 0)
	(compare:CC (match_dup 4)
		    (const_int 0)))]
  "")

(define_insn "*rotldi3_internal6"
  [(set (match_operand:CC 4 "cc_reg_operand" "=x,?y")
	(compare:CC (and:DI
		     (rotate:DI (match_operand:DI 1 "gpc_reg_operand" "r,r")
				(match_operand:DI 2 "reg_or_cint_operand" "rn,rn"))
		     (match_operand:DI 3 "mask64_operand" "n,n"))
		    (const_int 0)))
   (set (match_operand:DI 0 "gpc_reg_operand" "=r,r")
	(and:DI (rotate:DI (match_dup 1) (match_dup 2)) (match_dup 3)))]
  "TARGET_64BIT"
  "@
   rld%I2c%B3. %0,%1,%H2,%S3
   #"
  [(set_attr "type" "shift")
   (set_attr "maybe_var_shift" "yes")
   (set_attr "dot" "yes")
   (set_attr "length" "4,8")])

(define_split
  [(set (match_operand:CC 4 "cc_reg_not_micro_cr0_operand" "")
	(compare:CC (and:DI
		     (rotate:DI (match_operand:DI 1 "gpc_reg_operand" "")
				(match_operand:DI 2 "reg_or_cint_operand" ""))
		     (match_operand:DI 3 "mask64_operand" ""))
		    (const_int 0)))
   (set (match_operand:DI 0 "gpc_reg_operand" "")
	(and:DI (rotate:DI (match_dup 1) (match_dup 2)) (match_dup 3)))]
  "TARGET_POWERPC64 && reload_completed"
  [(set (match_dup 0)
	(and:DI (rotate:DI (match_dup 1) (match_dup 2)) (match_dup 3)))
   (set (match_dup 4)
	(compare:CC (match_dup 0)
		    (const_int 0)))]
  "")

(define_insn "*rotldi3_internal7le"
  [(set (match_operand:DI 0 "gpc_reg_operand" "=r")
	(zero_extend:DI
	 (subreg:QI
	  (rotate:DI (match_operand:DI 1 "gpc_reg_operand" "r")
		     (match_operand:DI 2 "reg_or_cint_operand" "rn")) 0)))]
  "TARGET_POWERPC64 && !BYTES_BIG_ENDIAN"
  "rld%I2cl %0,%1,%H2,56"
  [(set_attr "type" "shift")
   (set_attr "maybe_var_shift" "yes")])

(define_insn "*rotldi3_internal7be"
  [(set (match_operand:DI 0 "gpc_reg_operand" "=r")
	(zero_extend:DI
	 (subreg:QI
	  (rotate:DI (match_operand:DI 1 "gpc_reg_operand" "r")
		     (match_operand:DI 2 "reg_or_cint_operand" "rn")) 7)))]
  "TARGET_POWERPC64 && BYTES_BIG_ENDIAN"
  "rld%I2cl %0,%1,%H2,56"
  [(set_attr "type" "shift")
   (set_attr "maybe_var_shift" "yes")])

(define_insn "*rotldi3_internal8le"
  [(set (match_operand:CC 0 "cc_reg_operand" "=x,?y")
	(compare:CC (zero_extend:DI
		     (subreg:QI
		      (rotate:DI (match_operand:DI 1 "gpc_reg_operand" "r,r")
				 (match_operand:DI 2 "reg_or_cint_operand" "rn,rn")) 0))
		    (const_int 0)))
   (clobber (match_scratch:DI 3 "=r,r"))]
  "TARGET_64BIT && !BYTES_BIG_ENDIAN"
  "@
   rld%I2cl. %3,%1,%H2,56
   #"
  [(set_attr "type" "shift")
   (set_attr "maybe_var_shift" "yes")
   (set_attr "dot" "yes")
   (set_attr "length" "4,8")])

(define_insn "*rotldi3_internal8be"
  [(set (match_operand:CC 0 "cc_reg_operand" "=x,?y")
	(compare:CC (zero_extend:DI
		     (subreg:QI
		      (rotate:DI (match_operand:DI 1 "gpc_reg_operand" "r,r")
				 (match_operand:DI 2 "reg_or_cint_operand" "rn,rn")) 7))
		    (const_int 0)))
   (clobber (match_scratch:DI 3 "=r,r"))]
  "TARGET_64BIT && BYTES_BIG_ENDIAN"
  "@
   rld%I2cl. %3,%1,%H2,56
   #"
  [(set_attr "type" "shift")
   (set_attr "maybe_var_shift" "yes")
   (set_attr "dot" "yes")
   (set_attr "length" "4,8")])

(define_split
  [(set (match_operand:CC 0 "cc_reg_not_micro_cr0_operand" "")
	(compare:CC (zero_extend:DI
		     (subreg:QI
		      (rotate:DI (match_operand:DI 1 "gpc_reg_operand" "")
				 (match_operand:DI 2 "reg_or_cint_operand" "")) 0))
		    (const_int 0)))
   (clobber (match_scratch:DI 3 ""))]
  "TARGET_POWERPC64 && !BYTES_BIG_ENDIAN && reload_completed"
  [(set (match_dup 3)
	(zero_extend:DI (subreg:QI
		      (rotate:DI (match_dup 1)
				 (match_dup 2)) 0)))
   (set (match_dup 0)
	(compare:CC (match_dup 3)
		    (const_int 0)))]
  "")

(define_split
  [(set (match_operand:CC 0 "cc_reg_not_micro_cr0_operand" "")
	(compare:CC (zero_extend:DI
		     (subreg:QI
		      (rotate:DI (match_operand:DI 1 "gpc_reg_operand" "")
				 (match_operand:DI 2 "reg_or_cint_operand" "")) 7))
		    (const_int 0)))
   (clobber (match_scratch:DI 3 ""))]
  "TARGET_POWERPC64 && BYTES_BIG_ENDIAN && reload_completed"
  [(set (match_dup 3)
	(zero_extend:DI (subreg:QI
		      (rotate:DI (match_dup 1)
				 (match_dup 2)) 7)))
   (set (match_dup 0)
	(compare:CC (match_dup 3)
		    (const_int 0)))]
  "")

(define_insn "*rotldi3_internal9le"
  [(set (match_operand:CC 3 "cc_reg_operand" "=x,?y")
	(compare:CC (zero_extend:DI
		     (subreg:QI
		      (rotate:DI (match_operand:DI 1 "gpc_reg_operand" "r,r")
				 (match_operand:DI 2 "reg_or_cint_operand" "rn,rn")) 0))
		    (const_int 0)))
   (set (match_operand:DI 0 "gpc_reg_operand" "=r,r")
	(zero_extend:DI (subreg:QI (rotate:DI (match_dup 1) (match_dup 2)) 0)))]
  "TARGET_64BIT && !BYTES_BIG_ENDIAN"
  "@
   rld%I2cl. %0,%1,%H2,56
   #"
  [(set_attr "type" "shift")
   (set_attr "maybe_var_shift" "yes")
   (set_attr "dot" "yes")
   (set_attr "length" "4,8")])

(define_insn "*rotldi3_internal9be"
  [(set (match_operand:CC 3 "cc_reg_operand" "=x,?y")
	(compare:CC (zero_extend:DI
		     (subreg:QI
		      (rotate:DI (match_operand:DI 1 "gpc_reg_operand" "r,r")
				 (match_operand:DI 2 "reg_or_cint_operand" "rn,rn")) 7))
		    (const_int 0)))
   (set (match_operand:DI 0 "gpc_reg_operand" "=r,r")
	(zero_extend:DI (subreg:QI (rotate:DI (match_dup 1) (match_dup 2)) 7)))]
  "TARGET_64BIT && BYTES_BIG_ENDIAN"
  "@
   rld%I2cl. %0,%1,%H2,56
   #"
  [(set_attr "type" "shift")
   (set_attr "maybe_var_shift" "yes")
   (set_attr "dot" "yes")
   (set_attr "length" "4,8")])

(define_split
  [(set (match_operand:CC 3 "cc_reg_not_micro_cr0_operand" "")
	(compare:CC (zero_extend:DI
		     (subreg:QI
		      (rotate:DI (match_operand:DI 1 "gpc_reg_operand" "")
				 (match_operand:DI 2 "reg_or_cint_operand" "")) 0))
		    (const_int 0)))
   (set (match_operand:DI 0 "gpc_reg_operand" "")
	(zero_extend:DI (subreg:QI (rotate:DI (match_dup 1) (match_dup 2)) 0)))]
  "TARGET_POWERPC64 && !BYTES_BIG_ENDIAN && reload_completed"
  [(set (match_dup 0)
	(zero_extend:DI (subreg:QI (rotate:DI (match_dup 1) (match_dup 2)) 0)))
   (set (match_dup 3)
	(compare:CC (match_dup 0)
		    (const_int 0)))]
  "")

(define_split
  [(set (match_operand:CC 3 "cc_reg_not_micro_cr0_operand" "")
	(compare:CC (zero_extend:DI
		     (subreg:QI
		      (rotate:DI (match_operand:DI 1 "gpc_reg_operand" "")
				 (match_operand:DI 2 "reg_or_cint_operand" "")) 7))
		    (const_int 0)))
   (set (match_operand:DI 0 "gpc_reg_operand" "")
	(zero_extend:DI (subreg:QI (rotate:DI (match_dup 1) (match_dup 2)) 7)))]
  "TARGET_POWERPC64 && BYTES_BIG_ENDIAN && reload_completed"
  [(set (match_dup 0)
	(zero_extend:DI (subreg:QI (rotate:DI (match_dup 1) (match_dup 2)) 7)))
   (set (match_dup 3)
	(compare:CC (match_dup 0)
		    (const_int 0)))]
  "")

(define_insn "*rotldi3_internal10le"
  [(set (match_operand:DI 0 "gpc_reg_operand" "=r")
	(zero_extend:DI
	 (subreg:HI
	  (rotate:DI (match_operand:DI 1 "gpc_reg_operand" "r")
		     (match_operand:DI 2 "reg_or_cint_operand" "rn")) 0)))]
  "TARGET_POWERPC64 && !BYTES_BIG_ENDIAN"
  "rld%I2cl %0,%1,%H2,48"
  [(set_attr "type" "shift")
   (set_attr "maybe_var_shift" "yes")])

(define_insn "*rotldi3_internal10be"
  [(set (match_operand:DI 0 "gpc_reg_operand" "=r")
	(zero_extend:DI
	 (subreg:HI
	  (rotate:DI (match_operand:DI 1 "gpc_reg_operand" "r")
		     (match_operand:DI 2 "reg_or_cint_operand" "rn")) 6)))]
  "TARGET_POWERPC64 && BYTES_BIG_ENDIAN"
  "rld%I2cl %0,%1,%H2,48"
  [(set_attr "type" "shift")
   (set_attr "maybe_var_shift" "yes")])

(define_insn "*rotldi3_internal11le"
  [(set (match_operand:CC 0 "cc_reg_operand" "=x,?y")
	(compare:CC (zero_extend:DI
		     (subreg:HI
		      (rotate:DI (match_operand:DI 1 "gpc_reg_operand" "r,r")
				 (match_operand:DI 2 "reg_or_cint_operand" "rn,rn")) 0))
		    (const_int 0)))
   (clobber (match_scratch:DI 3 "=r,r"))]
  "TARGET_64BIT && !BYTES_BIG_ENDIAN"
  "@
   rld%I2cl. %3,%1,%H2,48
   #"
  [(set_attr "type" "shift")
   (set_attr "maybe_var_shift" "yes")
   (set_attr "dot" "yes")
   (set_attr "length" "4,8")])

(define_insn "*rotldi3_internal11be"
  [(set (match_operand:CC 0 "cc_reg_operand" "=x,?y")
	(compare:CC (zero_extend:DI
		     (subreg:HI
		      (rotate:DI (match_operand:DI 1 "gpc_reg_operand" "r,r")
				 (match_operand:DI 2 "reg_or_cint_operand" "rn,rn")) 6))
		    (const_int 0)))
   (clobber (match_scratch:DI 3 "=r,r"))]
  "TARGET_64BIT && BYTES_BIG_ENDIAN"
  "@
   rld%I2cl. %3,%1,%H2,48
   #"
  [(set_attr "type" "shift")
   (set_attr "maybe_var_shift" "yes")
   (set_attr "dot" "yes")
   (set_attr "length" "4,8")])

(define_split
  [(set (match_operand:CC 0 "cc_reg_not_micro_cr0_operand" "")
	(compare:CC (zero_extend:DI
		     (subreg:HI
		      (rotate:DI (match_operand:DI 1 "gpc_reg_operand" "")
				 (match_operand:DI 2 "reg_or_cint_operand" "")) 0))
		    (const_int 0)))
   (clobber (match_scratch:DI 3 ""))]
  "TARGET_POWERPC64 && !BYTES_BIG_ENDIAN && reload_completed"
  [(set (match_dup 3)
	(zero_extend:DI (subreg:HI
		      (rotate:DI (match_dup 1)
				 (match_dup 2)) 0)))
   (set (match_dup 0)
	(compare:CC (match_dup 3)
		    (const_int 0)))]
  "")

(define_split
  [(set (match_operand:CC 0 "cc_reg_not_micro_cr0_operand" "")
	(compare:CC (zero_extend:DI
		     (subreg:HI
		      (rotate:DI (match_operand:DI 1 "gpc_reg_operand" "")
				 (match_operand:DI 2 "reg_or_cint_operand" "")) 6))
		    (const_int 0)))
   (clobber (match_scratch:DI 3 ""))]
  "TARGET_POWERPC64 && BYTES_BIG_ENDIAN && reload_completed"
  [(set (match_dup 3)
	(zero_extend:DI (subreg:HI
		      (rotate:DI (match_dup 1)
				 (match_dup 2)) 6)))
   (set (match_dup 0)
	(compare:CC (match_dup 3)
		    (const_int 0)))]
  "")

(define_insn "*rotldi3_internal12le"
  [(set (match_operand:CC 3 "cc_reg_operand" "=x,?y")
	(compare:CC (zero_extend:DI
		     (subreg:HI
		      (rotate:DI (match_operand:DI 1 "gpc_reg_operand" "r,r")
				 (match_operand:DI 2 "reg_or_cint_operand" "rn,rn")) 0))
		    (const_int 0)))
   (set (match_operand:DI 0 "gpc_reg_operand" "=r,r")
	(zero_extend:DI (subreg:HI (rotate:DI (match_dup 1) (match_dup 2)) 0)))]
  "TARGET_64BIT && !BYTES_BIG_ENDIAN"
  "@
   rld%I2cl. %0,%1,%H2,48
   #"
  [(set_attr "type" "shift")
   (set_attr "maybe_var_shift" "yes")
   (set_attr "dot" "yes")
   (set_attr "length" "4,8")])

(define_insn "*rotldi3_internal12be"
  [(set (match_operand:CC 3 "cc_reg_operand" "=x,?y")
	(compare:CC (zero_extend:DI
		     (subreg:HI
		      (rotate:DI (match_operand:DI 1 "gpc_reg_operand" "r,r")
				 (match_operand:DI 2 "reg_or_cint_operand" "rn,rn")) 6))
		    (const_int 0)))
   (set (match_operand:DI 0 "gpc_reg_operand" "=r,r")
	(zero_extend:DI (subreg:HI (rotate:DI (match_dup 1) (match_dup 2)) 6)))]
  "TARGET_64BIT && BYTES_BIG_ENDIAN"
  "@
   rld%I2cl. %0,%1,%H2,48
   #"
  [(set_attr "type" "shift")
   (set_attr "maybe_var_shift" "yes")
   (set_attr "dot" "yes")
   (set_attr "length" "4,8")])

(define_split
  [(set (match_operand:CC 3 "cc_reg_not_micro_cr0_operand" "")
	(compare:CC (zero_extend:DI
		     (subreg:HI
		      (rotate:DI (match_operand:DI 1 "gpc_reg_operand" "")
				 (match_operand:DI 2 "reg_or_cint_operand" "")) 0))
		    (const_int 0)))
   (set (match_operand:DI 0 "gpc_reg_operand" "")
	(zero_extend:DI (subreg:HI (rotate:DI (match_dup 1) (match_dup 2)) 0)))]
  "TARGET_POWERPC64 && !BYTES_BIG_ENDIAN && reload_completed"
  [(set (match_dup 0)
	(zero_extend:DI (subreg:HI (rotate:DI (match_dup 1) (match_dup 2)) 0)))
   (set (match_dup 3)
	(compare:CC (match_dup 0)
		    (const_int 0)))]
  "")

(define_split
  [(set (match_operand:CC 3 "cc_reg_not_micro_cr0_operand" "")
	(compare:CC (zero_extend:DI
		     (subreg:HI
		      (rotate:DI (match_operand:DI 1 "gpc_reg_operand" "")
				 (match_operand:DI 2 "reg_or_cint_operand" "")) 6))
		    (const_int 0)))
   (set (match_operand:DI 0 "gpc_reg_operand" "")
	(zero_extend:DI (subreg:HI (rotate:DI (match_dup 1) (match_dup 2)) 6)))]
  "TARGET_POWERPC64 && BYTES_BIG_ENDIAN && reload_completed"
  [(set (match_dup 0)
	(zero_extend:DI (subreg:HI (rotate:DI (match_dup 1) (match_dup 2)) 6)))
   (set (match_dup 3)
	(compare:CC (match_dup 0)
		    (const_int 0)))]
  "")

(define_insn "*rotldi3_internal13le"
  [(set (match_operand:DI 0 "gpc_reg_operand" "=r")
	(zero_extend:DI
	 (subreg:SI
	  (rotate:DI (match_operand:DI 1 "gpc_reg_operand" "r")
		     (match_operand:DI 2 "reg_or_cint_operand" "rn")) 0)))]
  "TARGET_POWERPC64 && !BYTES_BIG_ENDIAN"
  "rld%I2cl %0,%1,%H2,32"
  [(set_attr "type" "shift")
   (set_attr "maybe_var_shift" "yes")])

(define_insn "*rotldi3_internal13be"
  [(set (match_operand:DI 0 "gpc_reg_operand" "=r")
	(zero_extend:DI
	 (subreg:SI
	  (rotate:DI (match_operand:DI 1 "gpc_reg_operand" "r")
		     (match_operand:DI 2 "reg_or_cint_operand" "rn")) 4)))]
  "TARGET_POWERPC64 && BYTES_BIG_ENDIAN"
  "rld%I2cl %0,%1,%H2,32"
  [(set_attr "type" "shift")
   (set_attr "maybe_var_shift" "yes")])

(define_insn "*rotldi3_internal14le"
  [(set (match_operand:CC 0 "cc_reg_operand" "=x,?y")
	(compare:CC (zero_extend:DI
		     (subreg:SI
		      (rotate:DI (match_operand:DI 1 "gpc_reg_operand" "r,r")
				 (match_operand:DI 2 "reg_or_cint_operand" "rn,rn")) 0))
		    (const_int 0)))
   (clobber (match_scratch:DI 3 "=r,r"))]
  "TARGET_64BIT && !BYTES_BIG_ENDIAN"
  "@
   rld%I2cl. %3,%1,%H2,32
   #"
  [(set_attr "type" "shift")
   (set_attr "maybe_var_shift" "yes")
   (set_attr "dot" "yes")
   (set_attr "length" "4,8")])

(define_insn "*rotldi3_internal14be"
  [(set (match_operand:CC 0 "cc_reg_operand" "=x,?y")
	(compare:CC (zero_extend:DI
		     (subreg:SI
		      (rotate:DI (match_operand:DI 1 "gpc_reg_operand" "r,r")
				 (match_operand:DI 2 "reg_or_cint_operand" "rn,rn")) 4))
		    (const_int 0)))
   (clobber (match_scratch:DI 3 "=r,r"))]
  "TARGET_64BIT && BYTES_BIG_ENDIAN"
  "@
   rld%I2cl. %3,%1,%H2,32
   #"
  [(set_attr "type" "shift")
   (set_attr "maybe_var_shift" "yes")
   (set_attr "dot" "yes")
   (set_attr "length" "4,8")])

(define_split
  [(set (match_operand:CC 0 "cc_reg_not_micro_cr0_operand" "")
	(compare:CC (zero_extend:DI
		     (subreg:SI
		      (rotate:DI (match_operand:DI 1 "gpc_reg_operand" "")
				 (match_operand:DI 2 "reg_or_cint_operand" "")) 0))
		    (const_int 0)))
   (clobber (match_scratch:DI 3 ""))]
  "TARGET_POWERPC64 && !BYTES_BIG_ENDIAN && reload_completed"
  [(set (match_dup 3)
	(zero_extend:DI (subreg:SI
		      (rotate:DI (match_dup 1)
				 (match_dup 2)) 0)))
   (set (match_dup 0)
	(compare:CC (match_dup 3)
		    (const_int 0)))]
  "")

(define_split
  [(set (match_operand:CC 0 "cc_reg_not_micro_cr0_operand" "")
	(compare:CC (zero_extend:DI
		     (subreg:SI
		      (rotate:DI (match_operand:DI 1 "gpc_reg_operand" "")
				 (match_operand:DI 2 "reg_or_cint_operand" "")) 4))
		    (const_int 0)))
   (clobber (match_scratch:DI 3 ""))]
  "TARGET_POWERPC64 && BYTES_BIG_ENDIAN && reload_completed"
  [(set (match_dup 3)
	(zero_extend:DI (subreg:SI
		      (rotate:DI (match_dup 1)
				 (match_dup 2)) 4)))
   (set (match_dup 0)
	(compare:CC (match_dup 3)
		    (const_int 0)))]
  "")

(define_insn "*rotldi3_internal15le"
  [(set (match_operand:CC 3 "cc_reg_operand" "=x,?y")
	(compare:CC (zero_extend:DI
		     (subreg:SI
		      (rotate:DI (match_operand:DI 1 "gpc_reg_operand" "r,r")
				 (match_operand:DI 2 "reg_or_cint_operand" "rn,rn")) 0))
		    (const_int 0)))
   (set (match_operand:DI 0 "gpc_reg_operand" "=r,r")
	(zero_extend:DI (subreg:SI (rotate:DI (match_dup 1) (match_dup 2)) 0)))]
  "TARGET_64BIT && !BYTES_BIG_ENDIAN"
  "@
   rld%I2cl. %0,%1,%H2,32
   #"
  [(set_attr "type" "shift")
   (set_attr "maybe_var_shift" "yes")
   (set_attr "dot" "yes")
   (set_attr "length" "4,8")])

(define_insn "*rotldi3_internal15be"
  [(set (match_operand:CC 3 "cc_reg_operand" "=x,?y")
	(compare:CC (zero_extend:DI
		     (subreg:SI
		      (rotate:DI (match_operand:DI 1 "gpc_reg_operand" "r,r")
				 (match_operand:DI 2 "reg_or_cint_operand" "rn,rn")) 4))
		    (const_int 0)))
   (set (match_operand:DI 0 "gpc_reg_operand" "=r,r")
	(zero_extend:DI (subreg:SI (rotate:DI (match_dup 1) (match_dup 2)) 4)))]
  "TARGET_64BIT && BYTES_BIG_ENDIAN"
  "@
   rld%I2cl. %0,%1,%H2,32
   #"
  [(set_attr "type" "shift")
   (set_attr "maybe_var_shift" "yes")
   (set_attr "dot" "yes")
   (set_attr "length" "4,8")])

(define_split
  [(set (match_operand:CC 3 "cc_reg_not_micro_cr0_operand" "")
	(compare:CC (zero_extend:DI
		     (subreg:SI
		      (rotate:DI (match_operand:DI 1 "gpc_reg_operand" "")
				 (match_operand:DI 2 "reg_or_cint_operand" "")) 0))
		    (const_int 0)))
   (set (match_operand:DI 0 "gpc_reg_operand" "")
	(zero_extend:DI (subreg:SI (rotate:DI (match_dup 1) (match_dup 2)) 0)))]
  "TARGET_POWERPC64 && !BYTES_BIG_ENDIAN && reload_completed"
  [(set (match_dup 0)
	(zero_extend:DI (subreg:SI (rotate:DI (match_dup 1) (match_dup 2)) 0)))
   (set (match_dup 3)
	(compare:CC (match_dup 0)
		    (const_int 0)))]
  "")

(define_split
  [(set (match_operand:CC 3 "cc_reg_not_micro_cr0_operand" "")
	(compare:CC (zero_extend:DI
		     (subreg:SI
		      (rotate:DI (match_operand:DI 1 "gpc_reg_operand" "")
				 (match_operand:DI 2 "reg_or_cint_operand" "")) 4))
		    (const_int 0)))
   (set (match_operand:DI 0 "gpc_reg_operand" "")
	(zero_extend:DI (subreg:SI (rotate:DI (match_dup 1) (match_dup 2)) 4)))]
  "TARGET_POWERPC64 && BYTES_BIG_ENDIAN && reload_completed"
  [(set (match_dup 0)
	(zero_extend:DI (subreg:SI (rotate:DI (match_dup 1) (match_dup 2)) 4)))
   (set (match_dup 3)
	(compare:CC (match_dup 0)
		    (const_int 0)))]
  "")

(define_insn "*ashldi3_internal4"
  [(set (match_operand:DI 0 "gpc_reg_operand" "=r")
	(and:DI (ashift:DI (match_operand:DI 1 "gpc_reg_operand" "r")
			   (match_operand:SI 2 "const_int_operand" "i"))
		(match_operand:DI 3 "const_int_operand" "n")))]
  "TARGET_POWERPC64 && includes_rldic_lshift_p (operands[2], operands[3])"
  "rldic %0,%1,%H2,%W3"
  [(set_attr "type" "shift")])

(define_insn "ashldi3_internal5"
  [(set (match_operand:CC 0 "cc_reg_operand" "=x,?y")
	(compare:CC
	 (and:DI (ashift:DI (match_operand:DI 1 "gpc_reg_operand" "r,r")
			    (match_operand:SI 2 "const_int_operand" "i,i"))
		 (match_operand:DI 3 "const_int_operand" "n,n"))
	 (const_int 0)))
   (clobber (match_scratch:DI 4 "=r,r"))]
  "TARGET_64BIT && includes_rldic_lshift_p (operands[2], operands[3])"
  "@
   rldic. %4,%1,%H2,%W3
   #"
  [(set_attr "type" "shift")
   (set_attr "dot" "yes")
   (set_attr "length" "4,8")])

(define_split
  [(set (match_operand:CC 0 "cc_reg_not_micro_cr0_operand" "")
	(compare:CC
	 (and:DI (ashift:DI (match_operand:DI 1 "gpc_reg_operand" "")
			    (match_operand:SI 2 "const_int_operand" ""))
		 (match_operand:DI 3 "const_int_operand" ""))
	 (const_int 0)))
   (clobber (match_scratch:DI 4 ""))]
  "TARGET_POWERPC64 && reload_completed
   && includes_rldic_lshift_p (operands[2], operands[3])"
  [(set (match_dup 4)
	(and:DI (ashift:DI (match_dup 1) (match_dup 2))
		(match_dup 3)))
   (set (match_dup 0)
	(compare:CC (match_dup 4)
		    (const_int 0)))]
  "")

(define_insn "*ashldi3_internal6"
  [(set (match_operand:CC 4 "cc_reg_operand" "=x,?y")
	(compare:CC
	 (and:DI (ashift:DI (match_operand:DI 1 "gpc_reg_operand" "r,r")
			    (match_operand:SI 2 "const_int_operand" "i,i"))
		    (match_operand:DI 3 "const_int_operand" "n,n"))
	 (const_int 0)))
   (set (match_operand:DI 0 "gpc_reg_operand" "=r,r")
	(and:DI (ashift:DI (match_dup 1) (match_dup 2)) (match_dup 3)))]
  "TARGET_64BIT && includes_rldic_lshift_p (operands[2], operands[3])"
  "@
   rldic. %0,%1,%H2,%W3
   #"
  [(set_attr "type" "shift")
   (set_attr "dot" "yes")
   (set_attr "length" "4,8")])

(define_split
  [(set (match_operand:CC 4 "cc_reg_not_micro_cr0_operand" "")
	(compare:CC
	 (and:DI (ashift:DI (match_operand:DI 1 "gpc_reg_operand" "")
			    (match_operand:SI 2 "const_int_operand" ""))
		 (match_operand:DI 3 "const_int_operand" ""))
	 (const_int 0)))
   (set (match_operand:DI 0 "gpc_reg_operand" "")
	(and:DI (ashift:DI (match_dup 1) (match_dup 2)) (match_dup 3)))]
  "TARGET_POWERPC64 && reload_completed
   && includes_rldic_lshift_p (operands[2], operands[3])"
  [(set (match_dup 0)
	(and:DI (ashift:DI (match_dup 1) (match_dup 2))
		(match_dup 3)))
   (set (match_dup 4)
	(compare:CC (match_dup 0)
		    (const_int 0)))]
  "")

(define_insn "*ashldi3_internal7"
  [(set (match_operand:DI 0 "gpc_reg_operand" "=r")
	(and:DI (ashift:DI (match_operand:DI 1 "gpc_reg_operand" "r")
			   (match_operand:SI 2 "const_int_operand" "i"))
		(match_operand:DI 3 "mask64_operand" "n")))]
  "TARGET_POWERPC64 && includes_rldicr_lshift_p (operands[2], operands[3])"
  "rldicr %0,%1,%H2,%S3"
  [(set_attr "type" "shift")])

(define_insn "ashldi3_internal8"
  [(set (match_operand:CC 0 "cc_reg_operand" "=x,?y")
	(compare:CC
	 (and:DI (ashift:DI (match_operand:DI 1 "gpc_reg_operand" "r,r")
			    (match_operand:SI 2 "const_int_operand" "i,i"))
		 (match_operand:DI 3 "mask64_operand" "n,n"))
	 (const_int 0)))
   (clobber (match_scratch:DI 4 "=r,r"))]
  "TARGET_64BIT && includes_rldicr_lshift_p (operands[2], operands[3])"
  "@
   rldicr. %4,%1,%H2,%S3
   #"
  [(set_attr "type" "shift")
   (set_attr "dot" "yes")
   (set_attr "length" "4,8")])

(define_split
  [(set (match_operand:CC 0 "cc_reg_not_micro_cr0_operand" "")
	(compare:CC
	 (and:DI (ashift:DI (match_operand:DI 1 "gpc_reg_operand" "")
			    (match_operand:SI 2 "const_int_operand" ""))
		 (match_operand:DI 3 "mask64_operand" ""))
	 (const_int 0)))
   (clobber (match_scratch:DI 4 ""))]
  "TARGET_POWERPC64 && reload_completed
   && includes_rldicr_lshift_p (operands[2], operands[3])"
  [(set (match_dup 4)
	(and:DI (ashift:DI (match_dup 1) (match_dup 2))
		(match_dup 3)))
   (set (match_dup 0)
	(compare:CC (match_dup 4)
		    (const_int 0)))]
  "")

(define_insn "*ashldi3_internal9"
  [(set (match_operand:CC 4 "cc_reg_operand" "=x,?y")
	(compare:CC
	 (and:DI (ashift:DI (match_operand:DI 1 "gpc_reg_operand" "r,r")
			    (match_operand:SI 2 "const_int_operand" "i,i"))
		    (match_operand:DI 3 "mask64_operand" "n,n"))
	 (const_int 0)))
   (set (match_operand:DI 0 "gpc_reg_operand" "=r,r")
	(and:DI (ashift:DI (match_dup 1) (match_dup 2)) (match_dup 3)))]
  "TARGET_64BIT && includes_rldicr_lshift_p (operands[2], operands[3])"
  "@
   rldicr. %0,%1,%H2,%S3
   #"
  [(set_attr "type" "shift")
   (set_attr "dot" "yes")
   (set_attr "length" "4,8")])

(define_split
  [(set (match_operand:CC 4 "cc_reg_not_micro_cr0_operand" "")
	(compare:CC
	 (and:DI (ashift:DI (match_operand:DI 1 "gpc_reg_operand" "")
			    (match_operand:SI 2 "const_int_operand" ""))
		 (match_operand:DI 3 "mask64_operand" ""))
	 (const_int 0)))
   (set (match_operand:DI 0 "gpc_reg_operand" "")
	(and:DI (ashift:DI (match_dup 1) (match_dup 2)) (match_dup 3)))]
  "TARGET_POWERPC64 && reload_completed
   && includes_rldicr_lshift_p (operands[2], operands[3])"
  [(set (match_dup 0)
	(and:DI (ashift:DI (match_dup 1) (match_dup 2))
		(match_dup 3)))
   (set (match_dup 4)
	(compare:CC (match_dup 0)
		    (const_int 0)))]
  "")


(define_insn_and_split "*anddi3_2rld"
  [(set (match_operand:DI 0 "gpc_reg_operand" "=r")
	(and:DI (match_operand:DI 1 "gpc_reg_operand" "%r")
		(match_operand:DI 2 "and_2rld_operand" "n")))]
  "TARGET_POWERPC64"
  "#"
  ""
  [(set (match_dup 0)
	(and:DI (rotate:DI (match_dup 1)
			   (match_dup 4))
		(match_dup 5)))
   (set (match_dup 0)
	(and:DI (rotate:DI (match_dup 0)
			   (match_dup 6))
		(match_dup 7)))]
{
  build_mask64_2_operands (operands[2], &operands[4]);
}
  [(set_attr "length" "8")])

(define_insn_and_split "*anddi3_2rld_dot"
  [(set (match_operand:CC 3 "cc_reg_operand" "=x,?y")
	(compare:CC (and:DI (match_operand:DI 1 "gpc_reg_operand" "%r,r")
			    (match_operand:DI 2 "and_2rld_operand" "n,n"))
		    (const_int 0)))
   (clobber (match_scratch:DI 0 "=r,r"))]
  "TARGET_64BIT && rs6000_gen_cell_microcode"
  "@
   #
   #"
  "&& reload_completed"
  [(set (match_dup 0)
	(and:DI (rotate:DI (match_dup 1)
			   (match_dup 4))
		(match_dup 5)))
   (parallel [(set (match_dup 3)
		   (compare:CC (and:DI (rotate:DI (match_dup 0)
						  (match_dup 6))
				       (match_dup 7))
			       (const_int 0)))
	      (clobber (match_dup 0))])]
{
  build_mask64_2_operands (operands[2], &operands[4]);
}
  [(set_attr "type" "two")
   (set_attr "dot" "yes")
   (set_attr "length" "8,12")])

(define_insn_and_split "*anddi3_2rld_dot2"
  [(set (match_operand:CC 3 "cc_reg_operand" "=x,?y")
	(compare:CC (and:DI (match_operand:DI 1 "gpc_reg_operand" "%r,r")
			    (match_operand:DI 2 "and_2rld_operand" "n,n"))
		    (const_int 0)))
   (set (match_operand:DI 0 "gpc_reg_operand" "=r,r")
	(and:DI (match_dup 1)
		(match_dup 2)))]
  "TARGET_64BIT && rs6000_gen_cell_microcode"
  "@
   #
   #"
  "&& reload_completed"
  [(set (match_dup 0)
	(and:DI (rotate:DI (match_dup 1)
			   (match_dup 4))
		(match_dup 5)))
   (parallel [(set (match_dup 3)
		   (compare:CC (and:DI (rotate:DI (match_dup 0)
						  (match_dup 6))
				       (match_dup 7))
			       (const_int 0)))
	      (set (match_dup 0)
		   (and:DI (rotate:DI (match_dup 0)
				      (match_dup 6))
			   (match_dup 7)))])]
{
  build_mask64_2_operands (operands[2], &operands[4]);
}
  [(set_attr "type" "two")
   (set_attr "dot" "yes")
   (set_attr "length" "8,12")])

;; 128-bit logical operations expanders

(define_expand "and<mode>3"
  [(set (match_operand:BOOL_128 0 "vlogical_operand" "")
	(and:BOOL_128 (match_operand:BOOL_128 1 "vlogical_operand" "")
		      (match_operand:BOOL_128 2 "vlogical_operand" "")))]
  ""
  "")

(define_expand "ior<mode>3"
  [(set (match_operand:BOOL_128 0 "vlogical_operand" "")
        (ior:BOOL_128 (match_operand:BOOL_128 1 "vlogical_operand" "")
		      (match_operand:BOOL_128 2 "vlogical_operand" "")))]
  ""
  "")

(define_expand "xor<mode>3"
  [(set (match_operand:BOOL_128 0 "vlogical_operand" "")
        (xor:BOOL_128 (match_operand:BOOL_128 1 "vlogical_operand" "")
		      (match_operand:BOOL_128 2 "vlogical_operand" "")))]
  ""
  "")

(define_expand "one_cmpl<mode>2"
  [(set (match_operand:BOOL_128 0 "vlogical_operand" "")
        (not:BOOL_128 (match_operand:BOOL_128 1 "vlogical_operand" "")))]
  ""
  "")

(define_expand "nor<mode>3"
  [(set (match_operand:BOOL_128 0 "vlogical_operand" "")
	(and:BOOL_128
	 (not:BOOL_128 (match_operand:BOOL_128 1 "vlogical_operand" ""))
	 (not:BOOL_128 (match_operand:BOOL_128 2 "vlogical_operand" ""))))]
  ""
  "")

(define_expand "andc<mode>3"
  [(set (match_operand:BOOL_128 0 "vlogical_operand" "")
        (and:BOOL_128
	 (not:BOOL_128 (match_operand:BOOL_128 2 "vlogical_operand" ""))
	 (match_operand:BOOL_128 1 "vlogical_operand" "")))]
  ""
  "")

;; Power8 vector logical instructions.
(define_expand "eqv<mode>3"
  [(set (match_operand:BOOL_128 0 "vlogical_operand" "")
	(not:BOOL_128
	 (xor:BOOL_128 (match_operand:BOOL_128 1 "vlogical_operand" "")
		       (match_operand:BOOL_128 2 "vlogical_operand" ""))))]
  "<MODE>mode == TImode || <MODE>mode == PTImode || TARGET_P8_VECTOR"
  "")

;; Rewrite nand into canonical form
(define_expand "nand<mode>3"
  [(set (match_operand:BOOL_128 0 "vlogical_operand" "")
	(ior:BOOL_128
	 (not:BOOL_128 (match_operand:BOOL_128 1 "vlogical_operand" ""))
	 (not:BOOL_128 (match_operand:BOOL_128 2 "vlogical_operand" ""))))]
  "<MODE>mode == TImode || <MODE>mode == PTImode || TARGET_P8_VECTOR"
  "")

;; The canonical form is to have the negated element first, so we need to
;; reverse arguments.
(define_expand "orc<mode>3"
  [(set (match_operand:BOOL_128 0 "vlogical_operand" "")
	(ior:BOOL_128
	 (not:BOOL_128 (match_operand:BOOL_128 2 "vlogical_operand" ""))
	 (match_operand:BOOL_128 1 "vlogical_operand" "")))]
  "<MODE>mode == TImode || <MODE>mode == PTImode || TARGET_P8_VECTOR"
  "")

;; 128-bit logical operations insns and split operations
(define_insn_and_split "*and<mode>3_internal"
  [(set (match_operand:BOOL_128 0 "vlogical_operand" "=<BOOL_REGS_OUTPUT>")
        (and:BOOL_128
	 (match_operand:BOOL_128 1 "vlogical_operand" "%<BOOL_REGS_OP1>")
	 (match_operand:BOOL_128 2 "vlogical_operand" "<BOOL_REGS_OP2>")))]
  ""
{
  if (TARGET_VSX && vsx_register_operand (operands[0], <MODE>mode))
    return "xxland %x0,%x1,%x2";

  if (TARGET_ALTIVEC && altivec_register_operand (operands[0], <MODE>mode))
    return "vand %0,%1,%2";

  return "#";
}
  "reload_completed && int_reg_operand (operands[0], <MODE>mode)"
  [(const_int 0)]
{
  rs6000_split_logical (operands, AND, false, false, false);
  DONE;
}
  [(set (attr "type")
      (if_then_else
	(match_test "vsx_register_operand (operands[0], <MODE>mode)")
	(const_string "vecsimple")
	(const_string "integer")))
   (set (attr "length")
      (if_then_else
	(match_test "vsx_register_operand (operands[0], <MODE>mode)")
	(const_string "4")
	(if_then_else
	 (match_test "TARGET_POWERPC64")
	 (const_string "8")
	 (const_string "16"))))])

;; 128-bit IOR/XOR
(define_insn_and_split "*bool<mode>3_internal"
  [(set (match_operand:BOOL_128 0 "vlogical_operand" "=<BOOL_REGS_OUTPUT>")
	(match_operator:BOOL_128 3 "boolean_or_operator"
	 [(match_operand:BOOL_128 1 "vlogical_operand" "%<BOOL_REGS_OP1>")
	  (match_operand:BOOL_128 2 "vlogical_operand" "<BOOL_REGS_OP2>")]))]
  ""
{
  if (TARGET_VSX && vsx_register_operand (operands[0], <MODE>mode))
    return "xxl%q3 %x0,%x1,%x2";

  if (TARGET_ALTIVEC && altivec_register_operand (operands[0], <MODE>mode))
    return "v%q3 %0,%1,%2";

  return "#";
}
  "reload_completed && int_reg_operand (operands[0], <MODE>mode)"
  [(const_int 0)]
{
  rs6000_split_logical (operands, GET_CODE (operands[3]), false, false, false);
  DONE;
}
  [(set (attr "type")
      (if_then_else
	(match_test "vsx_register_operand (operands[0], <MODE>mode)")
	(const_string "vecsimple")
	(const_string "integer")))
   (set (attr "length")
      (if_then_else
	(match_test "vsx_register_operand (operands[0], <MODE>mode)")
	(const_string "4")
	(if_then_else
	 (match_test "TARGET_POWERPC64")
	 (const_string "8")
	 (const_string "16"))))])

;; 128-bit ANDC/ORC
(define_insn_and_split "*boolc<mode>3_internal1"
  [(set (match_operand:BOOL_128 0 "vlogical_operand" "=<BOOL_REGS_OUTPUT>")
	(match_operator:BOOL_128 3 "boolean_operator"
	 [(not:BOOL_128
	   (match_operand:BOOL_128 2 "vlogical_operand" "<BOOL_REGS_OP2>"))
	  (match_operand:BOOL_128 1 "vlogical_operand" "<BOOL_REGS_OP1>")]))]
  "TARGET_P8_VECTOR || (GET_CODE (operands[3]) == AND)"
{
  if (TARGET_VSX && vsx_register_operand (operands[0], <MODE>mode))
    return "xxl%q3 %x0,%x1,%x2";

  if (TARGET_ALTIVEC && altivec_register_operand (operands[0], <MODE>mode))
    return "v%q3 %0,%1,%2";

  return "#";
}
  "(TARGET_P8_VECTOR || (GET_CODE (operands[3]) == AND))
   && reload_completed && int_reg_operand (operands[0], <MODE>mode)"
  [(const_int 0)]
{
  rs6000_split_logical (operands, GET_CODE (operands[3]), false, false, true);
  DONE;
}
  [(set (attr "type")
      (if_then_else
	(match_test "vsx_register_operand (operands[0], <MODE>mode)")
	(const_string "vecsimple")
	(const_string "integer")))
   (set (attr "length")
      (if_then_else
	(match_test "vsx_register_operand (operands[0], <MODE>mode)")
	(const_string "4")
	(if_then_else
	 (match_test "TARGET_POWERPC64")
	 (const_string "8")
	 (const_string "16"))))])

(define_insn_and_split "*boolc<mode>3_internal2"
  [(set (match_operand:TI2 0 "int_reg_operand" "=&r,r,r")
	(match_operator:TI2 3 "boolean_operator"
	 [(not:TI2
	   (match_operand:TI2 2 "int_reg_operand" "r,0,r"))
	  (match_operand:TI2 1 "int_reg_operand" "r,r,0")]))]
  "!TARGET_P8_VECTOR && (GET_CODE (operands[3]) != AND)"
  "#"
  "reload_completed && !TARGET_P8_VECTOR && (GET_CODE (operands[3]) != AND)"
  [(const_int 0)]
{
  rs6000_split_logical (operands, GET_CODE (operands[3]), false, false, true);
  DONE;
}
  [(set_attr "type" "integer")
   (set (attr "length")
	(if_then_else
	 (match_test "TARGET_POWERPC64")
	 (const_string "8")
	 (const_string "16")))])

;; 128-bit NAND/NOR
(define_insn_and_split "*boolcc<mode>3_internal1"
  [(set (match_operand:BOOL_128 0 "vlogical_operand" "=<BOOL_REGS_OUTPUT>")
	(match_operator:BOOL_128 3 "boolean_operator"
	 [(not:BOOL_128
	   (match_operand:BOOL_128 1 "vlogical_operand" "<BOOL_REGS_OP1>"))
	  (not:BOOL_128
	   (match_operand:BOOL_128 2 "vlogical_operand" "<BOOL_REGS_OP2>"))]))]
  "TARGET_P8_VECTOR || (GET_CODE (operands[3]) == AND)"
{
  if (TARGET_VSX && vsx_register_operand (operands[0], <MODE>mode))
    return "xxl%q3 %x0,%x1,%x2";

  if (TARGET_ALTIVEC && altivec_register_operand (operands[0], <MODE>mode))
    return "v%q3 %0,%1,%2";

  return "#";
}
  "(TARGET_P8_VECTOR || (GET_CODE (operands[3]) == AND))
   && reload_completed && int_reg_operand (operands[0], <MODE>mode)"
  [(const_int 0)]
{
  rs6000_split_logical (operands, GET_CODE (operands[3]), false, true, true);
  DONE;
}
  [(set (attr "type")
      (if_then_else
	(match_test "vsx_register_operand (operands[0], <MODE>mode)")
	(const_string "vecsimple")
	(const_string "integer")))
   (set (attr "length")
      (if_then_else
	(match_test "vsx_register_operand (operands[0], <MODE>mode)")
	(const_string "4")
	(if_then_else
	 (match_test "TARGET_POWERPC64")
	 (const_string "8")
	 (const_string "16"))))])

(define_insn_and_split "*boolcc<mode>3_internal2"
  [(set (match_operand:TI2 0 "int_reg_operand" "=&r,r,r")
	(match_operator:TI2 3 "boolean_operator"
	 [(not:TI2
	   (match_operand:TI2 1 "int_reg_operand" "r,0,r"))
	  (not:TI2
	   (match_operand:TI2 2 "int_reg_operand" "r,r,0"))]))]
  "!TARGET_P8_VECTOR && (GET_CODE (operands[3]) != AND)"
  "#"
  "reload_completed && !TARGET_P8_VECTOR && (GET_CODE (operands[3]) != AND)"
  [(const_int 0)]
{
  rs6000_split_logical (operands, GET_CODE (operands[3]), false, true, true);
  DONE;
}
  [(set_attr "type" "integer")
   (set (attr "length")
	(if_then_else
	 (match_test "TARGET_POWERPC64")
	 (const_string "8")
	 (const_string "16")))])


;; 128-bit EQV
(define_insn_and_split "*eqv<mode>3_internal1"
  [(set (match_operand:BOOL_128 0 "vlogical_operand" "=<BOOL_REGS_OUTPUT>")
	(not:BOOL_128
	 (xor:BOOL_128
	  (match_operand:BOOL_128 1 "vlogical_operand" "<BOOL_REGS_OP1>")
	  (match_operand:BOOL_128 2 "vlogical_operand" "<BOOL_REGS_OP2>"))))]
  "TARGET_P8_VECTOR"
{
  if (vsx_register_operand (operands[0], <MODE>mode))
    return "xxleqv %x0,%x1,%x2";

  return "#";
}
  "TARGET_P8_VECTOR && reload_completed
   && int_reg_operand (operands[0], <MODE>mode)"
  [(const_int 0)]
{
  rs6000_split_logical (operands, XOR, true, false, false);
  DONE;
}
  [(set (attr "type")
      (if_then_else
	(match_test "vsx_register_operand (operands[0], <MODE>mode)")
	(const_string "vecsimple")
	(const_string "integer")))
   (set (attr "length")
      (if_then_else
	(match_test "vsx_register_operand (operands[0], <MODE>mode)")
	(const_string "4")
	(if_then_else
	 (match_test "TARGET_POWERPC64")
	 (const_string "8")
	 (const_string "16"))))])

(define_insn_and_split "*eqv<mode>3_internal2"
  [(set (match_operand:TI2 0 "int_reg_operand" "=&r,r,r")
	(not:TI2
	 (xor:TI2
	  (match_operand:TI2 1 "int_reg_operand" "r,0,r")
	  (match_operand:TI2 2 "int_reg_operand" "r,r,0"))))]
  "!TARGET_P8_VECTOR"
  "#"
  "reload_completed && !TARGET_P8_VECTOR"
  [(const_int 0)]
{
  rs6000_split_logical (operands, XOR, true, false, false);
  DONE;
}
  [(set_attr "type" "integer")
   (set (attr "length")
	(if_then_else
	 (match_test "TARGET_POWERPC64")
	 (const_string "8")
	 (const_string "16")))])

;; 128-bit one's complement
(define_insn_and_split "*one_cmpl<mode>3_internal"
  [(set (match_operand:BOOL_128 0 "vlogical_operand" "=<BOOL_REGS_OUTPUT>")
	(not:BOOL_128
	  (match_operand:BOOL_128 1 "vlogical_operand" "<BOOL_REGS_UNARY>")))]
  ""
{
  if (TARGET_VSX && vsx_register_operand (operands[0], <MODE>mode))
    return "xxlnor %x0,%x1,%x1";

  if (TARGET_ALTIVEC && altivec_register_operand (operands[0], <MODE>mode))
    return "vnor %0,%1,%1";

  return "#";
}
  "reload_completed && int_reg_operand (operands[0], <MODE>mode)"
  [(const_int 0)]
{
  rs6000_split_logical (operands, NOT, false, false, false);
  DONE;
}
  [(set (attr "type")
      (if_then_else
	(match_test "vsx_register_operand (operands[0], <MODE>mode)")
	(const_string "vecsimple")
	(const_string "integer")))
   (set (attr "length")
      (if_then_else
	(match_test "vsx_register_operand (operands[0], <MODE>mode)")
	(const_string "4")
	(if_then_else
	 (match_test "TARGET_POWERPC64")
	 (const_string "8")
	 (const_string "16"))))])


;; Now define ways of moving data around.

;; Set up a register with a value from the GOT table

(define_expand "movsi_got"
  [(set (match_operand:SI 0 "gpc_reg_operand" "")
	(unspec:SI [(match_operand:SI 1 "got_operand" "")
		    (match_dup 2)] UNSPEC_MOVSI_GOT))]
  "DEFAULT_ABI == ABI_V4 && flag_pic == 1"
  "
{
  if (GET_CODE (operands[1]) == CONST)
    {
      rtx offset = const0_rtx;
      HOST_WIDE_INT value;

      operands[1] = eliminate_constant_term (XEXP (operands[1], 0), &offset);
      value = INTVAL (offset);
      if (value != 0)
	{
	  rtx tmp = (!can_create_pseudo_p ()
		     ? operands[0]
		     : gen_reg_rtx (Pmode));
	  emit_insn (gen_movsi_got (tmp, operands[1]));
	  emit_insn (gen_addsi3 (operands[0], tmp, offset));
	  DONE;
	}
    }

  operands[2] = rs6000_got_register (operands[1]);
}")

(define_insn "*movsi_got_internal"
  [(set (match_operand:SI 0 "gpc_reg_operand" "=r")
	(unspec:SI [(match_operand:SI 1 "got_no_const_operand" "")
		    (match_operand:SI 2 "gpc_reg_operand" "b")]
		   UNSPEC_MOVSI_GOT))]
  "DEFAULT_ABI == ABI_V4 && flag_pic == 1"
  "lwz %0,%a1@got(%2)"
  [(set_attr "type" "load")])

;; Used by sched, shorten_branches and final when the GOT pseudo reg
;; didn't get allocated to a hard register.
(define_split
  [(set (match_operand:SI 0 "gpc_reg_operand" "")
	(unspec:SI [(match_operand:SI 1 "got_no_const_operand" "")
		    (match_operand:SI 2 "memory_operand" "")]
		   UNSPEC_MOVSI_GOT))]
  "DEFAULT_ABI == ABI_V4
    && flag_pic == 1
    && (reload_in_progress || reload_completed)"
  [(set (match_dup 0) (match_dup 2))
   (set (match_dup 0) (unspec:SI [(match_dup 1)(match_dup 0)]
				 UNSPEC_MOVSI_GOT))]
  "")

;; For SI, we special-case integers that can't be loaded in one insn.  We
;; do the load 16-bits at a time.  We could do this by loading from memory,
;; and this is even supposed to be faster, but it is simpler not to get
;; integers in the TOC.
(define_insn "movsi_low"
  [(set (match_operand:SI 0 "gpc_reg_operand" "=r")
        (mem:SI (lo_sum:SI (match_operand:SI 1 "gpc_reg_operand" "b")
                           (match_operand 2 "" ""))))]
  "TARGET_MACHO && ! TARGET_64BIT"
  "lwz %0,lo16(%2)(%1)"
  [(set_attr "type" "load")
   (set_attr "length" "4")])

(define_insn "*movsi_internal1"
  [(set (match_operand:SI 0 "rs6000_nonimmediate_operand" "=r,r,r,m,r,r,r,r,*c*l,*h,*h")
	(match_operand:SI 1 "input_operand" "r,U,m,r,I,L,n,*h,r,r,0"))]
  "!TARGET_SINGLE_FPU &&
   (gpc_reg_operand (operands[0], SImode) || gpc_reg_operand (operands[1], SImode))"
  "@
   mr %0,%1
   la %0,%a1
   lwz%U1%X1 %0,%1
   stw%U0%X0 %1,%0
   li %0,%1
   lis %0,%v1
   #
   mf%1 %0
   mt%0 %1
   mt%0 %1
   nop"
  [(set_attr "type" "*,*,load,store,*,*,*,mfjmpr,mtjmpr,*,*")
   (set_attr "length" "4,4,4,4,4,4,8,4,4,4,4")])

(define_insn "*movsi_internal1_single"
  [(set (match_operand:SI 0 "rs6000_nonimmediate_operand" "=r,r,r,m,r,r,r,r,*c*l,*h,*h,m,*f")
        (match_operand:SI 1 "input_operand" "r,U,m,r,I,L,n,*h,r,r,0,f,m"))]
  "TARGET_SINGLE_FPU &&
   (gpc_reg_operand (operands[0], SImode) || gpc_reg_operand (operands[1], SImode))"
  "@
   mr %0,%1
   la %0,%a1
   lwz%U1%X1 %0,%1
   stw%U0%X0 %1,%0
   li %0,%1
   lis %0,%v1
   #
   mf%1 %0
   mt%0 %1
   mt%0 %1
   nop
   stfs%U0%X0 %1,%0
   lfs%U1%X1 %0,%1"
  [(set_attr "type" "*,*,load,store,*,*,*,mfjmpr,mtjmpr,*,*,fpstore,fpload")
   (set_attr "length" "4,4,4,4,4,4,8,4,4,4,4,4,4")])

;; Split a load of a large constant into the appropriate two-insn
;; sequence.

(define_split
  [(set (match_operand:SI 0 "gpc_reg_operand" "")
	(match_operand:SI 1 "const_int_operand" ""))]
  "(unsigned HOST_WIDE_INT) (INTVAL (operands[1]) + 0x8000) >= 0x10000
   && (INTVAL (operands[1]) & 0xffff) != 0"
  [(set (match_dup 0)
	(match_dup 2))
   (set (match_dup 0)
	(ior:SI (match_dup 0)
		(match_dup 3)))]
  "
{
  if (rs6000_emit_set_const (operands[0], operands[1]))
    DONE;
  else
    FAIL;
}")

(define_insn "*mov<mode>_internal2"
  [(set (match_operand:CC 2 "cc_reg_operand" "=y,x,?y")
	(compare:CC (match_operand:P 1 "gpc_reg_operand" "0,r,r")
		    (const_int 0)))
   (set (match_operand:P 0 "gpc_reg_operand" "=r,r,r") (match_dup 1))]
  ""
  "@
   cmp<wd>i %2,%0,0
   mr. %0,%1
   #"
  [(set_attr "type" "cmp,logical,cmp")
   (set_attr "dot" "yes")
   (set_attr "length" "4,4,8")])

(define_split
  [(set (match_operand:CC 2 "cc_reg_not_micro_cr0_operand" "")
	(compare:CC (match_operand:P 1 "gpc_reg_operand" "")
		    (const_int 0)))
   (set (match_operand:P 0 "gpc_reg_operand" "") (match_dup 1))]
  "reload_completed"
  [(set (match_dup 0) (match_dup 1))
   (set (match_dup 2)
	(compare:CC (match_dup 0)
		    (const_int 0)))]
  "")

(define_insn "*movhi_internal"
  [(set (match_operand:HI 0 "nonimmediate_operand" "=r,r,m,r,r,*c*l,*h")
	(match_operand:HI 1 "input_operand" "r,m,r,i,*h,r,0"))]
  "gpc_reg_operand (operands[0], HImode)
   || gpc_reg_operand (operands[1], HImode)"
  "@
   mr %0,%1
   lhz%U1%X1 %0,%1
   sth%U0%X0 %1,%0
   li %0,%w1
   mf%1 %0
   mt%0 %1
   nop"
  [(set_attr "type" "*,load,store,*,mfjmpr,mtjmpr,*")])

(define_expand "mov<mode>"
  [(set (match_operand:INT 0 "general_operand" "")
	(match_operand:INT 1 "any_operand" ""))]
  ""
  "{ rs6000_emit_move (operands[0], operands[1], <MODE>mode); DONE; }")

(define_insn "*movqi_internal"
  [(set (match_operand:QI 0 "nonimmediate_operand" "=r,r,m,r,r,*c*l,*h")
	(match_operand:QI 1 "input_operand" "r,m,r,i,*h,r,0"))]
  "gpc_reg_operand (operands[0], QImode)
   || gpc_reg_operand (operands[1], QImode)"
  "@
   mr %0,%1
   lbz%U1%X1 %0,%1
   stb%U0%X0 %1,%0
   li %0,%1
   mf%1 %0
   mt%0 %1
   nop"
  [(set_attr "type" "*,load,store,*,mfjmpr,mtjmpr,*")])

;; Here is how to move condition codes around.  When we store CC data in
;; an integer register or memory, we store just the high-order 4 bits.
;; This lets us not shift in the most common case of CR0.
(define_expand "movcc"
  [(set (match_operand:CC 0 "nonimmediate_operand" "")
	(match_operand:CC 1 "nonimmediate_operand" ""))]
  ""
  "")

(define_insn "*movcc_internal1"
  [(set (match_operand:CC 0 "nonimmediate_operand" "=y,x,?y,y,r,r,r,r,r,cl,r,m")
	(match_operand:CC 1 "general_operand" "y,r,r,O,x,y,r,I,h,r,m,r"))]
  "register_operand (operands[0], CCmode)
   || register_operand (operands[1], CCmode)"
  "@
   mcrf %0,%1
   mtcrf 128,%1
   rlwinm %1,%1,%F0,0xffffffff\;mtcrf %R0,%1\;rlwinm %1,%1,%f0,0xffffffff
   crxor %0,%0,%0
   mfcr %0%Q1
   mfcr %0%Q1\;rlwinm %0,%0,%f1,0xf0000000
   mr %0,%1
   li %0,%1
   mf%1 %0
   mt%0 %1
   lwz%U1%X1 %0,%1
   stw%U0%X0 %1,%0"
  [(set (attr "type")
     (cond [(eq_attr "alternative" "0,3")
		(const_string "cr_logical")
	    (eq_attr "alternative" "1,2")
		(const_string "mtcr")
	    (eq_attr "alternative" "6,7")
		(const_string "integer")
	    (eq_attr "alternative" "8")
		(const_string "mfjmpr")
	    (eq_attr "alternative" "9")
		(const_string "mtjmpr")
	    (eq_attr "alternative" "10")
		(const_string "load")
	    (eq_attr "alternative" "11")
		(const_string "store")
	    (match_test "TARGET_MFCRF")
		(const_string "mfcrf")
	   ]
	(const_string "mfcr")))
   (set_attr "length" "4,4,12,4,4,8,4,4,4,4,4,4")])

;; For floating-point, we normally deal with the floating-point registers
;; unless -msoft-float is used.  The sole exception is that parameter passing
;; can produce floating-point values in fixed-point registers.  Unless the
;; value is a simple constant or already in memory, we deal with this by
;; allocating memory and copying the value explicitly via that memory location.

;; Move 32-bit binary/decimal floating point
(define_expand "mov<mode>"
  [(set (match_operand:FMOVE32 0 "nonimmediate_operand" "")
	(match_operand:FMOVE32 1 "any_operand" ""))]
  "<fmove_ok>"
  "{ rs6000_emit_move (operands[0], operands[1], <MODE>mode); DONE; }")

(define_split
  [(set (match_operand:FMOVE32 0 "gpc_reg_operand" "")
	(match_operand:FMOVE32 1 "const_double_operand" ""))]
  "reload_completed
   && ((GET_CODE (operands[0]) == REG && REGNO (operands[0]) <= 31)
       || (GET_CODE (operands[0]) == SUBREG
	   && GET_CODE (SUBREG_REG (operands[0])) == REG
	   && REGNO (SUBREG_REG (operands[0])) <= 31))"
  [(set (match_dup 2) (match_dup 3))]
  "
{
  long l;
  REAL_VALUE_TYPE rv;

  REAL_VALUE_FROM_CONST_DOUBLE (rv, operands[1]);
  <real_value_to_target> (rv, l);

  if (! TARGET_POWERPC64)
    operands[2] = operand_subword (operands[0], 0, 0, <MODE>mode);
  else
    operands[2] = gen_lowpart (SImode, operands[0]);

  operands[3] = gen_int_mode (l, SImode);
}")

(define_insn "mov<mode>_hardfloat"
  [(set (match_operand:FMOVE32 0 "nonimmediate_operand" "=!r,!r,m,f,<f32_vsx>,<f32_vsx>,!r,<f32_lr>,<f32_sm>,<f32_av>,Z,?<f32_dm>,?r,*c*l,!r,*h")
	(match_operand:FMOVE32 1 "input_operand" "r,m,r,f,<f32_vsx>,j,j,<f32_lm>,<f32_sr>,Z,<f32_av>,r,<f32_dm>,r,h,0"))]
  "(gpc_reg_operand (operands[0], <MODE>mode)
   || gpc_reg_operand (operands[1], <MODE>mode))
   && (TARGET_HARD_FLOAT && TARGET_FPRS && TARGET_SINGLE_FLOAT)"
  "@
   mr %0,%1
   lwz%U1%X1 %0,%1
   stw%U0%X0 %1,%0
   fmr %0,%1
   xscpsgndp %x0,%x1,%x1
   xxlxor %x0,%x0,%x0
   li %0,0
   <f32_li>
   <f32_si>
   <f32_lv>
   <f32_sv>
   mtvsrwz %x0,%1
   mfvsrwz %0,%x1
   mt%0 %1
   mf%1 %0
   nop"
  [(set_attr "type" "*,load,store,fp,fp,vecsimple,integer,fpload,fpstore,fpload,fpstore,mftgpr,mffgpr,mtjmpr,mfjmpr,*")
   (set_attr "length" "4")])

(define_insn "*mov<mode>_softfloat"
  [(set (match_operand:FMOVE32 0 "nonimmediate_operand" "=r,cl,r,r,m,r,r,r,r,*h")
	(match_operand:FMOVE32 1 "input_operand" "r,r,h,m,r,I,L,G,Fn,0"))]
  "(gpc_reg_operand (operands[0], <MODE>mode)
   || gpc_reg_operand (operands[1], <MODE>mode))
   && (TARGET_SOFT_FLOAT || !TARGET_FPRS)"
  "@
   mr %0,%1
   mt%0 %1
   mf%1 %0
   lwz%U1%X1 %0,%1
   stw%U0%X0 %1,%0
   li %0,%1
   lis %0,%v1
   #
   #
   nop"
  [(set_attr "type" "*,mtjmpr,mfjmpr,load,store,*,*,*,*,*")
   (set_attr "length" "4,4,4,4,4,4,4,4,8,4")])


;; Move 64-bit binary/decimal floating point
(define_expand "mov<mode>"
  [(set (match_operand:FMOVE64 0 "nonimmediate_operand" "")
	(match_operand:FMOVE64 1 "any_operand" ""))]
  ""
  "{ rs6000_emit_move (operands[0], operands[1], <MODE>mode); DONE; }")

(define_split
  [(set (match_operand:FMOVE64 0 "gpc_reg_operand" "")
	(match_operand:FMOVE64 1 "const_int_operand" ""))]
  "! TARGET_POWERPC64 && reload_completed
   && ((GET_CODE (operands[0]) == REG && REGNO (operands[0]) <= 31)
       || (GET_CODE (operands[0]) == SUBREG
	   && GET_CODE (SUBREG_REG (operands[0])) == REG
	   && REGNO (SUBREG_REG (operands[0])) <= 31))"
  [(set (match_dup 2) (match_dup 4))
   (set (match_dup 3) (match_dup 1))]
  "
{
  int endian = (WORDS_BIG_ENDIAN == 0);
  HOST_WIDE_INT value = INTVAL (operands[1]);

  operands[2] = operand_subword (operands[0], endian, 0, <MODE>mode);
  operands[3] = operand_subword (operands[0], 1 - endian, 0, <MODE>mode);
  operands[4] = GEN_INT (value >> 32);
  operands[1] = GEN_INT (((value & 0xffffffff) ^ 0x80000000) - 0x80000000);
}")

(define_split
  [(set (match_operand:FMOVE64 0 "gpc_reg_operand" "")
	(match_operand:FMOVE64 1 "const_double_operand" ""))]
  "! TARGET_POWERPC64 && reload_completed
   && ((GET_CODE (operands[0]) == REG && REGNO (operands[0]) <= 31)
       || (GET_CODE (operands[0]) == SUBREG
	   && GET_CODE (SUBREG_REG (operands[0])) == REG
	   && REGNO (SUBREG_REG (operands[0])) <= 31))"
  [(set (match_dup 2) (match_dup 4))
   (set (match_dup 3) (match_dup 5))]
  "
{
  int endian = (WORDS_BIG_ENDIAN == 0);
  long l[2];
  REAL_VALUE_TYPE rv;

  REAL_VALUE_FROM_CONST_DOUBLE (rv, operands[1]);
  <real_value_to_target> (rv, l);

  operands[2] = operand_subword (operands[0], endian, 0, <MODE>mode);
  operands[3] = operand_subword (operands[0], 1 - endian, 0, <MODE>mode);
  operands[4] = gen_int_mode (l[endian], SImode);
  operands[5] = gen_int_mode (l[1 - endian], SImode);
}")

(define_split
  [(set (match_operand:FMOVE64 0 "gpc_reg_operand" "")
	(match_operand:FMOVE64 1 "const_double_operand" ""))]
  "TARGET_POWERPC64 && reload_completed
   && ((GET_CODE (operands[0]) == REG && REGNO (operands[0]) <= 31)
       || (GET_CODE (operands[0]) == SUBREG
	   && GET_CODE (SUBREG_REG (operands[0])) == REG
	   && REGNO (SUBREG_REG (operands[0])) <= 31))"
  [(set (match_dup 2) (match_dup 3))]
  "
{
  int endian = (WORDS_BIG_ENDIAN == 0);
  long l[2];
  REAL_VALUE_TYPE rv;
  HOST_WIDE_INT val;

  REAL_VALUE_FROM_CONST_DOUBLE (rv, operands[1]);
  <real_value_to_target> (rv, l);

  operands[2] = gen_lowpart (DImode, operands[0]);
  /* HIGHPART is lower memory address when WORDS_BIG_ENDIAN.  */
  val = ((HOST_WIDE_INT)(unsigned long)l[endian] << 32
         | ((HOST_WIDE_INT)(unsigned long)l[1 - endian]));

  operands[3] = gen_int_mode (val, DImode);
}")

;; Don't have reload use general registers to load a constant.  It is
;; less efficient than loading the constant into an FP register, since
;; it will probably be used there.

;; The move constraints are ordered to prefer floating point registers before
;; general purpose registers to avoid doing a store and a load to get the value
;; into a floating point register when it is needed for a floating point
;; operation.  Prefer traditional floating point registers over VSX registers,
;; since the D-form version of the memory instructions does not need a GPR for
;; reloading.

;; If we have FPR registers, rs6000_emit_move has moved all constants to memory,
;; except for 0.0 which can be created on VSX with an xor instruction.

(define_insn "*mov<mode>_hardfloat32"
  [(set (match_operand:FMOVE64 0 "nonimmediate_operand" "=m,d,d,<f64_av>,Z,<f64_vsx>,<f64_vsx>,!r,Y,r,!r")
	(match_operand:FMOVE64 1 "input_operand" "d,m,d,Z,<f64_av>,<f64_vsx>,j,j,r,Y,r"))]
  "! TARGET_POWERPC64 && TARGET_HARD_FLOAT && TARGET_FPRS && TARGET_DOUBLE_FLOAT 
   && (gpc_reg_operand (operands[0], <MODE>mode)
       || gpc_reg_operand (operands[1], <MODE>mode))"
  "@
   stfd%U0%X0 %1,%0
   lfd%U1%X1 %0,%1
   fmr %0,%1
   lxsd%U1x %x0,%y1
   stxsd%U0x %x1,%y0
   xxlor %x0,%x1,%x1
   xxlxor %x0,%x0,%x0
   #
   #
   #
   #"
  [(set_attr "type" "fpstore,fpload,fp,fpload,fpstore,vecsimple,vecsimple,two,store,load,two")
   (set_attr "length" "4,4,4,4,4,4,4,8,8,8,8")])

(define_insn "*mov<mode>_softfloat32"
  [(set (match_operand:FMOVE64 0 "nonimmediate_operand" "=Y,r,r,r,r,r")
	(match_operand:FMOVE64 1 "input_operand" "r,Y,r,G,H,F"))]
  "! TARGET_POWERPC64 
   && ((TARGET_FPRS && TARGET_SINGLE_FLOAT) 
       || TARGET_SOFT_FLOAT || TARGET_E500_SINGLE
       || (<MODE>mode == DDmode && TARGET_E500_DOUBLE))
   && (gpc_reg_operand (operands[0], <MODE>mode)
       || gpc_reg_operand (operands[1], <MODE>mode))"
  "#"
  [(set_attr "type" "store,load,two,*,*,*")
   (set_attr "length" "8,8,8,8,12,16")])

; ld/std require word-aligned displacements -> 'Y' constraint.
; List Y->r and r->Y before r->r for reload.
(define_insn "*mov<mode>_hardfloat64"
  [(set (match_operand:FMOVE64 0 "nonimmediate_operand" "=m,d,d,<f64_av>,Z,<f64_vsx>,<f64_vsx>,!r,Y,r,!r,*c*l,!r,*h,r,wg,r,<f64_dm>")
	(match_operand:FMOVE64 1 "input_operand" "d,m,d,Z,<f64_av>,<f64_vsx>,j,j,r,Y,r,r,h,0,wg,r,<f64_dm>,r"))]
  "TARGET_POWERPC64 && TARGET_HARD_FLOAT && TARGET_FPRS && TARGET_DOUBLE_FLOAT
   && (gpc_reg_operand (operands[0], <MODE>mode)
       || gpc_reg_operand (operands[1], <MODE>mode))"
  "@
   stfd%U0%X0 %1,%0
   lfd%U1%X1 %0,%1
   fmr %0,%1
   lxsd%U1x %x0,%y1
   stxsd%U0x %x1,%y0
   xxlor %x0,%x1,%x1
   xxlxor %x0,%x0,%x0
   li %0,0
   std%U0%X0 %1,%0
   ld%U1%X1 %0,%1
   mr %0,%1
   mt%0 %1
   mf%1 %0
   nop
   mftgpr %0,%1
   mffgpr %0,%1
   mfvsrd %0,%x1
   mtvsrd %x0,%1"
  [(set_attr "type" "fpstore,fpload,fp,fpload,fpstore,vecsimple,vecsimple,integer,store,load,*,mtjmpr,mfjmpr,*,mftgpr,mffgpr,mftgpr,mffgpr")
   (set_attr "length" "4")])

(define_insn "*mov<mode>_softfloat64"
  [(set (match_operand:FMOVE64 0 "nonimmediate_operand" "=Y,r,r,cl,r,r,r,r,*h")
	(match_operand:FMOVE64 1 "input_operand" "r,Y,r,r,h,G,H,F,0"))]
  "TARGET_POWERPC64 && (TARGET_SOFT_FLOAT || !TARGET_FPRS)
   && (gpc_reg_operand (operands[0], <MODE>mode)
       || gpc_reg_operand (operands[1], <MODE>mode))"
  "@
   std%U0%X0 %1,%0
   ld%U1%X1 %0,%1
   mr %0,%1
   mt%0 %1
   mf%1 %0
   #
   #
   #
   nop"
  [(set_attr "type" "store,load,*,mtjmpr,mfjmpr,*,*,*,*")
   (set_attr "length" "4,4,4,4,4,8,12,16,4")])

(define_expand "mov<mode>"
  [(set (match_operand:FMOVE128 0 "general_operand" "")
	(match_operand:FMOVE128 1 "any_operand" ""))]
  ""
  "{ rs6000_emit_move (operands[0], operands[1], <MODE>mode); DONE; }")

;; It's important to list Y->r and r->Y before r->r because otherwise
;; reload, given m->r, will try to pick r->r and reload it, which
;; doesn't make progress.

;; We can't split little endian direct moves of TDmode, because the words are
;; not swapped like they are for TImode or TFmode.  Subregs therefore are
;; problematical.  Don't allow direct move for this case.

(define_insn_and_split "*mov<mode>_64bit_dm"
  [(set (match_operand:FMOVE128 0 "nonimmediate_operand" "=m,d,d,ws,Y,r,r,r,wm")
	(match_operand:FMOVE128 1 "input_operand" "d,m,d,j,r,jY,r,wm,r"))]
  "TARGET_HARD_FLOAT && TARGET_FPRS && TARGET_POWERPC64
   && (<MODE>mode != TDmode || WORDS_BIG_ENDIAN)
   && (gpc_reg_operand (operands[0], <MODE>mode)
       || gpc_reg_operand (operands[1], <MODE>mode))"
  "#"
  "&& reload_completed"
  [(pc)]
{ rs6000_split_multireg_move (operands[0], operands[1]); DONE; }
  [(set_attr "length" "8,8,8,8,12,12,8,8,8")])

(define_insn_and_split "*movtd_64bit_nodm"
  [(set (match_operand:TD 0 "nonimmediate_operand" "=m,d,d,ws,Y,r,r")
	(match_operand:TD 1 "input_operand" "d,m,d,j,r,jY,r"))]
  "TARGET_HARD_FLOAT && TARGET_FPRS && TARGET_POWERPC64 && !WORDS_BIG_ENDIAN
   && (gpc_reg_operand (operands[0], TDmode)
       || gpc_reg_operand (operands[1], TDmode))"
  "#"
  "&& reload_completed"
  [(pc)]
{ rs6000_split_multireg_move (operands[0], operands[1]); DONE; }
  [(set_attr "length" "8,8,8,8,12,12,8")])

(define_insn_and_split "*mov<mode>_32bit"
  [(set (match_operand:FMOVE128 0 "nonimmediate_operand" "=m,d,d,ws,Y,r,r")
	(match_operand:FMOVE128 1 "input_operand" "d,m,d,j,r,jY,r"))]
  "TARGET_HARD_FLOAT && TARGET_FPRS && !TARGET_POWERPC64
   && (gpc_reg_operand (operands[0], <MODE>mode)
       || gpc_reg_operand (operands[1], <MODE>mode))"
  "#"
  "&& reload_completed"
  [(pc)]
{ rs6000_split_multireg_move (operands[0], operands[1]); DONE; }
  [(set_attr "length" "8,8,8,8,20,20,16")])

(define_insn_and_split "*mov<mode>_softfloat"
  [(set (match_operand:FMOVE128 0 "rs6000_nonimmediate_operand" "=Y,r,r")
	(match_operand:FMOVE128 1 "input_operand" "r,YGHF,r"))]
  "(TARGET_SOFT_FLOAT || !TARGET_FPRS)
   && (gpc_reg_operand (operands[0], <MODE>mode)
       || gpc_reg_operand (operands[1], <MODE>mode))"
  "#"
  "&& reload_completed"
  [(pc)]
{ rs6000_split_multireg_move (operands[0], operands[1]); DONE; }
  [(set_attr "length" "20,20,16")])

(define_expand "extenddftf2"
  [(set (match_operand:TF 0 "nonimmediate_operand" "")
	(float_extend:TF (match_operand:DF 1 "input_operand" "")))]
  "!TARGET_IEEEQUAD
   && TARGET_HARD_FLOAT
   && (TARGET_FPRS || TARGET_E500_DOUBLE)
   && TARGET_LONG_DOUBLE_128"
{
  if (TARGET_E500_DOUBLE)
    emit_insn (gen_spe_extenddftf2 (operands[0], operands[1]));
  else
    emit_insn (gen_extenddftf2_fprs (operands[0], operands[1]));
  DONE;
})

(define_expand "extenddftf2_fprs"
  [(parallel [(set (match_operand:TF 0 "nonimmediate_operand" "")
		   (float_extend:TF (match_operand:DF 1 "input_operand" "")))
	      (use (match_dup 2))])]
  "!TARGET_IEEEQUAD
   && TARGET_HARD_FLOAT && TARGET_FPRS && TARGET_DOUBLE_FLOAT 
   && TARGET_LONG_DOUBLE_128"
{
  /* VSX can create 0.0 directly, otherwise let rs6000_emit_move create
     the proper constant.  */
  if (TARGET_VSX)
    operands[2] = CONST0_RTX (DFmode);
  else
    {
      operands[2] = gen_reg_rtx (DFmode);
      rs6000_emit_move (operands[2], CONST0_RTX (DFmode), DFmode);
    }
})

(define_insn_and_split "*extenddftf2_internal"
  [(set (match_operand:TF 0 "nonimmediate_operand" "=m,Y,ws,d,&d")
       (float_extend:TF (match_operand:DF 1 "input_operand" "d,r,md,md,md")))
   (use (match_operand:DF 2 "zero_reg_mem_operand" "d,r,j,m,d"))]
  "!TARGET_IEEEQUAD
   && TARGET_HARD_FLOAT && TARGET_FPRS && TARGET_DOUBLE_FLOAT 
   && TARGET_LONG_DOUBLE_128"
  "#"
  "&& reload_completed"
  [(pc)]
{
  const int lo_word = LONG_DOUBLE_LARGE_FIRST ? GET_MODE_SIZE (DFmode) : 0;
  const int hi_word = LONG_DOUBLE_LARGE_FIRST ? 0 : GET_MODE_SIZE (DFmode);
  emit_move_insn (simplify_gen_subreg (DFmode, operands[0], TFmode, hi_word),
		  operands[1]);
  emit_move_insn (simplify_gen_subreg (DFmode, operands[0], TFmode, lo_word),
		  operands[2]);
  DONE;
})

(define_expand "extendsftf2"
  [(set (match_operand:TF 0 "nonimmediate_operand" "")
	(float_extend:TF (match_operand:SF 1 "gpc_reg_operand" "")))]
  "!TARGET_IEEEQUAD
   && TARGET_HARD_FLOAT
   && (TARGET_FPRS || TARGET_E500_DOUBLE)
   && TARGET_LONG_DOUBLE_128"
{
  rtx tmp = gen_reg_rtx (DFmode);
  emit_insn (gen_extendsfdf2 (tmp, operands[1]));
  emit_insn (gen_extenddftf2 (operands[0], tmp));
  DONE;
})

(define_expand "trunctfdf2"
  [(set (match_operand:DF 0 "gpc_reg_operand" "")
	(float_truncate:DF (match_operand:TF 1 "gpc_reg_operand" "")))]
  "!TARGET_IEEEQUAD
   && TARGET_HARD_FLOAT
   && (TARGET_FPRS || TARGET_E500_DOUBLE)
   && TARGET_LONG_DOUBLE_128"
  "")

(define_insn_and_split "trunctfdf2_internal1"
  [(set (match_operand:DF 0 "gpc_reg_operand" "=d,?d")
	(float_truncate:DF (match_operand:TF 1 "gpc_reg_operand" "0,d")))]
  "!TARGET_IEEEQUAD && !TARGET_XL_COMPAT
   && TARGET_HARD_FLOAT && TARGET_FPRS && TARGET_LONG_DOUBLE_128"
  "@
   #
   fmr %0,%1"
  "&& reload_completed && REGNO (operands[0]) == REGNO (operands[1])"
  [(const_int 0)]
{
  emit_note (NOTE_INSN_DELETED);
  DONE;
}
  [(set_attr "type" "fp")])

(define_insn "trunctfdf2_internal2"
  [(set (match_operand:DF 0 "gpc_reg_operand" "=d")
	(float_truncate:DF (match_operand:TF 1 "gpc_reg_operand" "d")))]
  "!TARGET_IEEEQUAD && TARGET_XL_COMPAT
   && TARGET_HARD_FLOAT && TARGET_FPRS && TARGET_DOUBLE_FLOAT 
   && TARGET_LONG_DOUBLE_128"
  "fadd %0,%1,%L1"
  [(set_attr "type" "fp")
   (set_attr "fp_type" "fp_addsub_d")])

(define_expand "trunctfsf2"
  [(set (match_operand:SF 0 "gpc_reg_operand" "")
	(float_truncate:SF (match_operand:TF 1 "gpc_reg_operand" "")))]
  "!TARGET_IEEEQUAD
   && TARGET_HARD_FLOAT
   && (TARGET_FPRS || TARGET_E500_DOUBLE)
   && TARGET_LONG_DOUBLE_128"
{
  if (TARGET_E500_DOUBLE)
    emit_insn (gen_spe_trunctfsf2 (operands[0], operands[1]));
  else
    emit_insn (gen_trunctfsf2_fprs (operands[0], operands[1]));
  DONE;
})

(define_insn_and_split "trunctfsf2_fprs"
  [(set (match_operand:SF 0 "gpc_reg_operand" "=f")
	(float_truncate:SF (match_operand:TF 1 "gpc_reg_operand" "d")))
   (clobber (match_scratch:DF 2 "=d"))]
  "!TARGET_IEEEQUAD
   && TARGET_HARD_FLOAT && TARGET_FPRS && TARGET_SINGLE_FLOAT 
   && TARGET_LONG_DOUBLE_128"
  "#"
  "&& reload_completed"
  [(set (match_dup 2)
	(float_truncate:DF (match_dup 1)))
   (set (match_dup 0)
	(float_truncate:SF (match_dup 2)))]
  "")

(define_expand "floatsitf2"
  [(set (match_operand:TF 0 "gpc_reg_operand" "")
        (float:TF (match_operand:SI 1 "gpc_reg_operand" "")))]
  "!TARGET_IEEEQUAD
   && TARGET_HARD_FLOAT
   && (TARGET_FPRS || TARGET_E500_DOUBLE)
   && TARGET_LONG_DOUBLE_128"
{
  rtx tmp = gen_reg_rtx (DFmode);
  expand_float (tmp, operands[1], false);
  emit_insn (gen_extenddftf2 (operands[0], tmp));
  DONE;
})

; fadd, but rounding towards zero.
; This is probably not the optimal code sequence.
(define_insn "fix_trunc_helper"
  [(set (match_operand:DF 0 "gpc_reg_operand" "=d")
	(unspec:DF [(match_operand:TF 1 "gpc_reg_operand" "d")]
		   UNSPEC_FIX_TRUNC_TF))
   (clobber (match_operand:DF 2 "gpc_reg_operand" "=&d"))]
  "TARGET_HARD_FLOAT && TARGET_FPRS && TARGET_DOUBLE_FLOAT"
  "mffs %2\n\tmtfsb1 31\n\tmtfsb0 30\n\tfadd %0,%1,%L1\n\tmtfsf 1,%2"
  [(set_attr "type" "fp")
   (set_attr "length" "20")])

(define_expand "fix_trunctfsi2"
  [(set (match_operand:SI 0 "gpc_reg_operand" "")
	(fix:SI (match_operand:TF 1 "gpc_reg_operand" "")))]
  "!TARGET_IEEEQUAD && TARGET_HARD_FLOAT
   && (TARGET_FPRS || TARGET_E500_DOUBLE) && TARGET_LONG_DOUBLE_128"
{
  if (TARGET_E500_DOUBLE)
    emit_insn (gen_spe_fix_trunctfsi2 (operands[0], operands[1]));
  else
    emit_insn (gen_fix_trunctfsi2_fprs (operands[0], operands[1]));
  DONE;
})

(define_expand "fix_trunctfsi2_fprs"
  [(parallel [(set (match_operand:SI 0 "gpc_reg_operand" "")
		   (fix:SI (match_operand:TF 1 "gpc_reg_operand" "")))
	      (clobber (match_dup 2))
	      (clobber (match_dup 3))
	      (clobber (match_dup 4))
	      (clobber (match_dup 5))])]
  "!TARGET_IEEEQUAD
   && TARGET_HARD_FLOAT && TARGET_FPRS && TARGET_LONG_DOUBLE_128"
{
  operands[2] = gen_reg_rtx (DFmode);
  operands[3] = gen_reg_rtx (DFmode);
  operands[4] = gen_reg_rtx (DImode);
  operands[5] = assign_stack_temp (DImode, GET_MODE_SIZE (DImode));
})

(define_insn_and_split "*fix_trunctfsi2_internal"
  [(set (match_operand:SI 0 "gpc_reg_operand" "=r")
        (fix:SI (match_operand:TF 1 "gpc_reg_operand" "d")))
   (clobber (match_operand:DF 2 "gpc_reg_operand" "=d"))
   (clobber (match_operand:DF 3 "gpc_reg_operand" "=&d"))
   (clobber (match_operand:DI 4 "gpc_reg_operand" "=d"))
   (clobber (match_operand:DI 5 "offsettable_mem_operand" "=o"))]
  "!TARGET_IEEEQUAD
   && TARGET_HARD_FLOAT && TARGET_FPRS && TARGET_LONG_DOUBLE_128"
  "#"
  ""
  [(pc)]
{
  rtx lowword;
  emit_insn (gen_fix_trunc_helper (operands[2], operands[1], operands[3]));

  gcc_assert (MEM_P (operands[5]));
  lowword = adjust_address (operands[5], SImode, WORDS_BIG_ENDIAN ? 4 : 0);

  emit_insn (gen_fctiwz_df (operands[4], operands[2]));
  emit_move_insn (operands[5], operands[4]);
  emit_move_insn (operands[0], lowword);
  DONE;
})

(define_expand "negtf2"
  [(set (match_operand:TF 0 "gpc_reg_operand" "")
	(neg:TF (match_operand:TF 1 "gpc_reg_operand" "")))]
  "!TARGET_IEEEQUAD
   && TARGET_HARD_FLOAT
   && (TARGET_FPRS || TARGET_E500_DOUBLE)
   && TARGET_LONG_DOUBLE_128"
  "")

(define_insn "negtf2_internal"
  [(set (match_operand:TF 0 "gpc_reg_operand" "=d")
	(neg:TF (match_operand:TF 1 "gpc_reg_operand" "d")))]
  "!TARGET_IEEEQUAD
   && TARGET_HARD_FLOAT && TARGET_FPRS && TARGET_LONG_DOUBLE_128"
  "*
{
  if (REGNO (operands[0]) == REGNO (operands[1]) + 1)
    return \"fneg %L0,%L1\;fneg %0,%1\";
  else
    return \"fneg %0,%1\;fneg %L0,%L1\";
}"
  [(set_attr "type" "fp")
   (set_attr "length" "8")])

(define_expand "abstf2"
  [(set (match_operand:TF 0 "gpc_reg_operand" "")
	(abs:TF (match_operand:TF 1 "gpc_reg_operand" "")))]
  "!TARGET_IEEEQUAD
   && TARGET_HARD_FLOAT
   && (TARGET_FPRS || TARGET_E500_DOUBLE)
   && TARGET_LONG_DOUBLE_128"
  "
{
  rtx label = gen_label_rtx ();
  if (TARGET_E500_DOUBLE)
    {
      if (flag_finite_math_only && !flag_trapping_math)
	emit_insn (gen_spe_abstf2_tst (operands[0], operands[1], label));
      else
	emit_insn (gen_spe_abstf2_cmp (operands[0], operands[1], label));
    }
  else
    emit_insn (gen_abstf2_internal (operands[0], operands[1], label));
  emit_label (label);
  DONE;
}")

(define_expand "abstf2_internal"
  [(set (match_operand:TF 0 "gpc_reg_operand" "")
	(match_operand:TF 1 "gpc_reg_operand" ""))
   (set (match_dup 3) (match_dup 5))
   (set (match_dup 5) (abs:DF (match_dup 5)))
   (set (match_dup 4) (compare:CCFP (match_dup 3) (match_dup 5)))
   (set (pc) (if_then_else (eq (match_dup 4) (const_int 0))
			   (label_ref (match_operand 2 "" ""))
			   (pc)))
   (set (match_dup 6) (neg:DF (match_dup 6)))]
  "!TARGET_IEEEQUAD
   && TARGET_HARD_FLOAT && TARGET_FPRS && TARGET_DOUBLE_FLOAT 
   && TARGET_LONG_DOUBLE_128"
  "
{
  const int hi_word = LONG_DOUBLE_LARGE_FIRST ? 0 : GET_MODE_SIZE (DFmode);
  const int lo_word = LONG_DOUBLE_LARGE_FIRST ? GET_MODE_SIZE (DFmode) : 0;
  operands[3] = gen_reg_rtx (DFmode);
  operands[4] = gen_reg_rtx (CCFPmode);
  operands[5] = simplify_gen_subreg (DFmode, operands[0], TFmode, hi_word);
  operands[6] = simplify_gen_subreg (DFmode, operands[0], TFmode, lo_word);
}")

;; Reload helper functions used by rs6000_secondary_reload.  The patterns all
;; must have 3 arguments, and scratch register constraint must be a single
;; constraint.

;; Reload patterns to support gpr load/store with misaligned mem.
;; and multiple gpr load/store at offset >= 0xfffc
(define_expand "reload_<mode>_store"
  [(parallel [(match_operand 0 "memory_operand" "=m")
              (match_operand 1 "gpc_reg_operand" "r")
              (match_operand:GPR 2 "register_operand" "=&b")])]
  ""
{
  rs6000_secondary_reload_gpr (operands[1], operands[0], operands[2], true);
  DONE;
})

(define_expand "reload_<mode>_load"
  [(parallel [(match_operand 0 "gpc_reg_operand" "=r")
              (match_operand 1 "memory_operand" "m")
              (match_operand:GPR 2 "register_operand" "=b")])]
  ""
{
  rs6000_secondary_reload_gpr (operands[0], operands[1], operands[2], false);
  DONE;
})


;; Reload patterns for various types using the vector registers.  We may need
;; an additional base register to convert the reg+offset addressing to reg+reg
;; for vector registers and reg+reg or (reg+reg)&(-16) addressing to just an
;; index register for gpr registers.
(define_expand "reload_<RELOAD:mode>_<P:mptrsize>_store"
  [(parallel [(match_operand:RELOAD 0 "memory_operand" "m")
              (match_operand:RELOAD 1 "gpc_reg_operand" "wa")
              (match_operand:P 2 "register_operand" "=b")])]
  "<P:tptrsize>"
{
  rs6000_secondary_reload_inner (operands[1], operands[0], operands[2], true);
  DONE;
})

(define_expand "reload_<RELOAD:mode>_<P:mptrsize>_load"
  [(parallel [(match_operand:RELOAD 0 "gpc_reg_operand" "wa")
              (match_operand:RELOAD 1 "memory_operand" "m")
              (match_operand:P 2 "register_operand" "=b")])]
  "<P:tptrsize>"
{
  rs6000_secondary_reload_inner (operands[0], operands[1], operands[2], false);
  DONE;
})


;; Reload sometimes tries to move the address to a GPR, and can generate
;; invalid RTL for addresses involving AND -16.  Allow addresses involving
;; reg+reg, reg+small constant, or just reg, all wrapped in an AND -16.

(define_insn_and_split "*vec_reload_and_plus_<mptrsize>"
  [(set (match_operand:P 0 "gpc_reg_operand" "=b")
	(and:P (plus:P (match_operand:P 1 "gpc_reg_operand" "r")
		       (match_operand:P 2 "reg_or_cint_operand" "rI"))
	       (const_int -16)))]
  "TARGET_ALTIVEC && (reload_in_progress || reload_completed)"
  "#"
  "&& reload_completed"
  [(set (match_dup 0)
	(plus:P (match_dup 1)
		(match_dup 2)))
   (set (match_dup 0)
	(and:P (match_dup 0)
	       (const_int -16)))])

;; Power8 merge instructions to allow direct move to/from floating point
;; registers in 32-bit mode.  We use TF mode to get two registers to move the
;; individual 32-bit parts across.  Subreg doesn't work too well on the TF
;; value, since it is allocated in reload and not all of the flow information
;; is setup for it.  We have two patterns to do the two moves between gprs and
;; fprs.  There isn't a dependancy between the two, but we could potentially
;; schedule other instructions between the two instructions.  TFmode is
;; currently limited to traditional FPR registers.  If/when this is changed, we
;; will need to revist %L to make sure it works with VSX registers, or add an
;; %x version of %L.

(define_insn "p8_fmrgow_<mode>"
  [(set (match_operand:FMOVE64X 0 "register_operand" "=d")
	(unspec:FMOVE64X [(match_operand:TF 1 "register_operand" "d")]
			 UNSPEC_P8V_FMRGOW))]
  "!TARGET_POWERPC64 && TARGET_DIRECT_MOVE"
  "fmrgow %0,%1,%L1"
  [(set_attr "type" "vecperm")])

(define_insn "p8_mtvsrwz_1"
  [(set (match_operand:TF 0 "register_operand" "=d")
	(unspec:TF [(match_operand:SI 1 "register_operand" "r")]
		   UNSPEC_P8V_MTVSRWZ))]
  "!TARGET_POWERPC64 && TARGET_DIRECT_MOVE"
  "mtvsrwz %x0,%1"
  [(set_attr "type" "mftgpr")])

(define_insn "p8_mtvsrwz_2"
  [(set (match_operand:TF 0 "register_operand" "+d")
	(unspec:TF [(match_dup 0)
		    (match_operand:SI 1 "register_operand" "r")]
		   UNSPEC_P8V_MTVSRWZ))]
  "!TARGET_POWERPC64 && TARGET_DIRECT_MOVE"
  "mtvsrwz %L0,%1"
  [(set_attr "type" "mftgpr")])

(define_insn_and_split "reload_fpr_from_gpr<mode>"
  [(set (match_operand:FMOVE64X 0 "register_operand" "=ws")
	(unspec:FMOVE64X [(match_operand:FMOVE64X 1 "register_operand" "r")]
			 UNSPEC_P8V_RELOAD_FROM_GPR))
   (clobber (match_operand:TF 2 "register_operand" "=d"))]
  "!TARGET_POWERPC64 && TARGET_DIRECT_MOVE"
  "#"
  "&& reload_completed"
  [(const_int 0)]
{
  rtx dest = operands[0];
  rtx src = operands[1];
  rtx tmp = operands[2];
  rtx gpr_hi_reg = gen_highpart (SImode, src);
  rtx gpr_lo_reg = gen_lowpart (SImode, src);

  emit_insn (gen_p8_mtvsrwz_1 (tmp, gpr_hi_reg));
  emit_insn (gen_p8_mtvsrwz_2 (tmp, gpr_lo_reg));
  emit_insn (gen_p8_fmrgow_<mode> (dest, tmp));
  DONE;
}
  [(set_attr "length" "12")
   (set_attr "type" "three")])

;; Move 128 bit values from GPRs to VSX registers in 64-bit mode
(define_insn "p8_mtvsrd_1"
  [(set (match_operand:TF 0 "register_operand" "=ws")
	(unspec:TF [(match_operand:DI 1 "register_operand" "r")]
		   UNSPEC_P8V_MTVSRD))]
  "TARGET_POWERPC64 && TARGET_DIRECT_MOVE"
  "mtvsrd %0,%1"
  [(set_attr "type" "mftgpr")])

(define_insn "p8_mtvsrd_2"
  [(set (match_operand:TF 0 "register_operand" "+ws")
	(unspec:TF [(match_dup 0)
		    (match_operand:DI 1 "register_operand" "r")]
		   UNSPEC_P8V_MTVSRD))]
  "TARGET_POWERPC64 && TARGET_DIRECT_MOVE"
  "mtvsrd %L0,%1"
  [(set_attr "type" "mftgpr")])

(define_insn "p8_xxpermdi_<mode>"
  [(set (match_operand:FMOVE128_GPR 0 "register_operand" "=wa")
	(unspec:FMOVE128_GPR [(match_operand:TF 1 "register_operand" "ws")]
			     UNSPEC_P8V_XXPERMDI))]
  "TARGET_POWERPC64 && TARGET_DIRECT_MOVE"
  "xxpermdi %x0,%1,%L1,0"
  [(set_attr "type" "vecperm")])

(define_insn_and_split "reload_vsx_from_gpr<mode>"
  [(set (match_operand:FMOVE128_GPR 0 "register_operand" "=wa")
	(unspec:FMOVE128_GPR
	 [(match_operand:FMOVE128_GPR 1 "register_operand" "r")]
	 UNSPEC_P8V_RELOAD_FROM_GPR))
   (clobber (match_operand:TF 2 "register_operand" "=ws"))]
  "TARGET_POWERPC64 && TARGET_DIRECT_MOVE"
  "#"
  "&& reload_completed"
  [(const_int 0)]
{
  rtx dest = operands[0];
  rtx src = operands[1];
  rtx tmp = operands[2];
  rtx gpr_hi_reg = gen_highpart (DImode, src);
  rtx gpr_lo_reg = gen_lowpart (DImode, src);

  emit_insn (gen_p8_mtvsrd_1 (tmp, gpr_hi_reg));
  emit_insn (gen_p8_mtvsrd_2 (tmp, gpr_lo_reg));
  emit_insn (gen_p8_xxpermdi_<mode> (dest, tmp));
}
  [(set_attr "length" "12")
   (set_attr "type" "three")])

(define_split
  [(set (match_operand:FMOVE128_GPR 0 "nonimmediate_operand" "")
	(match_operand:FMOVE128_GPR 1 "input_operand" ""))]
  "reload_completed
   && (int_reg_operand (operands[0], <MODE>mode)
       || int_reg_operand (operands[1], <MODE>mode))"
  [(pc)]
{ rs6000_split_multireg_move (operands[0], operands[1]); DONE; })

;; Move SFmode to a VSX from a GPR register.  Because scalar floating point
;; type is stored internally as double precision in the VSX registers, we have
;; to convert it from the vector format.

(define_insn_and_split "reload_vsx_from_gprsf"
  [(set (match_operand:SF 0 "register_operand" "=wa")
	(unspec:SF [(match_operand:SF 1 "register_operand" "r")]
		   UNSPEC_P8V_RELOAD_FROM_GPR))
   (clobber (match_operand:DI 2 "register_operand" "=r"))]
  "TARGET_POWERPC64 && TARGET_DIRECT_MOVE"
  "#"
  "&& reload_completed"
  [(const_int 0)]
{
  rtx op0 = operands[0];
  rtx op1 = operands[1];
  rtx op2 = operands[2];
  /* Also use the destination register to hold the unconverted DImode value.
     This is conceptually a separate value from OP0, so we use gen_rtx_REG
     rather than simplify_gen_subreg.  */
  rtx op0_di = gen_rtx_REG (DImode, REGNO (op0));
  rtx op1_di = simplify_gen_subreg (DImode, op1, SFmode, 0);

  /* Move SF value to upper 32-bits for xscvspdpn.  */
  emit_insn (gen_ashldi3 (op2, op1_di, GEN_INT (32)));
  emit_move_insn (op0_di, op2);
  emit_insn (gen_vsx_xscvspdpn_directmove (op0, op0_di));
  DONE;
}
  [(set_attr "length" "8")
   (set_attr "type" "two")])

;; Move 128 bit values from VSX registers to GPRs in 64-bit mode by doing a
;; normal 64-bit move, followed by an xxpermdi to get the bottom 64-bit value,
;; and then doing a move of that.
(define_insn "p8_mfvsrd_3_<mode>"
  [(set (match_operand:DF 0 "register_operand" "=r")
	(unspec:DF [(match_operand:FMOVE128_GPR 1 "register_operand" "wa")]
		   UNSPEC_P8V_RELOAD_FROM_VSX))]
  "TARGET_POWERPC64 && TARGET_DIRECT_MOVE"
  "mfvsrd %0,%x1"
  [(set_attr "type" "mftgpr")])

(define_insn_and_split "reload_gpr_from_vsx<mode>"
  [(set (match_operand:FMOVE128_GPR 0 "register_operand" "=r")
	(unspec:FMOVE128_GPR
	 [(match_operand:FMOVE128_GPR 1 "register_operand" "wa")]
	 UNSPEC_P8V_RELOAD_FROM_VSX))
   (clobber (match_operand:FMOVE128_GPR 2 "register_operand" "=wa"))]
  "TARGET_POWERPC64 && TARGET_DIRECT_MOVE"
  "#"
  "&& reload_completed"
  [(const_int 0)]
{
  rtx dest = operands[0];
  rtx src = operands[1];
  rtx tmp = operands[2];
  rtx gpr_hi_reg = gen_highpart (DFmode, dest);
  rtx gpr_lo_reg = gen_lowpart (DFmode, dest);

  emit_insn (gen_p8_mfvsrd_3_<mode> (gpr_hi_reg, src));
  emit_insn (gen_vsx_xxpermdi_<mode> (tmp, src, src, GEN_INT (3)));
  emit_insn (gen_p8_mfvsrd_3_<mode> (gpr_lo_reg, tmp));
}
  [(set_attr "length" "12")
   (set_attr "type" "three")])

;; Move SFmode to a GPR from a VSX register.  Because scalar floating point
;; type is stored internally as double precision, we have to convert it to the
;; vector format.

(define_insn_and_split "reload_gpr_from_vsxsf"
  [(set (match_operand:SF 0 "register_operand" "=r")
	(unspec:SF [(match_operand:SF 1 "register_operand" "wa")]
		   UNSPEC_P8V_RELOAD_FROM_VSX))
   (clobber (match_operand:V4SF 2 "register_operand" "=wa"))]
  "TARGET_POWERPC64 && TARGET_DIRECT_MOVE"
  "#"
  "&& reload_completed"
  [(const_int 0)]
{
  rtx op0 = operands[0];
  rtx op1 = operands[1];
  rtx op2 = operands[2];
  rtx diop0 = simplify_gen_subreg (DImode, op0, SFmode, 0);

  emit_insn (gen_vsx_xscvdpspn_scalar (op2, op1));
  emit_insn (gen_p8_mfvsrd_4_disf (diop0, op2));
  emit_insn (gen_lshrdi3 (diop0, diop0, GEN_INT (32)));
  DONE;
}
  [(set_attr "length" "12")
   (set_attr "type" "three")])

(define_insn "p8_mfvsrd_4_disf"
  [(set (match_operand:DI 0 "register_operand" "=r")
	(unspec:DI [(match_operand:V4SF 1 "register_operand" "wa")]
		   UNSPEC_P8V_RELOAD_FROM_VSX))]
  "TARGET_POWERPC64 && TARGET_DIRECT_MOVE"
  "mfvsrd %0,%x1"
  [(set_attr "type" "mftgpr")])


;; Next come the multi-word integer load and store and the load and store
;; multiple insns.

;; List r->r after r->Y, otherwise reload will try to reload a
;; non-offsettable address by using r->r which won't make progress.
;; Use of fprs is disparaged slightly otherwise reload prefers to reload
;; a gpr into a fpr instead of reloading an invalid 'Y' address
(define_insn "*movdi_internal32"
  [(set (match_operand:DI 0 "rs6000_nonimmediate_operand" "=Y,r,r,?m,?*d,?*d,r")
	(match_operand:DI 1 "input_operand" "r,Y,r,d,m,d,IJKnGHF"))]
  "! TARGET_POWERPC64
   && (gpc_reg_operand (operands[0], DImode)
       || gpc_reg_operand (operands[1], DImode))"
  "@
   #
   #
   #
   stfd%U0%X0 %1,%0
   lfd%U1%X1 %0,%1
   fmr %0,%1
   #"
  [(set_attr "type" "store,load,*,fpstore,fpload,fp,*")])

(define_split
  [(set (match_operand:DI 0 "gpc_reg_operand" "")
	(match_operand:DI 1 "const_int_operand" ""))]
  "! TARGET_POWERPC64 && reload_completed
   && gpr_or_gpr_p (operands[0], operands[1])
   && !direct_move_p (operands[0], operands[1])"
  [(set (match_dup 2) (match_dup 4))
   (set (match_dup 3) (match_dup 1))]
  "
{
  HOST_WIDE_INT value = INTVAL (operands[1]);
  operands[2] = operand_subword_force (operands[0], WORDS_BIG_ENDIAN == 0,
				       DImode);
  operands[3] = operand_subword_force (operands[0], WORDS_BIG_ENDIAN != 0,
				       DImode);
  operands[4] = GEN_INT (value >> 32);
  operands[1] = GEN_INT (((value & 0xffffffff) ^ 0x80000000) - 0x80000000);
}")

(define_split
  [(set (match_operand:DIFD 0 "rs6000_nonimmediate_operand" "")
        (match_operand:DIFD 1 "input_operand" ""))]
  "reload_completed && !TARGET_POWERPC64
   && gpr_or_gpr_p (operands[0], operands[1])
   && !direct_move_p (operands[0], operands[1])"
  [(pc)]
{ rs6000_split_multireg_move (operands[0], operands[1]); DONE; })

(define_insn "*movdi_internal64"
  [(set (match_operand:DI 0 "nonimmediate_operand" "=Y,r,r,r,r,r,?m,?*d,?*d,r,*h,*h,r,?*wg,r,?*wj,?*wi")
	(match_operand:DI 1 "input_operand" "r,Y,r,I,L,nF,d,m,d,*h,r,0,*wg,r,*wj,r,O"))]
  "TARGET_POWERPC64
   && (gpc_reg_operand (operands[0], DImode)
       || gpc_reg_operand (operands[1], DImode))"
  "@
   std%U0%X0 %1,%0
   ld%U1%X1 %0,%1
   mr %0,%1
   li %0,%1
   lis %0,%v1
   #
   stfd%U0%X0 %1,%0
   lfd%U1%X1 %0,%1
   fmr %0,%1
   mf%1 %0
   mt%0 %1
   nop
   mftgpr %0,%1
   mffgpr %0,%1
   mfvsrd %0,%x1
   mtvsrd %x0,%1
   xxlxor %x0,%x0,%x0"
  [(set_attr "type" "store,load,*,*,*,*,fpstore,fpload,fp,mfjmpr,mtjmpr,*,mftgpr,mffgpr,mftgpr,mffgpr,vecsimple")
   (set_attr "length" "4,4,4,4,4,20,4,4,4,4,4,4,4,4,4,4,4")])

;; Generate all one-bits and clear left or right.
;; Use (and:DI (rotate:DI ...)) to avoid anddi3 unnecessary clobber.
(define_split
  [(set (match_operand:DI 0 "gpc_reg_operand" "")
	(match_operand:DI 1 "mask64_operand" ""))]
  "TARGET_POWERPC64 && num_insns_constant (operands[1], DImode) > 1"
  [(set (match_dup 0) (const_int -1))
   (set (match_dup 0)
	(and:DI (rotate:DI (match_dup 0)
			   (const_int 0))
		(match_dup 1)))]
  "")

;; Split a load of a large constant into the appropriate five-instruction
;; sequence.  Handle anything in a constant number of insns.
;; When non-easy constants can go in the TOC, this should use
;; easy_fp_constant predicate.
(define_split
  [(set (match_operand:DI 0 "gpc_reg_operand" "")
	(match_operand:DI 1 "const_int_operand" ""))]
  "TARGET_POWERPC64 && num_insns_constant (operands[1], DImode) > 1"
  [(set (match_dup 0) (match_dup 2))
   (set (match_dup 0) (plus:DI (match_dup 0) (match_dup 3)))]
  "
{
  if (rs6000_emit_set_const (operands[0], operands[1]))
    DONE;
  else
    FAIL;
}")

(define_split
  [(set (match_operand:DI 0 "gpc_reg_operand" "")
	(match_operand:DI 1 "const_scalar_int_operand" ""))]
  "TARGET_POWERPC64 && num_insns_constant (operands[1], DImode) > 1"
  [(set (match_dup 0) (match_dup 2))
   (set (match_dup 0) (plus:DI (match_dup 0) (match_dup 3)))]
  "
{
  if (rs6000_emit_set_const (operands[0], operands[1]))
    DONE;
  else
    FAIL;
}")

;; TImode/PTImode is similar, except that we usually want to compute the
;; address into a register and use lsi/stsi (the exception is during reload).

(define_insn "*mov<mode>_string"
  [(set (match_operand:TI2 0 "reg_or_mem_operand" "=Q,Y,????r,????r,????r,r")
	(match_operand:TI2 1 "input_operand" "r,r,Q,Y,r,n"))]
  "! TARGET_POWERPC64
   && (<MODE>mode != TImode || VECTOR_MEM_NONE_P (TImode))
   && (gpc_reg_operand (operands[0], <MODE>mode)
       || gpc_reg_operand (operands[1], <MODE>mode))"
  "*
{
  switch (which_alternative)
    {
    default:
      gcc_unreachable ();
    case 0:
      if (TARGET_STRING)
        return \"stswi %1,%P0,16\";
    case 1:
      return \"#\";
    case 2:
      /* If the address is not used in the output, we can use lsi.  Otherwise,
	 fall through to generating four loads.  */
      if (TARGET_STRING
          && ! reg_overlap_mentioned_p (operands[0], operands[1]))
	return \"lswi %0,%P1,16\";
      /* ... fall through ...  */
    case 3:
    case 4:
    case 5:
      return \"#\";
    }
}"
  [(set_attr "type" "store,store,load,load,*,*")
   (set_attr "update" "yes")
   (set_attr "indexed" "yes")
   (set (attr "cell_micro") (if_then_else (match_test "TARGET_STRING")
   			                  (const_string "always")
					  (const_string "conditional")))])

(define_insn "*mov<mode>_ppc64"
  [(set (match_operand:TI2 0 "nonimmediate_operand" "=wQ,Y,r,r,r,r")
	(match_operand:TI2 1 "input_operand" "r,r,wQ,Y,r,n"))]
  "(TARGET_POWERPC64 && VECTOR_MEM_NONE_P (<MODE>mode)
   && (gpc_reg_operand (operands[0], <MODE>mode)
       || gpc_reg_operand (operands[1], <MODE>mode)))"
{
  return rs6000_output_move_128bit (operands);
}
  [(set_attr "type" "store,store,load,load,*,*")
   (set_attr "length" "8")])

(define_split
  [(set (match_operand:TI2 0 "int_reg_operand" "")
	(match_operand:TI2 1 "const_scalar_int_operand" ""))]
  "TARGET_POWERPC64
   && (VECTOR_MEM_NONE_P (<MODE>mode)
       || (reload_completed && INT_REGNO_P (REGNO (operands[0]))))"
  [(set (match_dup 2) (match_dup 4))
   (set (match_dup 3) (match_dup 5))]
  "
{
  operands[2] = operand_subword_force (operands[0], WORDS_BIG_ENDIAN == 0,
				       <MODE>mode);
  operands[3] = operand_subword_force (operands[0], WORDS_BIG_ENDIAN != 0,
				       <MODE>mode);
  if (CONST_WIDE_INT_P (operands[1]))
    {
      operands[4] = GEN_INT (CONST_WIDE_INT_ELT (operands[1], 1));
      operands[5] = GEN_INT (CONST_WIDE_INT_ELT (operands[1], 0));
    }
  else if (CONST_INT_P (operands[1]))
    {
      operands[4] = GEN_INT (- (INTVAL (operands[1]) < 0));
      operands[5] = operands[1];
    }
  else
    FAIL;
}")

(define_split
  [(set (match_operand:TI2 0 "nonimmediate_operand" "")
        (match_operand:TI2 1 "input_operand" ""))]
  "reload_completed
   && gpr_or_gpr_p (operands[0], operands[1])
   && !direct_move_p (operands[0], operands[1])
   && !quad_load_store_p (operands[0], operands[1])"
  [(pc)]
{ rs6000_split_multireg_move (operands[0], operands[1]); DONE; })

(define_expand "load_multiple"
  [(match_par_dup 3 [(set (match_operand:SI 0 "" "")
			  (match_operand:SI 1 "" ""))
		     (use (match_operand:SI 2 "" ""))])]
  "TARGET_STRING && !TARGET_POWERPC64"
  "
{
  int regno;
  int count;
  rtx op1;
  int i;

  /* Support only loading a constant number of fixed-point registers from
     memory and only bother with this if more than two; the machine
     doesn't support more than eight.  */
  if (GET_CODE (operands[2]) != CONST_INT
      || INTVAL (operands[2]) <= 2
      || INTVAL (operands[2]) > 8
      || GET_CODE (operands[1]) != MEM
      || GET_CODE (operands[0]) != REG
      || REGNO (operands[0]) >= 32)
    FAIL;

  count = INTVAL (operands[2]);
  regno = REGNO (operands[0]);

  operands[3] = gen_rtx_PARALLEL (VOIDmode, rtvec_alloc (count));
  op1 = replace_equiv_address (operands[1],
			       force_reg (SImode, XEXP (operands[1], 0)));

  for (i = 0; i < count; i++)
    XVECEXP (operands[3], 0, i)
      = gen_rtx_SET (VOIDmode, gen_rtx_REG (SImode, regno + i),
		     adjust_address_nv (op1, SImode, i * 4));
}")

(define_insn "*ldmsi8"
  [(match_parallel 0 "load_multiple_operation"
    [(set (match_operand:SI 2 "gpc_reg_operand" "")
          (mem:SI (match_operand:SI 1 "gpc_reg_operand" "b")))
     (set (match_operand:SI 3 "gpc_reg_operand" "")
          (mem:SI (plus:SI (match_dup 1) (const_int 4))))
     (set (match_operand:SI 4 "gpc_reg_operand" "")
          (mem:SI (plus:SI (match_dup 1) (const_int 8))))
     (set (match_operand:SI 5 "gpc_reg_operand" "")
          (mem:SI (plus:SI (match_dup 1) (const_int 12))))
     (set (match_operand:SI 6 "gpc_reg_operand" "")
          (mem:SI (plus:SI (match_dup 1) (const_int 16))))
     (set (match_operand:SI 7 "gpc_reg_operand" "")
          (mem:SI (plus:SI (match_dup 1) (const_int 20))))
     (set (match_operand:SI 8 "gpc_reg_operand" "")
          (mem:SI (plus:SI (match_dup 1) (const_int 24))))
     (set (match_operand:SI 9 "gpc_reg_operand" "")
          (mem:SI (plus:SI (match_dup 1) (const_int 28))))])]
  "TARGET_STRING && XVECLEN (operands[0], 0) == 8"
  "*
{ return rs6000_output_load_multiple (operands); }"
  [(set_attr "type" "load")
   (set_attr "update" "yes")
   (set_attr "indexed" "yes")
   (set_attr "length" "32")])

(define_insn "*ldmsi7"
  [(match_parallel 0 "load_multiple_operation"
    [(set (match_operand:SI 2 "gpc_reg_operand" "")
          (mem:SI (match_operand:SI 1 "gpc_reg_operand" "b")))
     (set (match_operand:SI 3 "gpc_reg_operand" "")
          (mem:SI (plus:SI (match_dup 1) (const_int 4))))
     (set (match_operand:SI 4 "gpc_reg_operand" "")
          (mem:SI (plus:SI (match_dup 1) (const_int 8))))
     (set (match_operand:SI 5 "gpc_reg_operand" "")
          (mem:SI (plus:SI (match_dup 1) (const_int 12))))
     (set (match_operand:SI 6 "gpc_reg_operand" "")
          (mem:SI (plus:SI (match_dup 1) (const_int 16))))
     (set (match_operand:SI 7 "gpc_reg_operand" "")
          (mem:SI (plus:SI (match_dup 1) (const_int 20))))
     (set (match_operand:SI 8 "gpc_reg_operand" "")
          (mem:SI (plus:SI (match_dup 1) (const_int 24))))])]
  "TARGET_STRING && XVECLEN (operands[0], 0) == 7"
  "*
{ return rs6000_output_load_multiple (operands); }"
  [(set_attr "type" "load")
   (set_attr "update" "yes")
   (set_attr "indexed" "yes")
   (set_attr "length" "32")])

(define_insn "*ldmsi6"
  [(match_parallel 0 "load_multiple_operation"
    [(set (match_operand:SI 2 "gpc_reg_operand" "")
          (mem:SI (match_operand:SI 1 "gpc_reg_operand" "b")))
     (set (match_operand:SI 3 "gpc_reg_operand" "")
          (mem:SI (plus:SI (match_dup 1) (const_int 4))))
     (set (match_operand:SI 4 "gpc_reg_operand" "")
          (mem:SI (plus:SI (match_dup 1) (const_int 8))))
     (set (match_operand:SI 5 "gpc_reg_operand" "")
          (mem:SI (plus:SI (match_dup 1) (const_int 12))))
     (set (match_operand:SI 6 "gpc_reg_operand" "")
          (mem:SI (plus:SI (match_dup 1) (const_int 16))))
     (set (match_operand:SI 7 "gpc_reg_operand" "")
          (mem:SI (plus:SI (match_dup 1) (const_int 20))))])]
  "TARGET_STRING && XVECLEN (operands[0], 0) == 6"
  "*
{ return rs6000_output_load_multiple (operands); }"
  [(set_attr "type" "load")
   (set_attr "update" "yes")
   (set_attr "indexed" "yes")
   (set_attr "length" "32")])

(define_insn "*ldmsi5"
  [(match_parallel 0 "load_multiple_operation"
    [(set (match_operand:SI 2 "gpc_reg_operand" "")
          (mem:SI (match_operand:SI 1 "gpc_reg_operand" "b")))
     (set (match_operand:SI 3 "gpc_reg_operand" "")
          (mem:SI (plus:SI (match_dup 1) (const_int 4))))
     (set (match_operand:SI 4 "gpc_reg_operand" "")
          (mem:SI (plus:SI (match_dup 1) (const_int 8))))
     (set (match_operand:SI 5 "gpc_reg_operand" "")
          (mem:SI (plus:SI (match_dup 1) (const_int 12))))
     (set (match_operand:SI 6 "gpc_reg_operand" "")
          (mem:SI (plus:SI (match_dup 1) (const_int 16))))])]
  "TARGET_STRING && XVECLEN (operands[0], 0) == 5"
  "*
{ return rs6000_output_load_multiple (operands); }"
  [(set_attr "type" "load")
   (set_attr "update" "yes")
   (set_attr "indexed" "yes")
   (set_attr "length" "32")])

(define_insn "*ldmsi4"
  [(match_parallel 0 "load_multiple_operation"
    [(set (match_operand:SI 2 "gpc_reg_operand" "")
          (mem:SI (match_operand:SI 1 "gpc_reg_operand" "b")))
     (set (match_operand:SI 3 "gpc_reg_operand" "")
          (mem:SI (plus:SI (match_dup 1) (const_int 4))))
     (set (match_operand:SI 4 "gpc_reg_operand" "")
          (mem:SI (plus:SI (match_dup 1) (const_int 8))))
     (set (match_operand:SI 5 "gpc_reg_operand" "")
          (mem:SI (plus:SI (match_dup 1) (const_int 12))))])]
  "TARGET_STRING && XVECLEN (operands[0], 0) == 4"
  "*
{ return rs6000_output_load_multiple (operands); }"
  [(set_attr "type" "load")
   (set_attr "update" "yes")
   (set_attr "indexed" "yes")
   (set_attr "length" "32")])

(define_insn "*ldmsi3"
  [(match_parallel 0 "load_multiple_operation"
    [(set (match_operand:SI 2 "gpc_reg_operand" "")
          (mem:SI (match_operand:SI 1 "gpc_reg_operand" "b")))
     (set (match_operand:SI 3 "gpc_reg_operand" "")
          (mem:SI (plus:SI (match_dup 1) (const_int 4))))
     (set (match_operand:SI 4 "gpc_reg_operand" "")
          (mem:SI (plus:SI (match_dup 1) (const_int 8))))])]
  "TARGET_STRING && XVECLEN (operands[0], 0) == 3"
  "*
{ return rs6000_output_load_multiple (operands); }"
  [(set_attr "type" "load")
   (set_attr "update" "yes")
   (set_attr "indexed" "yes")
   (set_attr "length" "32")])

(define_expand "store_multiple"
  [(match_par_dup 3 [(set (match_operand:SI 0 "" "")
			  (match_operand:SI 1 "" ""))
		     (clobber (scratch:SI))
		     (use (match_operand:SI 2 "" ""))])]
  "TARGET_STRING && !TARGET_POWERPC64"
  "
{
  int regno;
  int count;
  rtx to;
  rtx op0;
  int i;

  /* Support only storing a constant number of fixed-point registers to
     memory and only bother with this if more than two; the machine
     doesn't support more than eight.  */
  if (GET_CODE (operands[2]) != CONST_INT
      || INTVAL (operands[2]) <= 2
      || INTVAL (operands[2]) > 8
      || GET_CODE (operands[0]) != MEM
      || GET_CODE (operands[1]) != REG
      || REGNO (operands[1]) >= 32)
    FAIL;

  count = INTVAL (operands[2]);
  regno = REGNO (operands[1]);

  operands[3] = gen_rtx_PARALLEL (VOIDmode, rtvec_alloc (count + 1));
  to = force_reg (SImode, XEXP (operands[0], 0));
  op0 = replace_equiv_address (operands[0], to);

  XVECEXP (operands[3], 0, 0)
    = gen_rtx_SET (VOIDmode, adjust_address_nv (op0, SImode, 0), operands[1]);
  XVECEXP (operands[3], 0, 1) = gen_rtx_CLOBBER (VOIDmode,
						 gen_rtx_SCRATCH (SImode));

  for (i = 1; i < count; i++)
    XVECEXP (operands[3], 0, i + 1)
      = gen_rtx_SET (VOIDmode,
		     adjust_address_nv (op0, SImode, i * 4),
		     gen_rtx_REG (SImode, regno + i));
}")

(define_insn "*stmsi8"
  [(match_parallel 0 "store_multiple_operation"
    [(set (mem:SI (match_operand:SI 1 "gpc_reg_operand" "b"))
	  (match_operand:SI 2 "gpc_reg_operand" "r"))
     (clobber (match_scratch:SI 3 "=X"))
     (set (mem:SI (plus:SI (match_dup 1) (const_int 4)))
	  (match_operand:SI 4 "gpc_reg_operand" "r"))
     (set (mem:SI (plus:SI (match_dup 1) (const_int 8)))
	  (match_operand:SI 5 "gpc_reg_operand" "r"))
     (set (mem:SI (plus:SI (match_dup 1) (const_int 12)))
	  (match_operand:SI 6 "gpc_reg_operand" "r"))
     (set (mem:SI (plus:SI (match_dup 1) (const_int 16)))
	  (match_operand:SI 7 "gpc_reg_operand" "r"))
     (set (mem:SI (plus:SI (match_dup 1) (const_int 20)))
	  (match_operand:SI 8 "gpc_reg_operand" "r"))
     (set (mem:SI (plus:SI (match_dup 1) (const_int 24)))
	  (match_operand:SI 9 "gpc_reg_operand" "r"))
     (set (mem:SI (plus:SI (match_dup 1) (const_int 28)))
	  (match_operand:SI 10 "gpc_reg_operand" "r"))])]
  "TARGET_STRING && XVECLEN (operands[0], 0) == 9"
  "stswi %2,%1,%O0"
  [(set_attr "type" "store")
   (set_attr "update" "yes")
   (set_attr "indexed" "yes")
   (set_attr "cell_micro" "always")])

(define_insn "*stmsi7"
  [(match_parallel 0 "store_multiple_operation"
    [(set (mem:SI (match_operand:SI 1 "gpc_reg_operand" "b"))
	  (match_operand:SI 2 "gpc_reg_operand" "r"))
     (clobber (match_scratch:SI 3 "=X"))
     (set (mem:SI (plus:SI (match_dup 1) (const_int 4)))
	  (match_operand:SI 4 "gpc_reg_operand" "r"))
     (set (mem:SI (plus:SI (match_dup 1) (const_int 8)))
	  (match_operand:SI 5 "gpc_reg_operand" "r"))
     (set (mem:SI (plus:SI (match_dup 1) (const_int 12)))
	  (match_operand:SI 6 "gpc_reg_operand" "r"))
     (set (mem:SI (plus:SI (match_dup 1) (const_int 16)))
	  (match_operand:SI 7 "gpc_reg_operand" "r"))
     (set (mem:SI (plus:SI (match_dup 1) (const_int 20)))
	  (match_operand:SI 8 "gpc_reg_operand" "r"))
     (set (mem:SI (plus:SI (match_dup 1) (const_int 24)))
	  (match_operand:SI 9 "gpc_reg_operand" "r"))])]
  "TARGET_STRING && XVECLEN (operands[0], 0) == 8"
  "stswi %2,%1,%O0"
  [(set_attr "type" "store")
   (set_attr "update" "yes")
   (set_attr "indexed" "yes")
   (set_attr "cell_micro" "always")])

(define_insn "*stmsi6"
  [(match_parallel 0 "store_multiple_operation"
    [(set (mem:SI (match_operand:SI 1 "gpc_reg_operand" "b"))
	  (match_operand:SI 2 "gpc_reg_operand" "r"))
     (clobber (match_scratch:SI 3 "=X"))
     (set (mem:SI (plus:SI (match_dup 1) (const_int 4)))
	  (match_operand:SI 4 "gpc_reg_operand" "r"))
     (set (mem:SI (plus:SI (match_dup 1) (const_int 8)))
	  (match_operand:SI 5 "gpc_reg_operand" "r"))
     (set (mem:SI (plus:SI (match_dup 1) (const_int 12)))
	  (match_operand:SI 6 "gpc_reg_operand" "r"))
     (set (mem:SI (plus:SI (match_dup 1) (const_int 16)))
	  (match_operand:SI 7 "gpc_reg_operand" "r"))
     (set (mem:SI (plus:SI (match_dup 1) (const_int 20)))
	  (match_operand:SI 8 "gpc_reg_operand" "r"))])]
  "TARGET_STRING && XVECLEN (operands[0], 0) == 7"
  "stswi %2,%1,%O0"
  [(set_attr "type" "store")
   (set_attr "update" "yes")
   (set_attr "indexed" "yes")
   (set_attr "cell_micro" "always")])

(define_insn "*stmsi5"
  [(match_parallel 0 "store_multiple_operation"
    [(set (mem:SI (match_operand:SI 1 "gpc_reg_operand" "b"))
	  (match_operand:SI 2 "gpc_reg_operand" "r"))
     (clobber (match_scratch:SI 3 "=X"))
     (set (mem:SI (plus:SI (match_dup 1) (const_int 4)))
	  (match_operand:SI 4 "gpc_reg_operand" "r"))
     (set (mem:SI (plus:SI (match_dup 1) (const_int 8)))
	  (match_operand:SI 5 "gpc_reg_operand" "r"))
     (set (mem:SI (plus:SI (match_dup 1) (const_int 12)))
	  (match_operand:SI 6 "gpc_reg_operand" "r"))
     (set (mem:SI (plus:SI (match_dup 1) (const_int 16)))
	  (match_operand:SI 7 "gpc_reg_operand" "r"))])]
  "TARGET_STRING && XVECLEN (operands[0], 0) == 6"
  "stswi %2,%1,%O0"
  [(set_attr "type" "store")
   (set_attr "update" "yes")
   (set_attr "indexed" "yes")
   (set_attr "cell_micro" "always")])

(define_insn "*stmsi4"
  [(match_parallel 0 "store_multiple_operation"
    [(set (mem:SI (match_operand:SI 1 "gpc_reg_operand" "b"))
	  (match_operand:SI 2 "gpc_reg_operand" "r"))
     (clobber (match_scratch:SI 3 "=X"))
     (set (mem:SI (plus:SI (match_dup 1) (const_int 4)))
	  (match_operand:SI 4 "gpc_reg_operand" "r"))
     (set (mem:SI (plus:SI (match_dup 1) (const_int 8)))
	  (match_operand:SI 5 "gpc_reg_operand" "r"))
     (set (mem:SI (plus:SI (match_dup 1) (const_int 12)))
	  (match_operand:SI 6 "gpc_reg_operand" "r"))])]
  "TARGET_STRING && XVECLEN (operands[0], 0) == 5"
  "stswi %2,%1,%O0"
  [(set_attr "type" "store")
   (set_attr "update" "yes")
   (set_attr "indexed" "yes")
   (set_attr "cell_micro" "always")])

(define_insn "*stmsi3"
  [(match_parallel 0 "store_multiple_operation"
    [(set (mem:SI (match_operand:SI 1 "gpc_reg_operand" "b"))
	  (match_operand:SI 2 "gpc_reg_operand" "r"))
     (clobber (match_scratch:SI 3 "=X"))
     (set (mem:SI (plus:SI (match_dup 1) (const_int 4)))
	  (match_operand:SI 4 "gpc_reg_operand" "r"))
     (set (mem:SI (plus:SI (match_dup 1) (const_int 8)))
	  (match_operand:SI 5 "gpc_reg_operand" "r"))])]
  "TARGET_STRING && XVECLEN (operands[0], 0) == 4"
  "stswi %2,%1,%O0"
  [(set_attr "type" "store")
   (set_attr "update" "yes")
   (set_attr "indexed" "yes")
   (set_attr "cell_micro" "always")])

(define_expand "setmemsi"
  [(parallel [(set (match_operand:BLK 0 "" "")
		   (match_operand 2 "const_int_operand" ""))
	      (use (match_operand:SI 1 "" ""))
	      (use (match_operand:SI 3 "" ""))])]
  ""
  "
{
  /* If value to set is not zero, use the library routine.  */
  if (operands[2] != const0_rtx)
    FAIL;

  if (expand_block_clear (operands))
    DONE;
  else
    FAIL;
}")

;; String/block move insn.
;; Argument 0 is the destination
;; Argument 1 is the source
;; Argument 2 is the length
;; Argument 3 is the alignment

(define_expand "movmemsi"
  [(parallel [(set (match_operand:BLK 0 "" "")
		   (match_operand:BLK 1 "" ""))
	      (use (match_operand:SI 2 "" ""))
	      (use (match_operand:SI 3 "" ""))])]
  ""
  "
{
  if (expand_block_move (operands))
    DONE;
  else
    FAIL;
}")

;; Move up to 32 bytes at a time.  The fixed registers are needed because the
;; register allocator doesn't have a clue about allocating 8 word registers.
;; rD/rS = r5 is preferred, efficient form.
(define_expand "movmemsi_8reg"
  [(parallel [(set (match_operand 0 "" "")
		   (match_operand 1 "" ""))
	      (use (match_operand 2 "" ""))
	      (use (match_operand 3 "" ""))
	      (clobber (reg:SI  5))
	      (clobber (reg:SI  6))
	      (clobber (reg:SI  7))
	      (clobber (reg:SI  8))
	      (clobber (reg:SI  9))
	      (clobber (reg:SI 10))
	      (clobber (reg:SI 11))
	      (clobber (reg:SI 12))
	      (clobber (match_scratch:SI 4 ""))])]
  "TARGET_STRING"
  "")

(define_insn ""
  [(set (mem:BLK (match_operand:P 0 "gpc_reg_operand" "b"))
	(mem:BLK (match_operand:P 1 "gpc_reg_operand" "b")))
   (use (match_operand:SI 2 "immediate_operand" "i"))
   (use (match_operand:SI 3 "immediate_operand" "i"))
   (clobber (match_operand:SI 4 "gpc_reg_operand" "=&r"))
   (clobber (reg:SI  6))
   (clobber (reg:SI  7))
   (clobber (reg:SI  8))
   (clobber (reg:SI  9))
   (clobber (reg:SI 10))
   (clobber (reg:SI 11))
   (clobber (reg:SI 12))
   (clobber (match_scratch:SI 5 "=X"))]
  "TARGET_STRING
   && ((INTVAL (operands[2]) > 24 && INTVAL (operands[2]) < 32)
       || INTVAL (operands[2]) == 0)
   && (REGNO (operands[0]) < 5 || REGNO (operands[0]) > 12)
   && (REGNO (operands[1]) < 5 || REGNO (operands[1]) > 12)
   && REGNO (operands[4]) == 5"
  "lswi %4,%1,%2\;stswi %4,%0,%2"
  [(set_attr "type" "store")
   (set_attr "update" "yes")
   (set_attr "indexed" "yes")
   (set_attr "cell_micro" "always")
   (set_attr "length" "8")])

;; Move up to 24 bytes at a time.  The fixed registers are needed because the
;; register allocator doesn't have a clue about allocating 6 word registers.
;; rD/rS = r5 is preferred, efficient form.
(define_expand "movmemsi_6reg"
  [(parallel [(set (match_operand 0 "" "")
		   (match_operand 1 "" ""))
	      (use (match_operand 2 "" ""))
	      (use (match_operand 3 "" ""))
	      (clobber (reg:SI  5))
	      (clobber (reg:SI  6))
	      (clobber (reg:SI  7))
	      (clobber (reg:SI  8))
	      (clobber (reg:SI  9))
	      (clobber (reg:SI 10))
	      (clobber (match_scratch:SI 4 ""))])]
  "TARGET_STRING"
  "")

(define_insn ""
  [(set (mem:BLK (match_operand:P 0 "gpc_reg_operand" "b"))
	(mem:BLK (match_operand:P 1 "gpc_reg_operand" "b")))
   (use (match_operand:SI 2 "immediate_operand" "i"))
   (use (match_operand:SI 3 "immediate_operand" "i"))
   (clobber (match_operand:SI 4 "gpc_reg_operand" "=&r"))
   (clobber (reg:SI  6))
   (clobber (reg:SI  7))
   (clobber (reg:SI  8))
   (clobber (reg:SI  9))
   (clobber (reg:SI 10))
   (clobber (match_scratch:SI 5 "=X"))]
  "TARGET_STRING
   && INTVAL (operands[2]) > 16 && INTVAL (operands[2]) <= 32
   && (REGNO (operands[0]) < 5 || REGNO (operands[0]) > 10)
   && (REGNO (operands[1]) < 5 || REGNO (operands[1]) > 10)
   && REGNO (operands[4]) == 5"
  "lswi %4,%1,%2\;stswi %4,%0,%2"
  [(set_attr "type" "store")
   (set_attr "update" "yes")
   (set_attr "indexed" "yes")
   (set_attr "cell_micro" "always")
   (set_attr "length" "8")])

;; Move up to 16 bytes at a time, using 4 fixed registers to avoid spill
;; problems with TImode.
;; rD/rS = r5 is preferred, efficient form.
(define_expand "movmemsi_4reg"
  [(parallel [(set (match_operand 0 "" "")
		   (match_operand 1 "" ""))
	      (use (match_operand 2 "" ""))
	      (use (match_operand 3 "" ""))
	      (clobber (reg:SI 5))
	      (clobber (reg:SI 6))
	      (clobber (reg:SI 7))
	      (clobber (reg:SI 8))
	      (clobber (match_scratch:SI 4 ""))])]
  "TARGET_STRING"
  "")

(define_insn ""
  [(set (mem:BLK (match_operand:P 0 "gpc_reg_operand" "b"))
	(mem:BLK (match_operand:P 1 "gpc_reg_operand" "b")))
   (use (match_operand:SI 2 "immediate_operand" "i"))
   (use (match_operand:SI 3 "immediate_operand" "i"))
   (clobber (match_operand:SI 4 "gpc_reg_operand" "=&r"))
   (clobber (reg:SI 6))
   (clobber (reg:SI 7))
   (clobber (reg:SI 8))
   (clobber (match_scratch:SI 5 "=X"))]
  "TARGET_STRING
   && INTVAL (operands[2]) > 8 && INTVAL (operands[2]) <= 16
   && (REGNO (operands[0]) < 5 || REGNO (operands[0]) > 8)
   && (REGNO (operands[1]) < 5 || REGNO (operands[1]) > 8)
   && REGNO (operands[4]) == 5"
  "lswi %4,%1,%2\;stswi %4,%0,%2"
  [(set_attr "type" "store")
   (set_attr "update" "yes")
   (set_attr "indexed" "yes")
   (set_attr "cell_micro" "always")
   (set_attr "length" "8")])

;; Move up to 8 bytes at a time.
(define_expand "movmemsi_2reg"
  [(parallel [(set (match_operand 0 "" "")
		   (match_operand 1 "" ""))
	      (use (match_operand 2 "" ""))
	      (use (match_operand 3 "" ""))
	      (clobber (match_scratch:DI 4 ""))
	      (clobber (match_scratch:SI 5 ""))])]
  "TARGET_STRING && ! TARGET_POWERPC64"
  "")

(define_insn ""
  [(set (mem:BLK (match_operand:SI 0 "gpc_reg_operand" "b"))
	(mem:BLK (match_operand:SI 1 "gpc_reg_operand" "b")))
   (use (match_operand:SI 2 "immediate_operand" "i"))
   (use (match_operand:SI 3 "immediate_operand" "i"))
   (clobber (match_scratch:DI 4 "=&r"))
   (clobber (match_scratch:SI 5 "=X"))]
  "TARGET_STRING && ! TARGET_POWERPC64
   && INTVAL (operands[2]) > 4 && INTVAL (operands[2]) <= 8"
  "lswi %4,%1,%2\;stswi %4,%0,%2"
  [(set_attr "type" "store")
   (set_attr "update" "yes")
   (set_attr "indexed" "yes")
   (set_attr "cell_micro" "always")
   (set_attr "length" "8")])

;; Move up to 4 bytes at a time.
(define_expand "movmemsi_1reg"
  [(parallel [(set (match_operand 0 "" "")
		   (match_operand 1 "" ""))
	      (use (match_operand 2 "" ""))
	      (use (match_operand 3 "" ""))
	      (clobber (match_scratch:SI 4 ""))
	      (clobber (match_scratch:SI 5 ""))])]
  "TARGET_STRING"
  "")

(define_insn ""
  [(set (mem:BLK (match_operand:P 0 "gpc_reg_operand" "b"))
	(mem:BLK (match_operand:P 1 "gpc_reg_operand" "b")))
   (use (match_operand:SI 2 "immediate_operand" "i"))
   (use (match_operand:SI 3 "immediate_operand" "i"))
   (clobber (match_scratch:SI 4 "=&r"))
   (clobber (match_scratch:SI 5 "=X"))]
  "TARGET_STRING && INTVAL (operands[2]) > 0 && INTVAL (operands[2]) <= 4"
  "lswi %4,%1,%2\;stswi %4,%0,%2"
  [(set_attr "type" "store")
   (set_attr "update" "yes")
   (set_attr "indexed" "yes")
   (set_attr "cell_micro" "always")
   (set_attr "length" "8")])

;; Define insns that do load or store with update.  Some of these we can
;; get by using pre-decrement or pre-increment, but the hardware can also
;; do cases where the increment is not the size of the object.
;;
;; In all these cases, we use operands 0 and 1 for the register being
;; incremented because those are the operands that local-alloc will
;; tie and these are the pair most likely to be tieable (and the ones
;; that will benefit the most).

(define_insn "*movdi_update1"
  [(set (match_operand:DI 3 "gpc_reg_operand" "=r,r")
	(mem:DI (plus:DI (match_operand:DI 1 "gpc_reg_operand" "0,0")
			 (match_operand:DI 2 "reg_or_aligned_short_operand" "r,I"))))
   (set (match_operand:DI 0 "gpc_reg_operand" "=b,b")
	(plus:DI (match_dup 1) (match_dup 2)))]
  "TARGET_POWERPC64 && TARGET_UPDATE
   && (!avoiding_indexed_address_p (DImode)
       || !gpc_reg_operand (operands[2], DImode))"
  "@
   ldux %3,%0,%2
   ldu %3,%2(%0)"
  [(set_attr "type" "load")
   (set_attr "update" "yes")
   (set_attr "indexed" "yes,no")])

(define_insn "movdi_<mode>_update"
  [(set (mem:DI (plus:P (match_operand:P 1 "gpc_reg_operand" "0,0")
			 (match_operand:P 2 "reg_or_aligned_short_operand" "r,I")))
	(match_operand:DI 3 "gpc_reg_operand" "r,r"))
   (set (match_operand:P 0 "gpc_reg_operand" "=b,b")
	(plus:P (match_dup 1) (match_dup 2)))]
  "TARGET_POWERPC64 && TARGET_UPDATE
   && (!avoiding_indexed_address_p (Pmode)
       || !gpc_reg_operand (operands[2], Pmode)
       || (REG_P (operands[0])
	   && REGNO (operands[0]) == STACK_POINTER_REGNUM))"
  "@
   stdux %3,%0,%2
   stdu %3,%2(%0)"
  [(set_attr "type" "store")
   (set_attr "update" "yes")
   (set_attr "indexed" "yes,no")])

;; This pattern is only conditional on TARGET_POWERPC64, as it is
;; needed for stack allocation, even if the user passes -mno-update.
(define_insn "movdi_<mode>_update_stack"
  [(set (mem:DI (plus:P (match_operand:P 1 "gpc_reg_operand" "0,0")
			 (match_operand:P 2 "reg_or_aligned_short_operand" "r,I")))
	(match_operand:DI 3 "gpc_reg_operand" "r,r"))
   (set (match_operand:P 0 "gpc_reg_operand" "=b,b")
	(plus:P (match_dup 1) (match_dup 2)))]
  "TARGET_POWERPC64"
  "@
   stdux %3,%0,%2
   stdu %3,%2(%0)"
  [(set_attr "type" "store")
   (set_attr "update" "yes")
   (set_attr "indexed" "yes,no")])

(define_insn "*movsi_update1"
  [(set (match_operand:SI 3 "gpc_reg_operand" "=r,r")
	(mem:SI (plus:SI (match_operand:SI 1 "gpc_reg_operand" "0,0")
			 (match_operand:SI 2 "reg_or_short_operand" "r,I"))))
   (set (match_operand:SI 0 "gpc_reg_operand" "=b,b")
	(plus:SI (match_dup 1) (match_dup 2)))]
  "TARGET_UPDATE
   && (!avoiding_indexed_address_p (SImode)
       || !gpc_reg_operand (operands[2], SImode))"
  "@
   lwzux %3,%0,%2
   lwzu %3,%2(%0)"
  [(set_attr "type" "load")
   (set_attr "update" "yes")
   (set_attr "indexed" "yes,no")])

(define_insn "*movsi_update2"
  [(set (match_operand:DI 3 "gpc_reg_operand" "=r")
	(sign_extend:DI
	 (mem:SI (plus:DI (match_operand:DI 1 "gpc_reg_operand" "0")
			  (match_operand:DI 2 "gpc_reg_operand" "r")))))
   (set (match_operand:DI 0 "gpc_reg_operand" "=b")
	(plus:DI (match_dup 1) (match_dup 2)))]
  "TARGET_POWERPC64 && rs6000_gen_cell_microcode
   && !avoiding_indexed_address_p (DImode)"
  "lwaux %3,%0,%2"
  [(set_attr "type" "load")
   (set_attr "sign_extend" "yes")
   (set_attr "update" "yes")
   (set_attr "indexed" "yes")])

(define_insn "movsi_update"
  [(set (mem:SI (plus:SI (match_operand:SI 1 "gpc_reg_operand" "0,0")
			 (match_operand:SI 2 "reg_or_short_operand" "r,I")))
	(match_operand:SI 3 "gpc_reg_operand" "r,r"))
   (set (match_operand:SI 0 "gpc_reg_operand" "=b,b")
	(plus:SI (match_dup 1) (match_dup 2)))]
  "TARGET_UPDATE
   && (!avoiding_indexed_address_p (SImode)
       || !gpc_reg_operand (operands[2], SImode)
       || (REG_P (operands[0])
	   && REGNO (operands[0]) == STACK_POINTER_REGNUM))"
  "@
   stwux %3,%0,%2
   stwu %3,%2(%0)"
  [(set_attr "type" "store")
   (set_attr "update" "yes")
   (set_attr "indexed" "yes,no")])

;; This is an unconditional pattern; needed for stack allocation, even
;; if the user passes -mno-update.
(define_insn "movsi_update_stack"
  [(set (mem:SI (plus:SI (match_operand:SI 1 "gpc_reg_operand" "0,0")
			 (match_operand:SI 2 "reg_or_short_operand" "r,I")))
	(match_operand:SI 3 "gpc_reg_operand" "r,r"))
   (set (match_operand:SI 0 "gpc_reg_operand" "=b,b")
	(plus:SI (match_dup 1) (match_dup 2)))]
  ""
  "@
   stwux %3,%0,%2
   stwu %3,%2(%0)"
  [(set_attr "type" "store")
   (set_attr "update" "yes")
   (set_attr "indexed" "yes,no")])

(define_insn "*movhi_update1"
  [(set (match_operand:HI 3 "gpc_reg_operand" "=r,r")
	(mem:HI (plus:SI (match_operand:SI 1 "gpc_reg_operand" "0,0")
			 (match_operand:SI 2 "reg_or_short_operand" "r,I"))))
   (set (match_operand:SI 0 "gpc_reg_operand" "=b,b")
	(plus:SI (match_dup 1) (match_dup 2)))]
  "TARGET_UPDATE
   && (!avoiding_indexed_address_p (SImode)
       || !gpc_reg_operand (operands[2], SImode))"
  "@
   lhzux %3,%0,%2
   lhzu %3,%2(%0)"
  [(set_attr "type" "load")
   (set_attr "update" "yes")
   (set_attr "indexed" "yes,no")])

(define_insn "*movhi_update2"
  [(set (match_operand:SI 3 "gpc_reg_operand" "=r,r")
	(zero_extend:SI
	 (mem:HI (plus:SI (match_operand:SI 1 "gpc_reg_operand" "0,0")
			  (match_operand:SI 2 "reg_or_short_operand" "r,I")))))
   (set (match_operand:SI 0 "gpc_reg_operand" "=b,b")
	(plus:SI (match_dup 1) (match_dup 2)))]
  "TARGET_UPDATE
   && (!avoiding_indexed_address_p (SImode)
       || !gpc_reg_operand (operands[2], SImode))"
  "@
   lhzux %3,%0,%2
   lhzu %3,%2(%0)"
  [(set_attr "type" "load")
   (set_attr "update" "yes")
   (set_attr "indexed" "yes,no")])

(define_insn "*movhi_update3"
  [(set (match_operand:SI 3 "gpc_reg_operand" "=r,r")
	(sign_extend:SI
	 (mem:HI (plus:SI (match_operand:SI 1 "gpc_reg_operand" "0,0")
			  (match_operand:SI 2 "reg_or_short_operand" "r,I")))))
   (set (match_operand:SI 0 "gpc_reg_operand" "=b,b")
	(plus:SI (match_dup 1) (match_dup 2)))]
  "TARGET_UPDATE && rs6000_gen_cell_microcode
   && (!avoiding_indexed_address_p (SImode)
       || !gpc_reg_operand (operands[2], SImode))"
  "@
   lhaux %3,%0,%2
   lhau %3,%2(%0)"
  [(set_attr "type" "load")
   (set_attr "sign_extend" "yes")
   (set_attr "update" "yes")
   (set_attr "indexed" "yes,no")])

(define_insn "*movhi_update4"
  [(set (mem:HI (plus:SI (match_operand:SI 1 "gpc_reg_operand" "0,0")
			 (match_operand:SI 2 "reg_or_short_operand" "r,I")))
	(match_operand:HI 3 "gpc_reg_operand" "r,r"))
   (set (match_operand:SI 0 "gpc_reg_operand" "=b,b")
	(plus:SI (match_dup 1) (match_dup 2)))]
  "TARGET_UPDATE
   && (!avoiding_indexed_address_p (SImode)
       || !gpc_reg_operand (operands[2], SImode))"
  "@
   sthux %3,%0,%2
   sthu %3,%2(%0)"
  [(set_attr "type" "store")
   (set_attr "update" "yes")
   (set_attr "indexed" "yes,no")])

(define_insn "*movqi_update1"
  [(set (match_operand:QI 3 "gpc_reg_operand" "=r,r")
	(mem:QI (plus:SI (match_operand:SI 1 "gpc_reg_operand" "0,0")
			 (match_operand:SI 2 "reg_or_short_operand" "r,I"))))
   (set (match_operand:SI 0 "gpc_reg_operand" "=b,b")
	(plus:SI (match_dup 1) (match_dup 2)))]
  "TARGET_UPDATE
   && (!avoiding_indexed_address_p (SImode)
       || !gpc_reg_operand (operands[2], SImode))"
  "@
   lbzux %3,%0,%2
   lbzu %3,%2(%0)"
  [(set_attr "type" "load")
   (set_attr "update" "yes")
   (set_attr "indexed" "yes,no")])

(define_insn "*movqi_update2"
  [(set (match_operand:SI 3 "gpc_reg_operand" "=r,r")
	(zero_extend:SI
	 (mem:QI (plus:SI (match_operand:SI 1 "gpc_reg_operand" "0,0")
			  (match_operand:SI 2 "reg_or_short_operand" "r,I")))))
   (set (match_operand:SI 0 "gpc_reg_operand" "=b,b")
	(plus:SI (match_dup 1) (match_dup 2)))]
  "TARGET_UPDATE
   && (!avoiding_indexed_address_p (SImode)
       || !gpc_reg_operand (operands[2], SImode))"
  "@
   lbzux %3,%0,%2
   lbzu %3,%2(%0)"
  [(set_attr "type" "load")
   (set_attr "update" "yes")
   (set_attr "indexed" "yes,no")])

(define_insn "*movqi_update3"
  [(set (mem:QI (plus:SI (match_operand:SI 1 "gpc_reg_operand" "0,0")
			 (match_operand:SI 2 "reg_or_short_operand" "r,I")))
	(match_operand:QI 3 "gpc_reg_operand" "r,r"))
   (set (match_operand:SI 0 "gpc_reg_operand" "=b,b")
	(plus:SI (match_dup 1) (match_dup 2)))]
  "TARGET_UPDATE
   && (!avoiding_indexed_address_p (SImode)
       || !gpc_reg_operand (operands[2], SImode))"
  "@
   stbux %3,%0,%2
   stbu %3,%2(%0)"
  [(set_attr "type" "store")
   (set_attr "update" "yes")
   (set_attr "indexed" "yes,no")])

(define_insn "*movsf_update1"
  [(set (match_operand:SF 3 "gpc_reg_operand" "=f,f")
	(mem:SF (plus:SI (match_operand:SI 1 "gpc_reg_operand" "0,0")
			 (match_operand:SI 2 "reg_or_short_operand" "r,I"))))
   (set (match_operand:SI 0 "gpc_reg_operand" "=b,b")
	(plus:SI (match_dup 1) (match_dup 2)))]
  "TARGET_HARD_FLOAT && TARGET_FPRS && TARGET_SINGLE_FLOAT && TARGET_UPDATE
   && (!avoiding_indexed_address_p (SImode)
       || !gpc_reg_operand (operands[2], SImode))"
  "@
   lfsux %3,%0,%2
   lfsu %3,%2(%0)"
  [(set_attr "type" "fpload")
   (set_attr "update" "yes")
   (set_attr "indexed" "yes,no")])

(define_insn "*movsf_update2"
  [(set (mem:SF (plus:SI (match_operand:SI 1 "gpc_reg_operand" "0,0")
			 (match_operand:SI 2 "reg_or_short_operand" "r,I")))
	(match_operand:SF 3 "gpc_reg_operand" "f,f"))
   (set (match_operand:SI 0 "gpc_reg_operand" "=b,b")
	(plus:SI (match_dup 1) (match_dup 2)))]
  "TARGET_HARD_FLOAT && TARGET_FPRS && TARGET_SINGLE_FLOAT && TARGET_UPDATE
   && (!avoiding_indexed_address_p (SImode)
       || !gpc_reg_operand (operands[2], SImode))"
  "@
   stfsux %3,%0,%2
   stfsu %3,%2(%0)"
  [(set_attr "type" "fpstore")
   (set_attr "update" "yes")
   (set_attr "indexed" "yes,no")])

(define_insn "*movsf_update3"
  [(set (match_operand:SF 3 "gpc_reg_operand" "=r,r")
	(mem:SF (plus:SI (match_operand:SI 1 "gpc_reg_operand" "0,0")
			 (match_operand:SI 2 "reg_or_short_operand" "r,I"))))
   (set (match_operand:SI 0 "gpc_reg_operand" "=b,b")
	(plus:SI (match_dup 1) (match_dup 2)))]
  "(TARGET_SOFT_FLOAT || !TARGET_FPRS) && TARGET_UPDATE
   && (!avoiding_indexed_address_p (SImode)
       || !gpc_reg_operand (operands[2], SImode))"
  "@
   lwzux %3,%0,%2
   lwzu %3,%2(%0)"
  [(set_attr "type" "load")
   (set_attr "update" "yes")
   (set_attr "indexed" "yes,no")])

(define_insn "*movsf_update4"
  [(set (mem:SF (plus:SI (match_operand:SI 1 "gpc_reg_operand" "0,0")
			 (match_operand:SI 2 "reg_or_short_operand" "r,I")))
	(match_operand:SF 3 "gpc_reg_operand" "r,r"))
   (set (match_operand:SI 0 "gpc_reg_operand" "=b,b")
	(plus:SI (match_dup 1) (match_dup 2)))]
  "(TARGET_SOFT_FLOAT || !TARGET_FPRS) && TARGET_UPDATE
   && (!avoiding_indexed_address_p (SImode)
       || !gpc_reg_operand (operands[2], SImode))"
  "@
   stwux %3,%0,%2
   stwu %3,%2(%0)"
  [(set_attr "type" "store")
   (set_attr "update" "yes")
   (set_attr "indexed" "yes,no")])

(define_insn "*movdf_update1"
  [(set (match_operand:DF 3 "gpc_reg_operand" "=d,d")
	(mem:DF (plus:SI (match_operand:SI 1 "gpc_reg_operand" "0,0")
			 (match_operand:SI 2 "reg_or_short_operand" "r,I"))))
   (set (match_operand:SI 0 "gpc_reg_operand" "=b,b")
	(plus:SI (match_dup 1) (match_dup 2)))]
  "TARGET_HARD_FLOAT && TARGET_FPRS && TARGET_DOUBLE_FLOAT && TARGET_UPDATE
   && (!avoiding_indexed_address_p (SImode)
       || !gpc_reg_operand (operands[2], SImode))"
  "@
   lfdux %3,%0,%2
   lfdu %3,%2(%0)"
  [(set_attr "type" "fpload")
   (set_attr "update" "yes")
   (set_attr "indexed" "yes,no")])

(define_insn "*movdf_update2"
  [(set (mem:DF (plus:SI (match_operand:SI 1 "gpc_reg_operand" "0,0")
			 (match_operand:SI 2 "reg_or_short_operand" "r,I")))
	(match_operand:DF 3 "gpc_reg_operand" "d,d"))
   (set (match_operand:SI 0 "gpc_reg_operand" "=b,b")
	(plus:SI (match_dup 1) (match_dup 2)))]
  "TARGET_HARD_FLOAT && TARGET_FPRS && TARGET_DOUBLE_FLOAT && TARGET_UPDATE
   && (!avoiding_indexed_address_p (SImode)
       || !gpc_reg_operand (operands[2], SImode))"
  "@
   stfdux %3,%0,%2
   stfdu %3,%2(%0)"
  [(set_attr "type" "fpstore")
   (set_attr "update" "yes")
   (set_attr "indexed" "yes,no")])


;; After inserting conditional returns we can sometimes have
;; unnecessary register moves.  Unfortunately we cannot have a
;; modeless peephole here, because some single SImode sets have early
;; clobber outputs.  Although those sets expand to multi-ppc-insn
;; sequences, using get_attr_length here will smash the operands
;; array.  Neither is there an early_cobbler_p predicate.
;; Disallow subregs for E500 so we don't munge frob_di_df_2.
;; Also this optimization interferes with scalars going into
;; altivec registers (the code does reloading through the FPRs).
(define_peephole2
  [(set (match_operand:DF 0 "gpc_reg_operand" "")
	(match_operand:DF 1 "any_operand" ""))
   (set (match_operand:DF 2 "gpc_reg_operand" "")
	(match_dup 0))]
  "!(TARGET_E500_DOUBLE && GET_CODE (operands[2]) == SUBREG)
   && !TARGET_UPPER_REGS_DF
   && peep2_reg_dead_p (2, operands[0])"
  [(set (match_dup 2) (match_dup 1))])

(define_peephole2
  [(set (match_operand:SF 0 "gpc_reg_operand" "")
	(match_operand:SF 1 "any_operand" ""))
   (set (match_operand:SF 2 "gpc_reg_operand" "")
	(match_dup 0))]
  "!TARGET_UPPER_REGS_SF
   && peep2_reg_dead_p (2, operands[0])"
  [(set (match_dup 2) (match_dup 1))])


;; TLS support.

;; Mode attributes for different ABIs.
(define_mode_iterator TLSmode [(SI "! TARGET_64BIT") (DI "TARGET_64BIT")])
(define_mode_attr tls_abi_suffix [(SI "32") (DI "64")])
(define_mode_attr tls_sysv_suffix [(SI "si") (DI "di")])
(define_mode_attr tls_insn_suffix [(SI "wz") (DI "d")])

(define_insn_and_split "tls_gd_aix<TLSmode:tls_abi_suffix>"
  [(set (match_operand:TLSmode 0 "gpc_reg_operand" "=b")
        (call (mem:TLSmode (match_operand:TLSmode 3 "symbol_ref_operand" "s"))
	      (match_operand 4 "" "g")))
   (unspec:TLSmode [(match_operand:TLSmode 1 "gpc_reg_operand" "b")
	 	    (match_operand:TLSmode 2 "rs6000_tls_symbol_ref" "")]
		   UNSPEC_TLSGD)
   (clobber (reg:SI LR_REGNO))]
  "HAVE_AS_TLS && (DEFAULT_ABI == ABI_AIX || DEFAULT_ABI == ABI_ELFv2)"
{
  if (TARGET_CMODEL != CMODEL_SMALL)
    return "addis %0,%1,%2@got@tlsgd@ha\;addi %0,%0,%2@got@tlsgd@l\;"
	   "bl %z3\;nop";
  else
    return "addi %0,%1,%2@got@tlsgd\;bl %z3\;nop";
}
  "&& TARGET_TLS_MARKERS"
  [(set (match_dup 0)
	(unspec:TLSmode [(match_dup 1)
			 (match_dup 2)]
			UNSPEC_TLSGD))
   (parallel [(set (match_dup 0)
   	     	   (call (mem:TLSmode (match_dup 3))
		   	 (match_dup 4)))
	      (unspec:TLSmode [(match_dup 2)] UNSPEC_TLSGD)
	      (clobber (reg:SI LR_REGNO))])]
  ""
  [(set_attr "type" "two")
   (set (attr "length")
     (if_then_else (ne (symbol_ref "TARGET_CMODEL") (symbol_ref "CMODEL_SMALL"))
     		   (const_int 16)
     		   (const_int 12)))])

(define_insn_and_split "tls_gd_sysv<TLSmode:tls_sysv_suffix>"
  [(set (match_operand:TLSmode 0 "gpc_reg_operand" "=b")
        (call (mem:TLSmode (match_operand:TLSmode 3 "symbol_ref_operand" "s"))
	      (match_operand 4 "" "g")))
   (unspec:TLSmode [(match_operand:TLSmode 1 "gpc_reg_operand" "b")
	 	    (match_operand:TLSmode 2 "rs6000_tls_symbol_ref" "")]
		   UNSPEC_TLSGD)
   (clobber (reg:SI LR_REGNO))]
  "HAVE_AS_TLS && DEFAULT_ABI == ABI_V4"
{
  if (flag_pic)
    {
      if (TARGET_SECURE_PLT && flag_pic == 2)
	return "addi %0,%1,%2@got@tlsgd\;bl %z3+32768@plt";
      else
	return "addi %0,%1,%2@got@tlsgd\;bl %z3@plt";
    }
  else
    return "addi %0,%1,%2@got@tlsgd\;bl %z3";
}
  "&& TARGET_TLS_MARKERS"
  [(set (match_dup 0)
	(unspec:TLSmode [(match_dup 1)
			 (match_dup 2)]
			UNSPEC_TLSGD))
   (parallel [(set (match_dup 0)
   	     	   (call (mem:TLSmode (match_dup 3))
		   	 (match_dup 4)))
	      (unspec:TLSmode [(match_dup 2)] UNSPEC_TLSGD)
	      (clobber (reg:SI LR_REGNO))])]
  ""
  [(set_attr "type" "two")
   (set_attr "length" "8")])

(define_insn_and_split "*tls_gd<TLSmode:tls_abi_suffix>"
  [(set (match_operand:TLSmode 0 "gpc_reg_operand" "=b")
	(unspec:TLSmode [(match_operand:TLSmode 1 "gpc_reg_operand" "b")
			 (match_operand:TLSmode 2 "rs6000_tls_symbol_ref" "")]
			UNSPEC_TLSGD))]
  "HAVE_AS_TLS && TARGET_TLS_MARKERS"
  "addi %0,%1,%2@got@tlsgd"
  "&& TARGET_CMODEL != CMODEL_SMALL"
  [(set (match_dup 3)
  	(high:TLSmode
	    (unspec:TLSmode [(match_dup 1) (match_dup 2)] UNSPEC_TLSGD)))
   (set (match_dup 0)
   	(lo_sum:TLSmode (match_dup 3)
	    (unspec:TLSmode [(match_dup 1) (match_dup 2)] UNSPEC_TLSGD)))]
  "
{
  operands[3] = gen_reg_rtx (TARGET_64BIT ? DImode : SImode);
}"
  [(set (attr "length")
     (if_then_else (ne (symbol_ref "TARGET_CMODEL") (symbol_ref "CMODEL_SMALL"))
     		   (const_int 8)
     		   (const_int 4)))])

(define_insn "*tls_gd_high<TLSmode:tls_abi_suffix>"
  [(set (match_operand:TLSmode 0 "gpc_reg_operand" "=b")
     (high:TLSmode
       (unspec:TLSmode [(match_operand:TLSmode 1 "gpc_reg_operand" "b")
			(match_operand:TLSmode 2 "rs6000_tls_symbol_ref" "")]
		       UNSPEC_TLSGD)))]
  "HAVE_AS_TLS && TARGET_TLS_MARKERS && TARGET_CMODEL != CMODEL_SMALL"
  "addis %0,%1,%2@got@tlsgd@ha"
  [(set_attr "length" "4")])

(define_insn "*tls_gd_low<TLSmode:tls_abi_suffix>"
  [(set (match_operand:TLSmode 0 "gpc_reg_operand" "=b")
     (lo_sum:TLSmode (match_operand:TLSmode 1 "gpc_reg_operand" "b")
       (unspec:TLSmode [(match_operand:TLSmode 3 "gpc_reg_operand" "b")
			(match_operand:TLSmode 2 "rs6000_tls_symbol_ref" "")]
		       UNSPEC_TLSGD)))]
  "HAVE_AS_TLS && TARGET_TLS_MARKERS && TARGET_CMODEL != CMODEL_SMALL"
  "addi %0,%1,%2@got@tlsgd@l"
  [(set_attr "length" "4")])

(define_insn "*tls_gd_call_aix<TLSmode:tls_abi_suffix>"
  [(set (match_operand:TLSmode 0 "gpc_reg_operand" "=b")
        (call (mem:TLSmode (match_operand:TLSmode 1 "symbol_ref_operand" "s"))
	      (match_operand 2 "" "g")))
   (unspec:TLSmode [(match_operand:TLSmode 3 "rs6000_tls_symbol_ref" "")]
		   UNSPEC_TLSGD)
   (clobber (reg:SI LR_REGNO))]
  "HAVE_AS_TLS && TARGET_TLS_MARKERS
   && (DEFAULT_ABI == ABI_AIX || DEFAULT_ABI == ABI_ELFv2)"
  "bl %z1(%3@tlsgd)\;nop"
  [(set_attr "type" "branch")
   (set_attr "length" "8")])

(define_insn "*tls_gd_call_sysv<TLSmode:tls_abi_suffix>"
  [(set (match_operand:TLSmode 0 "gpc_reg_operand" "=b")
        (call (mem:TLSmode (match_operand:TLSmode 1 "symbol_ref_operand" "s"))
	      (match_operand 2 "" "g")))
   (unspec:TLSmode [(match_operand:TLSmode 3 "rs6000_tls_symbol_ref" "")]
		   UNSPEC_TLSGD)
   (clobber (reg:SI LR_REGNO))]
  "HAVE_AS_TLS && DEFAULT_ABI == ABI_V4 && TARGET_TLS_MARKERS"
{
  if (flag_pic)
    {
      if (TARGET_SECURE_PLT && flag_pic == 2)
	return "bl %z1+32768(%3@tlsgd)@plt";
      return "bl %z1(%3@tlsgd)@plt";
    }
  return "bl %z1(%3@tlsgd)";
}
  [(set_attr "type" "branch")
   (set_attr "length" "4")])

(define_insn_and_split "tls_ld_aix<TLSmode:tls_abi_suffix>"
  [(set (match_operand:TLSmode 0 "gpc_reg_operand" "=b")
        (call (mem:TLSmode (match_operand:TLSmode 2 "symbol_ref_operand" "s"))
	      (match_operand 3 "" "g")))
   (unspec:TLSmode [(match_operand:TLSmode 1 "gpc_reg_operand" "b")]
		   UNSPEC_TLSLD)
   (clobber (reg:SI LR_REGNO))]
  "HAVE_AS_TLS && (DEFAULT_ABI == ABI_AIX || DEFAULT_ABI == ABI_ELFv2)"
{
  if (TARGET_CMODEL != CMODEL_SMALL)
    return "addis %0,%1,%&@got@tlsld@ha\;addi %0,%0,%&@got@tlsld@l\;"
	   "bl %z2\;nop";
  else
    return "addi %0,%1,%&@got@tlsld\;bl %z2\;nop";
}
  "&& TARGET_TLS_MARKERS"
  [(set (match_dup 0)
	(unspec:TLSmode [(match_dup 1)]
			UNSPEC_TLSLD))
   (parallel [(set (match_dup 0)
   	     	   (call (mem:TLSmode (match_dup 2))
		   	 (match_dup 3)))
	      (unspec:TLSmode [(const_int 0)] UNSPEC_TLSLD)
	      (clobber (reg:SI LR_REGNO))])]
  ""
  [(set_attr "type" "two")
   (set (attr "length")
     (if_then_else (ne (symbol_ref "TARGET_CMODEL") (symbol_ref "CMODEL_SMALL"))
     		   (const_int 16)
     		   (const_int 12)))])

(define_insn_and_split "tls_ld_sysv<TLSmode:tls_sysv_suffix>"
  [(set (match_operand:TLSmode 0 "gpc_reg_operand" "=b")
        (call (mem:TLSmode (match_operand:TLSmode 2 "symbol_ref_operand" "s"))
	      (match_operand 3 "" "g")))
   (unspec:TLSmode [(match_operand:TLSmode 1 "gpc_reg_operand" "b")]
		   UNSPEC_TLSLD)
   (clobber (reg:SI LR_REGNO))]
  "HAVE_AS_TLS && DEFAULT_ABI == ABI_V4"
{
  if (flag_pic)
    {
      if (TARGET_SECURE_PLT && flag_pic == 2)
	return "addi %0,%1,%&@got@tlsld\;bl %z2+32768@plt";
      else
	return "addi %0,%1,%&@got@tlsld\;bl %z2@plt";
    }
  else
    return "addi %0,%1,%&@got@tlsld\;bl %z2";
}
  "&& TARGET_TLS_MARKERS"
  [(set (match_dup 0)
	(unspec:TLSmode [(match_dup 1)]
			UNSPEC_TLSLD))
   (parallel [(set (match_dup 0)
   	     	   (call (mem:TLSmode (match_dup 2))
		   	 (match_dup 3)))
	      (unspec:TLSmode [(const_int 0)] UNSPEC_TLSLD)
	      (clobber (reg:SI LR_REGNO))])]
  ""
  [(set_attr "length" "8")])

(define_insn_and_split "*tls_ld<TLSmode:tls_abi_suffix>"
  [(set (match_operand:TLSmode 0 "gpc_reg_operand" "=b")
	(unspec:TLSmode [(match_operand:TLSmode 1 "gpc_reg_operand" "b")]
			UNSPEC_TLSLD))]
  "HAVE_AS_TLS && TARGET_TLS_MARKERS"
  "addi %0,%1,%&@got@tlsld"
  "&& TARGET_CMODEL != CMODEL_SMALL"
  [(set (match_dup 2)
  	(high:TLSmode
	    (unspec:TLSmode [(const_int 0) (match_dup 1)] UNSPEC_TLSLD)))
   (set (match_dup 0)
   	(lo_sum:TLSmode (match_dup 2)
	    (unspec:TLSmode [(const_int 0) (match_dup 1)] UNSPEC_TLSLD)))]
  "
{
  operands[2] = gen_reg_rtx (TARGET_64BIT ? DImode : SImode);
}"
  [(set (attr "length")
     (if_then_else (ne (symbol_ref "TARGET_CMODEL") (symbol_ref "CMODEL_SMALL"))
     		   (const_int 8)
     		   (const_int 4)))])

(define_insn "*tls_ld_high<TLSmode:tls_abi_suffix>"
  [(set (match_operand:TLSmode 0 "gpc_reg_operand" "=b")
     (high:TLSmode
       (unspec:TLSmode [(const_int 0)
			(match_operand:TLSmode 1 "gpc_reg_operand" "b")]
		       UNSPEC_TLSLD)))]
  "HAVE_AS_TLS && TARGET_TLS_MARKERS && TARGET_CMODEL != CMODEL_SMALL"
  "addis %0,%1,%&@got@tlsld@ha"
  [(set_attr "length" "4")])

(define_insn "*tls_ld_low<TLSmode:tls_abi_suffix>"
  [(set (match_operand:TLSmode 0 "gpc_reg_operand" "=b")
     (lo_sum:TLSmode (match_operand:TLSmode 1 "gpc_reg_operand" "b")
       (unspec:TLSmode [(const_int 0)
                        (match_operand:TLSmode 2 "gpc_reg_operand" "b")]
                       UNSPEC_TLSLD)))]
  "HAVE_AS_TLS && TARGET_TLS_MARKERS && TARGET_CMODEL != CMODEL_SMALL"
  "addi %0,%1,%&@got@tlsld@l"
  [(set_attr "length" "4")])

(define_insn "*tls_ld_call_aix<TLSmode:tls_abi_suffix>"
  [(set (match_operand:TLSmode 0 "gpc_reg_operand" "=b")
        (call (mem:TLSmode (match_operand:TLSmode 1 "symbol_ref_operand" "s"))
	      (match_operand 2 "" "g")))
   (unspec:TLSmode [(const_int 0)] UNSPEC_TLSLD)
   (clobber (reg:SI LR_REGNO))]
  "HAVE_AS_TLS && TARGET_TLS_MARKERS
   && (DEFAULT_ABI == ABI_AIX || DEFAULT_ABI == ABI_ELFv2)"
  "bl %z1(%&@tlsld)\;nop"
  [(set_attr "type" "branch")
   (set_attr "length" "8")])

(define_insn "*tls_ld_call_sysv<TLSmode:tls_abi_suffix>"
  [(set (match_operand:TLSmode 0 "gpc_reg_operand" "=b")
        (call (mem:TLSmode (match_operand:TLSmode 1 "symbol_ref_operand" "s"))
	      (match_operand 2 "" "g")))
   (unspec:TLSmode [(const_int 0)] UNSPEC_TLSLD)
   (clobber (reg:SI LR_REGNO))]
  "HAVE_AS_TLS && DEFAULT_ABI == ABI_V4 && TARGET_TLS_MARKERS"
{
  if (flag_pic)
    {
      if (TARGET_SECURE_PLT && flag_pic == 2)
	return "bl %z1+32768(%&@tlsld)@plt";
      return "bl %z1(%&@tlsld)@plt";
    }
  return "bl %z1(%&@tlsld)";
}
  [(set_attr "type" "branch")
   (set_attr "length" "4")])

(define_insn "tls_dtprel_<TLSmode:tls_abi_suffix>"
  [(set (match_operand:TLSmode 0 "gpc_reg_operand" "=r")
	(unspec:TLSmode [(match_operand:TLSmode 1 "gpc_reg_operand" "b")
			 (match_operand:TLSmode 2 "rs6000_tls_symbol_ref" "")]
			UNSPEC_TLSDTPREL))]
  "HAVE_AS_TLS"
  "addi %0,%1,%2@dtprel")

(define_insn "tls_dtprel_ha_<TLSmode:tls_abi_suffix>"
  [(set (match_operand:TLSmode 0 "gpc_reg_operand" "=r")
	(unspec:TLSmode [(match_operand:TLSmode 1 "gpc_reg_operand" "b")
			 (match_operand:TLSmode 2 "rs6000_tls_symbol_ref" "")]
			UNSPEC_TLSDTPRELHA))]
  "HAVE_AS_TLS"
  "addis %0,%1,%2@dtprel@ha")

(define_insn "tls_dtprel_lo_<TLSmode:tls_abi_suffix>"
  [(set (match_operand:TLSmode 0 "gpc_reg_operand" "=r")
	(unspec:TLSmode [(match_operand:TLSmode 1 "gpc_reg_operand" "b")
			 (match_operand:TLSmode 2 "rs6000_tls_symbol_ref" "")]
			UNSPEC_TLSDTPRELLO))]
  "HAVE_AS_TLS"
  "addi %0,%1,%2@dtprel@l")

(define_insn_and_split "tls_got_dtprel_<TLSmode:tls_abi_suffix>"
  [(set (match_operand:TLSmode 0 "gpc_reg_operand" "=r")
	(unspec:TLSmode [(match_operand:TLSmode 1 "gpc_reg_operand" "b")
			 (match_operand:TLSmode 2 "rs6000_tls_symbol_ref" "")]
			UNSPEC_TLSGOTDTPREL))]
  "HAVE_AS_TLS"
  "l<TLSmode:tls_insn_suffix> %0,%2@got@dtprel(%1)"
  "&& TARGET_CMODEL != CMODEL_SMALL"
  [(set (match_dup 3)
	(high:TLSmode
	    (unspec:TLSmode [(match_dup 1) (match_dup 2)] UNSPEC_TLSGOTDTPREL)))
   (set (match_dup 0)
	(lo_sum:TLSmode (match_dup 3)
	    (unspec:TLSmode [(match_dup 1) (match_dup 2)] UNSPEC_TLSGOTDTPREL)))]
  "
{
  operands[3] = gen_reg_rtx (TARGET_64BIT ? DImode : SImode);
}"
  [(set (attr "length")
     (if_then_else (ne (symbol_ref "TARGET_CMODEL") (symbol_ref "CMODEL_SMALL"))
     		   (const_int 8)
     		   (const_int 4)))])

(define_insn "*tls_got_dtprel_high<TLSmode:tls_abi_suffix>"
  [(set (match_operand:TLSmode 0 "gpc_reg_operand" "=b")
     (high:TLSmode
       (unspec:TLSmode [(match_operand:TLSmode 1 "gpc_reg_operand" "b")
			(match_operand:TLSmode 2 "rs6000_tls_symbol_ref" "")]
		       UNSPEC_TLSGOTDTPREL)))]
  "HAVE_AS_TLS && TARGET_CMODEL != CMODEL_SMALL"
  "addis %0,%1,%2@got@dtprel@ha"
  [(set_attr "length" "4")])

(define_insn "*tls_got_dtprel_low<TLSmode:tls_abi_suffix>"
  [(set (match_operand:TLSmode 0 "gpc_reg_operand" "=r")
     (lo_sum:TLSmode (match_operand:TLSmode 1 "gpc_reg_operand" "b")
	 (unspec:TLSmode [(match_operand:TLSmode 3 "gpc_reg_operand" "b")
			  (match_operand:TLSmode 2 "rs6000_tls_symbol_ref" "")]
			 UNSPEC_TLSGOTDTPREL)))]
  "HAVE_AS_TLS && TARGET_CMODEL != CMODEL_SMALL"
  "l<TLSmode:tls_insn_suffix> %0,%2@got@dtprel@l(%1)"
  [(set_attr "length" "4")])

(define_insn "tls_tprel_<TLSmode:tls_abi_suffix>"
  [(set (match_operand:TLSmode 0 "gpc_reg_operand" "=r")
	(unspec:TLSmode [(match_operand:TLSmode 1 "gpc_reg_operand" "b")
			 (match_operand:TLSmode 2 "rs6000_tls_symbol_ref" "")]
			UNSPEC_TLSTPREL))]
  "HAVE_AS_TLS"
  "addi %0,%1,%2@tprel")

(define_insn "tls_tprel_ha_<TLSmode:tls_abi_suffix>"
  [(set (match_operand:TLSmode 0 "gpc_reg_operand" "=r")
	(unspec:TLSmode [(match_operand:TLSmode 1 "gpc_reg_operand" "b")
			 (match_operand:TLSmode 2 "rs6000_tls_symbol_ref" "")]
			UNSPEC_TLSTPRELHA))]
  "HAVE_AS_TLS"
  "addis %0,%1,%2@tprel@ha")

(define_insn "tls_tprel_lo_<TLSmode:tls_abi_suffix>"
  [(set (match_operand:TLSmode 0 "gpc_reg_operand" "=r")
	(unspec:TLSmode [(match_operand:TLSmode 1 "gpc_reg_operand" "b")
			 (match_operand:TLSmode 2 "rs6000_tls_symbol_ref" "")]
			UNSPEC_TLSTPRELLO))]
  "HAVE_AS_TLS"
  "addi %0,%1,%2@tprel@l")

;; "b" output constraint here and on tls_tls input to support linker tls
;; optimization.  The linker may edit the instructions emitted by a
;; tls_got_tprel/tls_tls pair to addis,addi.
(define_insn_and_split "tls_got_tprel_<TLSmode:tls_abi_suffix>"
  [(set (match_operand:TLSmode 0 "gpc_reg_operand" "=b")
	(unspec:TLSmode [(match_operand:TLSmode 1 "gpc_reg_operand" "b")
			 (match_operand:TLSmode 2 "rs6000_tls_symbol_ref" "")]
			UNSPEC_TLSGOTTPREL))]
  "HAVE_AS_TLS"
  "l<TLSmode:tls_insn_suffix> %0,%2@got@tprel(%1)"
  "&& TARGET_CMODEL != CMODEL_SMALL"
  [(set (match_dup 3)
	(high:TLSmode
	    (unspec:TLSmode [(match_dup 1) (match_dup 2)] UNSPEC_TLSGOTTPREL)))
   (set (match_dup 0)
	(lo_sum:TLSmode (match_dup 3)
	    (unspec:TLSmode [(match_dup 1) (match_dup 2)] UNSPEC_TLSGOTTPREL)))]
  "
{
  operands[3] = gen_reg_rtx (TARGET_64BIT ? DImode : SImode);
}"
  [(set (attr "length")
     (if_then_else (ne (symbol_ref "TARGET_CMODEL") (symbol_ref "CMODEL_SMALL"))
     		   (const_int 8)
     		   (const_int 4)))])

(define_insn "*tls_got_tprel_high<TLSmode:tls_abi_suffix>"
  [(set (match_operand:TLSmode 0 "gpc_reg_operand" "=b")
     (high:TLSmode
       (unspec:TLSmode [(match_operand:TLSmode 1 "gpc_reg_operand" "b")
			(match_operand:TLSmode 2 "rs6000_tls_symbol_ref" "")]
		       UNSPEC_TLSGOTTPREL)))]
  "HAVE_AS_TLS && TARGET_CMODEL != CMODEL_SMALL"
  "addis %0,%1,%2@got@tprel@ha"
  [(set_attr "length" "4")])

(define_insn "*tls_got_tprel_low<TLSmode:tls_abi_suffix>"
  [(set (match_operand:TLSmode 0 "gpc_reg_operand" "=r")
     (lo_sum:TLSmode (match_operand:TLSmode 1 "gpc_reg_operand" "b")
	 (unspec:TLSmode [(match_operand:TLSmode 3 "gpc_reg_operand" "b")
			  (match_operand:TLSmode 2 "rs6000_tls_symbol_ref" "")]
			 UNSPEC_TLSGOTTPREL)))]
  "HAVE_AS_TLS && TARGET_CMODEL != CMODEL_SMALL"
  "l<TLSmode:tls_insn_suffix> %0,%2@got@tprel@l(%1)"
  [(set_attr "length" "4")])

(define_insn "tls_tls_<TLSmode:tls_abi_suffix>"
  [(set (match_operand:TLSmode 0 "gpc_reg_operand" "=r")
	(unspec:TLSmode [(match_operand:TLSmode 1 "gpc_reg_operand" "b")
			 (match_operand:TLSmode 2 "rs6000_tls_symbol_ref" "")]
			UNSPEC_TLSTLS))]
  "TARGET_ELF && HAVE_AS_TLS"
  "add %0,%1,%2@tls")

(define_expand "tls_get_tpointer"
  [(set (match_operand:SI 0 "gpc_reg_operand" "")
	(unspec:SI [(const_int 0)] UNSPEC_TLSTLS))]
  "TARGET_XCOFF && HAVE_AS_TLS"
  "
{
  emit_insn (gen_tls_get_tpointer_internal ());
  emit_move_insn (operands[0], gen_rtx_REG (SImode, 3));
  DONE;
}")

(define_insn "tls_get_tpointer_internal"
  [(set (reg:SI 3)
	(unspec:SI [(const_int 0)] UNSPEC_TLSTLS))
   (clobber (reg:SI LR_REGNO))]
  "TARGET_XCOFF && HAVE_AS_TLS"
  "bla __get_tpointer")

(define_expand "tls_get_addr<mode>"
  [(set (match_operand:P 0 "gpc_reg_operand" "")
	(unspec:P [(match_operand:P 1 "gpc_reg_operand" "")
                   (match_operand:P 2 "gpc_reg_operand" "")] UNSPEC_TLSTLS))]
  "TARGET_XCOFF && HAVE_AS_TLS"
  "
{
  emit_move_insn (gen_rtx_REG (Pmode, 3), operands[1]);
  emit_move_insn (gen_rtx_REG (Pmode, 4), operands[2]);
  emit_insn (gen_tls_get_addr_internal<mode> ());
  emit_move_insn (operands[0], gen_rtx_REG (Pmode, 3));
  DONE;
}")

(define_insn "tls_get_addr_internal<mode>"
  [(set (reg:P 3)
	(unspec:P [(reg:P 3) (reg:P 4)] UNSPEC_TLSTLS))
   (clobber (reg:P 0))
   (clobber (reg:P 4))
   (clobber (reg:P 5))
   (clobber (reg:P 11))
   (clobber (reg:CC CR0_REGNO))
   (clobber (reg:P LR_REGNO))]
  "TARGET_XCOFF && HAVE_AS_TLS"
  "bla __tls_get_addr")

;; Next come insns related to the calling sequence.
;;
;; First, an insn to allocate new stack space for dynamic use (e.g., alloca).
;; We move the back-chain and decrement the stack pointer.

(define_expand "allocate_stack"
  [(set (match_operand 0 "gpc_reg_operand" "")
	(minus (reg 1) (match_operand 1 "reg_or_short_operand" "")))
   (set (reg 1)
	(minus (reg 1) (match_dup 1)))]
  ""
  "
{ rtx chain = gen_reg_rtx (Pmode);
  rtx stack_bot = gen_rtx_MEM (Pmode, stack_pointer_rtx);
  rtx neg_op0;
  rtx insn, par, set, mem;

  emit_move_insn (chain, stack_bot);

  /* Check stack bounds if necessary.  */
  if (crtl->limit_stack)
    {
      rtx available;
      available = expand_binop (Pmode, sub_optab,
				stack_pointer_rtx, stack_limit_rtx,
				NULL_RTX, 1, OPTAB_WIDEN);
      emit_insn (gen_cond_trap (LTU, available, operands[1], const0_rtx));
    }

  if (GET_CODE (operands[1]) != CONST_INT
      || INTVAL (operands[1]) < -32767
      || INTVAL (operands[1]) > 32768)
    {
      neg_op0 = gen_reg_rtx (Pmode);
      if (TARGET_32BIT)
	emit_insn (gen_negsi2 (neg_op0, operands[1]));
      else
	emit_insn (gen_negdi2 (neg_op0, operands[1]));
    }
  else
    neg_op0 = GEN_INT (- INTVAL (operands[1]));

  insn = emit_insn ((* ((TARGET_32BIT) ? gen_movsi_update_stack
				       : gen_movdi_di_update_stack))
			(stack_pointer_rtx, stack_pointer_rtx, neg_op0,
			 chain));
  /* Since we didn't use gen_frame_mem to generate the MEM, grab
     it now and set the alias set/attributes. The above gen_*_update
     calls will generate a PARALLEL with the MEM set being the first
     operation. */
  par = PATTERN (insn);
  gcc_assert (GET_CODE (par) == PARALLEL);
  set = XVECEXP (par, 0, 0);
  gcc_assert (GET_CODE (set) == SET);
  mem = SET_DEST (set);
  gcc_assert (MEM_P (mem));
  MEM_NOTRAP_P (mem) = 1;
  set_mem_alias_set (mem, get_frame_alias_set ());

  emit_move_insn (operands[0], virtual_stack_dynamic_rtx);
  DONE;
}")

;; These patterns say how to save and restore the stack pointer.  We need not
;; save the stack pointer at function level since we are careful to
;; preserve the backchain.  At block level, we have to restore the backchain
;; when we restore the stack pointer.
;;
;; For nonlocal gotos, we must save both the stack pointer and its
;; backchain and restore both.  Note that in the nonlocal case, the
;; save area is a memory location.

(define_expand "save_stack_function"
  [(match_operand 0 "any_operand" "")
   (match_operand 1 "any_operand" "")]
  ""
  "DONE;")

(define_expand "restore_stack_function"
  [(match_operand 0 "any_operand" "")
   (match_operand 1 "any_operand" "")]
  ""
  "DONE;")

;; Adjust stack pointer (op0) to a new value (op1).
;; First copy old stack backchain to new location, and ensure that the
;; scheduler won't reorder the sp assignment before the backchain write.
(define_expand "restore_stack_block"
  [(set (match_dup 2) (match_dup 3))
   (set (match_dup 4) (match_dup 2))
   (match_dup 5)
   (set (match_operand 0 "register_operand" "")
	(match_operand 1 "register_operand" ""))]
  ""
  "
{
  rtvec p;

  operands[1] = force_reg (Pmode, operands[1]);
  operands[2] = gen_reg_rtx (Pmode);
  operands[3] = gen_frame_mem (Pmode, operands[0]);
  operands[4] = gen_frame_mem (Pmode, operands[1]);
  p = rtvec_alloc (1);
  RTVEC_ELT (p, 0) = gen_rtx_SET (VOIDmode,
				  gen_frame_mem (BLKmode, operands[0]),
				  const0_rtx);
  operands[5] = gen_rtx_PARALLEL (VOIDmode, p);
}")

(define_expand "save_stack_nonlocal"
  [(set (match_dup 3) (match_dup 4))
   (set (match_operand 0 "memory_operand" "") (match_dup 3))
   (set (match_dup 2) (match_operand 1 "register_operand" ""))]
  ""
  "
{
  int units_per_word = (TARGET_32BIT) ? 4 : 8;

  /* Copy the backchain to the first word, sp to the second.  */
  operands[0] = adjust_address_nv (operands[0], Pmode, 0);
  operands[2] = adjust_address_nv (operands[0], Pmode, units_per_word);
  operands[3] = gen_reg_rtx (Pmode);
  operands[4] = gen_frame_mem (Pmode, operands[1]);
}")

(define_expand "restore_stack_nonlocal"
  [(set (match_dup 2) (match_operand 1 "memory_operand" ""))
   (set (match_dup 3) (match_dup 4))
   (set (match_dup 5) (match_dup 2))
   (match_dup 6)
   (set (match_operand 0 "register_operand" "") (match_dup 3))]
  ""
  "
{
  int units_per_word = (TARGET_32BIT) ? 4 : 8;
  rtvec p;

  /* Restore the backchain from the first word, sp from the second.  */
  operands[2] = gen_reg_rtx (Pmode);
  operands[3] = gen_reg_rtx (Pmode);
  operands[1] = adjust_address_nv (operands[1], Pmode, 0);
  operands[4] = adjust_address_nv (operands[1], Pmode, units_per_word);
  operands[5] = gen_frame_mem (Pmode, operands[3]);
  p = rtvec_alloc (1);
  RTVEC_ELT (p, 0) = gen_rtx_SET (VOIDmode,
				  gen_frame_mem (BLKmode, operands[0]),
				  const0_rtx);
  operands[6] = gen_rtx_PARALLEL (VOIDmode, p);
}")

;; TOC register handling.

;; Code to initialize the TOC register...

(define_insn "load_toc_aix_si"
  [(parallel [(set (match_operand:SI 0 "gpc_reg_operand" "=r")
		   (unspec:SI [(const_int 0)] UNSPEC_TOC))
	      (use (reg:SI 2))])]
  "(DEFAULT_ABI == ABI_AIX || DEFAULT_ABI == ABI_ELFv2) && TARGET_32BIT"
  "*
{
  char buf[30];
  ASM_GENERATE_INTERNAL_LABEL (buf, \"LCTOC\", 1);
  operands[1] = gen_rtx_SYMBOL_REF (Pmode, ggc_strdup (buf));
  operands[2] = gen_rtx_REG (Pmode, 2);
  return \"lwz %0,%1(%2)\";
}"
  [(set_attr "type" "load")
   (set_attr "update" "no")
   (set_attr "indexed" "no")])

(define_insn "load_toc_aix_di"
  [(parallel [(set (match_operand:DI 0 "gpc_reg_operand" "=r")
		   (unspec:DI [(const_int 0)] UNSPEC_TOC))
	      (use (reg:DI 2))])]
  "(DEFAULT_ABI == ABI_AIX || DEFAULT_ABI == ABI_ELFv2) && TARGET_64BIT"
  "*
{
  char buf[30];
#ifdef TARGET_RELOCATABLE
  ASM_GENERATE_INTERNAL_LABEL (buf, \"LCTOC\",
			       !TARGET_MINIMAL_TOC || TARGET_RELOCATABLE);
#else
  ASM_GENERATE_INTERNAL_LABEL (buf, \"LCTOC\", 1);
#endif
  if (TARGET_ELF)
    strcat (buf, \"@toc\");
  operands[1] = gen_rtx_SYMBOL_REF (Pmode, ggc_strdup (buf));
  operands[2] = gen_rtx_REG (Pmode, 2);
  return \"ld %0,%1(%2)\";
}"
  [(set_attr "type" "load")
   (set_attr "update" "no")
   (set_attr "indexed" "no")])

(define_insn "load_toc_v4_pic_si"
  [(set (reg:SI LR_REGNO)
	(unspec:SI [(const_int 0)] UNSPEC_TOC))]
  "DEFAULT_ABI == ABI_V4 && flag_pic == 1 && TARGET_32BIT"
  "bl _GLOBAL_OFFSET_TABLE_@local-4"
  [(set_attr "type" "branch")
   (set_attr "length" "4")])

(define_expand "load_toc_v4_PIC_1"
  [(parallel [(set (reg:SI LR_REGNO)
		   (match_operand:SI 0 "immediate_operand" "s"))
	      (use (unspec [(match_dup 0)] UNSPEC_TOC))])]
  "TARGET_ELF && DEFAULT_ABI == ABI_V4
   && (flag_pic == 2 || (flag_pic && TARGET_SECURE_PLT))"
  "")

(define_insn "load_toc_v4_PIC_1_normal"
  [(set (reg:SI LR_REGNO)
	(match_operand:SI 0 "immediate_operand" "s"))
   (use (unspec [(match_dup 0)] UNSPEC_TOC))]
  "!TARGET_LINK_STACK && TARGET_ELF && DEFAULT_ABI == ABI_V4
   && (flag_pic == 2 || (flag_pic && TARGET_SECURE_PLT))"
  "bcl 20,31,%0\\n%0:"
  [(set_attr "type" "branch")
   (set_attr "length" "4")])

(define_insn "load_toc_v4_PIC_1_476"
  [(set (reg:SI LR_REGNO)
	(match_operand:SI 0 "immediate_operand" "s"))
   (use (unspec [(match_dup 0)] UNSPEC_TOC))]
  "TARGET_LINK_STACK && TARGET_ELF && DEFAULT_ABI == ABI_V4
   && (flag_pic == 2 || (flag_pic && TARGET_SECURE_PLT))"
  "*
{
  char name[32];
  static char templ[32];

  get_ppc476_thunk_name (name);
  sprintf (templ, \"bl %s\\n%%0:\", name);
  return templ;
}"
  [(set_attr "type" "branch")
   (set_attr "length" "4")])

(define_expand "load_toc_v4_PIC_1b"
  [(parallel [(set (reg:SI LR_REGNO)
		   (unspec:SI [(match_operand:SI 0 "immediate_operand" "s")
			       (label_ref (match_operand 1 "" ""))]
		           UNSPEC_TOCPTR))
	      (match_dup 1)])]
  "TARGET_ELF && DEFAULT_ABI == ABI_V4 && flag_pic == 2"
  "")

(define_insn "load_toc_v4_PIC_1b_normal"
  [(set (reg:SI LR_REGNO)
	(unspec:SI [(match_operand:SI 0 "immediate_operand" "s")
		    (label_ref (match_operand 1 "" ""))]
		UNSPEC_TOCPTR))
   (match_dup 1)]
  "!TARGET_LINK_STACK && TARGET_ELF && DEFAULT_ABI == ABI_V4 && flag_pic == 2"
  "bcl 20,31,$+8\;.long %0-$"
  [(set_attr "type" "branch")
   (set_attr "length" "8")])

(define_insn "load_toc_v4_PIC_1b_476"
  [(set (reg:SI LR_REGNO)
	(unspec:SI [(match_operand:SI 0 "immediate_operand" "s")
		    (label_ref (match_operand 1 "" ""))]
		UNSPEC_TOCPTR))
   (match_dup 1)]
  "TARGET_LINK_STACK && TARGET_ELF && DEFAULT_ABI == ABI_V4 && flag_pic == 2"
  "*
{
  char name[32];
  static char templ[32];

  get_ppc476_thunk_name (name);
  sprintf (templ, \"bl %s\\n\\tb $+8\\n\\t.long %%0-$\", name);
  return templ;
}"
  [(set_attr "type" "branch")
   (set_attr "length" "16")])

(define_insn "load_toc_v4_PIC_2"
  [(set (match_operand:SI 0 "gpc_reg_operand" "=r")
	(mem:SI (plus:SI (match_operand:SI 1 "gpc_reg_operand" "b")
		   (minus:SI (match_operand:SI 2 "immediate_operand" "s")
			     (match_operand:SI 3 "immediate_operand" "s")))))]
  "TARGET_ELF && DEFAULT_ABI == ABI_V4 && flag_pic == 2"
  "lwz %0,%2-%3(%1)"
  [(set_attr "type" "load")])

(define_insn "load_toc_v4_PIC_3b"
  [(set (match_operand:SI 0 "gpc_reg_operand" "=r")
	(plus:SI (match_operand:SI 1 "gpc_reg_operand" "b")
		 (high:SI
		   (minus:SI (match_operand:SI 2 "symbol_ref_operand" "s")
			     (match_operand:SI 3 "symbol_ref_operand" "s")))))]
  "TARGET_ELF && TARGET_SECURE_PLT && DEFAULT_ABI == ABI_V4 && flag_pic"
  "addis %0,%1,%2-%3@ha")

(define_insn "load_toc_v4_PIC_3c"
  [(set (match_operand:SI 0 "gpc_reg_operand" "=r")
	(lo_sum:SI (match_operand:SI 1 "gpc_reg_operand" "b")
		   (minus:SI (match_operand:SI 2 "symbol_ref_operand" "s")
			     (match_operand:SI 3 "symbol_ref_operand" "s"))))]
  "TARGET_ELF && TARGET_SECURE_PLT && DEFAULT_ABI == ABI_V4 && flag_pic"
  "addi %0,%1,%2-%3@l")

;; If the TOC is shared over a translation unit, as happens with all
;; the kinds of PIC that we support, we need to restore the TOC
;; pointer only when jumping over units of translation.
;; On Darwin, we need to reload the picbase.

(define_expand "builtin_setjmp_receiver"
  [(use (label_ref (match_operand 0 "" "")))]
  "(DEFAULT_ABI == ABI_V4 && flag_pic == 1)
   || (TARGET_TOC && TARGET_MINIMAL_TOC)
   || (DEFAULT_ABI == ABI_DARWIN && flag_pic)"
  "
{
#if TARGET_MACHO
  if (DEFAULT_ABI == ABI_DARWIN)
    {
      rtx picrtx = gen_rtx_SYMBOL_REF (Pmode, MACHOPIC_FUNCTION_BASE_NAME);
      rtx picreg = gen_rtx_REG (Pmode, RS6000_PIC_OFFSET_TABLE_REGNUM);
      rtx tmplabrtx;
      char tmplab[20];

      crtl->uses_pic_offset_table = 1;
      ASM_GENERATE_INTERNAL_LABEL(tmplab, \"LSJR\",
				  CODE_LABEL_NUMBER (operands[0]));
      tmplabrtx = gen_rtx_SYMBOL_REF (Pmode, ggc_strdup (tmplab));

      emit_insn (gen_load_macho_picbase (tmplabrtx));
      emit_move_insn (picreg, gen_rtx_REG (Pmode, LR_REGNO));
      emit_insn (gen_macho_correct_pic (picreg, picreg, picrtx, tmplabrtx));
    }
  else
#endif
    rs6000_emit_load_toc_table (FALSE);
  DONE;
}")

;; Largetoc support
(define_insn "*largetoc_high"
  [(set (match_operand:DI 0 "gpc_reg_operand" "=b*r")
        (high:DI
	  (unspec [(match_operand:DI 1 "" "")
		   (match_operand:DI 2 "gpc_reg_operand" "b")]
		  UNSPEC_TOCREL)))]
   "TARGET_ELF && TARGET_CMODEL != CMODEL_SMALL"
   "addis %0,%2,%1@toc@ha")

(define_insn "*largetoc_high_aix<mode>"
  [(set (match_operand:P 0 "gpc_reg_operand" "=b*r")
        (high:P
	  (unspec [(match_operand:P 1 "" "")
		   (match_operand:P 2 "gpc_reg_operand" "b")]
		  UNSPEC_TOCREL)))]
   "TARGET_XCOFF && TARGET_CMODEL != CMODEL_SMALL"
   "addis %0,%1@u(%2)")

(define_insn "*largetoc_high_plus"
  [(set (match_operand:DI 0 "gpc_reg_operand" "=b*r")
        (high:DI
	  (plus:DI
	    (unspec [(match_operand:DI 1 "" "")
		     (match_operand:DI 2 "gpc_reg_operand" "b")]
		    UNSPEC_TOCREL)
	    (match_operand:DI 3 "add_cint_operand" "n"))))]
   "TARGET_ELF && TARGET_CMODEL != CMODEL_SMALL"
   "addis %0,%2,%1+%3@toc@ha")

(define_insn "*largetoc_high_plus_aix<mode>"
  [(set (match_operand:P 0 "gpc_reg_operand" "=b*r")
        (high:P
	  (plus:P
	    (unspec [(match_operand:P 1 "" "")
		     (match_operand:P 2 "gpc_reg_operand" "b")]
		    UNSPEC_TOCREL)
	    (match_operand:P 3 "add_cint_operand" "n"))))]
   "TARGET_XCOFF && TARGET_CMODEL != CMODEL_SMALL"
   "addis %0,%1+%3@u(%2)")

(define_insn "*largetoc_low"
  [(set (match_operand:DI 0 "gpc_reg_operand" "=r")
        (lo_sum:DI (match_operand:DI 1 "gpc_reg_operand" "b")
	           (match_operand:DI 2 "" "")))]
   "TARGET_ELF && TARGET_CMODEL != CMODEL_SMALL"
   "addi %0,%1,%2@l")

(define_insn "*largetoc_low_aix<mode>"
  [(set (match_operand:P 0 "gpc_reg_operand" "=r")
        (lo_sum:P (match_operand:P 1 "gpc_reg_operand" "b")
	           (match_operand:P 2 "" "")))]
   "TARGET_XCOFF && TARGET_CMODEL != CMODEL_SMALL"
   "la %0,%2@l(%1)")

(define_insn_and_split "*tocref<mode>"
  [(set (match_operand:P 0 "gpc_reg_operand" "=b")
	(match_operand:P 1 "small_toc_ref" "R"))]
   "TARGET_TOC"
   "la %0,%a1"
   "&& TARGET_CMODEL != CMODEL_SMALL && reload_completed"
  [(set (match_dup 0) (high:P (match_dup 1)))
   (set (match_dup 0) (lo_sum:P (match_dup 0) (match_dup 1)))])

;; Elf specific ways of loading addresses for non-PIC code.
;; The output of this could be r0, but we make a very strong
;; preference for a base register because it will usually
;; be needed there.
(define_insn "elf_high"
  [(set (match_operand:SI 0 "gpc_reg_operand" "=b*r")
	(high:SI (match_operand 1 "" "")))]
  "TARGET_ELF && ! TARGET_64BIT"
  "lis %0,%1@ha")

(define_insn "elf_low"
  [(set (match_operand:SI 0 "gpc_reg_operand" "=r")
	(lo_sum:SI (match_operand:SI 1 "gpc_reg_operand" "b")
		   (match_operand 2 "" "")))]
   "TARGET_ELF && ! TARGET_64BIT"
   "la %0,%2@l(%1)")

;; Call and call_value insns
(define_expand "call"
  [(parallel [(call (mem:SI (match_operand 0 "address_operand" ""))
		    (match_operand 1 "" ""))
	      (use (match_operand 2 "" ""))
	      (clobber (reg:SI LR_REGNO))])]
  ""
  "
{
#if TARGET_MACHO
  if (MACHOPIC_INDIRECT)
    operands[0] = machopic_indirect_call_target (operands[0]);
#endif

  gcc_assert (GET_CODE (operands[0]) == MEM);
  gcc_assert (GET_CODE (operands[1]) == CONST_INT);

  operands[0] = XEXP (operands[0], 0);

  if (DEFAULT_ABI == ABI_AIX || DEFAULT_ABI == ABI_ELFv2)
    {
      rs6000_call_aix (NULL_RTX, operands[0], operands[1], operands[2]);
      DONE;
    }

  if (GET_CODE (operands[0]) != SYMBOL_REF
      || (DEFAULT_ABI != ABI_DARWIN && (INTVAL (operands[2]) & CALL_LONG) != 0))
    {
      if (INTVAL (operands[2]) & CALL_LONG)
	operands[0] = rs6000_longcall_ref (operands[0]);

      switch (DEFAULT_ABI)
        {
	case ABI_V4:
	case ABI_DARWIN:
	  operands[0] = force_reg (Pmode, operands[0]);
	  break;

	default:
	  gcc_unreachable ();
	}
    }
}")

(define_expand "call_value"
  [(parallel [(set (match_operand 0 "" "")
		   (call (mem:SI (match_operand 1 "address_operand" ""))
			 (match_operand 2 "" "")))
	      (use (match_operand 3 "" ""))
	      (clobber (reg:SI LR_REGNO))])]
  ""
  "
{
#if TARGET_MACHO
  if (MACHOPIC_INDIRECT)
    operands[1] = machopic_indirect_call_target (operands[1]);
#endif

  gcc_assert (GET_CODE (operands[1]) == MEM);
  gcc_assert (GET_CODE (operands[2]) == CONST_INT);

  operands[1] = XEXP (operands[1], 0);

  if (DEFAULT_ABI == ABI_AIX || DEFAULT_ABI == ABI_ELFv2)
    {
      rs6000_call_aix (operands[0], operands[1], operands[2], operands[3]);
      DONE;
    }

  if (GET_CODE (operands[1]) != SYMBOL_REF
      || (DEFAULT_ABI != ABI_DARWIN && (INTVAL (operands[3]) & CALL_LONG) != 0))
    {
      if (INTVAL (operands[3]) & CALL_LONG)
	operands[1] = rs6000_longcall_ref (operands[1]);

      switch (DEFAULT_ABI)
        {
	case ABI_V4:
	case ABI_DARWIN:
	  operands[1] = force_reg (Pmode, operands[1]);
	  break;

	default:
	  gcc_unreachable ();
	}
    }
}")

;; Call to function in current module.  No TOC pointer reload needed.
;; Operand2 is nonzero if we are using the V.4 calling sequence and
;; either the function was not prototyped, or it was prototyped as a
;; variable argument function.  It is > 0 if FP registers were passed
;; and < 0 if they were not.

(define_insn "*call_local32"
  [(call (mem:SI (match_operand:SI 0 "current_file_function_operand" "s,s"))
	 (match_operand 1 "" "g,g"))
   (use (match_operand:SI 2 "immediate_operand" "O,n"))
   (clobber (reg:SI LR_REGNO))]
  "(INTVAL (operands[2]) & CALL_LONG) == 0"
  "*
{
  if (INTVAL (operands[2]) & CALL_V4_SET_FP_ARGS)
    output_asm_insn (\"crxor 6,6,6\", operands);

  else if (INTVAL (operands[2]) & CALL_V4_CLEAR_FP_ARGS)
    output_asm_insn (\"creqv 6,6,6\", operands);

  return (DEFAULT_ABI == ABI_V4 && flag_pic) ? \"bl %z0@local\" : \"bl %z0\";
}"
  [(set_attr "type" "branch")
   (set_attr "length" "4,8")])

(define_insn "*call_local64"
  [(call (mem:SI (match_operand:DI 0 "current_file_function_operand" "s,s"))
	 (match_operand 1 "" "g,g"))
   (use (match_operand:SI 2 "immediate_operand" "O,n"))
   (clobber (reg:SI LR_REGNO))]
  "TARGET_64BIT && (INTVAL (operands[2]) & CALL_LONG) == 0"
  "*
{
  if (INTVAL (operands[2]) & CALL_V4_SET_FP_ARGS)
    output_asm_insn (\"crxor 6,6,6\", operands);

  else if (INTVAL (operands[2]) & CALL_V4_CLEAR_FP_ARGS)
    output_asm_insn (\"creqv 6,6,6\", operands);

  return (DEFAULT_ABI == ABI_V4 && flag_pic) ? \"bl %z0@local\" : \"bl %z0\";
}"
  [(set_attr "type" "branch")
   (set_attr "length" "4,8")])

(define_insn "*call_value_local32"
  [(set (match_operand 0 "" "")
	(call (mem:SI (match_operand:SI 1 "current_file_function_operand" "s,s"))
	      (match_operand 2 "" "g,g")))
   (use (match_operand:SI 3 "immediate_operand" "O,n"))
   (clobber (reg:SI LR_REGNO))]
  "(INTVAL (operands[3]) & CALL_LONG) == 0"
  "*
{
  if (INTVAL (operands[3]) & CALL_V4_SET_FP_ARGS)
    output_asm_insn (\"crxor 6,6,6\", operands);

  else if (INTVAL (operands[3]) & CALL_V4_CLEAR_FP_ARGS)
    output_asm_insn (\"creqv 6,6,6\", operands);

  return (DEFAULT_ABI == ABI_V4 && flag_pic) ? \"bl %z1@local\" : \"bl %z1\";
}"
  [(set_attr "type" "branch")
   (set_attr "length" "4,8")])


(define_insn "*call_value_local64"
  [(set (match_operand 0 "" "")
	(call (mem:SI (match_operand:DI 1 "current_file_function_operand" "s,s"))
	      (match_operand 2 "" "g,g")))
   (use (match_operand:SI 3 "immediate_operand" "O,n"))
   (clobber (reg:SI LR_REGNO))]
  "TARGET_64BIT && (INTVAL (operands[3]) & CALL_LONG) == 0"
  "*
{
  if (INTVAL (operands[3]) & CALL_V4_SET_FP_ARGS)
    output_asm_insn (\"crxor 6,6,6\", operands);

  else if (INTVAL (operands[3]) & CALL_V4_CLEAR_FP_ARGS)
    output_asm_insn (\"creqv 6,6,6\", operands);

  return (DEFAULT_ABI == ABI_V4 && flag_pic) ? \"bl %z1@local\" : \"bl %z1\";
}"
  [(set_attr "type" "branch")
   (set_attr "length" "4,8")])


;; A function pointer under System V is just a normal pointer
;; operands[0] is the function pointer
;; operands[1] is the stack size to clean up
;; operands[2] is the value FUNCTION_ARG returns for the VOID argument
;; which indicates how to set cr1

(define_insn "*call_indirect_nonlocal_sysv<mode>"
  [(call (mem:SI (match_operand:P 0 "register_operand" "c,*l,c,*l"))
	 (match_operand 1 "" "g,g,g,g"))
   (use (match_operand:SI 2 "immediate_operand" "O,O,n,n"))
   (clobber (reg:SI LR_REGNO))]
  "DEFAULT_ABI == ABI_V4
   || DEFAULT_ABI == ABI_DARWIN"
{
  if (INTVAL (operands[2]) & CALL_V4_SET_FP_ARGS)
    output_asm_insn ("crxor 6,6,6", operands);

  else if (INTVAL (operands[2]) & CALL_V4_CLEAR_FP_ARGS)
    output_asm_insn ("creqv 6,6,6", operands);

  return "b%T0l";
}
  [(set_attr "type" "jmpreg,jmpreg,jmpreg,jmpreg")
   (set_attr "length" "4,4,8,8")])

(define_insn_and_split "*call_nonlocal_sysv<mode>"
  [(call (mem:SI (match_operand:P 0 "symbol_ref_operand" "s,s"))
	 (match_operand 1 "" "g,g"))
   (use (match_operand:SI 2 "immediate_operand" "O,n"))
   (clobber (reg:SI LR_REGNO))]
  "(DEFAULT_ABI == ABI_DARWIN
   || (DEFAULT_ABI == ABI_V4
       && (INTVAL (operands[2]) & CALL_LONG) == 0))"
{
  if (INTVAL (operands[2]) & CALL_V4_SET_FP_ARGS)
    output_asm_insn ("crxor 6,6,6", operands);

  else if (INTVAL (operands[2]) & CALL_V4_CLEAR_FP_ARGS)
    output_asm_insn ("creqv 6,6,6", operands);

#if TARGET_MACHO
  return output_call(insn, operands, 0, 2);
#else
  if (DEFAULT_ABI == ABI_V4 && flag_pic)
    {
      gcc_assert (!TARGET_SECURE_PLT);
      return "bl %z0@plt";
    }
  else
    return "bl %z0";
#endif
}
  "DEFAULT_ABI == ABI_V4
   && TARGET_SECURE_PLT && flag_pic && !SYMBOL_REF_LOCAL_P (operands[0])
   && (INTVAL (operands[2]) & CALL_LONG) == 0"
  [(parallel [(call (mem:SI (match_dup 0))
		    (match_dup 1))
	      (use (match_dup 2))
	      (use (match_dup 3))
	      (clobber (reg:SI LR_REGNO))])]
{
  operands[3] = pic_offset_table_rtx;
}
  [(set_attr "type" "branch,branch")
   (set_attr "length" "4,8")])

(define_insn "*call_nonlocal_sysv_secure<mode>"
  [(call (mem:SI (match_operand:P 0 "symbol_ref_operand" "s,s"))
	 (match_operand 1 "" "g,g"))
   (use (match_operand:SI 2 "immediate_operand" "O,n"))
   (use (match_operand:SI 3 "register_operand" "r,r"))
   (clobber (reg:SI LR_REGNO))]
  "(DEFAULT_ABI == ABI_V4
    && TARGET_SECURE_PLT && flag_pic && !SYMBOL_REF_LOCAL_P (operands[0])
    && (INTVAL (operands[2]) & CALL_LONG) == 0)"
{
  if (INTVAL (operands[2]) & CALL_V4_SET_FP_ARGS)
    output_asm_insn ("crxor 6,6,6", operands);

  else if (INTVAL (operands[2]) & CALL_V4_CLEAR_FP_ARGS)
    output_asm_insn ("creqv 6,6,6", operands);

  if (flag_pic == 2)
    /* The magic 32768 offset here and in the other sysv call insns
       corresponds to the offset of r30 in .got2, as given by LCTOC1.
       See sysv4.h:toc_section.  */
    return "bl %z0+32768@plt";
  else
    return "bl %z0@plt";
}
  [(set_attr "type" "branch,branch")
   (set_attr "length" "4,8")])

(define_insn "*call_value_indirect_nonlocal_sysv<mode>"
  [(set (match_operand 0 "" "")
	(call (mem:SI (match_operand:P 1 "register_operand" "c,*l,c,*l"))
	      (match_operand 2 "" "g,g,g,g")))
   (use (match_operand:SI 3 "immediate_operand" "O,O,n,n"))
   (clobber (reg:SI LR_REGNO))]
  "DEFAULT_ABI == ABI_V4
   || DEFAULT_ABI == ABI_DARWIN"
{
  if (INTVAL (operands[3]) & CALL_V4_SET_FP_ARGS)
    output_asm_insn ("crxor 6,6,6", operands);

  else if (INTVAL (operands[3]) & CALL_V4_CLEAR_FP_ARGS)
    output_asm_insn ("creqv 6,6,6", operands);

  return "b%T1l";
}
  [(set_attr "type" "jmpreg,jmpreg,jmpreg,jmpreg")
   (set_attr "length" "4,4,8,8")])

(define_insn_and_split "*call_value_nonlocal_sysv<mode>"
  [(set (match_operand 0 "" "")
	(call (mem:SI (match_operand:P 1 "symbol_ref_operand" "s,s"))
	      (match_operand 2 "" "g,g")))
   (use (match_operand:SI 3 "immediate_operand" "O,n"))
   (clobber (reg:SI LR_REGNO))]
  "(DEFAULT_ABI == ABI_DARWIN
   || (DEFAULT_ABI == ABI_V4
       && (INTVAL (operands[3]) & CALL_LONG) == 0))"
{
  if (INTVAL (operands[3]) & CALL_V4_SET_FP_ARGS)
    output_asm_insn ("crxor 6,6,6", operands);

  else if (INTVAL (operands[3]) & CALL_V4_CLEAR_FP_ARGS)
    output_asm_insn ("creqv 6,6,6", operands);

#if TARGET_MACHO
  return output_call(insn, operands, 1, 3);
#else
  if (DEFAULT_ABI == ABI_V4 && flag_pic)
    {
      gcc_assert (!TARGET_SECURE_PLT);
      return "bl %z1@plt";
    }
  else
    return "bl %z1";
#endif
}
  "DEFAULT_ABI == ABI_V4
   && TARGET_SECURE_PLT && flag_pic && !SYMBOL_REF_LOCAL_P (operands[1])
   && (INTVAL (operands[3]) & CALL_LONG) == 0"
  [(parallel [(set (match_dup 0)
		   (call (mem:SI (match_dup 1))
			 (match_dup 2)))
	      (use (match_dup 3))
	      (use (match_dup 4))
	      (clobber (reg:SI LR_REGNO))])]
{
  operands[4] = pic_offset_table_rtx;
}
  [(set_attr "type" "branch,branch")
   (set_attr "length" "4,8")])

(define_insn "*call_value_nonlocal_sysv_secure<mode>"
  [(set (match_operand 0 "" "")
	(call (mem:SI (match_operand:P 1 "symbol_ref_operand" "s,s"))
	      (match_operand 2 "" "g,g")))
   (use (match_operand:SI 3 "immediate_operand" "O,n"))
   (use (match_operand:SI 4 "register_operand" "r,r"))
   (clobber (reg:SI LR_REGNO))]
  "(DEFAULT_ABI == ABI_V4
    && TARGET_SECURE_PLT && flag_pic && !SYMBOL_REF_LOCAL_P (operands[1])
    && (INTVAL (operands[3]) & CALL_LONG) == 0)"
{
  if (INTVAL (operands[3]) & CALL_V4_SET_FP_ARGS)
    output_asm_insn ("crxor 6,6,6", operands);

  else if (INTVAL (operands[3]) & CALL_V4_CLEAR_FP_ARGS)
    output_asm_insn ("creqv 6,6,6", operands);

  if (flag_pic == 2)
    return "bl %z1+32768@plt";
  else
    return "bl %z1@plt";
}
  [(set_attr "type" "branch,branch")
   (set_attr "length" "4,8")])


;; Call to AIX abi function in the same module.

(define_insn "*call_local_aix<mode>"
  [(call (mem:SI (match_operand:P 0 "current_file_function_operand" "s"))
	 (match_operand 1 "" "g"))
   (clobber (reg:P LR_REGNO))]
  "DEFAULT_ABI == ABI_AIX || DEFAULT_ABI == ABI_ELFv2"
  "bl %z0"
  [(set_attr "type" "branch")
   (set_attr "length" "4")])

(define_insn "*call_value_local_aix<mode>"
  [(set (match_operand 0 "" "")
	(call (mem:SI (match_operand:P 1 "current_file_function_operand" "s"))
	      (match_operand 2 "" "g")))
   (clobber (reg:P LR_REGNO))]
  "DEFAULT_ABI == ABI_AIX || DEFAULT_ABI == ABI_ELFv2"
  "bl %z1"
  [(set_attr "type" "branch")
   (set_attr "length" "4")])

;; Call to AIX abi function which may be in another module.
;; Restore the TOC pointer (r2) after the call.

(define_insn "*call_nonlocal_aix<mode>"
  [(call (mem:SI (match_operand:P 0 "symbol_ref_operand" "s"))
	 (match_operand 1 "" "g"))
   (clobber (reg:P LR_REGNO))]
  "DEFAULT_ABI == ABI_AIX || DEFAULT_ABI == ABI_ELFv2"
  "bl %z0\;nop"
  [(set_attr "type" "branch")
   (set_attr "length" "8")])

(define_insn "*call_value_nonlocal_aix<mode>"
  [(set (match_operand 0 "" "")
	(call (mem:SI (match_operand:P 1 "symbol_ref_operand" "s"))
	      (match_operand 2 "" "g")))
   (clobber (reg:P LR_REGNO))]
  "DEFAULT_ABI == ABI_AIX || DEFAULT_ABI == ABI_ELFv2"
  "bl %z1\;nop"
  [(set_attr "type" "branch")
   (set_attr "length" "8")])

;; Call to indirect functions with the AIX abi using a 3 word descriptor.
;; Operand0 is the addresss of the function to call
;; Operand2 is the location in the function descriptor to load r2 from
;; Operand3 is the offset of the stack location holding the current TOC pointer

(define_insn "*call_indirect_aix<mode>"
  [(call (mem:SI (match_operand:P 0 "register_operand" "c,*l"))
	 (match_operand 1 "" "g,g"))
   (use (match_operand:P 2 "memory_operand" "<ptrm>,<ptrm>"))
   (set (reg:P TOC_REGNUM) (unspec [(match_operand:P 3 "const_int_operand" "n,n")] UNSPEC_TOCSLOT))
   (clobber (reg:P LR_REGNO))]
  "DEFAULT_ABI == ABI_AIX"
  "<ptrload> 2,%2\;b%T0l\;<ptrload> 2,%3(1)"
  [(set_attr "type" "jmpreg")
   (set_attr "length" "12")])

(define_insn "*call_value_indirect_aix<mode>"
  [(set (match_operand 0 "" "")
	(call (mem:SI (match_operand:P 1 "register_operand" "c,*l"))
	      (match_operand 2 "" "g,g")))
   (use (match_operand:P 3 "memory_operand" "<ptrm>,<ptrm>"))
   (set (reg:P TOC_REGNUM) (unspec [(match_operand:P 4 "const_int_operand" "n,n")] UNSPEC_TOCSLOT))
   (clobber (reg:P LR_REGNO))]
  "DEFAULT_ABI == ABI_AIX"
  "<ptrload> 2,%3\;b%T1l\;<ptrload> 2,%4(1)"
  [(set_attr "type" "jmpreg")
   (set_attr "length" "12")])

;; Call to indirect functions with the ELFv2 ABI.
;; Operand0 is the addresss of the function to call
;; Operand2 is the offset of the stack location holding the current TOC pointer

(define_insn "*call_indirect_elfv2<mode>"
  [(call (mem:SI (match_operand:P 0 "register_operand" "c,*l"))
	 (match_operand 1 "" "g,g"))
   (set (reg:P TOC_REGNUM) (unspec [(match_operand:P 2 "const_int_operand" "n,n")] UNSPEC_TOCSLOT))
   (clobber (reg:P LR_REGNO))]
  "DEFAULT_ABI == ABI_ELFv2"
  "b%T0l\;<ptrload> 2,%2(1)"
  [(set_attr "type" "jmpreg")
   (set_attr "length" "8")])

(define_insn "*call_value_indirect_elfv2<mode>"
  [(set (match_operand 0 "" "")
	(call (mem:SI (match_operand:P 1 "register_operand" "c,*l"))
	      (match_operand 2 "" "g,g")))
   (set (reg:P TOC_REGNUM) (unspec [(match_operand:P 3 "const_int_operand" "n,n")] UNSPEC_TOCSLOT))
   (clobber (reg:P LR_REGNO))]
  "DEFAULT_ABI == ABI_ELFv2"
  "b%T1l\;<ptrload> 2,%3(1)"
  [(set_attr "type" "jmpreg")
   (set_attr "length" "8")])


;; Call subroutine returning any type.
(define_expand "untyped_call"
  [(parallel [(call (match_operand 0 "" "")
		    (const_int 0))
	      (match_operand 1 "" "")
	      (match_operand 2 "" "")])]
  ""
  "
{
  int i;

  emit_call_insn (GEN_CALL (operands[0], const0_rtx, const0_rtx, const0_rtx));

  for (i = 0; i < XVECLEN (operands[2], 0); i++)
    {
      rtx set = XVECEXP (operands[2], 0, i);
      emit_move_insn (SET_DEST (set), SET_SRC (set));
    }

  /* The optimizer does not know that the call sets the function value
     registers we stored in the result block.  We avoid problems by
     claiming that all hard registers are used and clobbered at this
     point.  */
  emit_insn (gen_blockage ());

  DONE;
}")

;; sibling call patterns
(define_expand "sibcall"
  [(parallel [(call (mem:SI (match_operand 0 "address_operand" ""))
		    (match_operand 1 "" ""))
	      (use (match_operand 2 "" ""))
	      (use (reg:SI LR_REGNO))
	      (simple_return)])]
  ""
  "
{
#if TARGET_MACHO
  if (MACHOPIC_INDIRECT)
    operands[0] = machopic_indirect_call_target (operands[0]);
#endif

  gcc_assert (GET_CODE (operands[0]) == MEM);
  gcc_assert (GET_CODE (operands[1]) == CONST_INT);

  operands[0] = XEXP (operands[0], 0);

  if (DEFAULT_ABI == ABI_AIX || DEFAULT_ABI == ABI_ELFv2)
    {
      rs6000_sibcall_aix (NULL_RTX, operands[0], operands[1], operands[2]);
      DONE;
    }
}")

(define_expand "sibcall_value"
  [(parallel [(set (match_operand 0 "register_operand" "")
		(call (mem:SI (match_operand 1 "address_operand" ""))
		      (match_operand 2 "" "")))
	      (use (match_operand 3 "" ""))
	      (use (reg:SI LR_REGNO))
	      (simple_return)])]
  ""
  "
{
#if TARGET_MACHO
  if (MACHOPIC_INDIRECT)
    operands[1] = machopic_indirect_call_target (operands[1]);
#endif

  gcc_assert (GET_CODE (operands[1]) == MEM);
  gcc_assert (GET_CODE (operands[2]) == CONST_INT);

  operands[1] = XEXP (operands[1], 0);

  if (DEFAULT_ABI == ABI_AIX || DEFAULT_ABI == ABI_ELFv2)
    {
      rs6000_sibcall_aix (operands[0], operands[1], operands[2], operands[3]);
      DONE;
    }
}")

;; this and similar patterns must be marked as using LR, otherwise
;; dataflow will try to delete the store into it.  This is true
;; even when the actual reg to jump to is in CTR, when LR was
;; saved and restored around the PIC-setting BCL.
(define_insn "*sibcall_local32"
  [(call (mem:SI (match_operand:SI 0 "current_file_function_operand" "s,s"))
	 (match_operand 1 "" "g,g"))
   (use (match_operand:SI 2 "immediate_operand" "O,n"))
   (use (reg:SI LR_REGNO))
   (simple_return)]
  "(INTVAL (operands[2]) & CALL_LONG) == 0"
  "*
{
  if (INTVAL (operands[2]) & CALL_V4_SET_FP_ARGS)
    output_asm_insn (\"crxor 6,6,6\", operands);

  else if (INTVAL (operands[2]) & CALL_V4_CLEAR_FP_ARGS)
    output_asm_insn (\"creqv 6,6,6\", operands);

  return (DEFAULT_ABI == ABI_V4 && flag_pic) ? \"b %z0@local\" : \"b %z0\";
}"
  [(set_attr "type" "branch")
   (set_attr "length" "4,8")])

(define_insn "*sibcall_local64"
  [(call (mem:SI (match_operand:DI 0 "current_file_function_operand" "s,s"))
	 (match_operand 1 "" "g,g"))
   (use (match_operand:SI 2 "immediate_operand" "O,n"))
   (use (reg:SI LR_REGNO))
   (simple_return)]
  "TARGET_64BIT && (INTVAL (operands[2]) & CALL_LONG) == 0"
  "*
{
  if (INTVAL (operands[2]) & CALL_V4_SET_FP_ARGS)
    output_asm_insn (\"crxor 6,6,6\", operands);

  else if (INTVAL (operands[2]) & CALL_V4_CLEAR_FP_ARGS)
    output_asm_insn (\"creqv 6,6,6\", operands);

  return (DEFAULT_ABI == ABI_V4 && flag_pic) ? \"b %z0@local\" : \"b %z0\";
}"
  [(set_attr "type" "branch")
   (set_attr "length" "4,8")])

(define_insn "*sibcall_value_local32"
  [(set (match_operand 0 "" "")
	(call (mem:SI (match_operand:SI 1 "current_file_function_operand" "s,s"))
	      (match_operand 2 "" "g,g")))
   (use (match_operand:SI 3 "immediate_operand" "O,n"))
   (use (reg:SI LR_REGNO))
   (simple_return)]
  "(INTVAL (operands[3]) & CALL_LONG) == 0"
  "*
{
  if (INTVAL (operands[3]) & CALL_V4_SET_FP_ARGS)
    output_asm_insn (\"crxor 6,6,6\", operands);

  else if (INTVAL (operands[3]) & CALL_V4_CLEAR_FP_ARGS)
    output_asm_insn (\"creqv 6,6,6\", operands);

  return (DEFAULT_ABI == ABI_V4 && flag_pic) ? \"b %z1@local\" : \"b %z1\";
}"
  [(set_attr "type" "branch")
   (set_attr "length" "4,8")])

(define_insn "*sibcall_value_local64"
  [(set (match_operand 0 "" "")
	(call (mem:SI (match_operand:DI 1 "current_file_function_operand" "s,s"))
	      (match_operand 2 "" "g,g")))
   (use (match_operand:SI 3 "immediate_operand" "O,n"))
   (use (reg:SI LR_REGNO))
   (simple_return)]
  "TARGET_64BIT && (INTVAL (operands[3]) & CALL_LONG) == 0"
  "*
{
  if (INTVAL (operands[3]) & CALL_V4_SET_FP_ARGS)
    output_asm_insn (\"crxor 6,6,6\", operands);

  else if (INTVAL (operands[3]) & CALL_V4_CLEAR_FP_ARGS)
    output_asm_insn (\"creqv 6,6,6\", operands);

  return (DEFAULT_ABI == ABI_V4 && flag_pic) ? \"b %z1@local\" : \"b %z1\";
}"
  [(set_attr "type" "branch")
   (set_attr "length" "4,8")])

(define_insn "*sibcall_nonlocal_sysv<mode>"
  [(call (mem:SI (match_operand:P 0 "call_operand" "s,s,c,c"))
	 (match_operand 1 "" ""))
   (use (match_operand 2 "immediate_operand" "O,n,O,n"))
   (use (reg:SI LR_REGNO))
   (simple_return)]
  "(DEFAULT_ABI == ABI_DARWIN
    || DEFAULT_ABI == ABI_V4)
   && (INTVAL (operands[2]) & CALL_LONG) == 0"
  "*
{
  if (INTVAL (operands[2]) & CALL_V4_SET_FP_ARGS)
    output_asm_insn (\"crxor 6,6,6\", operands);

  else if (INTVAL (operands[2]) & CALL_V4_CLEAR_FP_ARGS)
    output_asm_insn (\"creqv 6,6,6\", operands);

  if (which_alternative >= 2)
    return \"b%T0\";
  else if (DEFAULT_ABI == ABI_V4 && flag_pic)
    {
      gcc_assert (!TARGET_SECURE_PLT);
      return \"b %z0@plt\";
    }
  else
    return \"b %z0\";
}"
  [(set_attr "type" "branch")
   (set_attr "length" "4,8,4,8")])

(define_insn "*sibcall_value_nonlocal_sysv<mode>"
  [(set (match_operand 0 "" "")
	(call (mem:SI (match_operand:P 1 "call_operand" "s,s,c,c"))
	      (match_operand 2 "" "")))
   (use (match_operand:SI 3 "immediate_operand" "O,n,O,n"))
   (use (reg:SI LR_REGNO))
   (simple_return)]
  "(DEFAULT_ABI == ABI_DARWIN
    || DEFAULT_ABI == ABI_V4)
   && (INTVAL (operands[3]) & CALL_LONG) == 0"
  "*
{
  if (INTVAL (operands[3]) & CALL_V4_SET_FP_ARGS)
    output_asm_insn (\"crxor 6,6,6\", operands);

  else if (INTVAL (operands[3]) & CALL_V4_CLEAR_FP_ARGS)
    output_asm_insn (\"creqv 6,6,6\", operands);

  if (which_alternative >= 2)
    return \"b%T1\";
  else if (DEFAULT_ABI == ABI_V4 && flag_pic)
    {
      gcc_assert (!TARGET_SECURE_PLT);
      return \"b %z1@plt\";
    }
  else
    return \"b %z1\";
}"
  [(set_attr "type" "branch")
   (set_attr "length" "4,8,4,8")])

;; AIX ABI sibling call patterns.

(define_insn "*sibcall_aix<mode>"
  [(call (mem:SI (match_operand:P 0 "call_operand" "s,c"))
	 (match_operand 1 "" "g,g"))
   (simple_return)]
  "DEFAULT_ABI == ABI_AIX || DEFAULT_ABI == ABI_ELFv2"
  "@
   b %z0
   b%T0"
  [(set_attr "type" "branch")
   (set_attr "length" "4")])

(define_insn "*sibcall_value_aix<mode>"
  [(set (match_operand 0 "" "")
	(call (mem:SI (match_operand:P 1 "call_operand" "s,c"))
	      (match_operand 2 "" "g,g")))
   (simple_return)]
  "DEFAULT_ABI == ABI_AIX || DEFAULT_ABI == ABI_ELFv2"
  "@
   b %z1
   b%T1"
  [(set_attr "type" "branch")
   (set_attr "length" "4")])

(define_expand "sibcall_epilogue"
  [(use (const_int 0))]
  ""
{
  if (!TARGET_SCHED_PROLOG)
    emit_insn (gen_blockage ());
  rs6000_emit_epilogue (TRUE);
  DONE;
})

;; UNSPEC_VOLATILE is considered to use and clobber all hard registers and
;; all of memory.  This blocks insns from being moved across this point.

(define_insn "blockage"
  [(unspec_volatile [(const_int 0)] UNSPECV_BLOCK)]
  ""
  "")

(define_expand "probe_stack_address"
  [(use (match_operand 0 "address_operand"))]
  ""
{
  operands[0] = gen_rtx_MEM (Pmode, operands[0]);
  MEM_VOLATILE_P (operands[0]) = 1;

  if (TARGET_64BIT)
    emit_insn (gen_probe_stack_di (operands[0]));
  else
    emit_insn (gen_probe_stack_si (operands[0]));
  DONE;
})

(define_insn "probe_stack_<mode>"
  [(set (match_operand:P 0 "memory_operand" "=m")
        (unspec:P [(const_int 0)] UNSPEC_PROBE_STACK))]
  ""
{
  operands[1] = gen_rtx_REG (Pmode, 0);
  return "st<wd>%U0%X0 %1,%0";
}
  [(set_attr "type" "store")
   (set (attr "update")
	(if_then_else (match_operand 0 "update_address_mem")
		      (const_string "yes")
		      (const_string "no")))
   (set (attr "indexed")
	(if_then_else (match_operand 0 "indexed_address_mem")
		      (const_string "yes")
		      (const_string "no")))
   (set_attr "length" "4")])

(define_insn "probe_stack_range<P:mode>"
  [(set (match_operand:P 0 "register_operand" "=r")
	(unspec_volatile:P [(match_operand:P 1 "register_operand" "0")
			    (match_operand:P 2 "register_operand" "r")]
			   UNSPECV_PROBE_STACK_RANGE))]
  ""
  "* return output_probe_stack_range (operands[0], operands[2]);"
  [(set_attr "type" "three")])

;; Compare insns are next.  Note that the RS/6000 has two types of compares,
;; signed & unsigned, and one type of branch.
;;
;; Start with the DEFINE_EXPANDs to generate the rtl for compares, scc
;; insns, and branches.

(define_expand "cbranch<mode>4"
  [(use (match_operator 0 "rs6000_cbranch_operator"
         [(match_operand:GPR 1 "gpc_reg_operand" "")
          (match_operand:GPR 2 "reg_or_short_operand" "")]))
   (use (match_operand 3 ""))]
  ""
  "
{
  /* Take care of the possibility that operands[2] might be negative but
     this might be a logical operation.  That insn doesn't exist.  */
  if (GET_CODE (operands[2]) == CONST_INT
      && INTVAL (operands[2]) < 0)
    {
      operands[2] = force_reg (<MODE>mode, operands[2]);
      operands[0] = gen_rtx_fmt_ee (GET_CODE (operands[0]),
				    GET_MODE (operands[0]),
				    operands[1], operands[2]);
   }

  rs6000_emit_cbranch (<MODE>mode, operands);
  DONE;
}")

(define_expand "cbranch<mode>4"
  [(use (match_operator 0 "rs6000_cbranch_operator"
         [(match_operand:FP 1 "gpc_reg_operand" "")
          (match_operand:FP 2 "gpc_reg_operand" "")]))
   (use (match_operand 3 ""))]
  ""
  "
{
  rs6000_emit_cbranch (<MODE>mode, operands);
  DONE;
}")

(define_expand "cstore<mode>4_unsigned"
  [(use (match_operator 1 "unsigned_comparison_operator"
         [(match_operand:P 2 "gpc_reg_operand" "")
          (match_operand:P 3 "reg_or_short_operand" "")]))
   (clobber (match_operand:P 0 "register_operand"))]
  ""
{
  enum rtx_code cond_code = GET_CODE (operands[1]);

  rtx op0 = operands[0];
  rtx op1 = operands[2];
  rtx op2 = operands[3];

  if (cond_code == GEU || cond_code == LTU)
    {
      cond_code = swap_condition (cond_code);
      op1 = operands[3];
      op2 = operands[2];
    }

  if (!gpc_reg_operand (op1, <MODE>mode))
    op1 = force_reg (<MODE>mode, op1);
  if (!reg_or_short_operand (op2, <MODE>mode))
    op2 = force_reg (<MODE>mode, op2);

  rtx tmp = gen_reg_rtx (<MODE>mode);
  rtx tmp2 = gen_reg_rtx (<MODE>mode);

  emit_insn (gen_subf<mode>3_carry (tmp, op1, op2));
  emit_insn (gen_subf<mode>3_carry_in_xx (tmp2));

  if (cond_code == LEU)
    emit_insn (gen_add<mode>3 (op0, tmp2, const1_rtx));
  else
    emit_insn (gen_neg<mode>2 (op0, tmp2));

  DONE;
})

(define_expand "cstore<mode>4"
  [(use (match_operator 1 "rs6000_cbranch_operator"
         [(match_operand:GPR 2 "gpc_reg_operand" "")
          (match_operand:GPR 3 "reg_or_short_operand" "")]))
   (clobber (match_operand:GPR 0 "register_operand"))]
  ""
{
  /* Use ISEL if the user asked for it.  */
  if (TARGET_ISEL)
    rs6000_emit_sISEL (<MODE>mode, operands);

  /* Expanding EQ and NE directly to some machine instructions does not help
     but does hurt combine.  So don't.  */
  else if (GET_CODE (operands[1]) == EQ)
    emit_insn (gen_eq<mode>3 (operands[0], operands[2], operands[3]));
  else if (<MODE>mode == Pmode
	   && GET_CODE (operands[1]) == NE)
    emit_insn (gen_ne<mode>3 (operands[0], operands[2], operands[3]));
  else if (GET_CODE (operands[1]) == NE)
    {
      rtx tmp = gen_reg_rtx (<MODE>mode);
      emit_insn (gen_eq<mode>3 (tmp, operands[2], operands[3]));
      emit_insn (gen_xor<mode>3 (operands[0], tmp, const1_rtx));
    }

  /* Expanding the unsigned comparisons however helps a lot: all the neg_ltu
     etc. combinations magically work out just right.  */
  else if (<MODE>mode == Pmode
	   && unsigned_comparison_operator (operands[1], VOIDmode))
    emit_insn (gen_cstore<mode>4_unsigned (operands[0], operands[1],
					   operands[2], operands[3]));

  /* The generic code knows tricks to compute signed comparisons against
     zero.  Let it do its thing.  */
  else if (operands[3] == const0_rtx
	   && signed_comparison_operator (operands[1], VOIDmode))
    FAIL;

  /* Everything else, use the mfcr brute force.  */
  else
    rs6000_emit_sCOND (<MODE>mode, operands);

  DONE;
})

(define_expand "cstore<mode>4"
  [(use (match_operator 1 "rs6000_cbranch_operator"
         [(match_operand:FP 2 "gpc_reg_operand" "")
          (match_operand:FP 3 "gpc_reg_operand" "")]))
   (clobber (match_operand:SI 0 "register_operand"))]
  ""
{
  rs6000_emit_sCOND (<MODE>mode, operands);
  DONE;
})


(define_expand "stack_protect_set"
  [(match_operand 0 "memory_operand" "")
   (match_operand 1 "memory_operand" "")]
  ""
{
#ifdef TARGET_THREAD_SSP_OFFSET
  rtx tlsreg = gen_rtx_REG (Pmode, TARGET_64BIT ? 13 : 2);
  rtx addr = gen_rtx_PLUS (Pmode, tlsreg, GEN_INT (TARGET_THREAD_SSP_OFFSET));
  operands[1] = gen_rtx_MEM (Pmode, addr);
#endif
  if (TARGET_64BIT)
    emit_insn (gen_stack_protect_setdi (operands[0], operands[1]));
  else
    emit_insn (gen_stack_protect_setsi (operands[0], operands[1]));
  DONE;
})

(define_insn "stack_protect_setsi"
  [(set (match_operand:SI 0 "memory_operand" "=m")
	(unspec:SI [(match_operand:SI 1 "memory_operand" "m")] UNSPEC_SP_SET))
   (set (match_scratch:SI 2 "=&r") (const_int 0))]
  "TARGET_32BIT"
  "lwz%U1%X1 %2,%1\;stw%U0%X0 %2,%0\;li %2,0"
  [(set_attr "type" "three")
   (set_attr "length" "12")])

(define_insn "stack_protect_setdi"
  [(set (match_operand:DI 0 "memory_operand" "=Y")
	(unspec:DI [(match_operand:DI 1 "memory_operand" "Y")] UNSPEC_SP_SET))
   (set (match_scratch:DI 2 "=&r") (const_int 0))]
  "TARGET_64BIT"
  "ld%U1%X1 %2,%1\;std%U0%X0 %2,%0\;li %2,0"
  [(set_attr "type" "three")
   (set_attr "length" "12")])

(define_expand "stack_protect_test"
  [(match_operand 0 "memory_operand" "")
   (match_operand 1 "memory_operand" "")
   (match_operand 2 "" "")]
  ""
{
  rtx test, op0, op1;
#ifdef TARGET_THREAD_SSP_OFFSET
  rtx tlsreg = gen_rtx_REG (Pmode, TARGET_64BIT ? 13 : 2);
  rtx addr = gen_rtx_PLUS (Pmode, tlsreg, GEN_INT (TARGET_THREAD_SSP_OFFSET));
  operands[1] = gen_rtx_MEM (Pmode, addr);
#endif
  op0 = operands[0];
  op1 = gen_rtx_UNSPEC (Pmode, gen_rtvec (1, operands[1]), UNSPEC_SP_TEST);
  test = gen_rtx_EQ (VOIDmode, op0, op1);
  emit_jump_insn (gen_cbranchsi4 (test, op0, op1, operands[2]));
  DONE;
})

(define_insn "stack_protect_testsi"
  [(set (match_operand:CCEQ 0 "cc_reg_operand" "=x,?y")
        (unspec:CCEQ [(match_operand:SI 1 "memory_operand" "m,m")
		      (match_operand:SI 2 "memory_operand" "m,m")]
		     UNSPEC_SP_TEST))
   (set (match_scratch:SI 4 "=r,r") (const_int 0))
   (clobber (match_scratch:SI 3 "=&r,&r"))]
  "TARGET_32BIT"
  "@
   lwz%U1%X1 %3,%1\;lwz%U2%X2 %4,%2\;xor. %3,%3,%4\;li %4,0
   lwz%U1%X1 %3,%1\;lwz%U2%X2 %4,%2\;cmplw %0,%3,%4\;li %3,0\;li %4,0"
  [(set_attr "length" "16,20")])

(define_insn "stack_protect_testdi"
  [(set (match_operand:CCEQ 0 "cc_reg_operand" "=x,?y")
        (unspec:CCEQ [(match_operand:DI 1 "memory_operand" "Y,Y")
		      (match_operand:DI 2 "memory_operand" "Y,Y")]
		     UNSPEC_SP_TEST))
   (set (match_scratch:DI 4 "=r,r") (const_int 0))
   (clobber (match_scratch:DI 3 "=&r,&r"))]
  "TARGET_64BIT"
  "@
   ld%U1%X1 %3,%1\;ld%U2%X2 %4,%2\;xor. %3,%3,%4\;li %4,0
   ld%U1%X1 %3,%1\;ld%U2%X2 %4,%2\;cmpld %0,%3,%4\;li %3,0\;li %4,0"
  [(set_attr "length" "16,20")])


;; Here are the actual compare insns.
(define_insn "*cmp<mode>_internal1"
  [(set (match_operand:CC 0 "cc_reg_operand" "=y")
	(compare:CC (match_operand:GPR 1 "gpc_reg_operand" "r")
		    (match_operand:GPR 2 "reg_or_short_operand" "rI")))]
  ""
  "cmp<wd>%I2 %0,%1,%2"
  [(set_attr "type" "cmp")])

;; If we are comparing a register for equality with a large constant,
;; we can do this with an XOR followed by a compare.  But this is profitable
;; only if the large constant is only used for the comparison (and in this
;; case we already have a register to reuse as scratch).
;;
;; For 64-bit registers, we could only do so if the constant's bit 15 is clear:
;; otherwise we'd need to XOR with FFFFFFFF????0000 which is not available.

(define_peephole2
  [(set (match_operand:SI 0 "register_operand")
        (match_operand:SI 1 "logical_const_operand" ""))
   (set (match_dup 0) (match_operator:SI 3 "boolean_or_operator"
		       [(match_dup 0)
			(match_operand:SI 2 "logical_const_operand" "")]))
   (set (match_operand:CC 4 "cc_reg_operand" "")
        (compare:CC (match_operand:SI 5 "gpc_reg_operand" "")
                    (match_dup 0)))
   (set (pc)
        (if_then_else (match_operator 6 "equality_operator"
                       [(match_dup 4) (const_int 0)])
                      (match_operand 7 "" "")
                      (match_operand 8 "" "")))]
  "peep2_reg_dead_p (3, operands[0])
   && peep2_reg_dead_p (4, operands[4])"
 [(set (match_dup 0) (xor:SI (match_dup 5) (match_dup 9)))
  (set (match_dup 4) (compare:CC (match_dup 0) (match_dup 10)))
  (set (pc) (if_then_else (match_dup 6) (match_dup 7) (match_dup 8)))]
 
{
  /* Get the constant we are comparing against, and see what it looks like
     when sign-extended from 16 to 32 bits.  Then see what constant we could
     XOR with SEXTC to get the sign-extended value.  */
  rtx cnst = simplify_const_binary_operation (GET_CODE (operands[3]),
					      SImode,
					      operands[1], operands[2]);
  HOST_WIDE_INT c = INTVAL (cnst);
  HOST_WIDE_INT sextc = ((c & 0xffff) ^ 0x8000) - 0x8000;
  HOST_WIDE_INT xorv = c ^ sextc;

  operands[9] = GEN_INT (xorv);
  operands[10] = GEN_INT (sextc);
})

(define_insn "*cmpsi_internal2"
  [(set (match_operand:CCUNS 0 "cc_reg_operand" "=y")
	(compare:CCUNS (match_operand:SI 1 "gpc_reg_operand" "r")
		       (match_operand:SI 2 "reg_or_u_short_operand" "rK")))]
  ""
  "cmplw%I2 %0,%1,%b2"
  [(set_attr "type" "cmp")])

(define_insn "*cmpdi_internal2"
  [(set (match_operand:CCUNS 0 "cc_reg_operand" "=y")
	(compare:CCUNS (match_operand:DI 1 "gpc_reg_operand" "r")
		       (match_operand:DI 2 "reg_or_u_short_operand" "rK")))]
  ""
  "cmpld%I2 %0,%1,%b2"
  [(set_attr "type" "cmp")])

;; The following two insns don't exist as single insns, but if we provide
;; them, we can swap an add and compare, which will enable us to overlap more
;; of the required delay between a compare and branch.  We generate code for
;; them by splitting.

(define_insn ""
  [(set (match_operand:CC 3 "cc_reg_operand" "=y")
	(compare:CC (match_operand:SI 1 "gpc_reg_operand" "r")
		    (match_operand:SI 2 "short_cint_operand" "i")))
   (set (match_operand:SI 0 "gpc_reg_operand" "=r")
	(plus:SI (match_dup 1) (match_operand:SI 4 "short_cint_operand" "i")))]
  ""
  "#"
  [(set_attr "length" "8")])

(define_insn ""
  [(set (match_operand:CCUNS 3 "cc_reg_operand" "=y")
	(compare:CCUNS (match_operand:SI 1 "gpc_reg_operand" "r")
		       (match_operand:SI 2 "u_short_cint_operand" "i")))
   (set (match_operand:SI 0 "gpc_reg_operand" "=r")
	(plus:SI (match_dup 1) (match_operand:SI 4 "short_cint_operand" "i")))]
  ""
  "#"
  [(set_attr "length" "8")])

(define_split
  [(set (match_operand:CC 3 "cc_reg_operand" "")
	(compare:CC (match_operand:SI 1 "gpc_reg_operand" "")
		    (match_operand:SI 2 "short_cint_operand" "")))
   (set (match_operand:SI 0 "gpc_reg_operand" "")
	(plus:SI (match_dup 1) (match_operand:SI 4 "short_cint_operand" "")))]
  ""
  [(set (match_dup 3) (compare:CC (match_dup 1) (match_dup 2)))
   (set (match_dup 0) (plus:SI (match_dup 1) (match_dup 4)))])

(define_split
  [(set (match_operand:CCUNS 3 "cc_reg_operand" "")
	(compare:CCUNS (match_operand:SI 1 "gpc_reg_operand" "")
		       (match_operand:SI 2 "u_short_cint_operand" "")))
   (set (match_operand:SI 0 "gpc_reg_operand" "")
	(plus:SI (match_dup 1) (match_operand:SI 4 "short_cint_operand" "")))]
  ""
  [(set (match_dup 3) (compare:CCUNS (match_dup 1) (match_dup 2)))
   (set (match_dup 0) (plus:SI (match_dup 1) (match_dup 4)))])

;; Only need to compare second words if first words equal
(define_insn "*cmptf_internal1"
  [(set (match_operand:CCFP 0 "cc_reg_operand" "=y")
	(compare:CCFP (match_operand:TF 1 "gpc_reg_operand" "d")
		      (match_operand:TF 2 "gpc_reg_operand" "d")))]
  "!TARGET_IEEEQUAD && !TARGET_XL_COMPAT
   && TARGET_HARD_FLOAT && TARGET_FPRS && TARGET_DOUBLE_FLOAT && TARGET_LONG_DOUBLE_128"
  "fcmpu %0,%1,%2\;bne %0,$+8\;fcmpu %0,%L1,%L2"
  [(set_attr "type" "fpcompare")
   (set_attr "length" "12")])

(define_insn_and_split "*cmptf_internal2"
  [(set (match_operand:CCFP 0 "cc_reg_operand" "=y")
	(compare:CCFP (match_operand:TF 1 "gpc_reg_operand" "d")
		      (match_operand:TF 2 "gpc_reg_operand" "d")))
    (clobber (match_scratch:DF 3 "=d"))
    (clobber (match_scratch:DF 4 "=d"))
    (clobber (match_scratch:DF 5 "=d"))
    (clobber (match_scratch:DF 6 "=d"))
    (clobber (match_scratch:DF 7 "=d"))
    (clobber (match_scratch:DF 8 "=d"))
    (clobber (match_scratch:DF 9 "=d"))
    (clobber (match_scratch:DF 10 "=d"))
    (clobber (match_scratch:GPR 11 "=b"))]
  "!TARGET_IEEEQUAD && TARGET_XL_COMPAT
   && TARGET_HARD_FLOAT && TARGET_FPRS && TARGET_DOUBLE_FLOAT && TARGET_LONG_DOUBLE_128"
  "#"
  "&& reload_completed"
  [(set (match_dup 3) (match_dup 14))
   (set (match_dup 4) (match_dup 15))
   (set (match_dup 9) (abs:DF (match_dup 5)))
   (set (match_dup 0) (compare:CCFP (match_dup 9) (match_dup 3)))
   (set (pc) (if_then_else (ne (match_dup 0) (const_int 0))
			   (label_ref (match_dup 12))
			   (pc)))
   (set (match_dup 0) (compare:CCFP (match_dup 5) (match_dup 7)))
   (set (pc) (label_ref (match_dup 13)))
   (match_dup 12)
   (set (match_dup 10) (minus:DF (match_dup 5) (match_dup 7)))
   (set (match_dup 9) (minus:DF (match_dup 6) (match_dup 8)))
   (set (match_dup 9) (plus:DF (match_dup 10) (match_dup 9)))
   (set (match_dup 0) (compare:CCFP (match_dup 9) (match_dup 4)))
   (match_dup 13)]
{
  REAL_VALUE_TYPE rv;
  const int lo_word = LONG_DOUBLE_LARGE_FIRST ? GET_MODE_SIZE (DFmode) : 0;
  const int hi_word = LONG_DOUBLE_LARGE_FIRST ? 0 : GET_MODE_SIZE (DFmode);

  operands[5] = simplify_gen_subreg (DFmode, operands[1], TFmode, hi_word);
  operands[6] = simplify_gen_subreg (DFmode, operands[1], TFmode, lo_word);
  operands[7] = simplify_gen_subreg (DFmode, operands[2], TFmode, hi_word);
  operands[8] = simplify_gen_subreg (DFmode, operands[2], TFmode, lo_word);
  operands[12] = gen_label_rtx ();
  operands[13] = gen_label_rtx ();
  real_inf (&rv);
  operands[14] = force_const_mem (DFmode,
				  CONST_DOUBLE_FROM_REAL_VALUE (rv, DFmode));
  operands[15] = force_const_mem (DFmode,
				  CONST_DOUBLE_FROM_REAL_VALUE (dconst0,
								DFmode));
  if (TARGET_TOC)
    {
      rtx tocref;
      tocref = create_TOC_reference (XEXP (operands[14], 0), operands[11]);
      operands[14] = gen_const_mem (DFmode, tocref);
      tocref = create_TOC_reference (XEXP (operands[15], 0), operands[11]);
      operands[15] = gen_const_mem (DFmode, tocref);
      set_mem_alias_set (operands[14], get_TOC_alias_set ());
      set_mem_alias_set (operands[15], get_TOC_alias_set ());
    }
})

;; Now we have the scc insns.  We can do some combinations because of the
;; way the machine works.
;;
;; Note that this is probably faster if we can put an insn between the
;; mfcr and rlinm, but this is tricky.  Let's leave it for now.  In most
;; cases the insns below which don't use an intermediate CR field will
;; be used instead.
(define_insn ""
  [(set (match_operand:SI 0 "gpc_reg_operand" "=r")
	(match_operator:SI 1 "scc_comparison_operator"
			   [(match_operand 2 "cc_reg_operand" "y")
			    (const_int 0)]))]
  ""
  "mfcr %0%Q2\;rlwinm %0,%0,%J1,1"
  [(set (attr "type")
     (cond [(match_test "TARGET_MFCRF")
		(const_string "mfcrf")
	   ]
	(const_string "mfcr")))
   (set_attr "length" "8")])

;; Same as above, but get the GT bit.
(define_insn "move_from_CR_gt_bit"
  [(set (match_operand:SI 0 "gpc_reg_operand" "=r")
	(unspec:SI [(match_operand 1 "cc_reg_operand" "y")] UNSPEC_MV_CR_GT))]
  "TARGET_HARD_FLOAT && !TARGET_FPRS"
  "mfcr %0\;rlwinm %0,%0,%D1,31,31"
  [(set_attr "type" "mfcr")
   (set_attr "length" "8")])

;; Same as above, but get the OV/ORDERED bit.
(define_insn "move_from_CR_ov_bit"
  [(set (match_operand:SI 0 "gpc_reg_operand" "=r")
	(unspec:SI [(match_operand:CC 1 "cc_reg_operand" "y")]
		   UNSPEC_MV_CR_OV))]
  "TARGET_ISEL"
  "mfcr %0\;rlwinm %0,%0,%t1,1"
  [(set_attr "type" "mfcr")
   (set_attr "length" "8")])

(define_insn ""
  [(set (match_operand:DI 0 "gpc_reg_operand" "=r")
	(match_operator:DI 1 "scc_comparison_operator"
			   [(match_operand 2 "cc_reg_operand" "y")
			    (const_int 0)]))]
  "TARGET_POWERPC64"
  "mfcr %0%Q2\;rlwinm %0,%0,%J1,1"
  [(set (attr "type")
     (cond [(match_test "TARGET_MFCRF")
		(const_string "mfcrf")
	   ]
	(const_string "mfcr")))
   (set_attr "length" "8")])

(define_insn ""
  [(set (match_operand:CC 0 "cc_reg_operand" "=x,?y")
	(compare:CC (match_operator:SI 1 "scc_comparison_operator"
				       [(match_operand 2 "cc_reg_operand" "y,y")
					(const_int 0)])
		    (const_int 0)))
   (set (match_operand:SI 3 "gpc_reg_operand" "=r,r")
	(match_op_dup 1 [(match_dup 2) (const_int 0)]))]
  "TARGET_32BIT"
  "@
   mfcr %3%Q2\;rlwinm. %3,%3,%J1,1
   #"
  [(set_attr "type" "shift")
   (set_attr "dot" "yes")
   (set_attr "length" "8,16")])

(define_split
  [(set (match_operand:CC 0 "cc_reg_not_cr0_operand" "")
	(compare:CC (match_operator:SI 1 "scc_comparison_operator"
				       [(match_operand 2 "cc_reg_operand" "")
					(const_int 0)])
		    (const_int 0)))
   (set (match_operand:SI 3 "gpc_reg_operand" "")
	(match_op_dup 1 [(match_dup 2) (const_int 0)]))]
  "TARGET_32BIT && reload_completed"
  [(set (match_dup 3)
	(match_op_dup 1 [(match_dup 2) (const_int 0)]))
   (set (match_dup 0)
	(compare:CC (match_dup 3)
		    (const_int 0)))]
  "")

(define_insn ""
  [(set (match_operand:SI 0 "gpc_reg_operand" "=r")
	(ashift:SI (match_operator:SI 1 "scc_comparison_operator"
				      [(match_operand 2 "cc_reg_operand" "y")
				       (const_int 0)])
		   (match_operand:SI 3 "const_int_operand" "n")))]
  ""
  "*
{
  int is_bit = ccr_bit (operands[1], 1);
  int put_bit = 31 - (INTVAL (operands[3]) & 31);
  int count;

  if (is_bit >= put_bit)
    count = is_bit - put_bit;
  else
    count = 32 - (put_bit - is_bit);

  operands[4] = GEN_INT (count);
  operands[5] = GEN_INT (put_bit);

  return \"mfcr %0%Q2\;rlwinm %0,%0,%4,%5,%5\";
}"
  [(set (attr "type")
     (cond [(match_test "TARGET_MFCRF")
		(const_string "mfcrf")
	   ]
	(const_string "mfcr")))
   (set_attr "length" "8")])

(define_insn ""
  [(set (match_operand:CC 0 "cc_reg_operand" "=x,?y")
	(compare:CC
	 (ashift:SI (match_operator:SI 1 "scc_comparison_operator"
				       [(match_operand 2 "cc_reg_operand" "y,y")
					(const_int 0)])
		    (match_operand:SI 3 "const_int_operand" "n,n"))
	 (const_int 0)))
   (set (match_operand:SI 4 "gpc_reg_operand" "=r,r")
	(ashift:SI (match_op_dup 1 [(match_dup 2) (const_int 0)])
		   (match_dup 3)))]
  ""
  "*
{
  int is_bit = ccr_bit (operands[1], 1);
  int put_bit = 31 - (INTVAL (operands[3]) & 31);
  int count;

  /* Force split for non-cc0 compare.  */
  if (which_alternative == 1)
     return \"#\";

  if (is_bit >= put_bit)
    count = is_bit - put_bit;
  else
    count = 32 - (put_bit - is_bit);

  operands[5] = GEN_INT (count);
  operands[6] = GEN_INT (put_bit);

  return \"mfcr %4%Q2\;rlwinm. %4,%4,%5,%6,%6\";
}"
  [(set_attr "type" "shift")
   (set_attr "dot" "yes")
   (set_attr "length" "8,16")])

(define_split
  [(set (match_operand:CC 0 "cc_reg_not_micro_cr0_operand" "")
	(compare:CC
	 (ashift:SI (match_operator:SI 1 "scc_comparison_operator"
				       [(match_operand 2 "cc_reg_operand" "")
					(const_int 0)])
		    (match_operand:SI 3 "const_int_operand" ""))
	 (const_int 0)))
   (set (match_operand:SI 4 "gpc_reg_operand" "")
	(ashift:SI (match_op_dup 1 [(match_dup 2) (const_int 0)])
		   (match_dup 3)))]
  "reload_completed"
  [(set (match_dup 4)
	(ashift:SI (match_op_dup 1 [(match_dup 2) (const_int 0)])
		   (match_dup 3)))
   (set (match_dup 0)
	(compare:CC (match_dup 4)
		    (const_int 0)))]
  "")

;; There is a 3 cycle delay between consecutive mfcr instructions
;; so it is useful to combine 2 scc instructions to use only one mfcr.

(define_peephole
  [(set (match_operand:SI 0 "gpc_reg_operand" "=r")
	(match_operator:SI 1 "scc_comparison_operator"
			   [(match_operand 2 "cc_reg_operand" "y")
			    (const_int 0)]))
   (set (match_operand:SI 3 "gpc_reg_operand" "=r")
	(match_operator:SI 4 "scc_comparison_operator"
			   [(match_operand 5 "cc_reg_operand" "y")
			    (const_int 0)]))]
  "REGNO (operands[2]) != REGNO (operands[5])"
  "mfcr %3\;rlwinm %0,%3,%J1,1\;rlwinm %3,%3,%J4,1"
  [(set_attr "type" "mfcr")
   (set_attr "length" "12")])

(define_peephole
  [(set (match_operand:DI 0 "gpc_reg_operand" "=r")
	(match_operator:DI 1 "scc_comparison_operator"
			   [(match_operand 2 "cc_reg_operand" "y")
			    (const_int 0)]))
   (set (match_operand:DI 3 "gpc_reg_operand" "=r")
	(match_operator:DI 4 "scc_comparison_operator"
			   [(match_operand 5 "cc_reg_operand" "y")
			    (const_int 0)]))]
  "TARGET_POWERPC64 && REGNO (operands[2]) != REGNO (operands[5])"
  "mfcr %3\;rlwinm %0,%3,%J1,1\;rlwinm %3,%3,%J4,1"
  [(set_attr "type" "mfcr")
   (set_attr "length" "12")])


(define_mode_attr scc_eq_op2 [(SI "rKLI")
			      (DI "rKJI")])

(define_insn_and_split "eq<mode>3"
  [(set (match_operand:GPR 0 "gpc_reg_operand" "=r")
	(eq:GPR (match_operand:GPR 1 "gpc_reg_operand" "r")
		(match_operand:GPR 2 "scc_eq_operand" "<scc_eq_op2>")))
   (clobber (match_scratch:GPR 3 "=r"))
   (clobber (match_scratch:GPR 4 "=r"))]
  ""
  "#"
  ""
  [(set (match_dup 4)
	(clz:GPR (match_dup 3)))
   (set (match_dup 0)
	(lshiftrt:GPR (match_dup 4)
		      (match_dup 5)))]
{
  operands[3] = rs6000_emit_eqne (<MODE>mode,
				  operands[1], operands[2], operands[3]);

  if (GET_CODE (operands[4]) == SCRATCH)
    operands[4] = gen_reg_rtx (<MODE>mode);

  operands[5] = GEN_INT (exact_log2 (GET_MODE_BITSIZE (<MODE>mode)));
}
  [(set (attr "length")
	(if_then_else (match_test "operands[2] == const0_rtx")
		      (const_string "8")
		      (const_string "12")))])

(define_insn_and_split "ne<mode>3"
  [(set (match_operand:P 0 "gpc_reg_operand" "=r")
	(ne:P (match_operand:P 1 "gpc_reg_operand" "r")
	      (match_operand:P 2 "scc_eq_operand" "<scc_eq_op2>")))
   (clobber (match_scratch:P 3 "=r"))
   (clobber (match_scratch:P 4 "=r"))
   (clobber (reg:P CA_REGNO))]
  "!TARGET_ISEL"
  "#"
  ""
  [(parallel [(set (match_dup 4)
		   (plus:P (match_dup 3)
			   (const_int -1)))
	      (set (reg:P CA_REGNO)
		   (ne:P (match_dup 3)
			 (const_int 0)))])
   (parallel [(set (match_dup 0)
		   (plus:P (plus:P (not:P (match_dup 4))
				   (reg:P CA_REGNO))
			   (match_dup 3)))
	      (clobber (reg:P CA_REGNO))])]
{
  operands[3] = rs6000_emit_eqne (<MODE>mode,
				  operands[1], operands[2], operands[3]);

  if (GET_CODE (operands[4]) == SCRATCH)
    operands[4] = gen_reg_rtx (<MODE>mode);
}
  [(set (attr "length")
	(if_then_else (match_test "operands[2] == const0_rtx")
		      (const_string "8")
		      (const_string "12")))])

(define_insn_and_split "*neg_eq_<mode>"
  [(set (match_operand:P 0 "gpc_reg_operand" "=r")
	(neg:P (eq:P (match_operand:P 1 "gpc_reg_operand" "r")
		     (match_operand:P 2 "scc_eq_operand" "<scc_eq_op2>"))))
   (clobber (match_scratch:P 3 "=r"))
   (clobber (match_scratch:P 4 "=r"))
   (clobber (reg:P CA_REGNO))]
  ""
  "#"
  ""
  [(parallel [(set (match_dup 4)
		   (plus:P (match_dup 3)
			   (const_int -1)))
	      (set (reg:P CA_REGNO)
		   (ne:P (match_dup 3)
			 (const_int 0)))])
   (parallel [(set (match_dup 0)
		   (plus:P (reg:P CA_REGNO)
			   (const_int -1)))
	      (clobber (reg:P CA_REGNO))])]
{
  operands[3] = rs6000_emit_eqne (<MODE>mode,
				  operands[1], operands[2], operands[3]);

  if (GET_CODE (operands[4]) == SCRATCH)
    operands[4] = gen_reg_rtx (<MODE>mode);
}
  [(set (attr "length")
	(if_then_else (match_test "operands[2] == const0_rtx")
		      (const_string "8")
		      (const_string "12")))])

(define_insn_and_split "*neg_ne_<mode>"
  [(set (match_operand:P 0 "gpc_reg_operand" "=r")
	(neg:P (ne:P (match_operand:P 1 "gpc_reg_operand" "r")
		     (match_operand:P 2 "scc_eq_operand" "<scc_eq_op2>"))))
   (clobber (match_scratch:P 3 "=r"))
   (clobber (match_scratch:P 4 "=r"))
   (clobber (reg:P CA_REGNO))]
  ""
  "#"
  ""
  [(parallel [(set (match_dup 4)
		   (neg:P (match_dup 3)))
	      (set (reg:P CA_REGNO)
		   (eq:P (match_dup 3)
			 (const_int 0)))])
   (parallel [(set (match_dup 0)
		   (plus:P (reg:P CA_REGNO)
			   (const_int -1)))
	      (clobber (reg:P CA_REGNO))])]
{
  operands[3] = rs6000_emit_eqne (<MODE>mode,
				  operands[1], operands[2], operands[3]);

  if (GET_CODE (operands[4]) == SCRATCH)
    operands[4] = gen_reg_rtx (<MODE>mode);
}
  [(set (attr "length")
	(if_then_else (match_test "operands[2] == const0_rtx")
		      (const_string "8")
		      (const_string "12")))])

(define_insn_and_split "*plus_eq_<mode>"
  [(set (match_operand:P 0 "gpc_reg_operand" "=r")
	(plus:P (eq:P (match_operand:P 1 "gpc_reg_operand" "r")
		      (match_operand:P 2 "scc_eq_operand" "<scc_eq_op2>"))
		(match_operand:P 3 "gpc_reg_operand" "r")))
   (clobber (match_scratch:P 4 "=r"))
   (clobber (match_scratch:P 5 "=r"))
   (clobber (reg:P CA_REGNO))]
  ""
  "#"
  ""
  [(parallel [(set (match_dup 5)
		   (neg:P (match_dup 4)))
	      (set (reg:P CA_REGNO)
		   (eq:P (match_dup 4)
			 (const_int 0)))])
   (parallel [(set (match_dup 0)
		   (plus:P (match_dup 3)
			   (reg:P CA_REGNO)))
	      (clobber (reg:P CA_REGNO))])]
{
  operands[4] = rs6000_emit_eqne (<MODE>mode,
				  operands[1], operands[2], operands[4]);

  if (GET_CODE (operands[5]) == SCRATCH)
    operands[5] = gen_reg_rtx (<MODE>mode);
}
  [(set (attr "length")
	(if_then_else (match_test "operands[2] == const0_rtx")
		      (const_string "8")
		      (const_string "12")))])

(define_insn_and_split "*plus_ne_<mode>"
  [(set (match_operand:P 0 "gpc_reg_operand" "=r")
	(plus:P (ne:P (match_operand:P 1 "gpc_reg_operand" "r")
		      (match_operand:P 2 "scc_eq_operand" "<scc_eq_op2>"))
		(match_operand:P 3 "gpc_reg_operand" "r")))
   (clobber (match_scratch:P 4 "=r"))
   (clobber (match_scratch:P 5 "=r"))
   (clobber (reg:P CA_REGNO))]
  ""
  "#"
  ""
  [(parallel [(set (match_dup 5)
		   (plus:P (match_dup 4)
			   (const_int -1)))
	      (set (reg:P CA_REGNO)
		   (ne:P (match_dup 4)
			 (const_int 0)))])
   (parallel [(set (match_dup 0)
		   (plus:P (match_dup 3)
			   (reg:P CA_REGNO)))
	      (clobber (reg:P CA_REGNO))])]
{
  operands[4] = rs6000_emit_eqne (<MODE>mode,
				  operands[1], operands[2], operands[4]);

  if (GET_CODE (operands[5]) == SCRATCH)
    operands[5] = gen_reg_rtx (<MODE>mode);
}
  [(set (attr "length")
	(if_then_else (match_test "operands[2] == const0_rtx")
		      (const_string "8")
		      (const_string "12")))])

(define_insn_and_split "*minus_eq_<mode>"
  [(set (match_operand:P 0 "gpc_reg_operand" "=r")
	(minus:P (match_operand:P 3 "gpc_reg_operand" "r")
		 (eq:P (match_operand:P 1 "gpc_reg_operand" "r")
		       (match_operand:P 2 "scc_eq_operand" "<scc_eq_op2>"))))
   (clobber (match_scratch:P 4 "=r"))
   (clobber (match_scratch:P 5 "=r"))
   (clobber (reg:P CA_REGNO))]
  ""
  "#"
  ""
  [(parallel [(set (match_dup 5)
		   (plus:P (match_dup 4)
			   (const_int -1)))
	      (set (reg:P CA_REGNO)
		   (ne:P (match_dup 4)
			 (const_int 0)))])
   (parallel [(set (match_dup 0)
		   (plus:P (plus:P (match_dup 3)
				   (reg:P CA_REGNO))
			   (const_int -1)))
	      (clobber (reg:P CA_REGNO))])]
{
  operands[4] = rs6000_emit_eqne (<MODE>mode,
				  operands[1], operands[2], operands[4]);

  if (GET_CODE (operands[5]) == SCRATCH)
    operands[5] = gen_reg_rtx (<MODE>mode);
}
  [(set (attr "length")
	(if_then_else (match_test "operands[2] == const0_rtx")
		      (const_string "8")
		      (const_string "12")))])

(define_insn_and_split "*minus_ne_<mode>"
  [(set (match_operand:P 0 "gpc_reg_operand" "=r")
	(minus:P (match_operand:P 3 "gpc_reg_operand" "r")
		 (ne:P (match_operand:P 1 "gpc_reg_operand" "r")
		       (match_operand:P 2 "scc_eq_operand" "<scc_eq_op2>"))))
   (clobber (match_scratch:P 4 "=r"))
   (clobber (match_scratch:P 5 "=r"))
   (clobber (reg:P CA_REGNO))]
  ""
  "#"
  ""
  [(parallel [(set (match_dup 5)
		   (neg:P (match_dup 4)))
	      (set (reg:P CA_REGNO)
		   (eq:P (match_dup 4)
			 (const_int 0)))])
   (parallel [(set (match_dup 0)
		   (plus:P (plus:P (match_dup 3)
				   (reg:P CA_REGNO))
			   (const_int -1)))
	      (clobber (reg:P CA_REGNO))])]
{
  operands[4] = rs6000_emit_eqne (<MODE>mode,
				  operands[1], operands[2], operands[4]);

  if (GET_CODE (operands[5]) == SCRATCH)
    operands[5] = gen_reg_rtx (<MODE>mode);
}
  [(set (attr "length")
	(if_then_else (match_test "operands[2] == const0_rtx")
		      (const_string "8")
		      (const_string "12")))])

(define_insn_and_split "*eqsi3_ext<mode>"
  [(set (match_operand:EXTSI 0 "gpc_reg_operand" "=r")
	(eq:EXTSI (match_operand:SI 1 "gpc_reg_operand" "r")
		  (match_operand:SI 2 "scc_eq_operand" "rKLI")))
   (clobber (match_scratch:SI 3 "=r"))
   (clobber (match_scratch:SI 4 "=r"))]
  ""
  "#"
  ""
  [(set (match_dup 4)
	(clz:SI (match_dup 3)))
   (set (match_dup 0)
	(zero_extend:EXTSI
	  (lshiftrt:SI (match_dup 4)
		       (const_int 5))))]
{
  operands[3] = rs6000_emit_eqne (SImode,
				  operands[1], operands[2], operands[3]);

  if (GET_CODE (operands[4]) == SCRATCH)
    operands[4] = gen_reg_rtx (SImode);
}
  [(set (attr "length")
	(if_then_else (match_test "operands[2] == const0_rtx")
		      (const_string "8")
		      (const_string "12")))])

(define_insn_and_split "*nesi3_ext<mode>"
  [(set (match_operand:EXTSI 0 "gpc_reg_operand" "=r")
	(ne:EXTSI (match_operand:SI 1 "gpc_reg_operand" "r")
		  (match_operand:SI 2 "scc_eq_operand" "rKLI")))
   (clobber (match_scratch:SI 3 "=r"))
   (clobber (match_scratch:SI 4 "=r"))
   (clobber (match_scratch:EXTSI 5 "=r"))]
  ""
  "#"
  ""
  [(set (match_dup 4)
	(clz:SI (match_dup 3)))
   (set (match_dup 5)
	(zero_extend:EXTSI
	  (lshiftrt:SI (match_dup 4)
		       (const_int 5))))
   (set (match_dup 0)
	(xor:EXTSI (match_dup 5)
		   (const_int 1)))]
{
  operands[3] = rs6000_emit_eqne (SImode,
				  operands[1], operands[2], operands[3]);

  if (GET_CODE (operands[4]) == SCRATCH)
    operands[4] = gen_reg_rtx (SImode);
  if (GET_CODE (operands[5]) == SCRATCH)
    operands[5] = gen_reg_rtx (<MODE>mode);
}
  [(set (attr "length")
	(if_then_else (match_test "operands[2] == const0_rtx")
		      (const_string "12")
		      (const_string "16")))])

;; Define both directions of branch and return.  If we need a reload
;; register, we'd rather use CR0 since it is much easier to copy a
;; register CC value to there.

(define_insn ""
  [(set (pc)
	(if_then_else (match_operator 1 "branch_comparison_operator"
				      [(match_operand 2
						      "cc_reg_operand" "y")
				       (const_int 0)])
		      (label_ref (match_operand 0 "" ""))
		      (pc)))]
  ""
  "*
{
  return output_cbranch (operands[1], \"%l0\", 0, insn);
}"
  [(set_attr "type" "branch")])

(define_insn ""
  [(set (pc)
	(if_then_else (match_operator 0 "branch_comparison_operator"
				      [(match_operand 1
						      "cc_reg_operand" "y")
				       (const_int 0)])
		      (any_return)
		      (pc)))]
  "<return_pred>"
  "*
{
  return output_cbranch (operands[0], NULL, 0, insn);
}"
  [(set_attr "type" "jmpreg")
   (set_attr "length" "4")])

(define_insn ""
  [(set (pc)
	(if_then_else (match_operator 1 "branch_comparison_operator"
				      [(match_operand 2
						      "cc_reg_operand" "y")
				       (const_int 0)])
		      (pc)
		      (label_ref (match_operand 0 "" ""))))]
  ""
  "*
{
  return output_cbranch (operands[1], \"%l0\", 1, insn);
}"
  [(set_attr "type" "branch")])

(define_insn ""
  [(set (pc)
	(if_then_else (match_operator 0 "branch_comparison_operator"
				      [(match_operand 1
						      "cc_reg_operand" "y")
				       (const_int 0)])
		      (pc)
		      (any_return)))]
  "<return_pred>"
  "*
{
  return output_cbranch (operands[0], NULL, 1, insn);
}"
  [(set_attr "type" "jmpreg")
   (set_attr "length" "4")])

;; Logic on condition register values.

; This pattern matches things like
; (set (reg:CCEQ 68) (compare:CCEQ (ior:SI (gt:SI (reg:CCFP 68) (const_int 0))
;					   (eq:SI (reg:CCFP 68) (const_int 0)))
;				   (const_int 1)))
; which are generated by the branch logic.
; Prefer destructive operations where BT = BB (for crXX BT,BA,BB)

(define_insn "*cceq_ior_compare"
  [(set (match_operand:CCEQ 0 "cc_reg_operand" "=y,?y")
        (compare:CCEQ (match_operator:SI 1 "boolean_operator"
	                [(match_operator:SI 2
				      "branch_positive_comparison_operator"
				      [(match_operand 3
						      "cc_reg_operand" "y,y")
				       (const_int 0)])
	                 (match_operator:SI 4
				      "branch_positive_comparison_operator"
				      [(match_operand 5
						      "cc_reg_operand" "0,y")
				       (const_int 0)])])
		      (const_int 1)))]
  ""
  "cr%q1 %E0,%j2,%j4"
  [(set_attr "type" "cr_logical,delayed_cr")])

; Why is the constant -1 here, but 1 in the previous pattern?
; Because ~1 has all but the low bit set.
(define_insn ""
  [(set (match_operand:CCEQ 0 "cc_reg_operand" "=y,?y")
        (compare:CCEQ (match_operator:SI 1 "boolean_or_operator"
	                [(not:SI (match_operator:SI 2
				      "branch_positive_comparison_operator"
				      [(match_operand 3
						      "cc_reg_operand" "y,y")
				       (const_int 0)]))
	                 (match_operator:SI 4
				"branch_positive_comparison_operator"
				[(match_operand 5
						"cc_reg_operand" "0,y")
				 (const_int 0)])])
		      (const_int -1)))]
  ""
  "cr%q1 %E0,%j2,%j4"
  [(set_attr "type" "cr_logical,delayed_cr")])

(define_insn "*cceq_rev_compare"
  [(set (match_operand:CCEQ 0 "cc_reg_operand" "=y,?y")
	(compare:CCEQ (match_operator:SI 1
				      "branch_positive_comparison_operator"
				      [(match_operand 2
						      "cc_reg_operand" "0,y")
				       (const_int 0)])
		      (const_int 0)))]
  ""
  "crnot %E0,%j1"
  [(set_attr "type" "cr_logical,delayed_cr")])

;; If we are comparing the result of two comparisons, this can be done
;; using creqv or crxor.

(define_insn_and_split ""
  [(set (match_operand:CCEQ 0 "cc_reg_operand" "=y")
	(compare:CCEQ (match_operator 1 "branch_comparison_operator"
			      [(match_operand 2 "cc_reg_operand" "y")
			       (const_int 0)])
		      (match_operator 3 "branch_comparison_operator"
			      [(match_operand 4 "cc_reg_operand" "y")
			       (const_int 0)])))]
  ""
  "#"
  ""
  [(set (match_dup 0) (compare:CCEQ (xor:SI (match_dup 1) (match_dup 3))
				    (match_dup 5)))]
  "
{
  int positive_1, positive_2;

  positive_1 = branch_positive_comparison_operator (operands[1],
						    GET_MODE (operands[1]));
  positive_2 = branch_positive_comparison_operator (operands[3],
						    GET_MODE (operands[3]));

  if (! positive_1)
    operands[1] = gen_rtx_fmt_ee (rs6000_reverse_condition (GET_MODE (operands[2]),
							    GET_CODE (operands[1])),
				  SImode,
				  operands[2], const0_rtx);
  else if (GET_MODE (operands[1]) != SImode)
    operands[1] = gen_rtx_fmt_ee (GET_CODE (operands[1]), SImode,
				  operands[2], const0_rtx);

  if (! positive_2)
    operands[3] = gen_rtx_fmt_ee (rs6000_reverse_condition (GET_MODE (operands[4]),
							    GET_CODE (operands[3])),
				  SImode,
				  operands[4], const0_rtx);
  else if (GET_MODE (operands[3]) != SImode)
    operands[3] = gen_rtx_fmt_ee (GET_CODE (operands[3]), SImode,
				  operands[4], const0_rtx);

  if (positive_1 == positive_2)
    {
      operands[1] = gen_rtx_NOT (SImode, operands[1]);
      operands[5] = constm1_rtx;
    }
  else
    {
      operands[5] = const1_rtx;
    }
}")

;; Unconditional branch and return.

(define_insn "jump"
  [(set (pc)
	(label_ref (match_operand 0 "" "")))]
  ""
  "b %l0"
  [(set_attr "type" "branch")])

(define_insn "<return_str>return"
  [(any_return)]
  "<return_pred>"
  "blr"
  [(set_attr "type" "jmpreg")])

(define_expand "indirect_jump"
  [(set (pc) (match_operand 0 "register_operand" ""))])

(define_insn "*indirect_jump<mode>"
  [(set (pc) (match_operand:P 0 "register_operand" "c,*l"))]
  ""
  "@
   bctr
   blr"
  [(set_attr "type" "jmpreg")])

;; Table jump for switch statements:
(define_expand "tablejump"
  [(use (match_operand 0 "" ""))
   (use (label_ref (match_operand 1 "" "")))]
  ""
  "
{
  if (TARGET_32BIT)
    emit_jump_insn (gen_tablejumpsi (operands[0], operands[1]));
  else
    emit_jump_insn (gen_tablejumpdi (operands[0], operands[1]));
  DONE;
}")

(define_expand "tablejumpsi"
  [(set (match_dup 3)
	(plus:SI (match_operand:SI 0 "" "")
		 (match_dup 2)))
   (parallel [(set (pc) (match_dup 3))
	      (use (label_ref (match_operand 1 "" "")))])]
  "TARGET_32BIT"
  "
{ operands[0] = force_reg (SImode, operands[0]);
  operands[2] = force_reg (SImode, gen_rtx_LABEL_REF (SImode, operands[1]));
  operands[3] = gen_reg_rtx (SImode);
}")

(define_expand "tablejumpdi"
  [(set (match_dup 4)
        (sign_extend:DI (match_operand:SI 0 "lwa_operand" "")))
   (set (match_dup 3)
	(plus:DI (match_dup 4)
		 (match_dup 2)))
   (parallel [(set (pc) (match_dup 3))
	      (use (label_ref (match_operand 1 "" "")))])]
  "TARGET_64BIT"
  "
{ operands[2] = force_reg (DImode, gen_rtx_LABEL_REF (DImode, operands[1]));
  operands[3] = gen_reg_rtx (DImode);
  operands[4] = gen_reg_rtx (DImode);
}")

(define_insn "*tablejump<mode>_internal1"
  [(set (pc)
	(match_operand:P 0 "register_operand" "c,*l"))
   (use (label_ref (match_operand 1 "" "")))]
  ""
  "@
   bctr
   blr"
  [(set_attr "type" "jmpreg")])

(define_insn "nop"
  [(const_int 0)]
  ""
  "nop")

(define_insn "group_ending_nop"
  [(unspec [(const_int 0)] UNSPEC_GRP_END_NOP)]
  ""
  "*
{
  if (rs6000_cpu_attr == CPU_POWER6)
    return \"ori 1,1,0\";
  return \"ori 2,2,0\";
}")

;; Define the subtract-one-and-jump insns, starting with the template
;; so loop.c knows what to generate.

(define_expand "doloop_end"
  [(use (match_operand 0 "" ""))	; loop pseudo
   (use (match_operand 1 "" ""))]	; label
  ""
  "
{
  if (TARGET_64BIT)
    {
      if (GET_MODE (operands[0]) != DImode)
	FAIL;
      emit_jump_insn (gen_ctrdi (operands[0], operands[1]));
    }
  else
    {
      if (GET_MODE (operands[0]) != SImode)
	FAIL;
      emit_jump_insn (gen_ctrsi (operands[0], operands[1]));
    }
  DONE;
}")

(define_expand "ctr<mode>"
  [(parallel [(set (pc)
		   (if_then_else (ne (match_operand:P 0 "register_operand" "")
				     (const_int 1))
				 (label_ref (match_operand 1 "" ""))
				 (pc)))
	      (set (match_dup 0)
		   (plus:P (match_dup 0)
			    (const_int -1)))
	      (clobber (match_scratch:CC 2 ""))
	      (clobber (match_scratch:P 3 ""))])]
  ""
  "")

;; We need to be able to do this for any operand, including MEM, or we
;; will cause reload to blow up since we don't allow output reloads on
;; JUMP_INSNs.
;; For the length attribute to be calculated correctly, the
;; label MUST be operand 0.

(define_insn "*ctr<mode>_internal1"
  [(set (pc)
	(if_then_else (ne (match_operand:P 1 "register_operand" "c,*b,*b,*b")
			  (const_int 1))
		      (label_ref (match_operand 0 "" ""))
		      (pc)))
   (set (match_operand:P 2 "nonimmediate_operand" "=1,*r,m,*c*l")
	(plus:P (match_dup 1)
		 (const_int -1)))
   (clobber (match_scratch:CC 3 "=X,&x,&x,&x"))
   (clobber (match_scratch:P 4 "=X,X,&r,r"))]
  ""
  "*
{
  if (which_alternative != 0)
    return \"#\";
  else if (get_attr_length (insn) == 4)
    return \"bdnz %l0\";
  else
    return \"bdz $+8\;b %l0\";
}"
  [(set_attr "type" "branch")
   (set_attr "length" "*,16,20,20")])

(define_insn "*ctr<mode>_internal2"
  [(set (pc)
	(if_then_else (ne (match_operand:P 1 "register_operand" "c,*b,*b,*b")
			  (const_int 1))
		      (pc)
		      (label_ref (match_operand 0 "" ""))))
   (set (match_operand:P 2 "nonimmediate_operand" "=1,*r,m,*c*l")
	(plus:P (match_dup 1)
		 (const_int -1)))
   (clobber (match_scratch:CC 3 "=X,&x,&x,&x"))
   (clobber (match_scratch:P 4 "=X,X,&r,r"))]
  ""
  "*
{
  if (which_alternative != 0)
    return \"#\";
  else if (get_attr_length (insn) == 4)
    return \"bdz %l0\";
  else
    return \"bdnz $+8\;b %l0\";
}"
  [(set_attr "type" "branch")
   (set_attr "length" "*,16,20,20")])

;; Similar but use EQ

(define_insn "*ctr<mode>_internal5"
  [(set (pc)
	(if_then_else (eq (match_operand:P 1 "register_operand" "c,*b,*b,*b")
			  (const_int 1))
		      (label_ref (match_operand 0 "" ""))
		      (pc)))
   (set (match_operand:P 2 "nonimmediate_operand" "=1,*r,m,*c*l")
	(plus:P (match_dup 1)
		 (const_int -1)))
   (clobber (match_scratch:CC 3 "=X,&x,&x,&x"))
   (clobber (match_scratch:P 4 "=X,X,&r,r"))]
  ""
  "*
{
  if (which_alternative != 0)
    return \"#\";
  else if (get_attr_length (insn) == 4)
    return \"bdz %l0\";
  else
    return \"bdnz $+8\;b %l0\";
}"
  [(set_attr "type" "branch")
   (set_attr "length" "*,16,20,20")])

(define_insn "*ctr<mode>_internal6"
  [(set (pc)
	(if_then_else (eq (match_operand:P 1 "register_operand" "c,*b,*b,*b")
			  (const_int 1))
		      (pc)
		      (label_ref (match_operand 0 "" ""))))
   (set (match_operand:P 2 "nonimmediate_operand" "=1,*r,m,*c*l")
	(plus:P (match_dup 1)
		 (const_int -1)))
   (clobber (match_scratch:CC 3 "=X,&x,&x,&x"))
   (clobber (match_scratch:P 4 "=X,X,&r,r"))]
  ""
  "*
{
  if (which_alternative != 0)
    return \"#\";
  else if (get_attr_length (insn) == 4)
    return \"bdnz %l0\";
  else
    return \"bdz $+8\;b %l0\";
}"
  [(set_attr "type" "branch")
   (set_attr "length" "*,16,20,20")])

;; Now the splitters if we could not allocate the CTR register

(define_split
  [(set (pc)
	(if_then_else (match_operator 2 "comparison_operator"
				      [(match_operand:P 1 "gpc_reg_operand" "")
				       (const_int 1)])
		      (match_operand 5 "" "")
		      (match_operand 6 "" "")))
   (set (match_operand:P 0 "gpc_reg_operand" "")
	(plus:P (match_dup 1) (const_int -1)))
   (clobber (match_scratch:CC 3 ""))
   (clobber (match_scratch:P 4 ""))]
  "reload_completed"
  [(set (match_dup 3)
	(compare:CC (match_dup 1)
		    (const_int 1)))
   (set (match_dup 0)
	(plus:P (match_dup 1)
		(const_int -1)))
   (set (pc) (if_then_else (match_dup 7)
			   (match_dup 5)
			   (match_dup 6)))]
  "
{ operands[7] = gen_rtx_fmt_ee (GET_CODE (operands[2]), VOIDmode,
				operands[3], const0_rtx); }")

(define_split
  [(set (pc)
	(if_then_else (match_operator 2 "comparison_operator"
				      [(match_operand:P 1 "gpc_reg_operand" "")
				       (const_int 1)])
		      (match_operand 5 "" "")
		      (match_operand 6 "" "")))
   (set (match_operand:P 0 "nonimmediate_operand" "")
	(plus:P (match_dup 1) (const_int -1)))
   (clobber (match_scratch:CC 3 ""))
   (clobber (match_scratch:P 4 ""))]
  "reload_completed && ! gpc_reg_operand (operands[0], SImode)"
  [(set (match_dup 3)
	(compare:CC (match_dup 1)
		    (const_int 1)))
   (set (match_dup 4)
	(plus:P (match_dup 1)
		(const_int -1)))
   (set (match_dup 0)
	(match_dup 4))
   (set (pc) (if_then_else (match_dup 7)
			   (match_dup 5)
			   (match_dup 6)))]
  "
{ operands[7] = gen_rtx_fmt_ee (GET_CODE (operands[2]), VOIDmode,
				operands[3], const0_rtx); }")

(define_insn "trap"
  [(trap_if (const_int 1) (const_int 0))]
  ""
  "trap"
  [(set_attr "type" "trap")])

(define_expand "ctrap<mode>4"
  [(trap_if (match_operator 0 "ordered_comparison_operator"
			    [(match_operand:GPR 1 "register_operand")
			     (match_operand:GPR 2 "reg_or_short_operand")])
	    (match_operand 3 "zero_constant" ""))]
  ""
  "")

(define_insn ""
  [(trap_if (match_operator 0 "ordered_comparison_operator"
                            [(match_operand:GPR 1 "register_operand" "r")
                             (match_operand:GPR 2 "reg_or_short_operand" "rI")])
	    (const_int 0))]
  ""
  "t<wd>%V0%I2 %1,%2"
  [(set_attr "type" "trap")])

;; Insns related to generating the function prologue and epilogue.

(define_expand "prologue"
  [(use (const_int 0))]
  ""
{
  rs6000_emit_prologue ();
  if (!TARGET_SCHED_PROLOG)
    emit_insn (gen_blockage ());
  DONE;
})

(define_insn "*movesi_from_cr_one"
  [(match_parallel 0 "mfcr_operation"
		   [(set (match_operand:SI 1 "gpc_reg_operand" "=r")
			 (unspec:SI [(match_operand:CC 2 "cc_reg_operand" "y")
				     (match_operand 3 "immediate_operand" "n")]
			  UNSPEC_MOVESI_FROM_CR))])]
  "TARGET_MFCRF"
  "*
{
  int mask = 0;
  int i;
  for (i = 0; i < XVECLEN (operands[0], 0); i++)
  {
    mask = INTVAL (XVECEXP (SET_SRC (XVECEXP (operands[0], 0, i)), 0, 1));
    operands[4] = GEN_INT (mask);
    output_asm_insn (\"mfcr %1,%4\", operands);
  }
  return \"\";
}"
  [(set_attr "type" "mfcrf")])

(define_insn "movesi_from_cr"
  [(set (match_operand:SI 0 "gpc_reg_operand" "=r")
        (unspec:SI [(reg:CC CR0_REGNO) (reg:CC CR1_REGNO)
		    (reg:CC CR2_REGNO) (reg:CC CR3_REGNO)
		    (reg:CC CR4_REGNO) (reg:CC CR5_REGNO)
		    (reg:CC CR6_REGNO) (reg:CC CR7_REGNO)]
		   UNSPEC_MOVESI_FROM_CR))]
  ""
  "mfcr %0"
  [(set_attr "type" "mfcr")])

(define_insn "*crsave"
  [(match_parallel 0 "crsave_operation"
		   [(set (match_operand:SI 1 "memory_operand" "=m")
			 (match_operand:SI 2 "gpc_reg_operand" "r"))])]
  ""
  "stw %2,%1"
  [(set_attr "type" "store")])

(define_insn "*stmw"
  [(match_parallel 0 "stmw_operation"
		   [(set (match_operand:SI 1 "memory_operand" "=m")
       			 (match_operand:SI 2 "gpc_reg_operand" "r"))])]
  "TARGET_MULTIPLE"
  "stmw %2,%1"
  [(set_attr "type" "store")
   (set_attr "update" "yes")
   (set_attr "indexed" "yes")])

; The following comment applies to:
;     save_gpregs_*
;     save_fpregs_*
;     restore_gpregs*
;     return_and_restore_gpregs*
;     return_and_restore_fpregs*
;     return_and_restore_fpregs_aix*
;
; The out-of-line save / restore functions expects one input argument.
; Since those are not standard call_insn's, we must avoid using
; MATCH_OPERAND for that argument. That way the register rename
; optimization will not try to rename this register.
; Each pattern is repeated for each possible register number used in 
; various ABIs (r11, r1, and for some functions r12)

(define_insn "*save_gpregs_<mode>_r11"
  [(match_parallel 0 "any_parallel_operand"
		   [(clobber (reg:P 65))
		    (use (match_operand:P 1 "symbol_ref_operand" "s"))
                    (use (reg:P 11))
		    (set (match_operand:P 2 "memory_operand" "=m")
			 (match_operand:P 3 "gpc_reg_operand" "r"))])]
  ""
  "bl %1"
  [(set_attr "type" "branch")
   (set_attr "length" "4")])

(define_insn "*save_gpregs_<mode>_r12"
  [(match_parallel 0 "any_parallel_operand"
		   [(clobber (reg:P 65))
		    (use (match_operand:P 1 "symbol_ref_operand" "s"))
                    (use (reg:P 12))
		    (set (match_operand:P 2 "memory_operand" "=m")
			 (match_operand:P 3 "gpc_reg_operand" "r"))])]
  ""
  "bl %1"
  [(set_attr "type" "branch")
   (set_attr "length" "4")])

(define_insn "*save_gpregs_<mode>_r1"
  [(match_parallel 0 "any_parallel_operand"
		   [(clobber (reg:P 65))
		    (use (match_operand:P 1 "symbol_ref_operand" "s"))
                    (use (reg:P 1))
		    (set (match_operand:P 2 "memory_operand" "=m")
			 (match_operand:P 3 "gpc_reg_operand" "r"))])]
  ""
  "bl %1"
  [(set_attr "type" "branch")
   (set_attr "length" "4")])

(define_insn "*save_fpregs_<mode>_r11"
  [(match_parallel 0 "any_parallel_operand"
		   [(clobber (reg:P 65))
		    (use (match_operand:P 1 "symbol_ref_operand" "s"))
                    (use (reg:P 11))
		    (set (match_operand:DF 2 "memory_operand" "=m")
			 (match_operand:DF 3 "gpc_reg_operand" "d"))])]
  ""
  "bl %1"
  [(set_attr "type" "branch")
   (set_attr "length" "4")])

(define_insn "*save_fpregs_<mode>_r12"
  [(match_parallel 0 "any_parallel_operand"
		   [(clobber (reg:P 65))
		    (use (match_operand:P 1 "symbol_ref_operand" "s"))
                    (use (reg:P 12))
		    (set (match_operand:DF 2 "memory_operand" "=m")
			 (match_operand:DF 3 "gpc_reg_operand" "d"))])]
  ""
  "bl %1"
  [(set_attr "type" "branch")
   (set_attr "length" "4")])

(define_insn "*save_fpregs_<mode>_r1"
  [(match_parallel 0 "any_parallel_operand"
		   [(clobber (reg:P 65))
		    (use (match_operand:P 1 "symbol_ref_operand" "s"))
                    (use (reg:P 1))
		    (set (match_operand:DF 2 "memory_operand" "=m")
			 (match_operand:DF 3 "gpc_reg_operand" "d"))])]
  ""
  "bl %1"
  [(set_attr "type" "branch")
   (set_attr "length" "4")])

; This is to explain that changes to the stack pointer should
; not be moved over loads from or stores to stack memory.
(define_insn "stack_tie"
  [(match_parallel 0 "tie_operand"
		   [(set (mem:BLK (reg 1)) (const_int 0))])]
  ""
  ""
  [(set_attr "length" "0")])

(define_expand "epilogue"
  [(use (const_int 0))]
  ""
{
  if (!TARGET_SCHED_PROLOG)
    emit_insn (gen_blockage ());
  rs6000_emit_epilogue (FALSE);
  DONE;
})

; On some processors, doing the mtcrf one CC register at a time is
; faster (like on the 604e).  On others, doing them all at once is
; faster; for instance, on the 601 and 750.

(define_expand "movsi_to_cr_one"
  [(set (match_operand:CC 0 "cc_reg_operand" "")
        (unspec:CC [(match_operand:SI 1 "gpc_reg_operand" "")
		    (match_dup 2)] UNSPEC_MOVESI_TO_CR))]
  ""
  "operands[2] = GEN_INT (1 << (75 - REGNO (operands[0])));")

(define_insn "*movsi_to_cr"
  [(match_parallel 0 "mtcrf_operation"
		   [(set (match_operand:CC 1 "cc_reg_operand" "=y")
			 (unspec:CC [(match_operand:SI 2 "gpc_reg_operand" "r")
				     (match_operand 3 "immediate_operand" "n")]
				    UNSPEC_MOVESI_TO_CR))])]
 ""
 "*
{
  int mask = 0;
  int i;
  for (i = 0; i < XVECLEN (operands[0], 0); i++)
    mask |= INTVAL (XVECEXP (SET_SRC (XVECEXP (operands[0], 0, i)), 0, 1));
  operands[4] = GEN_INT (mask);
  return \"mtcrf %4,%2\";
}"
  [(set_attr "type" "mtcr")])

(define_insn "*mtcrfsi"
  [(set (match_operand:CC 0 "cc_reg_operand" "=y")
        (unspec:CC [(match_operand:SI 1 "gpc_reg_operand" "r")
		    (match_operand 2 "immediate_operand" "n")]
		   UNSPEC_MOVESI_TO_CR))]
  "GET_CODE (operands[0]) == REG
   && CR_REGNO_P (REGNO (operands[0]))
   && GET_CODE (operands[2]) == CONST_INT
   && INTVAL (operands[2]) == 1 << (75 - REGNO (operands[0]))"
  "mtcrf %R0,%1"
  [(set_attr "type" "mtcr")])

; The load-multiple instructions have similar properties.
; Note that "load_multiple" is a name known to the machine-independent
; code that actually corresponds to the PowerPC load-string.

(define_insn "*lmw"
  [(match_parallel 0 "lmw_operation"
		   [(set (match_operand:SI 1 "gpc_reg_operand" "=r")
       			 (match_operand:SI 2 "memory_operand" "m"))])]
  "TARGET_MULTIPLE"
  "lmw %1,%2"
  [(set_attr "type" "load")
   (set_attr "update" "yes")
   (set_attr "indexed" "yes")
   (set_attr "cell_micro" "always")])

(define_insn "*return_internal_<mode>"
  [(simple_return)
   (use (match_operand:P 0 "register_operand" "lc"))]
  ""
  "b%T0"
  [(set_attr "type" "jmpreg")])

; FIXME: This would probably be somewhat simpler if the Cygnus sibcall
; stuff was in GCC.  Oh, and "any_parallel_operand" is a bit flexible...

; The following comment applies to:
;     save_gpregs_*
;     save_fpregs_*
;     restore_gpregs*
;     return_and_restore_gpregs*
;     return_and_restore_fpregs*
;     return_and_restore_fpregs_aix*
;
; The out-of-line save / restore functions expects one input argument.
; Since those are not standard call_insn's, we must avoid using
; MATCH_OPERAND for that argument. That way the register rename
; optimization will not try to rename this register.
; Each pattern is repeated for each possible register number used in 
; various ABIs (r11, r1, and for some functions r12)

(define_insn "*restore_gpregs_<mode>_r11"
 [(match_parallel 0 "any_parallel_operand"
                  [(clobber (match_operand:P 1 "register_operand" "=l"))
                   (use (match_operand:P 2 "symbol_ref_operand" "s"))
                   (use (reg:P 11))
		   (set (match_operand:P 3 "gpc_reg_operand" "=r")
			(match_operand:P 4 "memory_operand" "m"))])]
 ""
 "bl %2"
 [(set_attr "type" "branch")
  (set_attr "length" "4")])

(define_insn "*restore_gpregs_<mode>_r12"
 [(match_parallel 0 "any_parallel_operand"
                  [(clobber (match_operand:P 1 "register_operand" "=l"))
                   (use (match_operand:P 2 "symbol_ref_operand" "s"))
                   (use (reg:P 12))
		   (set (match_operand:P 3 "gpc_reg_operand" "=r")
			(match_operand:P 4 "memory_operand" "m"))])]
 ""
 "bl %2"
 [(set_attr "type" "branch")
  (set_attr "length" "4")])

(define_insn "*restore_gpregs_<mode>_r1"
 [(match_parallel 0 "any_parallel_operand"
                  [(clobber (match_operand:P 1 "register_operand" "=l"))
                   (use (match_operand:P 2 "symbol_ref_operand" "s"))
                   (use (reg:P 1))
		   (set (match_operand:P 3 "gpc_reg_operand" "=r")
			(match_operand:P 4 "memory_operand" "m"))])]
 ""
 "bl %2"
 [(set_attr "type" "branch")
  (set_attr "length" "4")])

(define_insn "*return_and_restore_gpregs_<mode>_r11"
 [(match_parallel 0 "any_parallel_operand"
                  [(return)
		   (clobber (match_operand:P 1 "register_operand" "=l"))
		   (use (match_operand:P 2 "symbol_ref_operand" "s"))
                   (use (reg:P 11))
		   (set (match_operand:P 3 "gpc_reg_operand" "=r")
			(match_operand:P 4 "memory_operand" "m"))])]
 ""
 "b %2"
 [(set_attr "type" "branch")
  (set_attr "length" "4")])

(define_insn "*return_and_restore_gpregs_<mode>_r12"
 [(match_parallel 0 "any_parallel_operand"
                  [(return)
		   (clobber (match_operand:P 1 "register_operand" "=l"))
		   (use (match_operand:P 2 "symbol_ref_operand" "s"))
                   (use (reg:P 12))
		   (set (match_operand:P 3 "gpc_reg_operand" "=r")
			(match_operand:P 4 "memory_operand" "m"))])]
 ""
 "b %2"
 [(set_attr "type" "branch")
  (set_attr "length" "4")])

(define_insn "*return_and_restore_gpregs_<mode>_r1"
 [(match_parallel 0 "any_parallel_operand"
                  [(return)
		   (clobber (match_operand:P 1 "register_operand" "=l"))
		   (use (match_operand:P 2 "symbol_ref_operand" "s"))
                   (use (reg:P 1))
		   (set (match_operand:P 3 "gpc_reg_operand" "=r")
			(match_operand:P 4 "memory_operand" "m"))])]
 ""
 "b %2"
 [(set_attr "type" "branch")
  (set_attr "length" "4")])

(define_insn "*return_and_restore_fpregs_<mode>_r11"
 [(match_parallel 0 "any_parallel_operand"
                  [(return)
		   (clobber (match_operand:P 1 "register_operand" "=l"))
		   (use (match_operand:P 2 "symbol_ref_operand" "s"))
                   (use (reg:P 11))
		   (set (match_operand:DF 3 "gpc_reg_operand" "=d")
			(match_operand:DF 4 "memory_operand" "m"))])]
 ""
 "b %2"
 [(set_attr "type" "branch")
  (set_attr "length" "4")])

(define_insn "*return_and_restore_fpregs_<mode>_r12"
 [(match_parallel 0 "any_parallel_operand"
                  [(return)
		   (clobber (match_operand:P 1 "register_operand" "=l"))
		   (use (match_operand:P 2 "symbol_ref_operand" "s"))
                   (use (reg:P 12))
		   (set (match_operand:DF 3 "gpc_reg_operand" "=d")
			(match_operand:DF 4 "memory_operand" "m"))])]
 ""
 "b %2"
 [(set_attr "type" "branch")
  (set_attr "length" "4")])

(define_insn "*return_and_restore_fpregs_<mode>_r1"
 [(match_parallel 0 "any_parallel_operand"
                  [(return)
		   (clobber (match_operand:P 1 "register_operand" "=l"))
		   (use (match_operand:P 2 "symbol_ref_operand" "s"))
                   (use (reg:P 1))
		   (set (match_operand:DF 3 "gpc_reg_operand" "=d")
			(match_operand:DF 4 "memory_operand" "m"))])]
 ""
 "b %2"
 [(set_attr "type" "branch")
  (set_attr "length" "4")])

(define_insn "*return_and_restore_fpregs_aix_<mode>_r11"
 [(match_parallel 0 "any_parallel_operand"
		  [(return)
		   (use (match_operand:P 1 "register_operand" "l"))
		   (use (match_operand:P 2 "symbol_ref_operand" "s"))
		   (use (reg:P 11))
		   (set (match_operand:DF 3 "gpc_reg_operand" "=d")
			(match_operand:DF 4 "memory_operand" "m"))])]
 ""
 "b %2"
 [(set_attr "type" "branch")
  (set_attr "length" "4")])

(define_insn "*return_and_restore_fpregs_aix_<mode>_r1"
 [(match_parallel 0 "any_parallel_operand"
		  [(return)
		   (use (match_operand:P 1 "register_operand" "l"))
		   (use (match_operand:P 2 "symbol_ref_operand" "s"))
		   (use (reg:P 1))
		   (set (match_operand:DF 3 "gpc_reg_operand" "=d")
			(match_operand:DF 4 "memory_operand" "m"))])]
 ""
 "b %2"
 [(set_attr "type" "branch")
  (set_attr "length" "4")])

; This is used in compiling the unwind routines.
(define_expand "eh_return"
  [(use (match_operand 0 "general_operand" ""))]
  ""
  "
{
  if (TARGET_32BIT)
    emit_insn (gen_eh_set_lr_si (operands[0]));
  else
    emit_insn (gen_eh_set_lr_di (operands[0]));
  DONE;
}")

; We can't expand this before we know where the link register is stored.
(define_insn "eh_set_lr_<mode>"
  [(unspec_volatile [(match_operand:P 0 "register_operand" "r")]
  		    UNSPECV_EH_RR)
   (clobber (match_scratch:P 1 "=&b"))]
  ""
  "#")

(define_split
  [(unspec_volatile [(match_operand 0 "register_operand" "")] UNSPECV_EH_RR)
   (clobber (match_scratch 1 ""))]
  "reload_completed"
  [(const_int 0)]
  "
{
  rs6000_emit_eh_reg_restore (operands[0], operands[1]);
  DONE;
}")

(define_insn "prefetch"
  [(prefetch (match_operand 0 "indexed_or_indirect_address" "a")
	     (match_operand:SI 1 "const_int_operand" "n")
	     (match_operand:SI 2 "const_int_operand" "n"))]
  ""
  "*
{
  if (GET_CODE (operands[0]) == REG)
    return INTVAL (operands[1]) ? \"dcbtst 0,%0\" : \"dcbt 0,%0\";
  return INTVAL (operands[1]) ? \"dcbtst %a0\" : \"dcbt %a0\";
}"
  [(set_attr "type" "load")])

(define_insn "bpermd_<mode>"
  [(set (match_operand:P 0 "gpc_reg_operand" "=r")
	(unspec:P [(match_operand:P 1 "gpc_reg_operand" "r")
		   (match_operand:P 2 "gpc_reg_operand" "r")] UNSPEC_BPERM))]
  "TARGET_POPCNTD"
  "bpermd %0,%1,%2"
  [(set_attr "type" "popcnt")])


;; Builtin fma support.  Handle 
;; Note that the conditions for expansion are in the FMA_F iterator.

(define_expand "fma<mode>4"
  [(set (match_operand:FMA_F 0 "register_operand" "")
	(fma:FMA_F
	  (match_operand:FMA_F 1 "register_operand" "")
	  (match_operand:FMA_F 2 "register_operand" "")
	  (match_operand:FMA_F 3 "register_operand" "")))]
  ""
  "")

(define_insn "*fma<mode>4_fpr"
  [(set (match_operand:SFDF 0 "gpc_reg_operand" "=<Ff>,<Fv>,<Fv>")
	(fma:SFDF
	  (match_operand:SFDF 1 "gpc_reg_operand" "%<Ff>,<Fv>,<Fv>")
	  (match_operand:SFDF 2 "gpc_reg_operand" "<Ff>,<Fv>,0")
	  (match_operand:SFDF 3 "gpc_reg_operand" "<Ff>,0,<Fv>")))]
  "TARGET_<MODE>_FPR"
  "@
   fmadd<Ftrad> %0,%1,%2,%3
   xsmadda<Fvsx> %x0,%x1,%x2
   xsmaddm<Fvsx> %x0,%x1,%x3"
  [(set_attr "type" "fp")
   (set_attr "fp_type" "fp_maddsub_<Fs>")])

; Altivec only has fma and nfms.
(define_expand "fms<mode>4"
  [(set (match_operand:FMA_F 0 "register_operand" "")
	(fma:FMA_F
	  (match_operand:FMA_F 1 "register_operand" "")
	  (match_operand:FMA_F 2 "register_operand" "")
	  (neg:FMA_F (match_operand:FMA_F 3 "register_operand" ""))))]
  "!VECTOR_UNIT_ALTIVEC_P (<MODE>mode)"
  "")

(define_insn "*fms<mode>4_fpr"
  [(set (match_operand:SFDF 0 "gpc_reg_operand" "=<Ff>,<Fv>,<Fv>")
	(fma:SFDF
	 (match_operand:SFDF 1 "gpc_reg_operand" "<Ff>,<Fv>,<Fv>")
	 (match_operand:SFDF 2 "gpc_reg_operand" "<Ff>,<Fv>,0")
	 (neg:SFDF (match_operand:SFDF 3 "gpc_reg_operand" "<Ff>,0,<Fv>"))))]
  "TARGET_<MODE>_FPR"
  "@
   fmsub<Ftrad> %0,%1,%2,%3
   xsmsuba<Fvsx> %x0,%x1,%x2
   xsmsubm<Fvsx> %x0,%x1,%x3"
  [(set_attr "type" "fp")
   (set_attr "fp_type" "fp_maddsub_<Fs>")])

;; If signed zeros are ignored, -(a * b - c) = -a * b + c.
(define_expand "fnma<mode>4"
  [(set (match_operand:FMA_F 0 "register_operand" "")
	(neg:FMA_F
	  (fma:FMA_F
	    (match_operand:FMA_F 1 "register_operand" "")
	    (match_operand:FMA_F 2 "register_operand" "")
	    (neg:FMA_F (match_operand:FMA_F 3 "register_operand" "")))))]
  "!HONOR_SIGNED_ZEROS (<MODE>mode)"
  "")

;; If signed zeros are ignored, -(a * b + c) = -a * b - c.
(define_expand "fnms<mode>4"
  [(set (match_operand:FMA_F 0 "register_operand" "")
	(neg:FMA_F
	  (fma:FMA_F
	    (match_operand:FMA_F 1 "register_operand" "")
	    (match_operand:FMA_F 2 "register_operand" "")
	    (match_operand:FMA_F 3 "register_operand" ""))))]
  "!HONOR_SIGNED_ZEROS (<MODE>mode) && !VECTOR_UNIT_ALTIVEC_P (<MODE>mode)"
  "")

; Not an official optab name, but used from builtins.
(define_expand "nfma<mode>4"
  [(set (match_operand:FMA_F 0 "register_operand" "")
	(neg:FMA_F
	  (fma:FMA_F
	    (match_operand:FMA_F 1 "register_operand" "")
	    (match_operand:FMA_F 2 "register_operand" "")
	    (match_operand:FMA_F 3 "register_operand" ""))))]
  "!VECTOR_UNIT_ALTIVEC_P (<MODE>mode)"
  "")

(define_insn "*nfma<mode>4_fpr"
  [(set (match_operand:SFDF 0 "gpc_reg_operand" "=<Ff>,<Fv>,<Fv>")
	(neg:SFDF
	 (fma:SFDF
	  (match_operand:SFDF 1 "gpc_reg_operand" "<Ff>,<Fv>,<Fv>")
	  (match_operand:SFDF 2 "gpc_reg_operand" "<Ff>,<Fv>,0")
	  (match_operand:SFDF 3 "gpc_reg_operand" "<Ff>,0,<Fv>"))))]
  "TARGET_<MODE>_FPR"
  "@
   fnmadd<Ftrad> %0,%1,%2,%3
   xsnmadda<Fvsx> %x0,%x1,%x2
   xsnmaddm<Fvsx> %x0,%x1,%x3"
  [(set_attr "type" "fp")
   (set_attr "fp_type" "fp_maddsub_<Fs>")])

; Not an official optab name, but used from builtins.
(define_expand "nfms<mode>4"
  [(set (match_operand:FMA_F 0 "register_operand" "")
	(neg:FMA_F
	  (fma:FMA_F
	    (match_operand:FMA_F 1 "register_operand" "")
	    (match_operand:FMA_F 2 "register_operand" "")
	    (neg:FMA_F (match_operand:FMA_F 3 "register_operand" "")))))]
  ""
  "")

(define_insn "*nfmssf4_fpr"
  [(set (match_operand:SFDF 0 "gpc_reg_operand" "=<Ff>,<Fv>,<Fv>")
	(neg:SFDF
	 (fma:SFDF
	  (match_operand:SFDF 1 "gpc_reg_operand" "<Ff>,<Fv>,<Fv>")
	  (match_operand:SFDF 2 "gpc_reg_operand" "<Ff>,<Fv>,0")
	  (neg:SFDF
	   (match_operand:SFDF 3 "gpc_reg_operand" "<Ff>,0,<Fv>")))))]
  "TARGET_<MODE>_FPR"
  "@
   fnmsub<Ftrad> %0,%1,%2,%3
   xsnmsuba<Fvsx> %x0,%x1,%x2
   xsnmsubm<Fvsx> %x0,%x1,%x3"
  [(set_attr "type" "fp")
   (set_attr "fp_type" "fp_maddsub_<Fs>")])


(define_expand "rs6000_get_timebase"
  [(use (match_operand:DI 0 "gpc_reg_operand" ""))]
  ""
{
  if (TARGET_POWERPC64)
    emit_insn (gen_rs6000_mftb_di (operands[0]));
  else
    emit_insn (gen_rs6000_get_timebase_ppc32 (operands[0]));
  DONE;
})

(define_insn "rs6000_get_timebase_ppc32"
  [(set (match_operand:DI 0 "gpc_reg_operand" "=r")
        (unspec_volatile:DI [(const_int 0)] UNSPECV_MFTB))
   (clobber (match_scratch:SI 1 "=r"))
   (clobber (match_scratch:CC 2 "=y"))]
  "!TARGET_POWERPC64"
{
  if (WORDS_BIG_ENDIAN)
    if (TARGET_MFCRF)
      {
        return "mfspr %0,269\;"
	       "mfspr %L0,268\;"
	       "mfspr %1,269\;"
	       "cmpw %2,%0,%1\;"
	       "bne- %2,$-16";
      }
    else
      {
        return "mftbu %0\;"
	       "mftb %L0\;"
	       "mftbu %1\;"
	       "cmpw %2,%0,%1\;"
	       "bne- %2,$-16";
      }
  else
    if (TARGET_MFCRF)
      {
        return "mfspr %L0,269\;"
	       "mfspr %0,268\;"
	       "mfspr %1,269\;"
	       "cmpw %2,%L0,%1\;"
	       "bne- %2,$-16";
      }
    else
      {
        return "mftbu %L0\;"
	       "mftb %0\;"
	       "mftbu %1\;"
	       "cmpw %2,%L0,%1\;"
	       "bne- %2,$-16";
      }
}
  [(set_attr "length" "20")])

(define_insn "rs6000_mftb_<mode>"
  [(set (match_operand:GPR 0 "gpc_reg_operand" "=r")
        (unspec_volatile:GPR [(const_int 0)] UNSPECV_MFTB))]
  ""
{
  if (TARGET_MFCRF)
    return "mfspr %0,268";
  else
    return "mftb %0";
})


(define_insn "rs6000_mffs"
  [(set (match_operand:DF 0 "gpc_reg_operand" "=d")
	(unspec_volatile:DF [(const_int 0)] UNSPECV_MFFS))]
  "TARGET_HARD_FLOAT && TARGET_FPRS"
  "mffs %0")

(define_insn "rs6000_mtfsf"
  [(unspec_volatile [(match_operand:SI 0 "const_int_operand" "i")
		     (match_operand:DF 1 "gpc_reg_operand" "d")]
		    UNSPECV_MTFSF)]
  "TARGET_HARD_FLOAT && TARGET_FPRS"
  "mtfsf %0,%1")


;; Power8 fusion support for fusing an addis instruction with a D-form load of
;; a GPR.  The addis instruction must be adjacent to the load, and use the same
;; register that is being loaded.  The fused ops must be physically adjacent.

;; Find cases where the addis that feeds into a load instruction is either used
;; once or is the same as the target register, and replace it with the fusion
;; insn

(define_peephole2
  [(set (match_operand:P 0 "base_reg_operand" "")
	(match_operand:P 1 "fusion_gpr_addis" ""))
   (set (match_operand:INT1 2 "base_reg_operand" "")
	(match_operand:INT1 3 "fusion_gpr_mem_load" ""))]
  "TARGET_P8_FUSION
   && fusion_gpr_load_p (operands[0], operands[1], operands[2],
			 operands[3])"
  [(const_int 0)]
{
  expand_fusion_gpr_load (operands);
  DONE;
})

;; Fusion insn, created by the define_peephole2 above (and eventually by
;; reload)

(define_insn "fusion_gpr_load_<mode>"
  [(set (match_operand:INT1 0 "base_reg_operand" "=&b")
	(unspec:INT1 [(match_operand:INT1 1 "fusion_gpr_mem_combo" "")]
		     UNSPEC_FUSION_GPR))]
  "TARGET_P8_FUSION"
{
  return emit_fusion_gpr_load (operands[0], operands[1]);
}
  [(set_attr "type" "load")
   (set_attr "length" "8")])


;; Miscellaneous ISA 2.06 (power7) instructions
(define_insn "addg6s"
  [(set (match_operand:SI 0 "register_operand" "=r")
	(unspec:SI [(match_operand:SI 1 "register_operand" "r")
		    (match_operand:SI 2 "register_operand" "r")]
		   UNSPEC_ADDG6S))]
  "TARGET_POPCNTD"
  "addg6s %0,%1,%2"
  [(set_attr "type" "integer")
   (set_attr "length" "4")])

(define_insn "cdtbcd"
  [(set (match_operand:SI 0 "register_operand" "=r")
	(unspec:SI [(match_operand:SI 1 "register_operand" "r")]
		   UNSPEC_CDTBCD))]
  "TARGET_POPCNTD"
  "cdtbcd %0,%1"
  [(set_attr "type" "integer")
   (set_attr "length" "4")])

(define_insn "cbcdtd"
  [(set (match_operand:SI 0 "register_operand" "=r")
	(unspec:SI [(match_operand:SI 1 "register_operand" "r")]
		   UNSPEC_CBCDTD))]
  "TARGET_POPCNTD"
  "cbcdtd %0,%1"
  [(set_attr "type" "integer")
   (set_attr "length" "4")])

(define_int_iterator UNSPEC_DIV_EXTEND [UNSPEC_DIVE
					UNSPEC_DIVEO
					UNSPEC_DIVEU
					UNSPEC_DIVEUO])

(define_int_attr div_extend [(UNSPEC_DIVE	"e")
			     (UNSPEC_DIVEO	"eo")
			     (UNSPEC_DIVEU	"eu")
			     (UNSPEC_DIVEUO	"euo")])

(define_insn "div<div_extend>_<mode>"
  [(set (match_operand:GPR 0 "register_operand" "=r")
	(unspec:GPR [(match_operand:GPR 1 "register_operand" "r")
		     (match_operand:GPR 2 "register_operand" "r")]
		    UNSPEC_DIV_EXTEND))]
  "TARGET_POPCNTD"
  "div<wd><div_extend> %0,%1,%2"
  [(set_attr "type" "div")
   (set_attr "size" "<bits>")])


;; Pack/unpack 128-bit floating point types that take 2 scalar registers

; Type of the 64-bit part when packing/unpacking 128-bit floating point types
(define_mode_attr FP128_64 [(TF "DF") (TD "DI")])

(define_expand "unpack<mode>"
  [(set (match_operand:<FP128_64> 0 "nonimmediate_operand" "")
	(unspec:<FP128_64>
	 [(match_operand:FMOVE128 1 "register_operand" "")
	  (match_operand:QI 2 "const_0_to_1_operand" "")]
	 UNSPEC_UNPACK_128BIT))]
  ""
  "")

(define_insn_and_split "unpack<mode>_dm"
  [(set (match_operand:<FP128_64> 0 "nonimmediate_operand" "=d,m,d,r,m")
	(unspec:<FP128_64>
	 [(match_operand:FMOVE128 1 "register_operand" "d,d,r,d,r")
	  (match_operand:QI 2 "const_0_to_1_operand" "i,i,i,i,i")]
	 UNSPEC_UNPACK_128BIT))]
  "TARGET_POWERPC64 && TARGET_DIRECT_MOVE"
  "#"
  "&& reload_completed"
  [(set (match_dup 0) (match_dup 3))]
{
  unsigned fp_regno = REGNO (operands[1]) + UINTVAL (operands[2]);

  if (REG_P (operands[0]) && REGNO (operands[0]) == fp_regno)
    {
      emit_note (NOTE_INSN_DELETED);
      DONE;
    }

  operands[3] = gen_rtx_REG (<FP128_64>mode, fp_regno);
}
  [(set_attr "type" "fp,fpstore,mffgpr,mftgpr,store")
   (set_attr "length" "4")])

(define_insn_and_split "unpack<mode>_nodm"
  [(set (match_operand:<FP128_64> 0 "nonimmediate_operand" "=d,m")
	(unspec:<FP128_64>
	 [(match_operand:FMOVE128 1 "register_operand" "d,d")
	  (match_operand:QI 2 "const_0_to_1_operand" "i,i")]
	 UNSPEC_UNPACK_128BIT))]
  "!TARGET_POWERPC64 || !TARGET_DIRECT_MOVE"
  "#"
  "&& reload_completed"
  [(set (match_dup 0) (match_dup 3))]
{
  unsigned fp_regno = REGNO (operands[1]) + UINTVAL (operands[2]);

  if (REG_P (operands[0]) && REGNO (operands[0]) == fp_regno)
    {
      emit_note (NOTE_INSN_DELETED);
      DONE;
    }

  operands[3] = gen_rtx_REG (<FP128_64>mode, fp_regno);
}
  [(set_attr "type" "fp,fpstore")
   (set_attr "length" "4")])

(define_insn_and_split "pack<mode>"
  [(set (match_operand:FMOVE128 0 "register_operand" "=d,&d")
	(unspec:FMOVE128
	 [(match_operand:<FP128_64> 1 "register_operand" "0,d")
	  (match_operand:<FP128_64> 2 "register_operand" "d,d")]
	 UNSPEC_PACK_128BIT))]
  ""
  "@
   fmr %L0,%2
   #"
  "&& reload_completed && REGNO (operands[0]) != REGNO (operands[1])"
  [(set (match_dup 3) (match_dup 1))
   (set (match_dup 4) (match_dup 2))]
{
  unsigned dest_hi = REGNO (operands[0]);
  unsigned dest_lo = dest_hi + 1;

  gcc_assert (!IN_RANGE (REGNO (operands[1]), dest_hi, dest_lo));
  gcc_assert (!IN_RANGE (REGNO (operands[2]), dest_hi, dest_lo));

  operands[3] = gen_rtx_REG (<FP128_64>mode, dest_hi);
  operands[4] = gen_rtx_REG (<FP128_64>mode, dest_lo);
}
  [(set_attr "type" "fp,fp")
   (set_attr "length" "4,8")])

(define_insn "unpackv1ti"
  [(set (match_operand:DI 0 "register_operand" "=d,d")
	(unspec:DI [(match_operand:V1TI 1 "register_operand" "0,wa")
		    (match_operand:QI 2 "const_0_to_1_operand" "O,i")]
	 UNSPEC_UNPACK_128BIT))]
  "TARGET_VSX"
{
  if (REGNO (operands[0]) == REGNO (operands[1]) && INTVAL (operands[2]) == 0)
    return ASM_COMMENT_START " xxpermdi to same register";

  operands[3] = GEN_INT (INTVAL (operands[2]) == 0 ? 0 : 3);
  return "xxpermdi %x0,%x1,%x1,%3";
}
  [(set_attr "type" "vecperm")
   (set_attr "length" "4")])

(define_insn "packv1ti"
  [(set (match_operand:V1TI 0 "register_operand" "=wa")
	(unspec:V1TI
	 [(match_operand:DI 1 "register_operand" "d")
	  (match_operand:DI 2 "register_operand" "d")]
	 UNSPEC_PACK_128BIT))]
  "TARGET_VSX"
  "xxpermdi %x0,%x1,%x2,0"
  [(set_attr "type" "vecperm")
   (set_attr "length" "4")])



(include "sync.md")
(include "vector.md")
(include "vsx.md")
(include "altivec.md")
(include "spe.md")
(include "dfp.md")
(include "paired.md")
(include "crypto.md")
(include "htm.md")
