vec_int64_ppc.h

Go to the documentation of this file.

/*
 Copyright (c) [2018] IBM Corporation.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
 You may obtain a copy of the License at
 
    http://www.apache.org/licenses/LICENSE-2.0
 
 Unless required by applicable law or agreed to in writing, software
 distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 
 vec_int32_ppc.h
 
 Contributors:
      IBM Corporation, Steven Munroe
      Created on: Mar 29, 2018
 */
 
#ifndef VEC_INT64_PPC_H_
#define VEC_INT64_PPC_H_
 
#include <pveclib/vec_int32_ppc.h>
 
static inline vb64_t vec_cmpgtsd (vi64_t a, vi64_t b);
static inline vb64_t vec_cmpequd (vui64_t a, vui64_t b);
static inline vb64_t vec_cmpgeud (vui64_t a, vui64_t b);
static inline vb64_t vec_cmpgtud (vui64_t a, vui64_t b);
static inline vb64_t vec_cmpneud (vui64_t a, vui64_t b);
static inline vui64_t vec_sldi (vui64_t vra, const unsigned int shb);
static inline vui64_t vec_maxud (vui64_t vra, vui64_t vrb);
static inline vui64_t vec_minud (vui64_t vra, vui64_t vrb);
static inline vui64_t vec_permdi (vui64_t vra, vui64_t vrb, const int ctl);
#ifndef vec_popcntd
static inline vui64_t vec_popcntd (vui64_t vra);
#else
/* Work around for GCC PR85830.  */
#undef vec_popcntd
#define vec_popcntd __builtin_vec_vpopcntd
#endif
static inline vi64_t vec_splat_s64 (const int sim);
static inline vui64_t vec_subudm (vui64_t a, vui64_t b);
static inline vui64_t
vec_vlsidx (const signed long long a, const unsigned long long *b);
static inline void
vec_vstsidx (vui64_t xs, const signed long long ra, unsigned long long *rb);
static inline vui64_t vec_xxspltd (vui64_t vra, const int ctl);
 
static inline vui64_t
vec_absdud (vui64_t vra, vui64_t vrb)
{
  return vec_subudm (vec_maxud (vra, vrb), vec_minud (vra, vrb));
}
 
static inline
vui64_t
vec_addudm(vui64_t a, vui64_t b)
{
  vui32_t r;
 
#ifdef _ARCH_PWR8
#if defined (vec_vaddudm)
  r = (vui32_t) vec_vaddudm (a, b);
#elif defined (__clang__)
  r = (vui32_t) vec_add (a, b);
#else
  __asm__(
      "vaddudm %0,%1,%2;"
      : "=v" (r)
      : "v" (a),
      "v" (b)
      : );
#endif
#else
  vui32_t c;
  vui32_t z= { 0,0,0,0};
  vui32_t cm= { 0,1,0,1};
 
  c = vec_vaddcuw ((vui32_t)a, (vui32_t)b);
  r = vec_vadduwm ((vui32_t)a, (vui32_t)b);
  c = vec_and (c, cm);
  c = vec_sld (c, z, 4);
  r = vec_vadduwm (r, c);
#endif
  return ((vui64_t) r);
}
 
static inline vui64_t
vec_clzd (vui64_t vra)
{
  vui64_t r;
#ifdef _ARCH_PWR8
#if defined (vec_vclzd)
  r = vec_vclzd (vra);
#elif defined (__clang__)
  r = vec_cntlz (vra);
#else
  __asm__(
      "vclzd %0,%1;"
      : "=v" (r)
      : "v" (vra)
      : );
#endif
#else
  vui32_t n, nt, y, x, m;
  vui32_t z = { 0, 0, 0, 0 };
  vui32_t dlwm = { 0, -1, 0, -1 };
 
  x = (vui32_t) vra;
 
  m = (vui32_t) vec_cmpgt (x, z);
  n = vec_sld (z, m, 12);
  y = vec_and (n, dlwm);
  nt = vec_or (x, y);
 
  n = vec_clzw (nt);
  r = (vui64_t) vec_vsum2sw ((vi32_t) n, (vi32_t) z);
#endif
  return (r);
}
 
static inline vui64_t
vec_ctzd (vui64_t vra)
{
  vui64_t r;
#ifdef _ARCH_PWR9
#if defined (vec_cnttz) || defined (__clang__)
  r = vec_cnttz (vra);
#else
  __asm__(
      "vctzd %0,%1;"
      : "=v" (r)
      : "v" (vra)
      : );
#endif
#else
// For _ARCH_PWR8 and earlier. Generate 1's for the trailing zeros
// and 0's otherwise. Then count (popcnt) the 1's. _ARCH_PWR8 uses
// the hardware vpopcntd instruction. _ARCH_PWR7 and earlier use the
// PVECLIB vec_popcntd implementation which runs ~24-33 instructions.
  const vui64_t ones = { -1, -1 };
  vui64_t tzmask;
  // tzmask = (!vra & (vra - 1))
  tzmask = vec_andc (vec_addudm (vra, ones), vra);
  // return = vec_popcnt (!vra & (vra - 1))
  r = vec_popcntd (tzmask);
#endif
  return ((vui64_t) r);
}
 
static inline
vb64_t
vec_cmpeqsd (vi64_t a, vi64_t b)
{
  /* vcmpequd works for both signed and unsigned compares.  */
  return vec_cmpequd ((vui64_t) a, (vui64_t) b);
}
 
static inline
vb64_t
vec_cmpequd (vui64_t a, vui64_t b)
{
  vb64_t result;
#ifdef _ARCH_PWR8
#if __GNUC__ >= 6
  result = vec_cmpeq(a, b);
#else
  __asm__(
      "vcmpequd %0,%1,%2;\n"
      : "=v" (result)
      : "v" (a),
      "v" (b)
      : );
#endif
#else
  /*
   * Don't have vector compare equal unsigned doubleword until power8.
   * So we have to compare word and unless all_eq we need to do some
   * extra work, ie the words may have different truth values.  So we
   * rotate each doubleword by 32-bits (here we use permute as we don't
   * have rotate doubleword either). Then vand the original word
   * compare and rotated value to get the final value.
   */
  vui8_t permute =
    { 0x04,0x05,0x6,0x7, 0x00,0x01,0x2,0x03, 0x0C,0x0D,0x0E,0x0F, 0x08,0x09,0x0A,0x0B};
  vui32_t r, rr;
  r = (vui32_t) vec_cmpeq ((vui32_t) a, (vui32_t) b);
  if (vec_any_ne ((vui32_t) a, (vui32_t) b))
    {
       rr = vec_perm (r, r, permute);
       r= vec_and (r, rr);
    }
  result = (vb64_t)r;
#endif
  return (result);
}
 
static inline
vb64_t
vec_cmpgesd (vi64_t a, vi64_t b)
{
  vb64_t r;
  /* vec_cmpge is implemented as the not of vec_cmplt. And vec_cmplt
     is implemented as vec_cmpgt with parms reversed.  */
  r = vec_cmpgtsd (b, a);
  return vec_nor (r, r);
}
 
static inline
vb64_t
vec_cmpgeud (vui64_t a, vui64_t b)
{
  vb64_t r;
  /* vec_cmpge is implemented as the not of vec_cmplt. And vec_cmplt
     is implemented as vec_cmpgt with parms reversed.  */
  r = vec_cmpgtud (b, a);
  return vec_nor (r, r);
}
 
static inline
vb64_t
vec_cmpgtsd (vi64_t a, vi64_t b)
{
  vb64_t result;
#ifdef _ARCH_PWR8
#if __GNUC__ >= 6
  result = vec_cmpgt(a, b);
#else
  __asm__(
      "vcmpgtsd %0,%1,%2;\n"
      : "=v" (result)
      : "v" (a),
      "v" (b)
      : );
#endif
#else
  vui64_t _A, _B;
  const vui64_t signmask = CONST_VINT128_DW(0x8000000000000000UL,
                                            0x8000000000000000UL);
  /* For a signed compare we can flip the sign bits, which give
     unsigned magnitudes, that retain the correct relative different.
   */
  _A = vec_xor ((vui64_t)a, signmask);
  _B = vec_xor ((vui64_t)b, signmask);
  result = vec_cmpgtud (_A, _B);
#endif
  return (result);
}
 
static inline
vb64_t
vec_cmpgtud (vui64_t a, vui64_t b)
{
  vb64_t result;
#ifdef _ARCH_PWR8
#if __GNUC__ >= 6
  result = vec_cmpgt(a, b);
#else
  __asm__(
      "vcmpgtud %0,%1,%2;\n"
      : "=v" (result)
      : "v" (a),
      "v" (b)
      : );
#endif
#else
  /*
   * Don't have vector compare greater than unsigned doubleword until
   * power8.  So we have to use compare word and logic to compute the
   * doubleword truth values.
   */
  __vector unsigned int r, x, y;
  __vector unsigned int c0, c1, c01;
  __vector unsigned int eq, gt, a32, b32;
 
  /* c10 = {0, -1, 0, -1}  */
  c0 = vec_splat_u32 (0);
  c1 = vec_splat_u32 (-1);
  c01 = vec_mergeh (c0, c1);
 
  a32 = (__vector unsigned int)a;
  b32 = (__vector unsigned int)b;
 
  gt = (__vector unsigned int)vec_cmpgt (a32, b32);
  eq = (__vector unsigned int)vec_cmpeq (a32, b32);
  /* GTxw = GThw | (EQhw & GTlw)  */
  x = vec_sld (gt, c0, 4);
  y = vec_and (eq, x);
  x = vec_or  (gt, y);
  /* Duplicate result word to dword width.  */
  y = vec_sld (c0, x, 12);
  r = vec_sel (x, y, c01);
  result = (vb64_t)r;
#endif
  return (result);
}
 
static inline
vb64_t
vec_cmplesd (vi64_t a, vi64_t b)
{
  vb64_t result;
  /* vec_cmple is implemented as the not of vec_cmpgt.   */
  result = vec_cmpgtsd (a, b);
  return vec_nor (result, result);
}
 
static inline
vb64_t
vec_cmpleud (vui64_t a, vui64_t b)
{
  vb64_t result;
  /* vec_cmple is implemented as the not of vec_cmpgt.   */
  result = vec_cmpgtud (a, b);
  return vec_nor (result, result);
}
 
static inline
vb64_t
vec_cmpltsd (vi64_t a, vi64_t b)
{
  return vec_cmpgtsd (b, a);
}
 
static inline
vb64_t
vec_cmpltud (vui64_t a, vui64_t b)
{
  return vec_cmpgtud (b, a);
}
 
static inline
vb64_t
vec_cmpnesd (vi64_t a, vi64_t b)
{
  return vec_cmpneud ((vui64_t) a, (vui64_t) b);
}
 
static inline
vb64_t
vec_cmpneud (vui64_t a, vui64_t b)
{
  vb64_t r;
  /* vec_cmpne is implemented as the not of vec_cmpeq.  */
  r = vec_cmpequd (a, b);
  return vec_nor (r, r);
}
 
static inline
int
vec_cmpsd_all_eq (vi64_t a, vi64_t b)
{
  int result;
#if defined (_ARCH_PWR8) && (__GNUC__ >= 6) && (__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__)
  result = vec_all_eq(a, b);
#else
  result = vec_all_eq((vui32_t)a, (vui32_t)b);
#endif
  return (result);
}
 
static inline
int
vec_cmpsd_all_ge (vi64_t a, vi64_t b)
{
  int result;
#if defined (_ARCH_PWR8) && (__GNUC__ >= 6) && (__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__)
  result = vec_all_ge(a, b);
#else
  vui32_t wt = { -1, -1, -1, -1};
  vb64_t gt_bool = vec_cmpgesd (a, b);
  result = vec_all_eq((vui32_t)gt_bool, wt);
#endif
  return (result);
}
 
static inline
int
vec_cmpsd_all_gt (vi64_t a, vi64_t b)
{
  int result;
#if defined (_ARCH_PWR8) && (__GNUC__ >= 6) && (__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__)
  result = vec_all_gt(a, b);
#else
  vui32_t wt = { -1, -1, -1, -1};
  vb64_t gt_bool = vec_cmpgtsd (a, b);
  result = vec_all_eq((vui32_t)gt_bool, wt);
#endif
  return (result);
}
 
static inline
int
vec_cmpsd_all_le (vi64_t a, vi64_t b)
{
  return vec_cmpsd_all_ge (b, a);
}
 
static inline
int
vec_cmpsd_all_lt (vi64_t a, vi64_t b)
{
  return vec_cmpsd_all_gt (b, a);
}
 
static inline
int
vec_cmpsd_all_ne (vi64_t a, vi64_t b)
{
  int result;
#if defined (_ARCH_PWR8) && (__GNUC__ >= 6) && (__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__)
  result = vec_all_ne(a, b);
#else
  vui32_t wt = { -1, -1, -1, -1};
  vb64_t gt_bool = vec_cmpneud ((vui64_t)a, (vui64_t)b);
  result = vec_all_eq((vui32_t)gt_bool, wt);
#endif
  return (result);
}
 
static inline
int
vec_cmpsd_any_eq (vi64_t a, vi64_t b)
{
  int result;
#if defined (_ARCH_PWR8) && (__GNUC__ >= 6) && (__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__)
  result = vec_any_eq(a, b);
#else
  vui32_t wt = { -1, -1, -1, -1};
  vb64_t gt_bool = vec_cmpequd ((vui64_t)a, (vui64_t)b);
  result = vec_any_eq((vui32_t)gt_bool, wt);
#endif
  return (result);
}
 
static inline
int
vec_cmpsd_any_ge (vi64_t a, vi64_t b)
{
  int result;
#if defined (_ARCH_PWR8) && (__GNUC__ >= 6) && (__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__)
  result = vec_any_ge(a, b);
#else
  vui32_t wt = { -1, -1, -1, -1};
  vb64_t gt_bool = vec_cmpgesd (a, b);
  result = vec_any_eq((vui32_t)gt_bool, wt);
#endif
  return (result);
}
 
static inline
int
vec_cmpsd_any_gt (vi64_t a, vi64_t b)
{
  int result;
#if defined (_ARCH_PWR8) && (__GNUC__ >= 6) && (__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__)
  result = vec_any_gt(a, b);
#else
  vui32_t wt = { -1, -1, -1, -1};
  vb64_t gt_bool = vec_cmpgtsd (a, b);
  result = vec_any_eq((vui32_t)gt_bool, wt);
#endif
  return (result);
}
 
static inline
int
vec_cmpsd_any_le (vi64_t a, vi64_t b)
{
  return vec_cmpsd_any_ge (b, a);
}
 
static inline
int
vec_cmpsd_any_lt (vi64_t a, vi64_t b)
{
  return vec_cmpsd_any_gt (b, a);
}
 
static inline
int
vec_cmpsd_any_ne (vi64_t a, vi64_t b)
{
  int result;
#if defined (_ARCH_PWR8) && (__GNUC__ >= 6) && (__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__)
  result = vec_any_ne(a, b);
#else
  vui32_t wt = { -1, -1, -1, -1};
  vb64_t gt_bool = vec_cmpneud ((vui64_t)a, (vui64_t)b);
  result = vec_any_eq((vui32_t)gt_bool, wt);
#endif
  return (result);
}
 
static inline
int
vec_cmpud_all_eq (vui64_t a, vui64_t b)
{
  int result;
#if defined (_ARCH_PWR8) && (__GNUC__ >= 6) && (__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__)
  result = vec_all_eq(a, b);
#else
  result = vec_all_eq((vui32_t)a, (vui32_t)b);
#endif
  return (result);
}
 
static inline
int
vec_cmpud_all_ge (vui64_t a, vui64_t b)
{
  int result;
#if defined (_ARCH_PWR8) && (__GNUC__ >= 6) && (__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__)
  result = vec_all_ge(a, b);
#else
  vui32_t wt = { -1, -1, -1, -1};
  vb64_t gt_bool = vec_cmpgeud (a, b);
  result = vec_all_eq((vui32_t)gt_bool, wt);
#endif
  return (result);
}
 
static inline
int
vec_cmpud_all_gt (vui64_t a, vui64_t b)
{
  int result;
#if defined (_ARCH_PWR8) && (__GNUC__ >= 6) && (__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__)
  result = vec_all_gt(a, b);
#else
  vui32_t wt = { -1, -1, -1, -1};
  vb64_t gt_bool = vec_cmpgtud (a, b);
  result = vec_all_eq((vui32_t)gt_bool, wt);
#endif
  return (result);
}
 
static inline
int
vec_cmpud_all_le (vui64_t a, vui64_t b)
{
  return vec_cmpud_all_ge (b, a);
}
 
static inline
int
vec_cmpud_all_lt (vui64_t a, vui64_t b)
{
  return vec_cmpud_all_gt (b, a);
}
 
static inline
int
vec_cmpud_all_ne (vui64_t a, vui64_t b)
{
  int result;
#if defined (_ARCH_PWR8) && (__GNUC__ >= 6) && (__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__)
  result = vec_all_ne(a, b);
#else
  vui32_t wt = { -1, -1, -1, -1};
  vb64_t gt_bool = vec_cmpneud (a, b);
  result = vec_all_eq((vui32_t)gt_bool, wt);
#endif
  return (result);
}
 
static inline
int
vec_cmpud_any_eq (vui64_t a, vui64_t b)
{
  int result;
#if defined (_ARCH_PWR8) && (__GNUC__ >= 6) && (__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__)
  result = vec_any_eq(a, b);
#else
  vui32_t wt = { -1, -1, -1, -1};
  vb64_t gt_bool = vec_cmpequd (a, b);
  result = vec_any_eq((vui32_t)gt_bool, wt);
#endif
  return (result);
}
 
static inline
int
vec_cmpud_any_ge (vui64_t a, vui64_t b)
{
  int result;
#if defined (_ARCH_PWR8) && (__GNUC__ >= 6) && (__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__)
  result = vec_any_ge(a, b);
#else
  vui32_t wt = { -1, -1, -1, -1};
  vb64_t gt_bool = vec_cmpgeud (a, b);
  result = vec_any_eq((vui32_t)gt_bool, wt);
#endif
  return (result);
}
 
static inline
int
vec_cmpud_any_gt (vui64_t a, vui64_t b)
{
  int result;
#if defined (_ARCH_PWR8) && (__GNUC__ >= 6) && (__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__)
  result = vec_any_gt(a, b);
#else
  vui32_t wt = { -1, -1, -1, -1};
  vb64_t gt_bool = vec_cmpgtud (a, b);
  result = vec_any_eq((vui32_t)gt_bool, wt);
#endif
  return (result);
}
 
static inline
int
vec_cmpud_any_le (vui64_t a, vui64_t b)
{
  return vec_cmpud_any_ge (b, a);
}
 
static inline
int
vec_cmpud_any_lt (vui64_t a, vui64_t b)
{
  return vec_cmpud_any_gt (b, a);
}
 
static inline
int
vec_cmpud_any_ne (vui64_t a, vui64_t b)
{
  int result;
#if defined (_ARCH_PWR8) && (__GNUC__ >= 6) && (__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__)
  result = vec_any_ne(a, b);
#else
  vui32_t wt = { -1, -1, -1, -1};
  vb64_t gt_bool = vec_cmpneud (a, b);
  result = vec_any_eq((vui32_t)gt_bool, wt);
#endif
  return (result);
}
 
static inline
vi64_t
vec_maxsd (vi64_t vra, vi64_t vrb)
{
  vi64_t r;
 
#ifdef _ARCH_PWR8
#if defined (vec_vmaxsd)
  r = vec_vmaxsd (vra, vrb);
#elif defined (__clang__)
  r = vec_max (vra, vrb);
#else
  __asm__(
      "vmaxsd %0,%1,%2;"
      : "=v" (r)
      : "v" (vra),
      "v" (vrb)
      : );
#endif
#else
  vb64_t maxmask;
 
  maxmask = vec_cmpgtsd ( vra, vrb );
  r = vec_sel (vrb, vra, maxmask);
#endif
  return r;
}
 
static inline
vui64_t
vec_maxud (vui64_t vra, vui64_t vrb)
{
  vui64_t r;
 
#ifdef _ARCH_PWR8
#if defined (vec_vmaxud)
  r = vec_vmaxud (vra, vrb);
#elif defined (__clang__)
  r = vec_max (vra, vrb);
#else
  __asm__(
      "vmaxud %0,%1,%2;"
      : "=v" (r)
      : "v" (vra),
      "v" (vrb)
      : );
#endif
#else
  vb64_t maxmask;
 
  maxmask = vec_cmpgtud ( vra, vrb );
  r = vec_sel (vrb, vra, maxmask);
#endif
  return r;
}
 
static inline
vi64_t
vec_minsd (vi64_t vra, vi64_t vrb)
{
  vi64_t r;
 
#ifdef _ARCH_PWR8
#if defined (vec_vminsd)
  r = vec_vminsd (vra, vrb);
#elif defined (__clang__)
  r = vec_min (vra, vrb);
#else
  __asm__(
      "vminsd %0,%1,%2;"
      : "=v" (r)
      : "v" (vra),
      "v" (vrb)
      : );
#endif
#else
  vb64_t minmask;
 
  minmask = vec_cmpgtsd ( vrb, vra );
  r = vec_sel (vrb, vra, minmask);
#endif
  return r;
}
 
static inline
vui64_t
vec_minud (vui64_t vra, vui64_t vrb)
{
  vui64_t r;
 
#ifdef _ARCH_PWR8
#if defined (vec_vminud)
  r = vec_vminud (vra, vrb);
#elif defined (__clang__)
  r = vec_min (vra, vrb);
#else
  __asm__(
      "vminud %0,%1,%2;"
      : "=v" (r)
      : "v" (vra),
      "v" (vrb)
      : );
#endif
#else
  vb64_t minmask;
 
  minmask = vec_cmpgtud ( vrb, vra );
  r = vec_sel (vrb, vra, minmask);
#endif
  return r;
}
 
static inline vui64_t
vec_mrgahd (vui128_t vra, vui128_t vrb)
{
  return vec_permdi ((vui64_t) vra, (vui64_t) vrb, 0);
}
 
static inline vui64_t
vec_mrgald (vui128_t vra, vui128_t vrb)
{
  return vec_permdi ((vui64_t) vra, (vui64_t) vrb, 3);
}
 
static inline vui64_t
vec_mrged (vui64_t __VA, vui64_t __VB)
{
  vui64_t result;
  /*
   result[0] = __VA[0];
   result[1] = __VB[0];
   */
#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
  result = vec_permdi (__VB, __VA, 3);
#else
  result = vec_permdi (__VA, __VB, 0);
#endif
  return (result);
}
 
static inline vui64_t
vec_mrghd (vui64_t __VA, vui64_t __VB)
{
  vui64_t result;
  /*
   result[0] = __VA[0];
   result[1] = __VB[0];
   */
#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
  result = vec_permdi (__VB, __VA, 3);
#else
  result = vec_permdi (__VA, __VB, 0);
#endif
  return (result);
}
 
static inline vui64_t
vec_mrgld (vui64_t __VA, vui64_t __VB)
{
  vui64_t result;
  /*
   result[0] = __VA[1];
   result[1] = __VB[1];
   */
#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
  result = vec_permdi (__VB, __VA, 0);
#else
  result = vec_permdi (__VA, __VB, 3);
#endif
 
  return (result);
}
 
static inline vui64_t
vec_mrgod (vui64_t __VA, vui64_t __VB)
{
  vui64_t result;
  /*
   result[0] = __VA[1];
   result[1] = __VB[1];
   */
#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
  result = vec_permdi (__VB, __VA, 0);
#else
  result = vec_permdi (__VA, __VB, 3);
#endif
 
  return (result);
}
 
static inline vui128_t
vec_msumudm (vui64_t a, vui64_t b, vui128_t c);
 
static inline vui128_t
vec_muleud (vui64_t a, vui64_t b);
 
static inline vui64_t
vec_mulhud (vui64_t vra, vui64_t vrb);
 
static inline vui128_t
vec_muloud (vui64_t a, vui64_t b);
 
static inline vui64_t
vec_muludm (vui64_t vra, vui64_t vrb);
 
static inline vui64_t
vec_pasted (vui64_t __VH, vui64_t __VL)
{
  vui64_t result;
  /*
   result[1] = __VH[1];
   result[0] = __VL[0];
   */
  result = vec_permdi (__VH, __VL, 1);
 
  return (result);
}
 
static inline vui64_t
vec_permdi (vui64_t vra, vui64_t vrb, const int ctl)
{
  vui64_t result;
#ifdef _ARCH_PWR7
  switch (ctl & 3)
    {
#if (__BYTE_ORDER__ == __ORDER_BIG_ENDIAN__) || (defined (__clang__) && (__clang_major__ < 7))
    case 0:
      result = vec_xxpermdi (vra, vrb, 0);
      break;
    case 1:
      result = vec_xxpermdi (vra, vrb, 1);
      break;
    case 2:
      result = vec_xxpermdi (vra, vrb, 2);
      break;
    case 3:
      result = vec_xxpermdi (vra, vrb, 3);
      break;
#else
    case 0:
      result = vec_xxpermdi (vrb, vra, 3);
      break;
    case 1:
      result = vec_xxpermdi (vrb, vra, 1);
      break;
    case 2:
      result = vec_xxpermdi (vrb, vra, 2);
      break;
    case 3:
      result = vec_xxpermdi (vrb, vra, 0);
      break;
#endif
    default:
      result = (vui64_t){ 0, 0 };
    }
#else
  /* Current compilers don't accept vector unsigned long int as vector
   * parms to vec_sld, so use vector unsigned int.  The vsldoi
   * instruction does not care).  */
  vui32_t temp;
  switch (ctl & 3)
    {
      case 0:
      temp = vec_sld ((vui32_t) vra, (vui32_t) vra, 8);
      result = (vui64_t) vec_sld (temp, (vui32_t) vrb, 8);
      break;
      case 1:
      temp = vec_sld ((vui32_t) vrb, (vui32_t) vra, 8);
      result = (vui64_t) vec_sld (temp, temp, 8);
      break;
      case 2:
      result = (vui64_t) vec_sld ((vui32_t) vra, (vui32_t) vrb, 8);
      break;
      case 3:
      temp = vec_sld ((vui32_t) vrb, (vui32_t) vrb, 8);
      result = (vui64_t) vec_sld ((vui32_t) vra, temp, 8);
      break;
    }
#endif
  return (result);
}
 
#ifndef vec_popcntd
static inline vui64_t
vec_popcntd (vui64_t vra)
{
  vui64_t r;
#ifdef _ARCH_PWR8
#if defined (vec_vpopcntd)
  r = vec_vpopcntd (vra);
#elif defined (__clang__)
  r = vec_popcnt (vra);
#else
  __asm__(
      "vpopcntd %0,%1;"
      : "=v" (r)
      : "v" (vra)
      : );
#endif
#else
  vui32_t z= { 0,0,0,0};
  vui32_t x;
  x = vec_popcntw ((vui32_t) vra);
  r = (vui64_t) vec_vsum2sw ((vi32_t) x, (vi32_t) z);
#endif
  return (r);
}
#else
/* Work around for GCC PR85830.  */
#undef vec_popcntd
#define vec_popcntd __builtin_vec_vpopcntd
#endif
 
static inline vui64_t
vec_revbd (vui64_t vra)
{
  vui64_t result;
 
#ifdef _ARCH_PWR9
#if defined (vec_revb) || defined (__clang__)
  result = vec_revb (vra);
#else
  __asm__(
      "xxbrd %x0,%x1;"
      : "=wa" (result)
      : "wa" (vra)
      : );
#endif
#else
#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
  const vui64_t vconstp =
        CONST_VINT64_DW(0x0706050403020100UL, 0x0F0E0D0C0B0A0908UL);
#else
  const vui64_t vconstp =
      CONST_VINT64_DW(0x08090A0B0C0D0E0FUL, 0x0001020304050607UL);
#endif
  result = (vui64_t) vec_perm ((vui8_t) vra, (vui8_t) vra, (vui8_t) vconstp);
#endif
 
  return (result);
}
 
#ifndef vec_vsld
static inline vui64_t vec_vrld (vui64_t vra, vui64_t vrb);
static inline vui64_t vec_vsld (vui64_t vra, vui64_t vrb);
static inline vui64_t vec_vsrd (vui64_t vra, vui64_t vrb);
static inline vi64_t vec_vsrad (vi64_t vra, vui64_t vrb);
#endif
 
static inline vb64_t
vec_setb_sd (vi64_t vra)
{
  vb64_t result;
 
#if defined (_ARCH_PWR10)  && (__GNUC__ >= 10)
  __asm__(
      "vexpanddm %0,%1;\n"
      : "=v" (result)
      : "v" (vra)
      : );
#elif defined (_ARCH_PWR8)
  // Compare signed doubleword less than zero
  const vi64_t zero = {0, 0};
  result = vec_cmpltsd (vra, zero);
#else // ARCH_PWR7 or older, without compare signed doubleword
  const vui8_t rshift =  vec_splat_u8( 7 );
  const vui8_t sperm = { 0,0,0,0, 0,0,0,0, 8,8,8,8, 8,8,8,8 };
  // Splat the high byte of each doubleword across.
  vui8_t splat = vec_perm ((vui8_t) vra, (vui8_t) vra, sperm);
  // Vector Shift Right Algebraic Bytes 7-bits.
  result = (vb64_t) vec_sra (splat, rshift);
#endif
  return result;
}
 
static inline vui64_t
vec_rldi (vui64_t vra, const unsigned int shb)
{
  vui64_t lshift;
  vui64_t result;
 
  if ((shb%64) != 0)
    {
      /* Load the rotate const in a vector.  The element rotates require
         a rotate amount for each element. For the immediate form the
         rotate constant is splatted to all elements of the
         rotate control.  */
      if (__builtin_constant_p (shb) && (shb < 16))
        lshift = (vui64_t) vec_splat_s32(shb);
      else
        lshift = vec_splats ((unsigned long long) shb);
 
      /* Vector Shift right bytes based on the lower 6-bits of
         corresponding element of lshift.  */
      result = vec_vrld (vra, lshift);
    }
  else
    { /* Rotation of 0 bits returns vra unchanged.  */
      result = vra;
    }
 
  return (vui64_t) result;
}
 
static inline vui64_t
vec_sldi (vui64_t vra, const unsigned int shb)
{
  vui64_t result;
 
  if (shb < 64)
    {
      /* Load the shift const in a vector.  The element shifts require
         a shift amount for each element. For the immediate form the
         shift constant is splatted to all elements of the
         shift control.  */
#ifdef _ARCH_PWR8
      vui64_t lshift;
 
      if (__builtin_constant_p (shb) && (shb < 16))
        lshift = (vui64_t) vec_splat_s32(shb);
      else
        lshift = vec_splats ((unsigned long long) shb);
      /* Vector Shift left doubleword from the lower 6-bits of
         corresponding element of lshift.  */
      result = vec_vsld (vra, lshift);
#else
      /*
       * POWER7 and earlier do not have vsld. So use the vector shift
       * left bit/octet instructions. But these may shift bits from
       * element 1 in the low bits of element 0. So generate a mask of
       * '1's, shifted left by the same shb and rotated into the
       * element 0 position.
       */
      vui8_t lshift;
 
      if (__builtin_constant_p (shb) && (shb < 16))
        lshift = vec_splat_u8(shb);
      else
        lshift = vec_splats ((unsigned char) shb);
 
      {
        vui8_t sl_a;
        vui8_t sl_m = (vui8_t) vec_splat_s8(-1);
 
        sl_a = ((vui8_t) vra);
        if (shb > 7)
          {
            /* Vector Shift Left By Octet by bits 121-124 of lshift.  */
            sl_m = vec_slo (sl_m, lshift);
            sl_a = vec_slo ((vui8_t) vra, lshift);
          }
        if ((shb & 7) != 0)
          {
            /* Vector Shift Left by bits 125-127 of lshift.  */
            sl_m = vec_sll (sl_m, lshift);
            sl_a = vec_sll (sl_a, lshift);
          }
        /* Rotate mask and clear low order bits of Element 0. */
        sl_m = vec_sld (sl_m, sl_m, 8);
        result = (vui64_t) vec_and (sl_a, sl_m);
      }
#endif
    }
  else
    { /* shifts greater then 63 bits return zeros.  */
      result = vec_xor ((vui64_t) vra, (vui64_t) vra);
    }
 
  return (vui64_t) result;
}
 
static inline vi64_t
vec_selsd (vi64_t vra, vi64_t vrb, vb64_t vrc)
{
  return (vi64_t) vec_sel ((vui32_t) vra, (vui32_t)vrb, (vui32_t)vrc);
}
 
static inline vui64_t
vec_selud (vui64_t vra, vui64_t vrb, vb64_t vrc)
{
  return (vui64_t) vec_sel ((vui32_t) vra, (vui32_t)vrb, (vui32_t)vrc);
}
 
static inline vui64_t
vec_splatd (vui64_t vra, const int ctl)
{
  vui64_t result;
  switch (ctl & 1)
    {
    case 0:
      /*
       result[1] = vra[0];
       result[0] = vra[0];
       */
      result = vec_mrged (vra, vra);
      break;
    case 1:
      /*
       result[1] = vra[1];
       result[0] = vra[1];
       */
      result = vec_mrgod (vra, vra);
      break;
    }
 
  return (result);
}
 
static inline vi64_t
vec_splat_s64 (const int sim)
{
  vi64_t result;
#ifdef _ARCH_PWR9
  result = vec_splats ((signed long long) sim);
#else
  if (__builtin_constant_p (sim) && ((sim >= -16) && (sim < 16)))
    {
      vi32_t vwi = vec_splat_s32 (sim);
 
      if (__builtin_constant_p (sim) && ((sim == 0) || (sim == -1)))
        {
          // Special case for -1 and 0. Skip vec_unpackl().
          result = (vi64_t) vwi;
        } else {
          // For P8 can use either vupklsh or vupklsw but P7 only has
          // vupklsh. Given the reduced range, Either works here.
          // Unpack signed HW works here because immediate value fits
          // into the low HW and sign extends to high HW of each word.
          // Unpack will expand the low HW to low word and high HW
          // (sign extend) into the high word of each DW.
          // Unpack low/high (or endian) will not change the result.
#if defined (__GNUC__) && (__GNUC__ == 8)
          // GCC 8 (AT12) handle this correctly.
          result = (vi64_t) vec_vupklsh ((vi16_t) vwi);
#else
          // But GCC 9+ optimized the above to be load from .rodata.
          // With a little register pressure it adds some gratuitous store/reloads.
          // So the following work-around is required.
          __asm__(
              "vupklsh %0,%1;"
              : "=v" (result)
              : "v" (vwi)
              : );
#endif
        }
    }
  else
    result = vec_splats ((signed long long) sim);
#endif
  return (result);
}
 
static inline vui64_t
vec_splat_u64 (const int sim)
{
  vui64_t result;
#ifdef _ARCH_PWR9
  result = vec_splats ((unsigned long long) sim);
#else
  if (__builtin_constant_p (sim) && ((sim >= 0) && (sim < 16)))
    {
      vui32_t vwi = vec_splat_u32 (sim);
 
      if (__builtin_constant_p (sim) && (sim == 0))
        {
          // Special case for -1 and 0. Skip vec_unpackl().
          result = (vui64_t) vwi;
        } else {
          // For P8 can use either vupklsh or vupklsw but P7 only has
          // vupklsh. Given the reduced range, Either works here.
          // Unpack unsigned HW works here because immediate value fits
          // into the low HW and zero extends to high HW of each word.
          // Unpack will expand the low HW to low word and high HW
          // (zero extended) into the high word of each DW.
          // Unpack low/high (or endian) will not change the result.
#if defined (__GNUC__) && (__GNUC__ == 8)
          // GCC 8 (AT12) handle this correctly.
          result = (vui64_t) vec_vupklsh ((vi16_t) vwi);
#else
          // But GCC 9+ optimized the above to be load from .rodata.
          // With a little register pressure it adds some gratuitous store/reloads.
          // So the following work-around is required.
          __asm__(
              "vupklsh %0,%1;"
              : "=v" (result)
              : "v" (vwi)
              : );
#endif
        }
    }
  else
    result = vec_splats ((unsigned long long) sim);
#endif
  return (result);
}
 
static inline vui64_t
vec_spltd (vui64_t vra, const int ctl)
{
  vui64_t result;
  /* Don't need to reverse the cases for LE because vec_permdi handles
     that.  */
  switch (ctl & 1)
    {
    case 0:
      /*
       result[1] = vra[0];
       result[0] = vra[0];
       */
      result = vec_permdi (vra, vra, 0);
      break;
    case 1:
      /*
       result[1] = vra[1];
       result[0] = vra[1];
       */
      result = vec_permdi (vra, vra, 3);
      break;
    }
 
  return (result);
}
 
static inline vui64_t
vec_srdi (vui64_t vra, const unsigned int shb)
{
  vui64_t result;
 
  if (shb < 64)
    {
      /* Load the shift const in a vector.  The element shifts require
         a shift amount for each element. For the immediate form the
         shift constant is splatted to all elements of the
         shift control.  */
#ifdef _ARCH_PWR8
      vui64_t rshift;
 
#if defined (__GNUC__) && (__GNUC__ < 8)
      if (__builtin_constant_p (shb) && (shb < 16))
        rshift = (vui64_t) vec_splat_s32(shb);
      else
        rshift = vec_splats ((unsigned long long) shb);
#else
      rshift = CONST_VINT128_DW (shb, shb);
#endif
      /* Vector Shift right bytes based on the lower 6-bits of
         corresponding element of rshift.  */
      result = vec_vsrd (vra, rshift);
#else
      /*
       * POWER7 and earlier do not have vsrd. So use the vector shift
       * right bit/octet instructions. But these may shift bits from
       * element 0 in the high bits of element 1. So generate a mask of
       * '1's, shifted right by the same shb and rotated into the
       * element 1 position.
       */
      vui8_t rshift;
 
      if (__builtin_constant_p (shb) && (shb < 16))
        rshift = vec_splat_u8(shb);
      else
        rshift = vec_splats ((unsigned char) shb);
 
      {
        vui8_t sr_a;
        vui8_t sr_m = (vui8_t) vec_splat_s8(-1);
 
        sr_a = ((vui8_t) vra);
        if (shb > 7)
          {
            /* Vector Shift Right By Octet by bits 121-124 of rshift.  */
            sr_m = vec_sro (sr_m, rshift);
            sr_a = vec_sro ((vui8_t) vra, rshift);
          }
        if ((shb & 7) != 0)
          {
            /* Vector Shift Right by bits 125-127 of rshift.  */
            sr_m = vec_srl (sr_m, rshift);
            sr_a = vec_srl (sr_a, rshift);
          }
        /* Rotate mask and clear high order bits of Element 1. */
        sr_m = vec_sld (sr_m, sr_m, 8);
        result = (vui64_t) vec_and (sr_a, sr_m);
      }
#endif
    }
  else
    { /* shifts greater then 63 bits return zeros.  */
      result = vec_xor ((vui64_t) vra, (vui64_t) vra);
    }
  return (vui64_t) result;
}
 
static inline vi64_t
vec_sradi (vi64_t vra, const unsigned int shb)
{
  vui64_t rshift;
  vi64_t result;
 
  if (shb < 64)
    {
      /* Load the shift const in a vector.  The element shifts require
         a shift amount for each element. For the immediate form the
         shift constant is splatted to all elements of the
         shift control.  */
#if defined (__GNUC__) && (__GNUC__ < 8)
      if (__builtin_constant_p (shb) && (shb < 16))
        rshift = (vui64_t) vec_splat_s32(shb);
      else
        rshift = vec_splats ((unsigned long long) shb);
#else
      rshift = CONST_VINT128_DW (shb, shb);
#endif
      /* Vector Shift Right Algebraic Doublewords based on the lower 6-bits
         of corresponding element of rshift.  */
      result = vec_vsrad (vra, rshift);
    }
  else
    { /* shifts greater then 63 bits returns the sign bit propagated to
         all bits.   This is equivalent to shift Right Algebraic of
         63 bits.  */
      rshift = (vui64_t) vec_splats(63);
      result = vec_vsrad (vra, rshift);
    }
 
  return (vi64_t) result;
}
 
static inline vui64_t
vec_subudm(vui64_t a, vui64_t b)
{
  vui32_t r;
 
#ifdef _ARCH_PWR8
#if defined (vec_vsubudm)
  r = (vui32_t) vec_vsubudm (a, b);
#elif defined (__clang__)
  r = (vui32_t) vec_sub (a, b);
#else
  __asm__(
      "vsubudm %0,%1,%2;"
      : "=v" (r)
      : "v" (a),
      "v" (b)
      : );
#endif
#else
  vui32_t c;
  vui32_t z= { 0,0,0,0};
  vui32_t cm= { 0,1,0,1};
 
  c = vec_vsubcuw ((vui32_t)a, (vui32_t)b);
  r = vec_vsubuwm ((vui32_t)a, (vui32_t)b);
  c = vec_andc (cm, c);
  c = vec_sld (c, z, 4);
  r = vec_vsubuwm (r, c);
#endif
  return ((vui64_t) r);
}
 
static inline vui64_t
vec_swapd (vui64_t vra)
{
  vui64_t result;
  /*
   result[1] = vra[0];
   result[0] = vra[1];
   */
  result = vec_permdi (vra, vra, 2);
 
  return (result);
}
 
static inline
vui64_t
vec_vgluddo (unsigned long long *array, vi64_t vra)
{
  vui64_t rese0, rese1;
 
#ifdef _ARCH_PWR8
  rese0 = vec_vlsidx (vra[VEC_DW_H], array);
  rese1 = vec_vlsidx (vra[VEC_DW_L], array);
#else
  // Need to explicitly manage the VR/GPR xfer for PWR7
  unsigned __int128 gprp = vec_transfer_vui128t_to_uint128 ((vui128_t) vra);
 
  rese0 = vec_vlsidx (scalar_extract_uint64_from_high_uint128(gprp), array);
  rese1 = vec_vlsidx (scalar_extract_uint64_from_low_uint128(gprp), array);
#endif
  return  vec_permdi (rese0, rese1, 0);
}
 
static inline vui64_t
vec_vgluddsx (unsigned long long *array, vi64_t vra,
             const unsigned char scale)
{
  vi64_t offset;
 
  offset = (vi64_t) vec_sldi ((vui64_t) vra, (3 + scale));
  return vec_vgluddo (array, offset);
}
 
static inline
vui64_t
vec_vgluddx (unsigned long long *array, vi64_t vra)
{
  vi64_t offset;
 
  offset = (vi64_t) vec_sldi ((vui64_t) vra, 3);
  return vec_vgluddo (array, offset);
}
 
static inline vui64_t
vec_vgludso (unsigned long long *array, const long long offset0,
             const long long offset1)
{
  vui64_t re0, re1, result;
 
  re0 = vec_vlsidx (offset0, array);
  re1 = vec_vlsidx (offset1, array);
  /* Need to handle endian as the vec_vlsidx result is always left
   * justified in VSR, while element [0] may be left or right. */
#if (__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__)
  result = vec_permdi (re1, re0, 0);
#else
  result = vec_permdi (re0, re1, 0);
#endif
  return result;
}
 
static inline vui64_t
vec_vlsidx (const signed long long ra, const unsigned long long *rb)
{
  vui64_t xt;
 
#if (defined(__clang__) && __clang_major__ < 8)
  __VEC_U_128 t;
  unsigned long long *p = (unsigned long long *)((char *)rb + ra);
  t.ulong.upper = *p;
  xt = t.vx1;
#else
  if (__builtin_constant_p (ra) && (ra <= 32760) && (ra >= -32768)
      && ((ra & 3) == 0))
    {
#if defined (_ARCH_PWR9)
      __asm__(
          "lxsd%X1 %0,%1;"
          : "=v" (xt)
          : "m" (*(unsigned long long *)((char *)rb + ra))
          : );
#else
      if (ra == 0)
        {
          __asm__(
              "lxsdx %x0,%y1;"
              : "=wa" (xt)
              : "Z" (*rb)
              : );
        } else {
          unsigned long long rt;
#if defined (_ARCH_PWR8)
          // For P8 better if li and lxsdx shared a single asm block
          // (enforcing consecutive instructions).
          // This enables instruction fusion for P8.
          __asm__(
              "li %0,%2;"
              "lxsdx %x1,%3,%0;"
              : "=&r" (rt), "=wa" (xt)
              : "I" (ra), "b" (rb), "Z" (*(unsigned long long *)((char *)rb+ra))
              : );
#else // _ARCH_PWR7
          // This generates operationally the same code, but the
          // compiler may rearrange/schedule the code.
          __asm__(
              "li %0,%1;"
              : "=r" (rt)
              : "I" (ra)
              : );
          __asm__(
              "lxsdx %x0,%y1;"
              : "=wa" (xt)
              : "Z" (*(unsigned long long *)((char *)rb+rt))
              : );
#endif
        }
#endif
    } else {
      __asm__(
          "lxsdx %x0,%y1;"
          : "=wa" (xt)
          : "Z" (*(unsigned long long *)((char *)rb+ra))
          : );
    }
#endif
  return xt;
}
 
static inline vui128_t
vec_vmadd2eud (vui64_t a, vui64_t b, vui64_t c, vui64_t d);
 
static inline vui128_t
vec_vmaddeud (vui64_t a, vui64_t b, vui64_t c);
 
static inline vui128_t
vec_vmadd2oud (vui64_t a, vui64_t b, vui64_t c, vui64_t d);
 
static inline vui128_t
vec_vmaddoud (vui64_t a, vui64_t b, vui64_t c);
 
static inline vui128_t
vec_vmuleud (vui64_t a, vui64_t b);
 
static inline vui128_t
vec_vmuloud (vui64_t a, vui64_t b);
 
static inline vui128_t
vec_vmsumeud (vui64_t a, vui64_t b, vui128_t c);
 
static inline vui128_t
vec_vmsumoud (vui64_t a, vui64_t b, vui128_t c);
 
#ifndef vec_vpkudum
// May be defined as inline function for clang
// But only for _ARCH_PWR8 or higher.
#if !defined(__clang__) || !defined(_ARCH_PWR8)
static inline vui32_t
vec_vpkudum (vui64_t vra, vui64_t vrb)
{
  vui32_t r;
#ifdef _ARCH_PWR8
  __asm__(
#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
      "vpkudum %0,%2,%1;\n"
#else
      "vpkudum %0,%1,%2;\n"
#endif
      : "=v" (r)
      : "v" (vra),
        "v" (vrb)
      : );
#else
  const vui32_t vconstp =
      CONST_VINT128_W(0x04050607, 0x0c0d0e0f, 0x14151617, 0x1c1d1e1f);
 
  r = vec_perm ((vui32_t) vra, (vui32_t) vrb, (vui8_t) vconstp);
#endif
  return (r);
}
#endif
#endif
 
#ifndef vec_vrld
static inline vui64_t
vec_vrld (vui64_t vra, vui64_t vrb)
{
  vui64_t r;
#ifdef _ARCH_PWR8
#ifdef __clang__
  r = vec_rl (vra, vrb);
#else
  __asm__(
      "vrld %0,%1,%2;"
      : "=v" (r)
      : "v" (vra),
        "v" (vrb)
      : );
#endif
#else
  vui64_t hd, ld;
  vui32_t t1, t2;
  vui8_t shh, shl;
 
  shh = vec_splat ((vui8_t) vrb, VEC_BYTE_L_DWH);
  shl = vec_splat ((vui8_t) vrb, VEC_BYTE_L_DWL);
  hd = vec_xxspltd (vra, VEC_DW_H);
  ld = vec_xxspltd (vra, VEC_DW_L);
  t1 = vec_vslo ((vui32_t)hd, shh);
  t2 = vec_vslo ((vui32_t)ld, shl);
  t1 = vec_vsl (t1, shh);
  t2 = vec_vsl (t2, shl);
  r = vec_mrghd ((vui64_t)t1, (vui64_t)t2);
#endif
  return (r);
}
#endif
 
#ifndef vec_vsld
static inline vui64_t
vec_vsld (vui64_t vra, vui64_t vrb)
{
  vui64_t result;
 
#ifdef _ARCH_PWR8
#ifdef __clang__
  result = vec_sl (vra, vrb);
#else
  __asm__(
      "vsld %0,%1,%2;"
      : "=v" (result)
      : "v" (vra),
      "v" (vrb)
      : );
#endif
#else
  vui8_t  vsh_h, vsh_l;
  vui8_t  vr_h, vr_l;
  vui64_t sel_mask = CONST_VINT128_DW(0, -1LL);
  vui64_t shft_mask = CONST_VINT128_DW(63, 63);
 
  /* constrain the dword shift amounts to 0-63.  */
  vsh_l = vec_and ((vui8_t) vrb, (vui8_t) shft_mask);
  /* Isolate the high dword so that bits from the low dword
   * do not contaminate the result.  */
  vr_h = vec_andc ((vui8_t) vra, (vui8_t) sel_mask);
  /* The low dword is just vra as the 128-bit shift left generates
   * '0's on the right and the final merge (vec_sel)
   * cleans up 64-bit overflow on the left.  */
  vr_l  = (vui8_t) vra;
  /* The vsr instruction only works correctly if the bit shift
   * value is splatted to each byte of the vector.  */
  vsh_h = vec_splat (vsh_l, VEC_BYTE_L_DWH);
  vsh_l = vec_splat (vsh_l, VEC_BYTE_L_DWL);
  /* Shift the high dword by vsh_h.  */
  vr_h = vec_vslo (vr_h,  vsh_h);
  vr_h = vec_vsl  (vr_h, vsh_h);
  /* Shift the low dword by vsh_l.  */
  vr_l = vec_vslo (vr_l,  vsh_l);
  vr_l = vec_vsl  (vr_l, vsh_l);
  /* Merge the dwords after shift.  */
  result = (vui64_t) vec_sel (vr_h, vr_l, (vui8_t) sel_mask);
#endif
  return ((vui64_t) result);
}
#endif
 
#ifndef vec_vsrad
static inline vi64_t
vec_vsrad (vi64_t vra, vui64_t vrb)
{
  vi64_t result;
 
#ifdef _ARCH_PWR8
#ifdef __clang__bad
// clang8/9 has code gen bug here, disabled for now
  result = vec_sra (vra, vrb);
#else
  __asm__(
      "vsrad %0,%1,%2;"
      : "=v" (result)
      : "v" (vra),
      "v" (vrb)
      : );
#endif
#else
  vui8_t  vsh_h, vsh_l;
  vui8_t  vr_h, vr_l;
  vi32_t exsa;
  vui32_t shw31 = CONST_VINT128_W (-1, -1, -1, -1);
  vui64_t exsah, exsal;
  vui64_t shft_mask = CONST_VINT128_DW(63, 63);
 
  /* Need to extend each signed long int to __int128. So the unsigned
   * (128-bit) shift right behaves as a arithmetic (64-bit) shift.  */
  exsa = vec_vsraw ((vi32_t) vra, shw31);
  exsah = (vui64_t) vec_vmrghw (exsa, exsa);
  exsal = (vui64_t) vec_vmrglw (exsa, exsa);
  /* constrain the dword shift amounts to 0-63.  */
  vsh_l = vec_and ((vui8_t) vrb, (vui8_t) shft_mask);
  /* The vsr instruction only works correctly if the bit shift
   * value is splatted to each byte of the vector.  */
  vsh_h = vec_splat (vsh_l, VEC_BYTE_L_DWH);
  vsh_l = vec_splat (vsh_l, VEC_BYTE_L_DWL);
  /* Merge the extended sign with high dword.  */
  exsah = vec_mrghd (exsah, (vui64_t) vra);
  /* Shift the high dword by vsh_h.  */
  vr_h = vec_vsro ((vui8_t) exsah,  vsh_h);
  vr_h = vec_vsr  (vr_h, vsh_h);
  /* Merge the extended sign with high dword.  */
  exsal = vec_pasted (exsal, (vui64_t) vra);
  /* Shift the low dword by vsh_l.  */
  vr_l = vec_vsro ((vui8_t) exsal, vsh_l);
  vr_l = vec_vsr  (vr_l, vsh_l);
  /* Merge the dwords after shift.  */
  result = (vi64_t) vec_mrgld ((vui64_t) vr_h, (vui64_t) vr_l);
#endif
  return ((vi64_t) result);
}
#endif
 
#ifndef vec_vsrd
static inline vui64_t
vec_vsrd (vui64_t vra, vui64_t vrb)
{
  vui64_t result;
 
#ifdef _ARCH_PWR8
#ifdef __clang__
  result = vec_sr (vra, vrb);
#else
  __asm__(
      "vsrd %0,%1,%2;"
      : "=v" (result)
      : "v" (vra),
      "v" (vrb)
      : );
#endif
#else
  vui8_t  vsh_h, vsh_l;
  vui8_t  vr_h, vr_l;
  vui64_t sel_mask = CONST_VINT128_DW(0, -1LL);
  vui64_t shft_mask = CONST_VINT128_DW(63, 63);
 
  /* constrain the dword shift amounts to 0-63.  */
  vsh_l = vec_and ((vui8_t) vrb, (vui8_t) shft_mask);
  /* Isolate the low dword so that bits from the high dword,
   * do not contaminate the result.  */
  vr_l  = vec_and ((vui8_t) vra, (vui8_t) sel_mask);
  /* The vsr instruction only works correctly if the bit shift
   * value is splatted to each byte of the vector.  */
  vsh_h = vec_splat (vsh_l, VEC_BYTE_L_DWH);
  vsh_l = vec_splat (vsh_l, VEC_BYTE_L_DWL);
  /* Shift the high dword by vsh_h.  */
  vr_h = vec_vsro ((vui8_t) vra,  vsh_h);
  vr_h = vec_vsr  (vr_h, vsh_h);
  /* Shift the low dword by vsh_l.  */
  vr_l = vec_vsro (vr_l,  vsh_l);
  vr_l = vec_vsr  (vr_l, vsh_l);
  /* Merge the dwords after shift.  */
  result = (vui64_t) vec_sel (vr_h, vr_l, (vui8_t) sel_mask);
#endif
  return ((vui64_t) result);
}
#endif
 
static inline void
vec_vsstuddo (vui64_t xs, unsigned long long *array, vi64_t vra)
{
  vui64_t xs1;
 
  xs1 = vec_xxspltd (xs, 1);
#ifdef _ARCH_PWR8
  vec_vstsidx (xs, vra[VEC_DW_H], array);
  vec_vstsidx (xs1, vra[VEC_DW_L], array);
#else
  // Need to explicitly manage the VR/GPR xfer for PWR7
  unsigned __int128 gprp = vec_transfer_vui128t_to_uint128 ((vui128_t) vra);
  vec_vstsidx (xs, scalar_extract_uint64_from_high_uint128(gprp), array);
  vec_vstsidx (xs1, scalar_extract_uint64_from_low_uint128(gprp), array);
#endif
}
 
static inline void
vec_vsstuddsx (vui64_t xs, unsigned long long *array,
            vi64_t vra, const unsigned char scale)
{
  vi64_t offset;
 
  offset = (vi64_t) vec_sldi ((vui64_t) vra, (3 + scale));
  vec_vsstuddo (xs, array, offset);
}
 
static inline void
vec_vsstuddx (vui64_t xs, unsigned long long *array,
            vi64_t vra)
{
  vi64_t offset;
 
  offset = (vi64_t) vec_sldi ((vui64_t) vra, 3);
  vec_vsstuddo (xs, array, offset);
}
 
static inline void
vec_vsstudso (vui64_t xs, unsigned long long *array,
              const long long offset0, const long long offset1)
{
  vui64_t xs1;
 
  xs1 = vec_xxspltd (xs, 1);
  /* Need to handle endian as vec_vstsfdux always left side of
   * the VR, while the element [0] may in the left or right. */
#if (__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__)
  vec_vstsidx (xs, offset1, array);
  vec_vstsidx (xs1, offset0, array);
#else
  vec_vstsidx (xs, offset0, array);
  vec_vstsidx (xs1, offset1, array);
#endif
}
 
static inline void
vec_vstsidx (vui64_t xs, const signed long long ra, unsigned long long *rb)
{
#if (defined(__clang__) && __clang_major__ < 8)
  __VEC_U_128 t;
  unsigned long long *p = (unsigned long long *)((char *)rb + ra);
  t.vx1 = xs;
  *p = t.ulong.upper;
#else
  if (__builtin_constant_p (ra) &&  (ra <= 32760) && (ra >= -32768)
      && ((ra & 3) == 0))
    {
#if defined (_ARCH_PWR9)
      __asm__(
          "stxsd%X0 %1,%0;"
          : "=m" (*((char *)rb + ra))
          : "v" (xs)
          : );
#else
      if (ra == 0)
        {
          __asm__(
              "stxsdx %x1,%y0;"
              : "=Z" (*rb)
              : "wa" (xs)
              : );
        } else {
          unsigned long long rt;
          __asm__(
              "li %0,%1;"
              : "=r" (rt)
              : "I" (ra)
              : );
          __asm__(
              "stxsdx %x1,%y0;"
              : "=Z" (*((char *)rb+rt))
              : "wa" (xs)
              : );
        }
#endif
    } else {
      __asm__(
          "stxsdx %x1,%y0;"
          : "=Z" (*((char *)rb+ra))
          : "wa" (xs)
          : );
    }
#endif
}
 
static inline vui64_t
vec_xxspltd (vui64_t vra, const int ctl)
{
  vui64_t result;
  /* Don't need to reverse the cases for LE because vec_permdi handles
     that.  */
  switch (ctl & 1)
    {
    case 0:
      result = vec_permdi (vra, vra, 0);
      break;
    case 1:
      result = vec_permdi (vra, vra, 3);
      break;
    }
 
  return (result);
}
 
static inline vui64_t
vec_vmaddeuw (vui32_t a, vui32_t b, vui32_t c)
{
  const vui32_t zero = { 0, 0, 0, 0 };
  vui64_t res;
  vui32_t c_euw = vec_mrgahw ((vui64_t) zero, (vui64_t) c);
  res = vec_vmuleuw (a, b);
  return vec_addudm (res, (vui64_t) c_euw);
}
 
static inline vui64_t
vec_vmadd2euw (vui32_t a, vui32_t b, vui32_t c, vui32_t d)
{
  const vui32_t zero = { 0, 0, 0, 0 };
  vui64_t res, sum;
  vui32_t c_euw = vec_mrgahw ((vui64_t) zero, (vui64_t) c);
  vui32_t d_euw = vec_mrgahw ((vui64_t) zero, (vui64_t) d);
  res = vec_vmuleuw (a, b);
  sum = vec_addudm ( (vui64_t) c_euw, (vui64_t) d_euw);
  return vec_addudm (res, sum);
}
 
static inline vui64_t
vec_vmaddouw (vui32_t a, vui32_t b, vui32_t c)
{
  const vui32_t zero = { 0, 0, 0, 0 };
  vui64_t res;
  vui32_t c_ouw = vec_mrgalw ((vui64_t) zero, (vui64_t) c);
  res = vec_vmulouw (a, b);
  return vec_addudm (res, (vui64_t) c_ouw);
}
 
static inline vui64_t
vec_vmadd2ouw (vui32_t a, vui32_t b, vui32_t c, vui32_t d)
{
  const vui32_t zero = { 0, 0, 0, 0 };
  vui64_t res, sum;
  vui32_t c_ouw = vec_mrgalw ((vui64_t) zero, (vui64_t) c);
  vui32_t d_ouw = vec_mrgalw ((vui64_t) zero, (vui64_t) d);
  res = vec_vmulouw (a, b);
  sum = vec_addudm ((vui64_t) c_ouw, (vui64_t) d_ouw);
  return vec_addudm (res, sum);
}
 
static inline vui64_t
vec_vmsumuwm (vui32_t vra, vui32_t vrb, vui64_t vrc)
{
  vui64_t peven, podd, psum;
 
  peven = vec_muleuw (vra, vrb);
  podd  = vec_mulouw (vra, vrb);
  psum  = vec_addudm (peven, podd);
 
  return vec_addudm (psum, vrc);
}
 
#endif /* VEC_INT64_PPC_H_ */