vec_f64_ppc.h

Go to the documentation of this file.

/*
 Copyright (c) [2017] IBM Corporation.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
 You may obtain a copy of the License at
 
    http://www.apache.org/licenses/LICENSE-2.0
 
 Unless required by applicable law or agreed to in writing, software
 distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 
 vec_f64_ppc.h
 
 Contributors:
      IBM Corporation, Steven Munroe
      Created on: Apr 12, 2016
 */
 
#ifndef VEC_F64_PPC_H_
#define VEC_F64_PPC_H_
 
#include <pveclib/vec_common_ppc.h>
#include <pveclib/vec_int128_ppc.h>
static inline vf64_t
vec_vlxsfdx (const signed long long ra, const double *rb);
static inline void
vec_vstxsfdx (vf64_t xs, const signed long long ra, double *rb);
 
static inline vf64_t
vec_absf64 (vf64_t vf64x)
{
#if _ARCH_PWR7
  /* Requires VSX but eliminates a const load. */
  return vec_abs (vf64x);
#else
  const vui32_t signmask = CONST_VINT128_DW (0x8000000000000000UL,
                                             0x8000000000000000UL);
  return (vf64_t)vec_andc ((vui32_t)vf64x, signmask);
#endif
}
 
static inline int
vec_all_isfinitef64 (vf64_t vf64)
{
  vui64_t tmp;
#if _ARCH_PWR9 && (__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__)
  const vui64_t vec_zero = CONST_VINT128_DW (0, 0);
#ifdef vec_test_data_class
  tmp = (vui64_t)vec_test_data_class (vf64, 0x70);
#else
  __asm__(
      "xvtstdcdp %x0,%x1,0x70;\n"
      : "=wa" (tmp)
      : "wa" (vf64)
      :);
#endif
  return vec_all_eq(tmp, vec_zero);
#else
  const vui64_t expmask = CONST_VINT128_DW (0x7ff0000000000000UL,
                                            0x7ff0000000000000UL);
  tmp = vec_and ((vui64_t)vf64, expmask);
  return !vec_cmpud_any_eq(tmp, expmask);
#endif
}
 
static inline int
vec_all_isinff64 (vf64_t vf64)
{
  vui64_t tmp;
 
#if _ARCH_PWR9 && !(defined(__clang__) && __clang_major__ < 9)
  const vui64_t vec_ones = CONST_VINT128_DW (-1, -1);
#ifdef vec_test_data_class
  tmp = (vui64_t)vec_test_data_class (vf64, 0x30);
#else
  __asm__(
      "xvtstdcdp %x0,%x1,0x30;\n"
      : "=wa" (tmp)
      : "wa" (vf64)
      :);
#endif
  return vec_all_eq(tmp, vec_ones);
#else
  const vui64_t signmask = CONST_VINT128_DW (0x8000000000000000UL,
                                             0x8000000000000000UL);
  const vui64_t expmask = CONST_VINT128_DW (0x7ff0000000000000UL,
                                            0x7ff0000000000000UL);
  tmp = vec_andc ((vui64_t)vf64, signmask);
  return vec_cmpud_all_eq(tmp, expmask);
#endif
}
 
static inline int
vec_all_isnanf64 (vf64_t vf64)
{
  vui64_t tmp;
 
#if _ARCH_PWR9  && !(defined(__clang__) && __clang_major__ < 9)
  const vui64_t vec_ones = CONST_VINT128_DW (-1, -1);
#ifdef vec_test_data_class
  tmp = (vui64_t)vec_test_data_class (vf64, 0x40);
#else
  __asm__(
      "xvtstdcdp %x0,%x1,0x40;\n"
      : "=wa" (tmp)
      : "wa" (vf64)
      :);
#endif
  return vec_all_eq(tmp, vec_ones);
#else
  const vui64_t signmask = CONST_VINT128_DW (0x8000000000000000UL,
                                             0x8000000000000000UL);
  const vui64_t expmask = CONST_VINT128_DW (0x7ff0000000000000UL,
                                            0x7ff0000000000000UL);
  tmp = vec_andc ((vui64_t)vf64, signmask);
  return vec_cmpud_all_gt(tmp, expmask);
#endif
}
 
static inline int
vec_all_isnormalf64 (vf64_t vf64)
{
  vui64_t tmp;
  const vui64_t vec_zero = CONST_VINT128_DW (0, 0);
#if _ARCH_PWR9 && (__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__)
#ifdef vec_test_data_class
  tmp = (vui64_t)vec_test_data_class (vf64, 0x7f);
#else
  __asm__(
      "xvtstdcdp %x0,%x1,0x7f;\n"
      : "=wa" (tmp)
      : "wa" (vf64)
      :);
#endif
  return vec_all_eq(tmp, vec_zero);
#else
  const vui64_t expmask = CONST_VINT128_DW (0x7ff0000000000000UL,
                                            0x7ff0000000000000UL);
  tmp = vec_and ((vui64_t) vf64, expmask);
  return !(vec_cmpud_any_eq (tmp, expmask)
        || vec_cmpud_any_eq (tmp, vec_zero));
#endif
}
 
static inline int
vec_all_issubnormalf64 (vf64_t vf64)
{
  vui64_t tmp;
 
#if _ARCH_PWR9 && (__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__)
  const vui64_t vec_ones = CONST_VINT128_DW (-1, -1);
#ifdef vec_test_data_class
  tmp = (vui64_t)vec_test_data_class (vf64, 0x03);
#else
  __asm__(
      "xvtstdcdp %x0,%x1,0x03;\n"
      : "=wa" (tmp)
      : "wa" (vf64)
      :);
#endif
  return vec_all_eq(tmp, vec_ones);
#else
  const vui64_t explow = CONST_VINT128_DW (0x0010000000000000,
                                           0x0010000000000000);
  const vui64_t signmask = CONST_VINT128_DW (0x8000000000000000UL,
                                             0x8000000000000000UL);
  const vui64_t vec_zero = CONST_VINT128_DW (0, 0);
 
  tmp = vec_andc ((vui64_t)vf64, signmask);
  return vec_cmpud_all_lt (tmp, explow) && vec_cmpud_all_ne (tmp, vec_zero);
#endif
}
 
static inline int
vec_all_iszerof64 (vf64_t vf64)
{
  vui64_t tmp;
 
#if _ARCH_PWR9 && (__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__)
  const vui64_t vec_ones = CONST_VINT128_DW (-1, -1);
#ifdef vec_test_data_class
  tmp = (vui64_t)vec_test_data_class (vf64, 0x0c);
#else
  __asm__(
      "xvtstdcdp %x0,%x1,0x0c;\n"
      : "=wa" (tmp)
      : "wa" (vf64)
      :);
#endif
  return vec_all_eq(tmp, vec_ones);
#else
  const vui64_t signmask = CONST_VINT128_DW (0x8000000000000000UL,
                                             0x8000000000000000UL);
  const vui64_t vec_zero = CONST_VINT128_DW (0, 0);
 
  tmp = vec_andc ((vui64_t)vf64, signmask);
  return vec_all_eq((vui32_t)tmp, (vui32_t)vec_zero);
#endif
}
 
static inline int
vec_any_isfinitef64 (vf64_t vf64)
{
  vui64_t tmp;
#if _ARCH_PWR9
  const vui64_t vec_zero = CONST_VINT128_DW (0, 0);
#ifdef vec_test_data_class
  tmp = (vui64_t)vec_test_data_class (vf64, 0x70);
#else
  __asm__(
      "xvtstdcdp %x0,%x1,0x70;\n"
      : "=wa" (tmp)
      : "wa" (vf64)
      :);
#endif
  return vec_any_eq(tmp, vec_zero);
#else
  const vui64_t expmask = CONST_VINT128_DW (0x7ff0000000000000UL,
                                            0x7ff0000000000000UL);
  tmp = vec_and ((vui64_t)vf64, expmask);
  return !vec_cmpud_all_eq(tmp, expmask);
#endif
}
 
static inline int
vec_any_isinff64 (vf64_t vf64)
{
  vui64_t tmp;
 
#if _ARCH_PWR9  && !(defined(__clang__) && __clang_major__ < 9)
  const vui64_t vec_ones = CONST_VINT128_DW (-1, -1);
#ifdef vec_test_data_class
  tmp = (vui64_t)vec_test_data_class (vf64, 0x30);
#else
  __asm__(
      "xvtstdcdp %x0,%x1,0x30;\n"
      : "=wa" (tmp)
      : "wa" (vf64)
      :);
#endif
  return vec_any_eq(tmp, vec_ones);
#else
  const vui64_t expmask = CONST_VINT128_DW (0x7ff0000000000000UL,
                                            0x7ff0000000000000UL);
  const vui64_t signmask = CONST_VINT128_DW (0x8000000000000000UL,
                                             0x8000000000000000UL);
  tmp = vec_andc ((vui64_t)vf64, signmask);
  return vec_cmpud_any_eq(tmp, expmask);
#endif
}
 
static inline int
vec_any_isnanf64 (vf64_t vf64)
{
  vui64_t tmp;
 
#if _ARCH_PWR9 && !(defined(__clang__) && __clang_major__ < 9)
  const vui64_t vec_ones = CONST_VINT128_DW (-1, -1);
#ifdef vec_test_data_class
  tmp = (vui64_t)vec_test_data_class (vf64, 0x40);
#else
  __asm__(
      "xvtstdcdp %x0,%x1,0x40;\n"
      : "=wa" (tmp)
      : "wa" (vf64)
      :);
#endif
  return vec_any_eq(tmp, vec_ones);
#else
  const vui64_t signmask = CONST_VINT128_DW (0x8000000000000000UL,
                                             0x8000000000000000UL);
  const vui64_t expmask = CONST_VINT128_DW (0x7ff0000000000000UL,
                                            0x7ff0000000000000UL);
  tmp = vec_andc ((vui64_t)vf64, signmask);
  return vec_cmpud_any_gt(tmp, expmask);
#endif
}
 
static inline int
vec_any_isnormalf64 (vf64_t vf64)
{
  const vui64_t vec_zero = CONST_VINT128_DW (0, 0);
  vui64_t tmp;
#if _ARCH_PWR9
#ifdef vec_test_data_class
  tmp = (vui64_t)vec_test_data_class (vf64, 0x7f);
#else
  __asm__(
      "xvtstdcdp %x0,%x1,0x7f;\n"
      : "=wa" (tmp)
      : "wa" (vf64)
      :);
#endif
  return vec_any_eq(tmp, vec_zero);
#else
  vui64_t res;
  const vui64_t expmask = CONST_VINT128_DW (0x7ff0000000000000UL,
                                            0x7ff0000000000000UL);
 
  tmp = vec_and ((vui64_t) vf64, expmask);
  res = (vui64_t) vec_nor (vec_cmpequd (tmp, expmask),
                           vec_cmpequd (tmp, vec_zero));
  return vec_cmpud_any_gt (res, vec_zero);
#endif
}
 
static inline int
vec_any_issubnormalf64 (vf64_t vf64)
{
  vui64_t tmp;
 
#if _ARCH_PWR9
  const vui64_t vec_ones = CONST_VINT128_DW (-1, -1);
#ifdef vec_test_data_class
  tmp = (vui64_t)vec_test_data_class (vf64, 0x03);
#else
  __asm__(
      "xvtstdcdp %x0,%x1,0x03;\n"
      : "=wa" (tmp)
      : "wa" (vf64)
      :);
#endif
  return vec_any_eq(tmp, vec_ones);
#else
  const vui64_t signmask = CONST_VINT128_DW (0x8000000000000000UL,
                                             0x8000000000000000UL);
  const vui64_t minnorm = CONST_VINT128_DW (0x0010000000000000UL,
                                            0x0010000000000000UL);
  const vui64_t vec_zero = CONST_VINT128_DW (0, 0);
  vui64_t tmpz, tmp2, vsubnorm;
 
  tmp2 = vec_andc ((vui64_t)vf64, signmask);
  tmp = (vui64_t) vec_cmpltud(tmp2, minnorm);
  tmpz = (vui64_t) vec_cmpequd (tmp2, vec_zero);
  vsubnorm = vec_andc (tmp, tmpz);
  return vec_cmpud_any_ne(vsubnorm, vec_zero);
#endif
}
 
static inline int
vec_any_iszerof64 (vf64_t vf64)
{
  vui64_t tmp;
 
#if _ARCH_PWR9
  const vui64_t vec_ones = CONST_VINT128_DW (-1, -1);
#ifdef vec_test_data_class
  tmp = (vui64_t)vec_test_data_class (vf64, 0x0c);
#else
  __asm__(
      "xvtstdcdp %x0,%x1,0x0c;\n"
      : "=wa" (tmp)
      : "wa" (vf64)
      :);
#endif
  return vec_any_eq(tmp, vec_ones);
#else
  const vui64_t signmask = CONST_VINT128_DW (0x8000000000000000UL,
                                             0x8000000000000000UL);
  const vui64_t vec_zero = CONST_VINT128_DW (0, 0);
  tmp = vec_andc ((vui64_t)vf64, signmask);
  return vec_cmpud_any_eq(tmp, vec_zero);
#endif
}
 
static inline vf64_t
vec_copysignf64 (vf64_t vf64x, vf64_t vf64y)
{
#if _ARCH_PWR7
  /* P9 has a 2 cycle xvcpsgndp and eliminates a const load. */
#ifdef PVECLIB_CPSGN_FIXED
  return (vec_cpsgn (vf64x, vf64y));
#else
  vf64_t result;
  __asm__(
      "xvcpsgndp %x0,%x1,%x2;\n"
      : "=wa" (result)
      : "wa" (vf64x), "wa" (vf64y)
      :);
  return (result);
#endif
#else
  const vui32_t signmask = CONST_VINT128_W(0x80000000, 0, 0x80000000, 0);
  vf64_t result;
 
  result = (vf64_t) vec_sel ((vui32_t) vf64y, (vui32_t) vf64x, signmask);
  return (result);
#endif
}
 
static inline vb64_t
vec_isfinitef64 (vf64_t vf64)
{
  vb64_t tmp2;
#if defined (_ARCH_PWR9)
#ifdef vec_test_data_class
  tmp2 = vec_test_data_class (vf64, 0x70);
#else
  __asm__(
      "xvtstdcdp %x0,%x1,0x70;\n"
      : "=wa" (tmp2)
      : "wa" (vf64)
      :);
#endif
  return vec_nor (tmp2, tmp2); // vec_not
#else
  const vui64_t expmask = CONST_VINT128_DW (0x7ff0000000000000UL,
                                            0x7ff0000000000000UL);
  vui64_t tmp;
 
  tmp = vec_and ((vui64_t)vf64, expmask);
  tmp2 = vec_cmpequd (tmp, expmask);
  return vec_nor (tmp2, tmp2); // vec_not
#endif
}
 
static inline vb64_t
vec_isinff64 (vf64_t vf64)
{
  vb64_t result;
 
#if _ARCH_PWR9 && !(defined(__clang__) && __clang_major__ < 9)
#ifdef vec_test_data_class
  result = vec_test_data_class (vf64, 0x30);
#else
  __asm__(
      "xvtstdcdp %x0,%x1,0x30;\n"
      : "=wa" (result)
      : "wa" (vf64)
      :);
#endif
#else
  vui64_t tmp;
  const vui64_t expmask = CONST_VINT128_DW (0x7ff0000000000000UL,
                                            0x7ff0000000000000UL);
  const vui64_t signmask = CONST_VINT128_DW (0x8000000000000000UL,
                                             0x8000000000000000UL);
  tmp = vec_andc ((vui64_t) vf64, signmask);
  result = (vb64_t)vec_cmpequd (tmp, expmask);
#endif
  return (result);
}
 
static inline vb64_t
vec_isnanf64 (vf64_t vf64)
{
  vb64_t result;
 
#if _ARCH_PWR9 && !(defined(__clang__) && __clang_major__ < 9)
#ifdef vec_test_data_class
  result = vec_test_data_class (vf64, 0x40);
#else
  __asm__(
      "xvtstdcdp %x0,%x1,0x40;\n"
      : "=wa" (result)
      : "wa" (vf64)
      :);
#endif
#else
  vui64_t tmp;
  const vui64_t expmask = CONST_VINT128_DW (0x7ff0000000000000UL,
                                            0x7ff0000000000000UL);
  const vui64_t signmask = CONST_VINT128_DW (0x8000000000000000UL,
                                             0x8000000000000000UL);
  tmp = vec_andc ((vui64_t)vf64, signmask);
  result = (vb64_t)vec_cmpgtud (tmp, expmask);
#endif
  return (result);
}
 
static inline vb64_t
vec_isnormalf64 (vf64_t vf64)
{
#if _ARCH_PWR9
  vb64_t tmp2;
#ifdef vec_test_data_class
  tmp2 = vec_test_data_class (vf64, 0x7f);
#else
  __asm__(
      "xvtstdcdp %x0,%x1,0x7f;\n"
      : "=wa" (tmp2)
      : "wa" (vf64)
      :);
#endif
  return vec_nor (tmp2, tmp2); // vec_not
#else
  const vui64_t expmask = CONST_VINT128_DW (0x7ff0000000000000UL,
                                            0x7ff0000000000000UL);
  const vui64_t veczero = CONST_VINT128_DW (0UL, 0UL);
  vui64_t tmp;
 
  tmp = vec_and ((vui64_t) vf64, expmask);
  return (vb64_t) vec_nor (vec_cmpequd (tmp, expmask),
                           vec_cmpequd (tmp, veczero));
#endif
}
 
static inline vb64_t
vec_issubnormalf64 (vf64_t vf64)
{
  vb64_t result;
 
#if _ARCH_PWR9
#ifdef vec_test_data_class
  result = vec_test_data_class (vf64, 0x03);
#else
  __asm__(
      "xvtstdcdp %x0,%x1,0x03;\n"
      : "=wa" (result)
      : "wa" (vf64)
      :);
#endif
#else
  vui64_t tmp;
  const vui64_t minnorm = CONST_VINT128_DW (0x0010000000000000UL,
                                            0x0010000000000000UL);
  const vui64_t vec_zero = CONST_VINT128_DW (0, 0);
  const vui64_t signmask = CONST_VINT128_DW (0x8000000000000000UL,
                                             0x8000000000000000UL);
  tmp = vec_andc ((vui64_t) vf64, signmask);
  result = vec_andc (vec_cmpltud (tmp, minnorm),
                     vec_cmpequd (tmp, vec_zero));
#endif
  return (result);
}
 
static inline vb64_t
vec_iszerof64 (vf64_t vf64)
{
  vb64_t result;
 
#if _ARCH_PWR9
#ifdef vec_test_data_class
  result = vec_test_data_class (vf64, 0x0c);
#else
  __asm__(
      "xvtstdcdp %x0,%x1,0x0c;\n"
      : "=wa" (result)
      : "wa" (vf64)
      :);
#endif
#else
  vui64_t tmp2;
  const vui64_t vec_zero = CONST_VINT128_DW (0, 0);
  const vui64_t signmask = CONST_VINT128_DW (0x8000000000000000UL,
                                             0x8000000000000000UL);
  tmp2 = vec_andc ((vui64_t)vf64, signmask);
  result = (vb64_t)vec_cmpequd (tmp2, vec_zero);
#endif
  return (result);
}
 
static inline long double
vec_pack_longdouble (vf64_t lval)
{
#ifdef __clang__
  __VEC_U_128 t;
  t.vf2 = lval;
  return (t.ldbl128);
#else
#ifdef _ARCH_PWR7
  long double t;
  __asm__(
      "xxlor %0,%x1,%x1;\n"
      "\txxswapd %L0,%x1;\n"
      : "=f" (t)
      : "wa" (lval)
      : );
  return (t);
#else
  __VEC_U_128 t;
  t.vf2 = lval;
  return (t.ldbl128);
#endif
#endif
}
 
static inline vb64_t
vec_setb_dp (vf64_t vra)
{
  return vec_setb_sd ((vi64_t) vra);
}
 
static inline vf64_t
vec_unpack_longdouble (long double lval)
{
#ifdef __clang__
  __VEC_U_128 t;
  t.ldbl128 = lval;
  return (t.vf2);
#else
#ifdef _ARCH_PWR7
  vf64_t t;
  __asm__(
      "xxmrghd %x0,%1,%L1;\n"
      : "=wa" (t)
      : "f" (lval)
      : );
  return (t);
#else
  __VEC_U_128 t;
  t.ldbl128 = lval;
  return (t.vf2);
#endif
#endif
}
 
static inline vf64_t
vec_vglfdso (double *array, const long long offset0,
             const long long offset1)
{
  vf64_t re0, re1, result;
 
  re0 = vec_vlxsfdx (offset0, array);
  re1 = vec_vlxsfdx (offset1, array);
  /* Need to handle endian as the vec_vlxsfdx result is always left
   * justified in VR, while element [0] may be left ot right. */
#if (__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__)
  result = (vf64_t) vec_permdi ((vui64_t) re1, (vui64_t) re0, 0);
#else
  result = (vf64_t) vec_permdi ((vui64_t) re0, (vui64_t) re1, 0);
#endif
  return result;
}
 
static inline vf64_t
vec_vglfddo (double *array, vi64_t vra)
{
  vf64_t rese0, rese1;
 
#ifdef _ARCH_PWR8
  rese0 = vec_vlxsfdx (vra[VEC_DW_H], array);
  rese1 = vec_vlxsfdx (vra[VEC_DW_L], array);
#else
  // Need to explicitly manage the VR/GPR xfer for PWR7
  unsigned __int128 gprp = vec_transfer_vui128t_to_uint128 ((vui128_t) vra);
 
  rese0 = vec_vlxsfdx (scalar_extract_uint64_from_high_uint128(gprp), array);
  rese1 = vec_vlxsfdx (scalar_extract_uint64_from_low_uint128(gprp), array);
#endif
  return (vf64_t) vec_permdi ((vui64_t) rese0, (vui64_t) rese1, 0);
}
 
static inline vf64_t
vec_vglfddsx (double *array, vi64_t vra,
             const unsigned char scale)
{
  vi64_t offset;
 
  offset = (vi64_t) vec_sldi ((vui64_t) vra, (3 + scale));
  return vec_vglfddo (array, offset);
}
 
static inline vf64_t
vec_vglfddx (double *array, vi64_t vra)
{
  vi64_t offset;
 
  offset = (vi64_t) vec_sldi ((vui64_t) vra, 3);
  return vec_vglfddo (array, offset);
}
 
static inline void
vec_vsstfdso (vf64_t xs, double *array,
              const long long offset0, const long long offset1)
{
  vf64_t xs1;
 
  xs1 = (vf64_t) vec_xxspltd ((vui64_t) xs, 1);
  /* Need to handle endian as vec_vstxsfdx always left side of
   * the VR, while the element [0] may in the left or right. */
#if (__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__)
  vec_vstxsfdx (xs, offset1, array);
  vec_vstxsfdx (xs1, offset0, array);
#else
  vec_vstxsfdx (xs, offset0, array);
  vec_vstxsfdx (xs1, offset1, array);
#endif
}
 
static inline void
vec_vsstfddo (vf64_t xs, double *array,
            vi64_t vra)
{
  vf64_t xs1 = (vf64_t) vec_xxspltd ((vui64_t) xs, 1);
#ifdef _ARCH_PWR8
  vec_vstxsfdx (xs, vra[VEC_DW_H], array);
  vec_vstxsfdx (xs1, vra[VEC_DW_L], array);
#else
  // Need to explicitly manage the VR/GPR xfer for PWR7
  unsigned __int128 gprp = vec_transfer_vui128t_to_uint128 ((vui128_t) vra);
  vec_vstxsfdx (xs, scalar_extract_uint64_from_high_uint128(gprp), array);
  vec_vstxsfdx (xs1, scalar_extract_uint64_from_low_uint128(gprp), array);
#endif
}
 
static inline void
vec_vsstfddsx (vf64_t xs, double *array,
            vi64_t vra, const unsigned char scale)
{
  vi64_t offset;
 
  offset = (vi64_t) vec_sldi ((vui64_t) vra, (3 + scale));
  vec_vsstfddo (xs, array, offset);
}
 
static inline void
vec_vsstfddx (vf64_t xs, double *array, vi64_t vra)
{
  vi64_t offset;
 
  offset = (vi64_t) vec_sldi ((vui64_t) vra, 3);
  vec_vsstfddo (xs, array, offset);
}
 
static inline vf64_t
vec_vlxsfdx (const signed long long ra, const double *rb)
{
  vf64_t xt;
 
#if defined (__clang__)
  __VEC_U_128 t;
  unsigned long long *p = (unsigned long long *)((char *)rb + ra);
  t.ulong.upper = *p;
  xt = t.vx1;
#else
  if (__builtin_constant_p (ra) && (ra < 32760) && (ra >= -32768)
      && ((ra & 3) == 0))
    {
#if defined (_ARCH_PWR9)
      __asm__(
          "lxsd%X1 %0,%1;"
          : "=v" (xt)
          : "m" (*(double*)((char *)rb + ra))
          : );
#else
      if (ra == 0)
        {
          __asm__(
              "lxsdx %x0,%y1;"
              : "=wa" (xt)
              : "Z" (*rb)
              : );
        } else {
          unsigned long long rt;
#if defined (_ARCH_PWR8)
          // For P8 better if li and lxsdx shared a single asm block
          // (enforcing consecutive instructions).
          // This enables instruction fusion for P8.
          __asm__(
              "li %0,%2;"
              "lxsdx %x1,%3,%0;"
              : "=&r" (rt), "=wa" (xt)
              : "I" (ra), "b" (rb), "m" (*(double*)((char *)rb+ra))
              : );
#else // _ARCH_PWR7
          // This generates operationally the same code, but the
          // compiler may rearrange/schedule the code.
          __asm__(
              "li %0,%1;"
              : "=r" (rt)
              : "I" (ra)
              : );
          __asm__(
              "lxsdx %x0,%y1;"
              : "=wa" (xt)
              : "Z" (*(double*)((char *)rb+rt))
              : );
#endif
        }
#endif
    } else {
      __asm__(
          "lxsdx %x0,%y1;"
          : "=wa" (xt)
          : "Z" (*(double*)((char *)rb+ra))
          : );
    }
#endif
  return xt;
}
 
static inline void
vec_vstxsfdx (vf64_t xs, const signed long long ra, double *rb)
{
#if defined (__clang__)
  __VEC_U_128 t;
  unsigned long long *p = (unsigned long long *)((char *)rb + ra);
  t.vx1 = xs;
  *p = t.ulong.upper;
#else
  if (__builtin_constant_p (ra) && (ra <= 32760) && (ra >= -32768)
      && ((ra & 3) == 0))
    {
#if defined (_ARCH_PWR9)
      __asm__(
          "stxsd%X0 %1,%0;"
          : "=m" (*(double*)((char *)rb + ra))
          : "v" (xs)
          : );
#else
      if (ra == 0)
        {
          __asm__(
              "stxsdx %x1,%y0;"
              : "=Z" (*rb)
              : "wa" (xs)
              : );
        } else {
          unsigned long long rt;
          __asm__(
              "li %0,%1;"
              : "=r" (rt)
              : "I" (ra)
              : );
          __asm__(
              "stxsdx %x1,%y0;"
              : "=Z" (*(double*)((char *)rb+rt))
              : "wa" (xs)
              : );
        }
#endif
    } else {
      __asm__(
          "stxsdx %x1,%y0;"
          : "=Z" (*(double*)((char *)rb+ra))
          : "wa" (xs)
          : );
    }
#endif
}
 
static inline vf64_t
vec_xviexpdp (vui64_t sig, vui64_t exp)
{
  vf64_t result;
#if defined (_ARCH_PWR9) && defined (__VSX__) && (__GNUC__ > 7)
#if defined (vec_insert_exp)
  result = vec_insert_exp (sig, exp);
#else
  __asm__(
      "xviexpdp %x0,%x1,%x2"
      : "=wa" (result)
      : "wa" (sig), "wa" (exp)
      : );
#endif
#else
  vui32_t tmp;
  const vui32_t expmask = CONST_VINT128_W(0x7ff00000, 0, 0x7ff00000, 0);
 
  tmp = (vui32_t) vec_slqi ((vui128_t) exp, 52);
  result = (vf64_t) vec_sel ((vui32_t) sig, tmp, expmask);
#endif
  return result;
}
 
static inline vui64_t
vec_xvxexpdp (vf64_t vrb)
{
  vui64_t result;
#if defined (_ARCH_PWR9) && defined (__VSX__) && (__GNUC__ > 7)
#if defined (vec_extract_exp)
  result = vec_extract_exp (vrb);
#else
  __asm__(
      "xvxexpdp %x0,%x1"
      : "=wa" (result)
      : "wa" (vrb)
      : );
#endif
#else
  vui32_t tmp;
  const vui32_t expmask = CONST_VINT128_W(0x7ff00000, 0, 0x7ff00000, 0);
 
  tmp = vec_and ((vui32_t) vrb, expmask);
  result = (vui64_t) vec_srqi ((vui128_t) tmp, 52);
#endif
  return result;
}
 
static inline vui64_t
vec_xvxsigdp (vf64_t vrb)
{
  vui64_t result;
#if defined (_ARCH_PWR9) && defined (__VSX__) && (__GNUC__ > 7)
#if defined (vec_extract_sig)
  result = vec_extract_sig (vrb);
#else
  __asm__(
      "xvxsigdp %x0,%x1"
      : "=wa" (result)
      : "wa" (vrb)
      : );
#endif
#else
  vui32_t t128, tmp;
  vui32_t normal;
  const vui32_t zero = CONST_VINT128_W(0, 0, 0, 0);
  const vui32_t sigmask = CONST_VINT128_W(0x000fffff, -1, 0x000fffff, -1);
  const vui32_t expmask = CONST_VINT128_W(0x7ff00000, 0, 0x7ff00000, 0);
  const vui32_t hidden = CONST_VINT128_W(0x00100000, 0, 0x00100000, 0);
 
  // Check if vrb is normal. Normal values need the hidden bit
  // restored to the significand. We use a simpler sequence here as
  // vec_isnormalf64 does more then we need.
  tmp = vec_and ((vui32_t) vrb, expmask);
  normal = (vui32_t) vec_nor (vec_cmpeq (tmp, expmask),
                              vec_cmpeq (tmp, zero));
  t128 = vec_and ((vui32_t) vrb, sigmask);
  result = (vui64_t) vec_sel (t128, normal, hidden);
#endif
  return result;
}
 
#endif /* VEC_F64_PPC_H_ */