aarch64: Add the ability to have three types in an sve/sme intrinsic name

The majority of sve/sme intrinsics have names which are defined by one type
(like svuint8_t svextq[_u8]) or two types (like svsub_za32[_f32]_vg1x2).
Some intrinsics now have three types (like svtmopa_lane_za32[_s8_u8]).
This change extends the number of type_suffix_indexes from two to three
to cover this case.

gcc/
	* config/aarch64/aarch64-sve-builtins-base.cc: (svmul_impl::fold):
	Replace use of type_suffix_pair with type_suffix_triple.
	* config/aarch64/aarch64-sve-builtins-shapes.cc: (parse_element_type):
	Handle third type suffix.
	(parse_type): Handle c2 in function signature. Add the u signature with
	the ability to pass a tuple with twice as many vectors as the base type.
	Calculate number of vectors against the type with the maximum number of
	bits rather than "the other one".
	(load_contiguous_base::resolve): Add argument to resolve_to call.
	(compare_scalar_def::resolve): Likewise.
	(ternary_mfloat8_def::resolve): Likewise.
	(ternary_mfloat8_lane_def::resolve): Likewise.
	(ternary_mfloat8_opt_n_def::resolve): Likewise.
	* config/aarch64/aarch64-sve-builtins.cc: (TYPES_all_pred,
	TYPES_all_count, TYPES_all_pred_count, TYPES_all_float,
	TYPES_all_signed, TYPES_all_float_and_signed, TYPES_all_unsigned,
	TYPES_all_integer, TYPES_all_arith, TYPES_all_data, TYPES_b, TYPES_c,
	TYPES_b_unsigned, TYPES_b_integer, TYPES_b_data, TYPES_bh_integer,
	TYPES_bs_unsigned, TYPES_bhs_signed, TYPES_bhs_unsigned,
	TYPES_bhs_integer, TYPES_bh_data, TYPES_bhs_data, TYPES_bhs_widen,
	TYPES_h_bfloat, TYPES_h_float, TYPES_h_integer, TYPES_h_data,
	TYPES_hs_signed, TYPES_hs_integer, TYPES_hs_float, TYPES_hs_data,
	TYPES_hd_unsigned, TYPES_hsd_signed, TYPES_hsd_integer, TYPES_hsd_data,
	TYPES_h_float_mf8, TYPES_s_float, TYPES_s_float_mf8,
	TYPES_s_float_hsd_integer, TYPES_s_float_sd_integer, TYPES_s_signed,
	TYPES_s_unsigned, TYPES_s_integer, TYPES_s_data, TYPES_sd_signed,
	TYPES_sd_unsigned, TYPES_sd_integer, TYPES_sd_data,
	TYPES_all_float_and_sd_integer, TYPES_d_float, TYPES_d_unsigned,
	TYPES_d_integer, TYPES_d_data, TYPES_cvt, TYPES_cvt_bfloat,
	TYPES_cvt_h_s_float, TYPES_cvt_f32_f16, TYPES_cvt_long,
	TYPES_cvt_narrow_s, TYPES_cvt_narrow, TYPES_cvt_s_s, TYPES_cvt_mf8,
	TYPES_cvtn_mf8, TYPES_cvtnx_mf8, TYPES_inc_dec_n, TYPES_qcvt_x2,
	TYPES_qcvt_x4, TYPES_qrshr_x2,TYPES_qrshru_x2, TYPES_qrshr_x4,
	TYPES_qrshru_x4, TYPES_reinterpret, TYPES_reinterpret_b,TYPES_while,
	TYPES_while_x, TYPES_while_x_c, TYPES_s_narrow_fsu,TYPES_all_za,
	TYPES_d_za, TYPES_za_bhsd_data,TYPES_za_all_data, TYPES_za_h_mf8,
	TYPES_za_hs_mf8, TYPES_za_h_bfloat, TYPES_za_h_float,
	TYPES_za_s_b_signed, TYPES_za_s_b_unsigned, TYPES_za_s_b_integer,
	TYPES_za_s_h_integer,TYPES_za_s_h_data, TYPES_za_s_unsigned,
	TYPES_za_s_integer, TYPES_za_s_mf8, TYPES_za_s_float, TYPES_za_s_data,
	TYPES_za_d_h_integer, TYPES_za_d_float, TYPES_za_d_integer,
	TYPES_mop_base, TYPES_mop_base_signed, TYPES_mop_base_unsigned,
	TYPES_mop_i16i64, TYPES_mop_i16i64_signed, TYPES_mop_i16i64_unsigned,
	ΤYPES_za): Extend defines to three arguments.
	(DEF_VECTOR_TYPE, DEF_DOUBLE_TYPE): Likewise.
	(DEF_TRIPLE_TYPE): Add new define.
	(DEF_SVE_TYPES_ARRAY): Redefine all types_ arrays into arrays of
	type_suffix_triple.
	(types_none): Likewise.
	(function_instance::hash): Add third type to hash calculation.
	(function_builder::get_name): Add third type to function name.
	(function_builder::add_overloaded_functions): Handle third type.
	(function_resolver::lookup_form): Likewise.
	(function_resolver::resolve_to): Likewise.
	(function_resolver::resolve_unary): Likewise.
	* config/aarch64/aarch64-sve-builtins.h: (type_suffix_triple): replace
	type_suffix_pair.
	(function_group_info::types): Likewise.
	(function_instance::ctor): Likewise.
	(function_instance::type_suffix_ids): Likewise.
	(function_resolver::lookup_form): Add third type argument.
	(function_resolver::resolve_to): Likewise.
	(function_instance::operator==): Add third type to equality calculation.

diff --git a/gcc/config/aarch64/aarch64-sve-builtins-base.cc b/gcc/config/aarch64/aarch64-sve-builtins-base.cc
index ca6e16578e8b7d8b4d417b2295f0480823bd3d90..c3638624af06f8c2b53f67dd023a888448b39e44 100644 (file)
--- a/gcc/config/aarch64/aarch64-sve-builtins-base.cc
+++ b/gcc/config/aarch64/aarch64-sve-builtins-base.cc
@@ -2341,9 +2341,11 @@ public:
            tree negated_op = op1;
            if (integer_minus_onep (op1))
              negated_op = op2;
-           type_suffix_pair signed_tsp =
-             {find_type_suffix (TYPE_signed, f.type_suffix (0).element_bits),
-               f.type_suffix_ids[1]};
+           type_suffix_triple signed_tsp = {
+             find_type_suffix (TYPE_signed, f.type_suffix (0).element_bits),
+             f.type_suffix_ids[1],
+             NUM_TYPE_SUFFIXES
+           };
            function_instance instance ("svneg", functions::svneg,
                                        shapes::unary, MODE_none, signed_tsp,
                                        GROUP_none, f.pred, FPM_unused);

diff --git a/gcc/config/aarch64/aarch64-sve-builtins-shapes.cc b/gcc/config/aarch64/aarch64-sve-builtins-shapes.cc
index 648f94395186eef8e9d0a49ea38135f564f06359..683e9603583b3912dee9502a16b673d422a22604 100644 (file)
--- a/gcc/config/aarch64/aarch64-sve-builtins-shapes.cc
+++ b/gcc/config/aarch64/aarch64-sve-builtins-shapes.cc
@@ -178,7 +178,7 @@ parse_element_type (const function_instance &instance, const char *&format)
                               type_suffixes[suffix].element_bits / 2);
     }
 
-  if (ch == '0' || ch == '1')
+  if (ch == '0' || ch == '1' || ch == '2')
     return instance.type_suffix_ids[ch - '0'];
 
   gcc_unreachable ();
@@ -194,11 +194,13 @@ parse_element_type (const function_instance &instance, const char *&format)
    b       - base vector type (from a _<m0>base suffix)
    c0      - the result of a conversion, based on type and group suffixes
    c1      - the source of a conversion, based on type and group suffixes
+   c2      - the source of a conversion, based on type and group suffixes
    d       - displacement vector type (from a _<m1>index or _<m1>offset suffix)
    e<name> - an enum with the given name
    s<elt>  - a scalar type with the given element suffix
    t<elt>  - a vector or tuple type with given element suffix [*1]
    T<elt>  - a vector or tuple type with given element suffix [*2]
+   u<elt>  - a vector or tuple type with given element suffix [*3]
    v<elt>  - a vector with the given element suffix
    D<elt>  - a 64 bit neon vector
    Q<elt>  - a 128 bit neon vector
@@ -208,6 +210,7 @@ parse_element_type (const function_instance &instance, const char *&format)
    [*1] the vectors_per_tuple function indicates whether the type should
        be a tuple, and if so, how many vectors it should contain.
    [*2] same as for [*1], but the tuple contains half as many vectors.
+   [*3] same as for [*1], but the tuple contains twice as many vectors.
 */
 static tree
 parse_type (const function_instance &instance, const char *&format)
@@ -235,16 +238,19 @@ parse_type (const function_instance &instance, const char *&format)
   if (ch == 'c')
     {
       int ch = *format++;
-      gcc_assert (ch == '0' || ch == '1');
-      unsigned int id = (ch == '0' ? 0 : 1);
+      gcc_assert (ch == '0' || ch == '1' || ch == '2');
+      unsigned int id = ch - '0';
       auto vector_type = instance.type_suffix (id).vector_type;
       unsigned int num_vectors = instance.group_suffix ().vectors_per_tuple;
       if (num_vectors != 1)
        {
-         unsigned int bits = instance.type_suffix (id).element_bits;
-         unsigned int other_bits = instance.type_suffix (1 - id).element_bits;
-         if (other_bits > bits)
-           num_vectors /= other_bits / bits;
+         unsigned int type_bits = instance.type_suffix (id).element_bits;
+         unsigned int max_bits = std::max ( std:: max (
+           instance.type_suffix (0).element_bits,
+           instance.type_suffix (1).element_bits),
+           instance.type_suffix (2).element_bits);
+         if (max_bits > type_bits)
+           num_vectors /= max_bits / type_bits;
        }
       return acle_vector_types[num_vectors - 1][vector_type];
     }
@@ -288,6 +294,14 @@ parse_type (const function_instance &instance, const char *&format)
       return acle_vector_types[num_vectors - 1][vector_type];
     }
 
+  if (ch == 'u')
+    {
+      type_suffix_index suffix = parse_element_type (instance, format);
+      vector_type_index vector_type = type_suffixes[suffix].vector_type;
+      unsigned int num_vectors = instance.vectors_per_tuple () * 2;
+      return acle_vector_types[num_vectors - 1][vector_type];
+    }
+
   if (ch == 'v')
     {
       type_suffix_index suffix = parse_element_type (instance, format);
@@ -850,7 +864,7 @@ struct load_contiguous_base : public overloaded_base<0>
       return error_mark_node;
 
     return r.resolve_to (r.mode_suffix_id, type, NUM_TYPE_SUFFIXES,
-                        r.group_suffix_id);
+                        NUM_TYPE_SUFFIXES, r.group_suffix_id);
   }
 };
 
@@ -2318,7 +2332,7 @@ struct compare_scalar_def : public overloaded_base<1>
       return error_mark_node;
 
     return r.resolve_to (r.mode_suffix_id, r.type_suffix_ids[0], type,
-                        r.group_suffix_id);
+                        NUM_TYPE_SUFFIXES, r.group_suffix_id);
   }
 };
 SHAPE (compare_scalar)
@@ -4167,7 +4181,8 @@ struct ternary_mfloat8_def
        || !r.require_scalar_type (3, "uint64_t"))
       return error_mark_node;
 
-    return r.resolve_to (r.mode_suffix_id, type, TYPE_SUFFIX_mf8, GROUP_none);
+    return r.resolve_to (r.mode_suffix_id, type, TYPE_SUFFIX_mf8,
+                        NUM_TYPE_SUFFIXES, GROUP_none);
   }
 };
 SHAPE (ternary_mfloat8)
@@ -4219,7 +4234,8 @@ struct ternary_mfloat8_lane_def
        || !r.require_scalar_type (4, "uint64_t"))
       return error_mark_node;
 
-    return r.resolve_to (r.mode_suffix_id, type, TYPE_SUFFIX_mf8, GROUP_none);
+    return r.resolve_to (r.mode_suffix_id, type, TYPE_SUFFIX_mf8,
+                        NUM_TYPE_SUFFIXES, GROUP_none);
   }
 };
 SHAPE (ternary_mfloat8_lane)
@@ -4290,7 +4306,8 @@ struct ternary_mfloat8_opt_n_def
     else if (!r.require_vector_type (2, VECTOR_TYPE_svmfloat8_t))
       return error_mark_node;
 
-    return r.resolve_to (mode, type, TYPE_SUFFIX_mf8, GROUP_none);
+    return r.resolve_to (mode, type, TYPE_SUFFIX_mf8, NUM_TYPE_SUFFIXES,
+                        GROUP_none);
   }
 };
 SHAPE (ternary_mfloat8_opt_n)

diff --git a/gcc/config/aarch64/aarch64-sve-builtins.cc b/gcc/config/aarch64/aarch64-sve-builtins.cc
index bf025adde890c97a92798677bb685c1702276245..79a1fca02917748f6a16237a470c659839390462 100644 (file)
--- a/gcc/config/aarch64/aarch64-sve-builtins.cc
+++ b/gcc/config/aarch64/aarch64-sve-builtins.cc
@@ -192,255 +192,255 @@ CONSTEXPR const group_suffix_info group_suffixes[] = {
    class ("b", "f", etc.) and a numerical bit count.  */
 
 /* _b8 _b16 _b32 _b64.  */
-#define TYPES_all_pred(S, D) \
+#define TYPES_all_pred(S, D, T) \
   S (b8), S (b16), S (b32), S (b64)
 
 /* _c8 _c16 _c32 _c64.  */
-#define TYPES_all_count(S, D) \
+#define TYPES_all_count(S, D, T) \
   S (c8), S (c16), S (c32), S (c64)
 
 /* _b8 _b16 _b32 _b64
    _c8 _c16 _c32 _c64.  */
-#define TYPES_all_pred_count(S, D) \
-  TYPES_all_pred (S, D), \
-  TYPES_all_count (S, D)
+#define TYPES_all_pred_count(S, D, T) \
+  TYPES_all_pred (S, D, T), \
+  TYPES_all_count (S, D, T)
 
 /* _f16 _f32 _f64.  */
-#define TYPES_all_float(S, D) \
+#define TYPES_all_float(S, D, T) \
   S (f16), S (f32), S (f64)
 
 /* _s8 _s16 _s32 _s64.  */
-#define TYPES_all_signed(S, D) \
+#define TYPES_all_signed(S, D, T) \
   S (s8), S (s16), S (s32), S (s64)
 
 /*     _f16 _f32 _f64
    _s8 _s16 _s32 _s64.  */
-#define TYPES_all_float_and_signed(S, D) \
-  TYPES_all_float (S, D), TYPES_all_signed (S, D)
+#define TYPES_all_float_and_signed(S, D, T) \
+  TYPES_all_float (S, D, T), TYPES_all_signed (S, D, T)
 
 /* _u8 _u16 _u32 _u64.  */
-#define TYPES_all_unsigned(S, D) \
+#define TYPES_all_unsigned(S, D, T) \
   S (u8), S (u16), S (u32), S (u64)
 
 /* _s8 _s16 _s32 _s64
    _u8 _u16 _u32 _u64.  */
-#define TYPES_all_integer(S, D) \
-  TYPES_all_signed (S, D), TYPES_all_unsigned (S, D)
+#define TYPES_all_integer(S, D, T) \
+  TYPES_all_signed (S, D, T), TYPES_all_unsigned (S, D, T)
 
 /*     _f16 _f32 _f64
    _s8 _s16 _s32 _s64
    _u8 _u16 _u32 _u64.  */
-#define TYPES_all_arith(S, D) \
-  TYPES_all_float (S, D), TYPES_all_integer (S, D)
+#define TYPES_all_arith(S, D, T) \
+  TYPES_all_float (S, D, T), TYPES_all_integer (S, D, T)
 
-#define TYPES_all_data(S, D) \
-  TYPES_b_data (S, D), \
-  TYPES_h_data (S, D), \
-  TYPES_s_data (S, D), \
-  TYPES_d_data (S, D)
+#define TYPES_all_data(S, D, T) \
+  TYPES_b_data (S, D, T), \
+  TYPES_h_data (S, D, T), \
+  TYPES_s_data (S, D, T), \
+  TYPES_d_data (S, D, T)
 
 /* _b only.  */
-#define TYPES_b(S, D) \
+#define TYPES_b(S, D, T) \
   S (b)
 
 /* _c only.  */
-#define TYPES_c(S, D) \
+#define TYPES_c(S, D, T) \
   S (c)
 
 /* _u8.  */
-#define TYPES_b_unsigned(S, D) \
+#define TYPES_b_unsigned(S, D, T) \
   S (u8)
 
 /* _s8
    _u8.  */
-#define TYPES_b_integer(S, D) \
-  S (s8), TYPES_b_unsigned (S, D)
+#define TYPES_b_integer(S, D, T) \
+  S (s8), TYPES_b_unsigned (S, D, T)
 
 /* _mf8
    _s8
    _u8.  */
-#define TYPES_b_data(S, D) \
-  S (mf8), TYPES_b_integer (S, D)
+#define TYPES_b_data(S, D, T) \
+  S (mf8), TYPES_b_integer (S, D, T)
 
 /* _s8 _s16
    _u8 _u16.  */
-#define TYPES_bh_integer(S, D) \
+#define TYPES_bh_integer(S, D, T) \
   S (s8), S (s16), S (u8), S (u16)
 
 /* _u8 _u32.  */
-#define TYPES_bs_unsigned(S, D) \
+#define TYPES_bs_unsigned(S, D, T) \
   S (u8), S (u32)
 
 /* _s8 _s16 _s32.  */
-#define TYPES_bhs_signed(S, D) \
+#define TYPES_bhs_signed(S, D, T) \
   S (s8), S (s16), S (s32)
 
 /* _u8 _u16 _u32.  */
-#define TYPES_bhs_unsigned(S, D) \
+#define TYPES_bhs_unsigned(S, D, T) \
   S (u8), S (u16), S (u32)
 
 /* _s8 _s16 _s32
    _u8 _u16 _u32.  */
-#define TYPES_bhs_integer(S, D) \
-  TYPES_bhs_signed (S, D), TYPES_bhs_unsigned (S, D)
+#define TYPES_bhs_integer(S, D, T) \
+  TYPES_bhs_signed (S, D, T), TYPES_bhs_unsigned (S, D, T)
 
-#define TYPES_bh_data(S, D)                    \
-  TYPES_b_data (S, D), \
-  TYPES_h_data (S, D)
+#define TYPES_bh_data(S, D, T)                 \
+  TYPES_b_data (S, D, T), \
+  TYPES_h_data (S, D, T)
 
-#define TYPES_bhs_data(S, D)                   \
-  TYPES_b_data (S, D), \
-  TYPES_h_data (S, D), \
-  TYPES_s_data (S, D)
+#define TYPES_bhs_data(S, D, T)                        \
+  TYPES_b_data (S, D, T), \
+  TYPES_h_data (S, D, T), \
+  TYPES_s_data (S, D, T)
 
 /* _s16_s8  _s32_s16  _s64_s32
    _u16_u8  _u32_u16  _u64_u32.  */
-#define TYPES_bhs_widen(S, D) \
+#define TYPES_bhs_widen(S, D, T) \
   D (s16, s8), D (s32, s16), D (s64, s32), \
   D (u16, u8), D (u32, u16), D (u64, u32)
 
 /* _bf16.  */
-#define TYPES_h_bfloat(S, D) \
+#define TYPES_h_bfloat(S, D, T) \
   S (bf16)
 
 /* _f16.  */
-#define TYPES_h_float(S, D) \
+#define TYPES_h_float(S, D, T) \
   S (f16)
 
 /* _s16
    _u16.  */
-#define TYPES_h_integer(S, D) \
+#define TYPES_h_integer(S, D, T) \
   S (s16), S (u16)
 
 /* _bf16
    _f16
    _s16
    _u16.  */
-#define TYPES_h_data(S, D) \
-  S (bf16), S (f16), TYPES_h_integer (S, D)
+#define TYPES_h_data(S, D, T) \
+  S (bf16), S (f16), TYPES_h_integer (S, D, T)
 
 /* _s16 _s32.  */
-#define TYPES_hs_signed(S, D) \
+#define TYPES_hs_signed(S, D, T) \
   S (s16), S (s32)
 
 /* _s16 _s32
    _u16 _u32.  */
-#define TYPES_hs_integer(S, D) \
-  TYPES_hs_signed (S, D), S (u16), S (u32)
+#define TYPES_hs_integer(S, D, T) \
+  TYPES_hs_signed (S, D, T), S (u16), S (u32)
 
 /* _f16 _f32.  */
-#define TYPES_hs_float(S, D) \
+#define TYPES_hs_float(S, D, T) \
   S (f16), S (f32)
 
-#define TYPES_hs_data(S, D) \
-  TYPES_h_data (S, D), \
-  TYPES_s_data (S, D)
+#define TYPES_hs_data(S, D, T) \
+  TYPES_h_data (S, D, T), \
+  TYPES_s_data (S, D, T)
 
 /* _u16 _u64.  */
-#define TYPES_hd_unsigned(S, D) \
+#define TYPES_hd_unsigned(S, D, T) \
   S (u16), S (u64)
 
 /* _s16 _s32 _s64.  */
-#define TYPES_hsd_signed(S, D) \
+#define TYPES_hsd_signed(S, D, T) \
   S (s16), S (s32), S (s64)
 
 /* _s16 _s32 _s64
    _u16 _u32 _u64.  */
-#define TYPES_hsd_integer(S, D) \
-  TYPES_hsd_signed (S, D), S (u16), S (u32), S (u64)
+#define TYPES_hsd_integer(S, D, T) \
+  TYPES_hsd_signed (S, D, T), S (u16), S (u32), S (u64)
 
-#define TYPES_hsd_data(S, D) \
-  TYPES_h_data (S, D), \
-  TYPES_s_data (S, D), \
-  TYPES_d_data (S, D)
+#define TYPES_hsd_data(S, D, T) \
+  TYPES_h_data (S, D, T), \
+  TYPES_s_data (S, D, T), \
+  TYPES_d_data (S, D, T)
 
 /* _f16_mf8.  */
-#define TYPES_h_float_mf8(S, D) \
+#define TYPES_h_float_mf8(S, D, T) \
   D (f16, mf8)
 
 /* _f32.  */
-#define TYPES_s_float(S, D) \
+#define TYPES_s_float(S, D, T) \
   S (f32)
 
 /* _f32_mf8.  */
-#define TYPES_s_float_mf8(S, D) \
+#define TYPES_s_float_mf8(S, D, T) \
   D (f32, mf8)
 
 /*      _f32
    _s16 _s32 _s64
    _u16 _u32 _u64.  */
-#define TYPES_s_float_hsd_integer(S, D) \
-  TYPES_s_float (S, D), TYPES_hsd_integer (S, D)
+#define TYPES_s_float_hsd_integer(S, D, T) \
+  TYPES_s_float (S, D, T), TYPES_hsd_integer (S, D, T)
 
 /* _f32
    _s32 _s64
    _u32 _u64.  */
-#define TYPES_s_float_sd_integer(S, D) \
-  TYPES_s_float (S, D), TYPES_sd_integer (S, D)
+#define TYPES_s_float_sd_integer(S, D, T) \
+  TYPES_s_float (S, D, T), TYPES_sd_integer (S, D, T)
 
 /* _s32.  */
-#define TYPES_s_signed(S, D) \
+#define TYPES_s_signed(S, D, T) \
   S (s32)
 
 /* _u32.  */
-#define TYPES_s_unsigned(S, D) \
+#define TYPES_s_unsigned(S, D, T) \
   S (u32)
 
 /* _s32
    _u32.  */
-#define TYPES_s_integer(S, D) \
-  TYPES_s_signed (S, D), TYPES_s_unsigned (S, D)
+#define TYPES_s_integer(S, D, T) \
+  TYPES_s_signed (S, D, T), TYPES_s_unsigned (S, D, T)
 
 /* _f32
    _s32
    _u32.  */
-#define TYPES_s_data(S, D) \
-  TYPES_s_float (S, D), TYPES_s_integer (S, D)
+#define TYPES_s_data(S, D, T) \
+  TYPES_s_float (S, D, T), TYPES_s_integer (S, D, T)
 
 /* _s32 _s64.  */
-#define TYPES_sd_signed(S, D) \
+#define TYPES_sd_signed(S, D, T) \
   S (s32), S (s64)
 
 /* _u32 _u64.  */
-#define TYPES_sd_unsigned(S, D) \
+#define TYPES_sd_unsigned(S, D, T) \
   S (u32), S (u64)
 
 /* _s32 _s64
    _u32 _u64.  */
-#define TYPES_sd_integer(S, D) \
-  TYPES_sd_signed (S, D), TYPES_sd_unsigned (S, D)
+#define TYPES_sd_integer(S, D, T) \
+  TYPES_sd_signed (S, D, T), TYPES_sd_unsigned (S, D, T)
 
-#define TYPES_sd_data(S, D) \
-  TYPES_s_data (S, D), \
-  TYPES_d_data (S, D)
+#define TYPES_sd_data(S, D, T) \
+  TYPES_s_data (S, D, T), \
+  TYPES_d_data (S, D, T)
 
 /* _f16 _f32 _f64
        _s32 _s64
        _u32 _u64.  */
-#define TYPES_all_float_and_sd_integer(S, D) \
-  TYPES_all_float (S, D), TYPES_sd_integer (S, D)
+#define TYPES_all_float_and_sd_integer(S, D, T) \
+  TYPES_all_float (S, D, T), TYPES_sd_integer (S, D, T)
 
 /* _f64.  */
-#define TYPES_d_float(S, D) \
+#define TYPES_d_float(S, D, T) \
   S (f64)
 
 /* _u64.  */
-#define TYPES_d_unsigned(S, D) \
+#define TYPES_d_unsigned(S, D, T) \
   S (u64)
 
 /* _s64
    _u64.  */
-#define TYPES_d_integer(S, D) \
-  S (s64), TYPES_d_unsigned (S, D)
+#define TYPES_d_integer(S, D, T) \
+  S (s64), TYPES_d_unsigned (S, D, T)
 
 /* _f64
    _s64
    _u64.  */
-#define TYPES_d_data(S, D) \
-  TYPES_d_float (S, D), TYPES_d_integer (S, D)
+#define TYPES_d_data(S, D, T) \
+  TYPES_d_float (S, D, T), TYPES_d_integer (S, D, T)
 
 /* All the type combinations allowed by svcvt.  */
-#define TYPES_cvt(S, D) \
+#define TYPES_cvt(S, D, T) \
   D (f16, f32), D (f16, f64), \
   D (f16, s16), D (f16, s32), D (f16, s64), \
   D (f16, u16), D (f16, u32), D (f16, u64), \
@@ -462,35 +462,35 @@ CONSTEXPR const group_suffix_info group_suffixes[] = {
   D (u64, f16), D (u64, f32), D (u64, f64)
 
 /* _bf16_f32.  */
-#define TYPES_cvt_bfloat(S, D) \
+#define TYPES_cvt_bfloat(S, D, T) \
   D (bf16, f32)
 
 /* { _bf16 _f16 } x _f32.  */
-#define TYPES_cvt_h_s_float(S, D) \
+#define TYPES_cvt_h_s_float(S, D, T) \
   D (bf16, f32), D (f16, f32)
 
 /* _f32_f16.  */
-#define TYPES_cvt_f32_f16(S, D) \
+#define TYPES_cvt_f32_f16(S, D, T) \
   D (f32, f16)
 
 /* _f32_f16
    _f64_f32.  */
-#define TYPES_cvt_long(S, D) \
+#define TYPES_cvt_long(S, D, T) \
   D (f32, f16), D (f64, f32)
 
 /* _f16_f32.  */
-#define TYPES_cvt_narrow_s(S, D) \
+#define TYPES_cvt_narrow_s(S, D, T) \
   D (f32, f64)
 
 /* _f16_f32
    _f32_f64.  */
-#define TYPES_cvt_narrow(S, D) \
-  D (f16, f32), TYPES_cvt_narrow_s (S, D)
+#define TYPES_cvt_narrow(S, D, T) \
+  D (f16, f32), TYPES_cvt_narrow_s (S, D, T)
 
 /* { _s32 _u32 } x _f32
 
    _f32 x { _s32 _u32 }.  */
-#define TYPES_cvt_s_s(S, D) \
+#define TYPES_cvt_s_s(S, D, T) \
   D (s32, f32), \
   D (u32, f32), \
   D (f32, s32), \
@@ -498,23 +498,23 @@ CONSTEXPR const group_suffix_info group_suffixes[] = {
 
 /* _f16_mf8
    _bf16_mf8.  */
-#define TYPES_cvt_mf8(S, D) \
+#define TYPES_cvt_mf8(S, D, T) \
   D (f16, mf8), D (bf16, mf8)
 
 /* _mf8_f16
    _mf8_bf16.  */
-#define TYPES_cvtn_mf8(S, D) \
+#define TYPES_cvtn_mf8(S, D, T) \
   D (mf8, f16), D (mf8, bf16)
 
 /* _mf8_f32.  */
-#define TYPES_cvtnx_mf8(S, D) \
+#define TYPES_cvtnx_mf8(S, D, T) \
   D (mf8, f32)
 
 /* { _s32 _s64 } x { _b8 _b16 _b32 _b64 }
    { _u32 _u64 }.  */
 #define TYPES_inc_dec_n1(D, A) \
   D (A, b8), D (A, b16), D (A, b32), D (A, b64)
-#define TYPES_inc_dec_n(S, D) \
+#define TYPES_inc_dec_n(S, D, T) \
   TYPES_inc_dec_n1 (D, s32), \
   TYPES_inc_dec_n1 (D, s64), \
   TYPES_inc_dec_n1 (D, u32), \
@@ -523,7 +523,7 @@ CONSTEXPR const group_suffix_info group_suffixes[] = {
 /* { _s16 _u16 } x _s32
 
    {      _u16 } x _u32.  */
-#define TYPES_qcvt_x2(S, D) \
+#define TYPES_qcvt_x2(S, D, T) \
   D (s16, s32), \
   D (u16, u32), \
   D (u16, s32)
@@ -535,7 +535,7 @@ CONSTEXPR const group_suffix_info group_suffixes[] = {
    { _s16 _u16 } x _s64
 
    {      _u16 } x _u64.  */
-#define TYPES_qcvt_x4(S, D) \
+#define TYPES_qcvt_x4(S, D, T) \
   D (s8, s32), \
   D (u8, u32), \
   D (u8, s32), \
@@ -545,19 +545,19 @@ CONSTEXPR const group_suffix_info group_suffixes[] = {
 
 /* _s16_s32
    _u16_u32.  */
-#define TYPES_qrshr_x2(S, D) \
+#define TYPES_qrshr_x2(S, D, T) \
   D (s16, s32), \
   D (u16, u32)
 
 /* _u16_s32.  */
-#define TYPES_qrshru_x2(S, D) \
+#define TYPES_qrshru_x2(S, D, T) \
   D (u16, s32)
 
 /* _s8_s32
    _s16_s64
    _u8_u32
    _u16_u64.  */
-#define TYPES_qrshr_x4(S, D) \
+#define TYPES_qrshr_x4(S, D, T) \
   D (s8, s32), \
   D (s16, s64), \
   D (u8, u32), \
@@ -565,7 +565,7 @@ CONSTEXPR const group_suffix_info group_suffixes[] = {
 
 /* _u8_s32
    _u16_s64.  */
-#define TYPES_qrshru_x4(S, D) \
+#define TYPES_qrshru_x4(S, D, T) \
   D (u8, s32), \
   D (u16, s64)
 
@@ -579,7 +579,7 @@ CONSTEXPR const group_suffix_info group_suffixes[] = {
   D (A, f16), D (A, f32), D (A, f64), \
   D (A, s8), D (A, s16), D (A, s32), D (A, s64), \
   D (A, u8), D (A, u16), D (A, u32), D (A, u64)
-#define TYPES_reinterpret(S, D) \
+#define TYPES_reinterpret(S, D, T) \
   TYPES_reinterpret1 (D, mf8), \
   TYPES_reinterpret1 (D, bf16), \
   TYPES_reinterpret1 (D, f16), \
@@ -596,7 +596,7 @@ CONSTEXPR const group_suffix_info group_suffixes[] = {
 
 /* _b_c
    _c_b.  */
-#define TYPES_reinterpret_b(S, D) \
+#define TYPES_reinterpret_b(S, D, T) \
   D (b, c), \
   D (c, b)
 
@@ -604,7 +604,7 @@ CONSTEXPR const group_suffix_info group_suffixes[] = {
                            { _u32 _u64 } */
 #define TYPES_while1(D, bn) \
   D (bn, s32), D (bn, s64), D (bn, u32), D (bn, u64)
-#define TYPES_while(S, D) \
+#define TYPES_while(S, D, T) \
   TYPES_while1 (D, b8), \
   TYPES_while1 (D, b16), \
   TYPES_while1 (D, b32), \
@@ -612,7 +612,7 @@ CONSTEXPR const group_suffix_info group_suffixes[] = {
 
 /* { _b8 _b16 _b32 _b64 } x { _s64 }
                            { _u64 } */
-#define TYPES_while_x(S, D) \
+#define TYPES_while_x(S, D, T) \
   D (b8, s64), D (b8, u64), \
   D (b16, s64), D (b16, u64), \
   D (b32, s64), D (b32, u64), \
@@ -620,7 +620,7 @@ CONSTEXPR const group_suffix_info group_suffixes[] = {
 
 /* { _c8 _c16 _c32 _c64 } x { _s64 }
                            { _u64 } */
-#define TYPES_while_x_c(S, D) \
+#define TYPES_while_x_c(S, D, T) \
   D (c8, s64), D (c8, u64), \
   D (c16, s64), D (c16, u64), \
   D (c32, s64), D (c32, u64), \
@@ -629,15 +629,15 @@ CONSTEXPR const group_suffix_info group_suffixes[] = {
 /* _f32_f16
    _s32_s16
    _u32_u16.  */
-#define TYPES_s_narrow_fsu(S, D) \
+#define TYPES_s_narrow_fsu(S, D, T) \
   D (f32, f16), D (s32, s16), D (u32, u16)
 
 /* _za8 _za16 _za32 _za64 _za128.  */
-#define TYPES_all_za(S, D) \
+#define TYPES_all_za(S, D, T) \
   S (za8), S (za16), S (za32), S (za64), S (za128)
 
 /* _za64.  */
-#define TYPES_d_za(S, D) \
+#define TYPES_d_za(S, D, T) \
   S (za64)
 
 /* {   _za8 } x {  _mf8       _s8  _u8 }
@@ -647,7 +647,7 @@ CONSTEXPR const group_suffix_info group_suffixes[] = {
    {  _za32 } x {       _f32 _s32 _u32 }
 
    {  _za64 } x {       _f64 _s64 _u64 }.  */
-#define TYPES_za_bhsd_data(S, D) \
+#define TYPES_za_bhsd_data(S, D, T) \
   D (za8, mf8), D (za8, s8), D (za8, u8), \
   D (za16, bf16), D (za16, f16), D (za16, s16), D (za16, u16), \
   D (za32, f32), D (za32, s32), D (za32, u32), \
@@ -660,128 +660,130 @@ CONSTEXPR const group_suffix_info group_suffixes[] = {
                { _s8   _s16 _s32 _s64 }
                { _u8   _u16 _u32 _u64 }.  */
 
-#define TYPES_za_all_data(S, D) \
-  TYPES_za_bhsd_data (S, D), \
+#define TYPES_za_all_data(S, D, T) \
+  TYPES_za_bhsd_data (S, D, T), \
   TYPES_reinterpret1 (D, za128)
 
 /* _za16_mf8.  */
-#define TYPES_za_h_mf8(S, D) \
+#define TYPES_za_h_mf8(S, D, T) \
   D (za16, mf8)
 
-/* _za32_mf8.  */
-#define TYPES_za_s_mf8(S, D) \
-  D (za32, mf8)
-
 /* { _za_16 _za_32 } x _mf8.  */
-#define TYPES_za_hs_mf8(S, D) \
+#define TYPES_za_hs_mf8(S, D, T) \
   D (za16, mf8), D (za32, mf8)
 
 /* _za16_bf16.  */
-#define TYPES_za_h_bfloat(S, D) \
+#define TYPES_za_h_bfloat(S, D, T) \
   D (za16, bf16)
 
 /* _za16_f16.  */
-#define TYPES_za_h_float(S, D) \
+#define TYPES_za_h_float(S, D, T) \
   D (za16, f16)
 
 /* _za32_s8.  */
-#define TYPES_za_s_b_signed(S, D) \
+#define TYPES_za_s_b_signed(S, D, T) \
    D (za32, s8)
 
 /* _za32_u8.  */
-#define TYPES_za_s_b_unsigned(S, D) \
+#define TYPES_za_s_b_unsigned(S, D, T) \
    D (za32, u8)
 
 /* _za32 x { _s8 _u8 }.  */
-#define TYPES_za_s_b_integer(S, D) \
+#define TYPES_za_s_b_integer(S, D, T) \
   D (za32, s8), D (za32, u8)
 
 /* _za32 x { _s16 _u16 }.  */
-#define TYPES_za_s_h_integer(S, D) \
+#define TYPES_za_s_h_integer(S, D, T) \
   D (za32, s16), D (za32, u16)
 
 /* _za32 x { _bf16 _f16 _s16 _u16 }.  */
-#define TYPES_za_s_h_data(S, D) \
+#define TYPES_za_s_h_data(S, D, T) \
   D (za32, bf16), D (za32, f16), D (za32, s16), D (za32, u16)
 
 /* _za32_u32.  */
-#define TYPES_za_s_unsigned(S, D) \
+#define TYPES_za_s_unsigned(S, D, T) \
   D (za32, u32)
 
 /* _za32 x { _s32 _u32 }.  */
-#define TYPES_za_s_integer(S, D) \
+#define TYPES_za_s_integer(S, D, T) \
   D (za32, s32), D (za32, u32)
 
 /* _za32_mf8.  */
-#define TYPES_za_s_mf8(S, D) \
+#define TYPES_za_s_mf8(S, D, T) \
   D (za32, mf8)
 
 /* _za32_f32.  */
-#define TYPES_za_s_float(S, D) \
+#define TYPES_za_s_float(S, D, T) \
   D (za32, f32)
 
 /* _za32 x { _f32 _s32 _u32 }.  */
-#define TYPES_za_s_data(S, D) \
+#define TYPES_za_s_data(S, D, T) \
   D (za32, f32), D (za32, s32), D (za32, u32)
 
 /* _za64 x { _s16 _u16 }.  */
-#define TYPES_za_d_h_integer(S, D) \
+#define TYPES_za_d_h_integer(S, D, T) \
   D (za64, s16), D (za64, u16)
 
 /* _za64_f64.  */
-#define TYPES_za_d_float(S, D) \
+#define TYPES_za_d_float(S, D, T) \
   D (za64, f64)
 
 /* _za64 x { _s64 _u64 }.  */
-#define TYPES_za_d_integer(S, D) \
+#define TYPES_za_d_integer(S, D, T) \
   D (za64, s64), D (za64, u64)
 
 /* _za32 x { _s8 _u8 _bf16 _f16 _f32 }.  */
-#define TYPES_mop_base(S, D) \
+#define TYPES_mop_base(S, D, T) \
   D (za32, s8), D (za32, u8), D (za32, bf16), D (za32, f16), D (za32, f32)
 
 /* _za32_s8.  */
-#define TYPES_mop_base_signed(S, D) \
+#define TYPES_mop_base_signed(S, D, T) \
   D (za32, s8)
 
 /* _za32_u8.  */
-#define TYPES_mop_base_unsigned(S, D) \
+#define TYPES_mop_base_unsigned(S, D, T) \
   D (za32, u8)
 
 /* _za64 x { _s16 _u16 }.  */
-#define TYPES_mop_i16i64(S, D) \
+#define TYPES_mop_i16i64(S, D, T) \
   D (za64, s16), D (za64, u16)
 
 /* _za64_s16.  */
-#define TYPES_mop_i16i64_signed(S, D) \
+#define TYPES_mop_i16i64_signed(S, D, T) \
   D (za64, s16)
 
 /* _za64_u16.  */
-#define TYPES_mop_i16i64_unsigned(S, D) \
+#define TYPES_mop_i16i64_unsigned(S, D, T) \
   D (za64, u16)
 
 /* _za.  */
-#define TYPES_za(S, D) \
+#define TYPES_za(S, D, T) \
   S (za)
 
-/* Describe a pair of type suffixes in which only the first is used.  */
-#define DEF_VECTOR_TYPE(X) { TYPE_SUFFIX_ ## X, NUM_TYPE_SUFFIXES }
+/* Describe a tuple of type suffixes in which only the first is used.  */
+#define DEF_VECTOR_TYPE(X) \
+  { TYPE_SUFFIX_ ## X, NUM_TYPE_SUFFIXES, NUM_TYPE_SUFFIXES }
+
+/* Describe a tuple of type suffixes in which only the first two are used.  */
+#define DEF_DOUBLE_TYPE(X, Y) \
+  { TYPE_SUFFIX_ ## X, TYPE_SUFFIX_ ## Y, NUM_TYPE_SUFFIXES }
 
-/* Describe a pair of type suffixes in which both are used.  */
-#define DEF_DOUBLE_TYPE(X, Y) { TYPE_SUFFIX_ ## X, TYPE_SUFFIX_ ## Y }
+/* Describe a tuple of type suffixes in which three elements are used.  */
+#define DEF_TRIPLE_TYPE(X, Y, Z) \
+  { TYPE_SUFFIX_ ## X, TYPE_SUFFIX_ ## Y, TYPE_SUFFIX_ ## Z }
 
 /* Create an array that can be used in aarch64-sve-builtins.def to
    select the type suffixes in TYPES_<NAME>.  */
 #define DEF_SVE_TYPES_ARRAY(NAME) \
-  static const type_suffix_pair types_##NAME[] = { \
-    TYPES_##NAME (DEF_VECTOR_TYPE, DEF_DOUBLE_TYPE), \
-    { NUM_TYPE_SUFFIXES, NUM_TYPE_SUFFIXES } \
+  static const type_suffix_triple types_##NAME[] = { \
+    TYPES_##NAME (DEF_VECTOR_TYPE, DEF_DOUBLE_TYPE, DEF_TRIPLE_TYPE), \
+    { NUM_TYPE_SUFFIXES, NUM_TYPE_SUFFIXES, NUM_TYPE_SUFFIXES } \
   }
 
 /* For functions that don't take any type suffixes.  */
-static const type_suffix_pair types_none[] = {
-  { NUM_TYPE_SUFFIXES, NUM_TYPE_SUFFIXES },
-  { NUM_TYPE_SUFFIXES, NUM_TYPE_SUFFIXES }
+static const type_suffix_triple types_none[] = {
+  { NUM_TYPE_SUFFIXES, NUM_TYPE_SUFFIXES, NUM_TYPE_SUFFIXES },
+  { NUM_TYPE_SUFFIXES, NUM_TYPE_SUFFIXES, NUM_TYPE_SUFFIXES }
 };
 
 /* Create an array for each TYPES_<combination> macro above.  */
@@ -1226,6 +1228,7 @@ function_instance::hash () const
   h.add_int (mode_suffix_id);
   h.add_int (type_suffix_ids[0]);
   h.add_int (type_suffix_ids[1]);
+  h.add_int (type_suffix_ids[2]);
   h.add_int (group_suffix_id);
   h.add_int (pred);
   h.add_int (fpm_mode);
@@ -1397,7 +1400,7 @@ function_builder::get_name (const function_instance &instance,
       }
   else
     append_name (instance.mode_suffix ().string);
-  for (unsigned int i = 0; i < 2; ++i)
+  for (unsigned int i = 0; i < 3; ++i)
     if (!overloaded_p || instance.shape->explicit_type_suffix_p (i))
       append_name (instance.type_suffix (i).string);
   if (!overloaded_p || instance.shape->explicit_group_suffix_p ())
@@ -1642,8 +1645,9 @@ function_builder::add_overloaded_functions (const function_group_info &group,
 {
   bool explicit_type0 = (*group.shape)->explicit_type_suffix_p (0);
   bool explicit_type1 = (*group.shape)->explicit_type_suffix_p (1);
+  bool explicit_type2 = (*group.shape)->explicit_type_suffix_p (2);
   bool explicit_group = (*group.shape)->explicit_group_suffix_p ();
-  auto add_function = [&](const type_suffix_pair &types,
+  auto add_function = [&](const type_suffix_triple &types,
                          group_suffix_index group_suffix_id,
                          unsigned int pi)
     {
@@ -1671,9 +1675,10 @@ function_builder::add_overloaded_functions (const function_group_info &group,
          {
            /* Stub out the types that are determined by overload
               resolution.  */
-           type_suffix_pair types = {
+           type_suffix_triple types = {
              explicit_type0 ? group.types[ti][0] : NUM_TYPE_SUFFIXES,
-             explicit_type1 ? group.types[ti][1] : NUM_TYPE_SUFFIXES
+             explicit_type1 ? group.types[ti][1] : NUM_TYPE_SUFFIXES,
+             explicit_type2 ? group.types[ti][2] : NUM_TYPE_SUFFIXES
            };
            add_function (types, group_suffix_id, pi);
          }
@@ -1824,9 +1829,10 @@ tree
 function_resolver::lookup_form (mode_suffix_index mode,
                                type_suffix_index type0,
                                type_suffix_index type1,
+                               type_suffix_index type2,
                                group_suffix_index group)
 {
-  type_suffix_pair types = { type0, type1 };
+  type_suffix_triple types = { type0, type1, type2 };
   function_instance instance (base_name, base, shape, mode, types, group, pred,
                              fpm_mode);
   registered_function *rfn
@@ -1846,13 +1852,14 @@ function_resolver::lookup_form (mode_suffix_index mode, sve_type type)
 {
   type_suffix_index type0 = type_suffix_ids[0];
   type_suffix_index type1 = type_suffix_ids[1];
+  type_suffix_index type2 = type_suffix_ids[2];
   (type0 == NUM_TYPE_SUFFIXES ? type0 : type1) = type.type;
 
   group_suffix_index group = group_suffix_id;
   if (group == GROUP_none && type.num_vectors != vectors_per_tuple ())
     group = num_vectors_to_group (type.num_vectors);
 
-  return lookup_form (mode, type0, type1, group);
+  return lookup_form (mode, type0, type1, type2, group);
 }
 
 /* Resolve the function to one with the mode suffix given by MODE, the
@@ -1863,9 +1870,10 @@ tree
 function_resolver::resolve_to (mode_suffix_index mode,
                               type_suffix_index type0,
                               type_suffix_index type1,
+                              type_suffix_index type2,
                               group_suffix_index group)
 {
-  tree res = lookup_form (mode, type0, type1, group);
+  tree res = lookup_form (mode, type0, type1, type2, group);
   if (!res)
     {
       if (type1 == NUM_TYPE_SUFFIXES)
@@ -3227,7 +3235,7 @@ function_resolver::resolve_unary (type_class_index merge_tclass,
      explicit.  */
   if (type_suffix_ids[0] != NUM_TYPE_SUFFIXES)
     return resolve_to (mode_suffix_id, type_suffix_ids[0], type,
-                      group_suffix_id);
+                      NUM_TYPE_SUFFIXES, group_suffix_id);
 
   return resolve_to (mode_suffix_id, type);
 }

diff --git a/gcc/config/aarch64/aarch64-sve-builtins.h b/gcc/config/aarch64/aarch64-sve-builtins.h
index 6098d8ff229ff7584742d0afa784a058ea606ed4..2d0fb0501da1b26ab5fdc448247f574341d4025c 100644 (file)
--- a/gcc/config/aarch64/aarch64-sve-builtins.h
+++ b/gcc/config/aarch64/aarch64-sve-builtins.h
@@ -38,7 +38,7 @@
    - mode_suffix_index represents the mode suffix
 
    - type_suffix_index represents individual type suffixes, while
-     type_suffix_pair represents a pair of them
+     type_suffix_triple represents a tuple of them
 
    - prediction_index extends the predication suffix with an additional
      alternative: PRED_implicit for implicitly-predicated operations
@@ -227,8 +227,8 @@ enum group_suffix_index
   NUM_GROUP_SUFFIXES
 };
 
-/* Combines two type suffixes.  */
-typedef enum type_suffix_index type_suffix_pair[2];
+/* Combines three type suffixes.  */
+typedef enum type_suffix_index type_suffix_triple[3];
 
 class function_base;
 class function_shape;
@@ -367,12 +367,12 @@ struct function_group_info
   /* A list of the available type suffixes, group suffixes, and predication
      types.  The function supports every combination of the three.
 
-     The list of type suffixes is terminated by two NUM_TYPE_SUFFIXES.
+     The list of type suffixes is terminated by three NUM_TYPE_SUFFIXES.
      It is lexicographically ordered based on the index value.
 
      The list of group suffixes is terminated by NUM_GROUP_SUFFIXES
      and the list of predication types is terminated by NUM_PREDS.  */
-  const type_suffix_pair *types;
+  const type_suffix_triple *types;
   const group_suffix_index *groups;
   const predication_index *preds;
 
@@ -390,7 +390,7 @@ class GTY((user)) function_instance
 public:
   function_instance (const char *, const function_base *,
                     const function_shape *, mode_suffix_index,
-                    const type_suffix_pair &, group_suffix_index,
+                    const type_suffix_triple &, group_suffix_index,
                     predication_index, fpm_mode_index);
 
   bool operator== (const function_instance &) const;
@@ -434,7 +434,7 @@ public:
   const function_base *base;
   const function_shape *shape;
   mode_suffix_index mode_suffix_id;
-  type_suffix_pair type_suffix_ids;
+  type_suffix_triple type_suffix_ids;
   group_suffix_index group_suffix_id;
   predication_index pred;
   fpm_mode_index fpm_mode;
@@ -525,11 +525,13 @@ public:
 
   tree report_no_such_form (sve_type);
   tree lookup_form (mode_suffix_index,
+                   type_suffix_index = NUM_TYPE_SUFFIXES,
                    type_suffix_index = NUM_TYPE_SUFFIXES,
                    type_suffix_index = NUM_TYPE_SUFFIXES,
                    group_suffix_index = GROUP_none);
   tree lookup_form (mode_suffix_index, sve_type);
   tree resolve_to (mode_suffix_index,
+                  type_suffix_index = NUM_TYPE_SUFFIXES,
                   type_suffix_index = NUM_TYPE_SUFFIXES,
                   type_suffix_index = NUM_TYPE_SUFFIXES,
                   group_suffix_index = GROUP_none);
@@ -907,7 +909,7 @@ inline function_instance::
 function_instance (const char *base_name_in, const function_base *base_in,
                   const function_shape *shape_in,
                   mode_suffix_index mode_suffix_id_in,
-                  const type_suffix_pair &type_suffix_ids_in,
+                  const type_suffix_triple &type_suffix_ids_in,
                   group_suffix_index group_suffix_id_in,
                   predication_index pred_in, fpm_mode_index fpm_mode_in)
   : base_name (base_name_in), base (base_in), shape (shape_in),
@@ -925,6 +927,7 @@ function_instance::operator== (const function_instance &other) const
          && mode_suffix_id == other.mode_suffix_id
          && type_suffix_ids[0] == other.type_suffix_ids[0]
          && type_suffix_ids[1] == other.type_suffix_ids[1]
+         && type_suffix_ids[2] == other.type_suffix_ids[2]
          && group_suffix_id == other.group_suffix_id
          && pred == other.pred
          && fpm_mode == other.fpm_mode);