Fix up several contrib modules that were using varlena datatypes in not-so-obvious

ways.  I'm not totally sure that I caught everything, but at least now they pass
their regression tests with VARSIZE/SET_VARSIZE defined to reverse byte order.

3 files changed, 17 insertions, 15 deletions

diff --git a/contrib/pg_trgm/trgm.h b/contrib/pg_trgm/trgm.h
index b7674c52b58..bc2a4dc44a8 100644
--- a/contrib/pg_trgm/trgm.h
+++ b/contrib/pg_trgm/trgm.h
@@ -31,12 +31,12 @@ typedef char trgm[3];
 
 typedef struct
 {
-	int4		len;
+	int32		vl_len_;		/* varlena header (do not touch directly!) */
 	uint8		flag;
 	char		data[1];
 }	TRGM;
 
-#define TRGMHRDSIZE		  (sizeof(int4)+sizeof(uint8))
+#define TRGMHDRSIZE		  (VARHDRSZ + sizeof(uint8))
 
 /* gist */
 #define BITBYTE 8
@@ -70,12 +70,13 @@ typedef char *BITVECP;
 #define ISSIGNKEY(x)	( ((TRGM*)x)->flag & SIGNKEY )
 #define ISALLTRUE(x)	( ((TRGM*)x)->flag & ALLISTRUE )
 
-#define CALCGTSIZE(flag, len) ( TRGMHRDSIZE + ( ( (flag) & ARRKEY ) ? ((len)*sizeof(trgm)) : (((flag) & ALLISTRUE) ? 0 : SIGLEN) ) )
-#define GETSIGN(x)		( (BITVECP)( (char*)x+TRGMHRDSIZE ) )
-#define GETARR(x)		( (trgm*)( (char*)x+TRGMHRDSIZE ) )
-#define ARRNELEM(x) ( ( ((TRGM*)x)->len - TRGMHRDSIZE )/sizeof(trgm) )
+#define CALCGTSIZE(flag, len) ( TRGMHDRSIZE + ( ( (flag) & ARRKEY ) ? ((len)*sizeof(trgm)) : (((flag) & ALLISTRUE) ? 0 : SIGLEN) ) )
+#define GETSIGN(x)		( (BITVECP)( (char*)x+TRGMHDRSIZE ) )
+#define GETARR(x)		( (trgm*)( (char*)x+TRGMHDRSIZE ) )
+#define ARRNELEM(x) ( ( VARSIZE(x) - TRGMHDRSIZE )/sizeof(trgm) )
 
 extern float4 trgm_limit;
+
 TRGM	   *generate_trgm(char *str, int slen);
 float4		cnt_sml(TRGM * trg1, TRGM * trg2);
 
diff --git a/contrib/pg_trgm/trgm_gist.c b/contrib/pg_trgm/trgm_gist.c
index 41a827ae0c3..476cb1b9763 100644
--- a/contrib/pg_trgm/trgm_gist.c
+++ b/contrib/pg_trgm/trgm_gist.c
@@ -120,7 +120,7 @@ gtrgm_compress(PG_FUNCTION_ARGS)
 
 		len = CALCGTSIZE(SIGNKEY | ALLISTRUE, 0);
 		res = (TRGM *) palloc(len);
-		res->len = len;
+		SET_VARSIZE(res, len);
 		res->flag = SIGNKEY | ALLISTRUE;
 
 		retval = (GISTENTRY *) palloc(sizeof(GISTENTRY));
@@ -235,10 +235,11 @@ gtrgm_union(PG_FUNCTION_ARGS)
 	flag |= SIGNKEY;
 	len = CALCGTSIZE(flag, 0);
 	result = (TRGM *) palloc(len);
-	*size = result->len = len;
+	SET_VARSIZE(result, len);
 	result->flag = flag;
 	if (!ISALLTRUE(result))
 		memcpy((void *) GETSIGN(result), (void *) base, sizeof(BITVEC));
+	*size = len;
 
 	PG_RETURN_POINTER(result);
 }
@@ -486,26 +487,26 @@ gtrgm_picksplit(PG_FUNCTION_ARGS)
 	if (cache[seed_1].allistrue)
 	{
 		datum_l = (TRGM *) palloc(CALCGTSIZE(SIGNKEY | ALLISTRUE, 0));
-		datum_l->len = CALCGTSIZE(SIGNKEY | ALLISTRUE, 0);
+		SET_VARSIZE(datum_l, CALCGTSIZE(SIGNKEY | ALLISTRUE, 0));
 		datum_l->flag = SIGNKEY | ALLISTRUE;
 	}
 	else
 	{
 		datum_l = (TRGM *) palloc(CALCGTSIZE(SIGNKEY, 0));
-		datum_l->len = CALCGTSIZE(SIGNKEY, 0);
+		SET_VARSIZE(datum_l, CALCGTSIZE(SIGNKEY, 0));
 		datum_l->flag = SIGNKEY;
 		memcpy((void *) GETSIGN(datum_l), (void *) cache[seed_1].sign, sizeof(BITVEC));
 	}
 	if (cache[seed_2].allistrue)
 	{
 		datum_r = (TRGM *) palloc(CALCGTSIZE(SIGNKEY | ALLISTRUE, 0));
-		datum_r->len = CALCGTSIZE(SIGNKEY | ALLISTRUE, 0);
+		SET_VARSIZE(datum_r, CALCGTSIZE(SIGNKEY | ALLISTRUE, 0));
 		datum_r->flag = SIGNKEY | ALLISTRUE;
 	}
 	else
 	{
 		datum_r = (TRGM *) palloc(CALCGTSIZE(SIGNKEY, 0));
-		datum_r->len = CALCGTSIZE(SIGNKEY, 0);
+		SET_VARSIZE(datum_r, CALCGTSIZE(SIGNKEY, 0));
 		datum_r->flag = SIGNKEY;
 		memcpy((void *) GETSIGN(datum_r), (void *) cache[seed_2].sign, sizeof(BITVEC));
 	}
diff --git a/contrib/pg_trgm/trgm_op.c b/contrib/pg_trgm/trgm_op.c
index e247756a399..e9afef4b21f 100644
--- a/contrib/pg_trgm/trgm_op.c
+++ b/contrib/pg_trgm/trgm_op.c
@@ -70,9 +70,9 @@ generate_trgm(char *str, int slen)
 	int			wl,
 				len;
 
-	trg = (TRGM *) palloc(TRGMHRDSIZE + sizeof(trgm) * (slen / 2 + 1) * 3);
+	trg = (TRGM *) palloc(TRGMHDRSIZE + sizeof(trgm) * (slen / 2 + 1) * 3);
 	trg->flag = ARRKEY;
-	trg->len = TRGMHRDSIZE;
+	SET_VARSIZE(trg, TRGMHDRSIZE);
 
 	if (slen + LPADDING + RPADDING < 3 || slen == 0)
 		return trg;
@@ -178,7 +178,7 @@ generate_trgm(char *str, int slen)
 		len = unique_array(GETARR(trg), len);
 	}
 
-	trg->len = CALCGTSIZE(ARRKEY, len);
+	SET_VARSIZE(trg, CALCGTSIZE(ARRKEY, len));
 
 	return trg;
 }