Tweak the core scanner so that it can be used by plpgsql too.

Changes:

Pass in the keyword lookup array instead of having it be hardwired.
(This incidentally allows elimination of some duplicate coding in ecpg.)

Re-order the token declarations in gram.y so that non-keyword tokens have
numbers that won't change when keywords are added or removed.

Add ".." and ":=" to the set of tokens recognized by scan.l.  (Since these
combinations are nowhere legal in core SQL, this does not change anything
except the precise wording of the error you get when you write this.)

4 files changed, 38 insertions, 70 deletions

diff --git a/src/interfaces/ecpg/preproc/c_keywords.c b/src/interfaces/ecpg/preproc/c_keywords.c
index 2eae2c769b5..36f72b537e9 100644
--- a/src/interfaces/ecpg/preproc/c_keywords.c
+++ b/src/interfaces/ecpg/preproc/c_keywords.c
@@ -1,10 +1,10 @@
 /*-------------------------------------------------------------------------
  *
- * keywords.c
+ * c_keywords.c
  *	  lexical token lookup for reserved words in postgres embedded SQL
  *
- * $PostgreSQL: pgsql/src/interfaces/ecpg/preproc/c_keywords.c,v 1.23 2009/06/11 14:49:13 momjian Exp $
- * §
+ * $PostgreSQL: pgsql/src/interfaces/ecpg/preproc/c_keywords.c,v 1.24 2009/07/14 20:24:10 tgl Exp $
+ *
  *-------------------------------------------------------------------------
  */
 #include "postgres_fe.h"
@@ -55,8 +55,31 @@ static const ScanKeyword ScanCKeywords[] = {
 	{"year", YEAR_P, 0},
 };
 
+
+/*
+ * Do a binary search using plain strcmp() comparison.  This is much like
+ * ScanKeywordLookup(), except we want case-sensitive matching.
+ */
 const ScanKeyword *
 ScanCKeywordLookup(const char *text)
 {
-	return DoLookup(text, &ScanCKeywords[0], endof(ScanCKeywords) - 1);
+	const ScanKeyword *low = &ScanCKeywords[0];
+	const ScanKeyword *high = &ScanCKeywords[lengthof(ScanCKeywords) - 1];
+
+	while (low <= high)
+	{
+		const ScanKeyword *middle;
+		int			difference;
+
+		middle = low + (high - low) / 2;
+		difference = strcmp(middle->name, text);
+		if (difference == 0)
+			return middle;
+		else if (difference < 0)
+			low = middle + 1;
+		else
+			high = middle - 1;
+	}
+
+	return NULL;
 }
diff --git a/src/interfaces/ecpg/preproc/ecpg_keywords.c b/src/interfaces/ecpg/preproc/ecpg_keywords.c
index 0aef8161697..c475bf9671a 100644
--- a/src/interfaces/ecpg/preproc/ecpg_keywords.c
+++ b/src/interfaces/ecpg/preproc/ecpg_keywords.c
@@ -4,7 +4,7 @@
  *	  lexical token lookup for reserved words in postgres embedded SQL
  *
  * IDENTIFICATION
- *	  $PostgreSQL: pgsql/src/interfaces/ecpg/preproc/ecpg_keywords.c,v 1.40 2009/06/11 14:49:13 momjian Exp $
+ *	  $PostgreSQL: pgsql/src/interfaces/ecpg/preproc/ecpg_keywords.c,v 1.41 2009/07/14 20:24:10 tgl Exp $
  *
  *-------------------------------------------------------------------------
  */
@@ -75,79 +75,26 @@ static const ScanKeyword ScanECPGKeywords[] = {
 	{"whenever", SQL_WHENEVER, 0},
 };
 
-/* This is all taken from src/backend/parser/keyword.c and adjusted for our needs. */
-/*
- * Do a binary search using plain strcmp() comparison.
- */
-const ScanKeyword *
-DoLookup(const char *word, const ScanKeyword *low, const ScanKeyword *high)
-{
-	while (low <= high)
-	{
-		const ScanKeyword *middle;
-		int			difference;
-
-		middle = low + (high - low) / 2;
-		difference = strcmp(middle->name, word);
-		if (difference == 0)
-			return middle;
-		else if (difference < 0)
-			low = middle + 1;
-		else
-			high = middle - 1;
-	}
-
-	return NULL;
-}
-
 /*
  * ScanECPGKeywordLookup - see if a given word is a keyword
  *
  * Returns a pointer to the ScanKeyword table entry, or NULL if no match.
- *
- * The match is done case-insensitively.  Note that we deliberately use a
- * dumbed-down case conversion that will only translate 'A'-'Z' into 'a'-'z',
- * even if we are in a locale where tolower() would produce more or different
- * translations.  This is to conform to the SQL99 spec, which says that
- * keywords are to be matched in this way even though non-keyword identifiers
- * receive a different case-normalization mapping.
+ * Keywords are matched using the same case-folding rules as in the backend.
  */
 const ScanKeyword *
 ScanECPGKeywordLookup(const char *text)
 {
-	int			len,
-				i;
-	char		word[NAMEDATALEN];
 	const ScanKeyword *res;
 
 	/* First check SQL symbols defined by the backend. */
-
-	res = ScanKeywordLookup(text);
+	res = ScanKeywordLookup(text, ScanKeywords, NumScanKeywords);
 	if (res)
 		return res;
 
-	len = strlen(text);
-	/* We assume all keywords are shorter than NAMEDATALEN. */
-	if (len >= NAMEDATALEN)
-		return NULL;
-
-	/*
-	 * Apply an ASCII-only downcasing. We must not use tolower() since it may
-	 * produce the wrong translation in some locales (eg, Turkish).
-	 */
-	for (i = 0; i < len; i++)
-	{
-		char		ch = text[i];
-
-		if (ch >= 'A' && ch <= 'Z')
-			ch += 'a' - 'A';
-		word[i] = ch;
-	}
-	word[len] = '\0';
-
-	/*
-	 * Now do a binary search using plain strcmp() comparison.
-	 */
+	/* Try ECPG-specific keywords. */
+	res = ScanKeywordLookup(text, ScanECPGKeywords, lengthof(ScanECPGKeywords));
+	if (res)
+		return res;
 
-	return DoLookup(word, &ScanECPGKeywords[0], endof(ScanECPGKeywords) - 1);
+	return NULL;
 }
diff --git a/src/interfaces/ecpg/preproc/extern.h b/src/interfaces/ecpg/preproc/extern.h
index a7a125c13b2..ff80a5f243b 100644
--- a/src/interfaces/ecpg/preproc/extern.h
+++ b/src/interfaces/ecpg/preproc/extern.h
@@ -1,4 +1,4 @@
-/* $PostgreSQL: pgsql/src/interfaces/ecpg/preproc/extern.h,v 1.73 2009/06/11 14:49:13 momjian Exp $ */
+/* $PostgreSQL: pgsql/src/interfaces/ecpg/preproc/extern.h,v 1.74 2009/07/14 20:24:10 tgl Exp $ */
 
 #ifndef _ECPG_PREPROC_EXTERN_H
 #define _ECPG_PREPROC_EXTERN_H
@@ -101,7 +101,6 @@ extern void remove_variables(int);
 extern struct variable *new_variable(const char *, struct ECPGtype *, int);
 extern const ScanKeyword *ScanCKeywordLookup(const char *);
 extern const ScanKeyword *ScanECPGKeywordLookup(const char *text);
-extern const ScanKeyword *DoLookup(const char *, const ScanKeyword *, const ScanKeyword *);
 extern void scanner_init(const char *);
 extern void parser_init(void);
 extern void scanner_finish(void);
diff --git a/src/interfaces/ecpg/preproc/keywords.c b/src/interfaces/ecpg/preproc/keywords.c
index 3f47ea14525..0dcac907fa2 100644
--- a/src/interfaces/ecpg/preproc/keywords.c
+++ b/src/interfaces/ecpg/preproc/keywords.c
@@ -9,7 +9,7 @@
  *
  *
  * IDENTIFICATION
- *	  $PostgreSQL: pgsql/src/interfaces/ecpg/preproc/keywords.c,v 1.88 2009/03/08 16:53:30 alvherre Exp $
+ *	  $PostgreSQL: pgsql/src/interfaces/ecpg/preproc/keywords.c,v 1.89 2009/07/14 20:24:10 tgl Exp $
  *
  *-------------------------------------------------------------------------
  */
@@ -26,5 +26,4 @@ const ScanKeyword ScanKeywords[] = {
 #include "parser/kwlist.h"
 };
 
-/* End of ScanKeywords, for use in kwlookup.c */
-const ScanKeyword *LastScanKeyword = endof(ScanKeywords);
+const int	NumScanKeywords = lengthof(ScanKeywords);