Merge scanner-opt-branch (r4121:r4221) to trunk.

git-svn-id: http://lm-sensors.org/svn/lm-sensors/trunk@4222 7894878c-1315-0410-8ee3-d5d059ff63e0

Makefile

@@ -187,6 +187,7 @@ endif
 SRCDIRS += kernel/include
 SRCDIRS += lib prog/detect prog/dump prog/eeprom prog/pwm \
            prog/sensors prog/xeon ${PROG_EXTRA:%=prog/%} etc
+SRCDIRS += lib/test
 
 # Some often-used commands with default options
 MKDIR := mkdir -p
@@ -476,4 +477,4 @@ manhtml:
 	$(BISON) -p sensors_yy -d $< -o $@
 
 %.c: %.l
-	$(FLEX) -Psensors_yy -t $< > $@
+	$(FLEX) -Psensors_yy -t -b -Cf $< > $@

lib/conf-lex.l

@@ -33,6 +33,8 @@ static char *buffer;
 
 char sensors_lex_error[100];
 
+int sensors_yylineno;
+
 #define buffer_malloc() sensors_malloc_array(&buffer,&buffer_count,\
                                              &buffer_max,1)
 #define buffer_free() sensors_free_array(&buffer,&buffer_count,\
@@ -46,155 +48,224 @@ char sensors_lex_error[100];
 
 %}
 
-/* Scanner for configuration files */
+ /* Scanner for configuration files */
 
 %option nodefault
 %option noyywrap
-%option yylineno
 %option nounput
 
-/* States. 'Normal' states STRING and MIDDLE share some rules; other states
-   have only their own rules */
-%s MIDDLE
+ /* All states are exclusive */
+
+%x MIDDLE
 %x STRING
 %x ERR
 
-/* Any whitespace-like character */
-BLANK		[[:space:]]
+ /* Any whitespace-like character */
+
+BLANK		[ \f\t\v]
 
 IDCHAR		[[:alnum:]_]
 
-/* Note: `10', `10.4' and `.4' are valid, `10.' is not */
+ /* Note: `10', `10.4' and `.4' are valid, `10.' is not */
+
 FLOAT   [[:digit:]]*\.?[[:digit:]]+
 
-/* Only positive whole numbers are recognized here */
+ /* Only positive whole numbers are recognized here */
+
 NUM	0|([1-9][[:digit:]]*)
 
-/* Only number between 1 and 255, octally represented. */
+ /* Only number between 1 and 255, octally represented. */
+
 OCTESC		(1[0-7]{0,2})|([2-7][0-7]?)|(0[1-7][0-7]?)|(00[1-7])
 
 
 %%
 
+ /*
+  * STATE: INITIAL
+  */
 
- /* End of line: It may be the end of this line. Same for End of file. */
-<MIDDLE>\n	|
-<MIDDLE><<EOF>> {
-		  BEGIN(INITIAL);
-		  return EOL;
+<INITIAL>{
+
+<<EOF>>		{ /* EOF from this state terminates */
+		  return 0;
 		}
 
- /* We want to match any blank, except End of line; that is why we have to
-    match whitespace one by one! */
-{BLANK}		/* Eat up a blank */
+{BLANK}+	; /* eat as many blanks as possible at once */
 
- /* Escaped End of line: eat and be happy */
-<MIDDLE>\\\n	/* Eat this! */
+{BLANK}*\n	{ /* eat a bare newline (possibly preceded by blanks) */
+		  sensors_yylineno++;
+		}
 
- /* Remove a comment; we do not change the state, this is done when the \n is
-    eaten */
-#[^\n]*		/* Eat this! */
+ /* comments */
 
- /* Some keywords at the beginning of lines */
-<INITIAL>"label" {
+#.*		; /* eat the rest of the line after comment char */
+
+#.*\n		{ /* eat the rest of the line after comment char */
+		  sensors_yylineno++;
+		}
+
+ /*
+  * Keywords must be followed by whitespace - eat that too.
+  * If there isn't trailing whitespace, we still need to
+  * accept it as lexically correct (even though the parser
+  * will reject it anyway.)
+  */
+
+label{BLANK}*	{
 		  sensors_yylval.line = sensors_yylineno;
 		  BEGIN(MIDDLE);
 		  return LABEL;
 		}
 
-<INITIAL>"set"	{
+set{BLANK}*	{
 		  sensors_yylval.line = sensors_yylineno;
 		  BEGIN(MIDDLE);
 		  return SET;
 		}
 
-<INITIAL>"compute" {
+compute{BLANK}*	{
 		  sensors_yylval.line = sensors_yylineno;
 		  BEGIN(MIDDLE);
 		  return COMPUTE;
 		}
 
-<INITIAL>"bus"	{
+bus{BLANK}*	{
 		  sensors_yylval.line = sensors_yylineno;
 		  BEGIN(MIDDLE);
 		  return BUS;
 		}
 
-<INITIAL>"chip"	{
+chip{BLANK}*	{
 		  sensors_yylval.line = sensors_yylineno;
 		  BEGIN(MIDDLE);
 		  return CHIP;
 		}
-<INITIAL>"ignore" {
+
+ignore{BLANK}*	{
 		  sensors_yylval.line = sensors_yylineno;
 		  BEGIN(MIDDLE);
 		  return IGNORE;
 		}
 
  /* Anything else at the beginning of a line is an error */
-<INITIAL>.	{
+
+[a-z]+		|
+.		{
 		  yymore();
 		  BEGIN(ERR);
-		}
-
-<ERR>[^\n]*\n	{
-		  BEGIN(INITIAL);
 		  strcpy(sensors_lex_error,"Invalid keyword");
 		  return ERROR;
 		}
+}
+
+ /*
+  * STATE: ERROR
+  */
+
+<ERR>{
+
+.*		; /* eat whatever is left on this line */
+
+\n		{
+		  BEGIN(INITIAL);
+		  sensors_yylineno++;
+		  return EOL;
+		}
+}
+
+ /*
+  * STATE: MIDDLE
+  */
+
+<MIDDLE>{
+
+{BLANK}+	; /* eat as many blanks as possible at once */
+
+\n		{ /* newline here sends EOL token to parser */
+		  BEGIN(INITIAL);
+		  sensors_yylineno++;
+		  return EOL;
+		}
+
+<<EOF>>		{ /* EOF here sends EOL token to parser also */
+		  BEGIN(INITIAL);
+		  return EOL;
+		}
+
+\\{BLANK}*\n	{ /* eat an escaped newline with no state change */
+		  sensors_yylineno++;
+		}
+
+ /* comments */
+
+#.*		; /* eat the rest of the line after comment char */
+
+#.*\n		{ /* eat the rest of the line after comment char */
+		  BEGIN(INITIAL);
+		  sensors_yylineno++;
+		  return EOL;
+		}
 
  /* A number */
-<MIDDLE>{FLOAT}	{
+
+{FLOAT}		{
 		  sensors_yylval.value = atof(sensors_yytext);
 		  return FLOAT;
 		}
 
  /* Some operators */
-<MIDDLE>"+"	{
-		  return '+';
-		}
 
-<MIDDLE>"-"	{
-		  return '-';
-		}
-
-<MIDDLE>"*"	{
-		  return '*';
-		}
-
-<MIDDLE>"/"	{
-		  return '/';
-		}
-
-<MIDDLE>"("	{
-		  return '(';
-		}
-
-<MIDDLE>")"	{
-		  return ')';
-		}
-<MIDDLE>","	{
-		  return ',';
-		}
-<MIDDLE>"@"	{
-		  return '@';
-		}
-<MIDDLE>"^"	{
-		  return '^';
-		}
-<MIDDLE>"`"	{
-		  return '`';
-		}
+"+"		return '+';
+"-"		return '-';
+"*"		return '*';
+"/"		return '/';
+"("		return '(';
+")"		return ')';
+","		return ',';
+"@"		return '@';
+"^"		return '^';
+"`"		return '`';
 
  /* Quoted string */
-<MIDDLE>\"	{
+
+\"		{
 		  buffer_malloc();
 		  BEGIN(STRING);
 		}
 
+ /* A normal, unquoted identifier */
+
+{IDCHAR}+	{
+		  sensors_yylval.name = strdup(sensors_yytext);
+		  if (! sensors_yylval.name)
+		    sensors_fatal_error("conf-lex.l",
+                                        "Allocating a new string");
+		  
+		  return NAME;
+		}
+
+ /* anything else is bogus */
+
+.		|
+[[:digit:]]*\.	|
+\\{BLANK}*	{
+		  yymore();
+		  BEGIN(ERR);
+		  return ERROR;
+		}
+}
+
+ /*
+  * STATE: STRING
+  */
+
+<STRING>{
+
  /* Oops, newline while in a string is not good */
-<STRING>\n	|
-<STRING>\\\n	{
+
+\n		|
+\\\n		{
 		  buffer_add_char("\0");
 		  strcpy(sensors_lex_error,"No matching double quote");
 		  buffer_free();
@@ -203,7 +274,8 @@ OCTESC		(1[0-7]{0,2})|([2-7][0-7]?)|(0[1-7][0-7]?)|(00[1-7])
 		}
 
  /* At the end */
-<STRING>\"	{
+
+\"		{
 		  buffer_add_char("\0");
 		  sensors_yylval.name = strdup(buffer);
 		  if (! sensors_yylval.name)
@@ -214,60 +286,61 @@ OCTESC		(1[0-7]{0,2})|([2-7][0-7]?)|(0[1-7][0-7]?)|(00[1-7])
 		  return NAME;
 		}
 
-<STRING>\\a	{
+\\a		{
 		  buffer_add_char("\a");
 		}
 
-<STRING>\\b	{
+\\b		{
 		  buffer_add_char("\b");
 		}
 
-<STRING>\\f	{
+\\f		{
 		  buffer_add_char("\f");
 		}
 
-<STRING>\\n	{
+\\n		{
 		  buffer_add_char("\n");
 		}
 
-<STRING>\\r	{
+\\r		{
 		  buffer_add_char("\r");
 		}
 
-<STRING>\\t	{
+\\t		{
 		  buffer_add_char("\t");
 		}
 
-<STRING>\\v	{
+\\v		{
 		  buffer_add_char("\v");
 		}
 
  /* We can't support \0, this would cause havoc! */
-<STRING>\\{OCTESC} {
+
+\\{OCTESC}	{
 		  int res;
 		  sscanf(sensors_yytext+1,"%o",&res);
 		  buffer_add_char(&res);
 		}
 
  /* Other escapes: just copy the character behind the slash */
-<STRING>\\.	{
+
+\\.		{
 		  buffer_add_char(&sensors_yytext[1]);
 		}
 
- /* Anything else */
-<STRING>[^\\\n\"]+ {
-		  buffer_add_string(sensors_yytext);
+ /* This prevents backing up; it is otherwise purely redundant */
+
+\\00		{
+		  buffer_add_char("00");
 		}
 
- /* A normal, unquoted identifier */
-<MIDDLE>{IDCHAR}+ {
-		  sensors_yylval.name = strdup(sensors_yytext);
-		  if (! sensors_yylval.name)
-		    sensors_fatal_error("conf-lex.l",
-                                        "Allocating a new string");
-		  
-		  return NAME;
+ /* Anything else (including a bare '\' which may be followed by EOF) */
+
+\\		|
+[^\\\n\"]+	{
+		  buffer_add_string(sensors_yytext);
 		}
+}
 
 %%
 
@@ -289,6 +362,7 @@ int sensors_scanner_init(FILE *input)
 		return -1;
 
 	sensors_yy_switch_to_buffer(scan_buf);
+	sensors_yylineno = 1;
 	return 0;
 }
 

lib/test/Module.mk

@@ -0,0 +1,26 @@
+LIB_DIR		:= lib
+LIB_TEST_DIR	:= lib/test
+
+LIB_TEST_TARGETS := $(LIB_TEST_DIR)/test-scanner
+LIB_TEST_SOURCES := $(LIB_TEST_DIR)/test-scanner.c
+
+LIBICONV := $(shell if /sbin/ldconfig -p | grep -q libiconv\\.so ; then echo \-liconv; else echo; fi)
+
+LIB_TEST_SCANNER_OBJS := \
+	$(LIB_TEST_DIR)/test-scanner.ro \
+	$(LIB_DIR)/conf-lex.ao \
+	$(LIB_DIR)/error.ao \
+	$(LIB_DIR)/general.ao
+
+$(LIB_TEST_DIR)/test-scanner: $(LIB_TEST_SCANNER_OBJS)
+	$(CC) $(EXLDFLAGS) -o $@ $(LIB_TEST_SCANNER_OBJS) $(LIBICONV) -Llib
+
+all-lib-test: $(LIB_TEST_TARGETS)
+user :: all-lib-test
+
+$(LIB_TEST_DIR)/test-scanner.ro: $(LIB_DIR)/data.h $(LIB_DIR)/conf.h $(LIB_DIR)/conf-parse.h $(LIB_DIR)/scanner.h
+
+clean-lib-test:
+	$(RM) $(LIB_TEST_DIR)/*.rd $(LIB_TEST_DIR)/*.ro 
+	$(RM) $(LIB_TEST_TARGETS)
+clean :: clean-lib-test

lib/test/comment-without-eol.conf

@@ -0,0 +1 @@
+# this is a comment without a trailing newline 

lib/test/comment-without-eol.conf.stdout

@@ -0,0 +1 @@
+1: EOF

lib/test/comment.conf

@@ -0,0 +1 @@
+# this is a comment

lib/test/comment.conf.stdout

@@ -0,0 +1 @@
+2: EOF

lib/test/empty.conf.stdout

@@ -0,0 +1 @@
+1: EOF

lib/test/keywords.conf

@@ -0,0 +1,42 @@
+# chip keyword on a line by itself
+chip
+
+# preceded by whitespace
+	 chip
+
+# with trailing whitespace
+chip	 
+
+# ditto for the rest of the keywords
+label
+
+	  label
+
+label	 	
+
+set
+
+ set
+
+set 
+
+compute
+
+	compute
+
+compute	
+
+bus
+
+	  	bus
+
+bus	  	
+
+ignore
+
+	ignore
+
+ignore	
+
+# keyword followed by EOL/EOF
+chip

lib/test/keywords.conf.stdout

@@ -0,0 +1,39 @@
+2: CHIP
+3: EOL
+5: CHIP
+6: EOL
+8: CHIP
+9: EOL
+11: LABEL
+12: EOL
+13: LABEL
+14: EOL
+15: LABEL
+16: EOL
+17: SET
+18: EOL
+19: SET
+20: EOL
+21: SET
+22: EOL
+23: COMPUTE
+24: EOL
+25: COMPUTE
+26: EOL
+27: COMPUTE
+28: EOL
+29: BUS
+30: EOL
+31: BUS
+32: EOL
+33: BUS
+34: EOL
+35: IGNORE
+36: EOL
+37: IGNORE
+38: EOL
+39: IGNORE
+40: EOL
+42: CHIP
+43: EOL
+43: EOF

lib/test/names-errors.conf

@@ -0,0 +1,5 @@
+# keyword with bogus names
+label ?foo
+label bar%
+label baz$foo
+label !

lib/test/names-errors.conf.stdout

@@ -0,0 +1,15 @@
+2: LABEL
+2: ERROR
+3: EOL
+3: LABEL
+3: NAME: bar
+3: ERROR
+4: EOL
+4: LABEL
+4: NAME: baz
+4: ERROR
+5: EOL
+5: LABEL
+5: ERROR
+6: EOL
+6: EOF

lib/test/names.conf

@@ -0,0 +1,25 @@
+# keyword with some unquoted names
+label foo bar
+
+# keyword with escaped newline, names, and whitespace
+label		\
+ foo bar
+
+# names with full range of alpha-numeric and underscores
+label \
+	abcdefg \
+	hijklmnop \
+	qrs \
+	tuv \
+	wx \
+	yz \
+	a0123456789 \
+	982lksdf \
+	_abcd \
+	1234_ \
+	_ \
+	foo_bar_baz \
+	liajesiajef82197fjadf
+
+# keyword with names, immediate EOF
+label foo bar

lib/test/names.conf.stdout

@@ -0,0 +1,28 @@
+2: LABEL
+2: NAME: foo
+2: NAME: bar
+3: EOL
+5: LABEL
+6: NAME: foo
+6: NAME: bar
+7: EOL
+9: LABEL
+10: NAME: abcdefg
+11: NAME: hijklmnop
+12: NAME: qrs
+13: NAME: tuv
+14: NAME: wx
+15: NAME: yz
+16: NAME: a0123456789
+17: NAME: 982lksdf
+18: NAME: _abcd
+19: NAME: 1234_
+20: NAME: _
+21: NAME: foo_bar_baz
+22: NAME: liajesiajef82197fjadf
+23: EOL
+25: LABEL
+25: NAME: foo
+25: NAME: bar
+25: EOL
+25: EOF

lib/test/non-keywords.conf

@@ -0,0 +1,8 @@
+# these are not keywords
+ignored
+
+labs
+
+extra
+stuff
+

lib/test/non-keywords.conf.stdout

@@ -0,0 +1,9 @@
+2: ERROR
+3: EOL
+4: ERROR
+5: EOL
+6: ERROR
+7: EOL
+7: ERROR
+8: EOL
+9: EOF

lib/test/test-scanner.c

@@ -0,0 +1,103 @@
+/*
+    test-scanner.c - Regression test driver for the libsensors config file scanner.
+    Copyright (C) 2006 Mark M. Hoffman <mhoffman@lightlink.com>
+
+    This program is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; version 2 of the License.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program; if not, write to the Free Software
+    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+*/
+
+#include <stdio.h>
+
+#include "../data.h"
+#include "../conf.h"
+#include "../conf-parse.h"
+#include "../scanner.h"
+
+YYSTYPE sensors_yylval;
+
+int main(void)
+{
+	int result;
+
+	/* init the scanner */
+	if ((result = sensors_scanner_init(stdin)))
+		return result;
+
+	do {
+		result = sensors_yylex();
+
+		printf("%d: ", sensors_yylineno);
+
+		switch (result) {
+
+			case 0:
+				printf("EOF\n");
+				break;
+
+			case NEG:
+				printf("NEG\n");
+				break;
+	
+			case EOL:
+				printf("EOL\n");
+				break;
+	
+			case BUS:
+				printf("BUS\n");
+				break;
+	
+			case LABEL:
+				printf("LABEL\n");
+				break;
+	
+			case SET:
+				printf("SET\n");
+				break;
+	
+			case CHIP:
+				printf("CHIP\n");
+				break;
+	
+			case COMPUTE:
+				printf("COMPUTE\n");
+				break;
+	
+			case IGNORE:
+				printf("IGNORE\n");
+				break;
+	
+			case FLOAT:
+				printf("FLOAT: %f\n", sensors_yylval.value);
+				break;
+	
+			case NAME:
+				printf("NAME: %s\n", sensors_yylval.name);
+				break;
+	
+			case ERROR:
+				printf("ERROR\n");
+				break;
+
+			default:
+				printf("%c\n", (char)result);
+				break;
+		}
+
+	} while (result);
+
+	/* clean up the scanner */
+	sensors_scanner_exit();
+
+	return 0;
+}
+

lib/test/test-scanner.pl

@@ -0,0 +1,89 @@
+#!/usr/bin/perl -w
+
+# test-scanner.pl - test script for the libsensors config-file scanner
+# Copyright (C) 2006 Mark M. Hoffman <mhoffman@lightlink.com>
+#
+#    This program is free software; you can redistribute it and/or modify
+#    it under the terms of the GNU General Public License as published by
+#    the Free Software Foundation; version 2 of the License.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU General Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License
+#    along with this program; if not, write to the Free Software
+#    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+
+require 5.004;
+
+use strict;
+use Test::More;
+use Test::Cmd;
+
+my ($scenario, $test);
+my @scenarios = (
+	{ base => 'empty', status => 0,
+		desc => 'empty file' },
+	{ base => 'comment', status => 0,
+		desc => 'one comment line' },
+	{ base => 'comment-without-eol', status => 0,
+		desc => 'one comment line, without a trailing newline' },
+	{ base => 'keywords', status => 0,
+		desc => 'keywords with various whitespace variations' },
+	{ base => 'non-keywords', status => 0,
+		desc => 'various invalid keyword scenarios' },
+	{ base => 'names', status => 0,
+		desc => 'normal, unquoted names' },
+	{ base => 'names-errors', status => 0,
+		desc => 'invalid, unquoted names' },
+);
+
+plan tests => ($#scenarios + 1) * 3;
+
+$test = Test::Cmd->new(prog => 'test-scanner', workdir => '');
+
+foreach $scenario (@scenarios) {
+	my ($filename, @stdin, @stdout, @expout, @stderr, @experr, @diff);
+
+	$filename = $scenario->{"base"} . ".conf";
+	open INPUT, "< $filename" or die "Cannot open $filename: $!";
+	@stdin = <INPUT>;
+	close INPUT or die "Cannot close $filename: $!";
+
+	$filename = $scenario->{"base"} . ".conf.stdout";
+	open OUTPUT, "< $filename" or die "Cannot open $filename: $!";
+	@expout = <OUTPUT>;
+	close OUTPUT or die "Cannot close $filename: $!";
+
+	# if stderr file is not present, assume none is expected
+	$filename = $scenario->{"base"} . ".conf.stderr";
+	if (open ERROR, "< $filename") {
+		@experr = <ERROR>;
+		close ERROR or die "Cannot close $filename: $!";
+	} else {
+		@experr = ();
+	}
+
+	$test->string($scenario->{"desc"});
+	$test->run(stdin => \@stdin);
+
+	# test return status
+	ok($scenario->{"status"} == $?, "status: " . $scenario->{"desc"});
+
+	# force the captured outputs into an array - for some reason, the
+	# 'standard invocation' of diff_exact() chokes without this
+	@stdout = $test->stdout;
+	@stderr = $test->stderr;
+
+	# test stdout
+	ok($test->diff_exact(\@stdout, \@expout, \@diff),
+		"stdout: " . $scenario->{"desc"}) or print @diff;
+
+	# test stderr
+	ok($test->diff_exact(\@stderr, \@experr, \@diff),
+		"stderr: " . $scenario->{"desc"}) or print @diff;
+}
+