Merge pull request #18 from tufanbarisyildirim/issue-17

Rework variable & keyword

full-example/formatting/issue17-formatted.conf

@@ -0,0 +1,40 @@
+location / {
+    set $serve_URL $fullurl${uri}index.html;
+    try_files $serve_URL $uri $uri/ /index.php$is_args$args;
+}
+# deny access to xmlrpc.php - https://kinsta.com/blog/xmlrpc-php/
+location ~* ^/xmlrpc.php$ {
+    return 403;
+}
+location ~ \.php$ {
+    include snippets/fastcgi-php.conf;
+    fastcgi_param PHP_VALUE "open_basedir=$document_root:/tmp;\nerror_log=/public_html/logs/demo1-php_errors.log;";
+    fastcgi_pass unix:/run/php/php7.4-fpm.sock;
+}
+location /wp-content/uploads/ {
+    location ~ .(aspx|php|jsp|cgi)$ {
+        return 410;
+    }
+}
+#location ~ \.pdf$ { rewrite .* /custom/pdf_auth.php; }
+location ~* \.(css|gif|ico|jpeg|jpg|js|png|woff|woff2|ttf|ttc|otf|eot)$ {
+    # https://nginx.org/en/docs/http/ngx_http_headers_module.html
+    expires 30d;
+    # https://nginx.org/en/docs/http/ngx_http_core_module.html#log_not_found
+    log_not_found off;
+}
+# deny access to .htaccess files
+location ~ /\.ht {
+    deny all;
+}
+location ~ ^/(status)$ {
+    # https://www.tecmint.com/enable-monitor-php-fpm-status-in-nginx/
+    allow 127.0.0.1;
+    fastcgi_param SCRIPT_FILENAME $document_root$fastcgi_script_name;
+    fastcgi_index index.php;
+    include fastcgi_params;
+    fastcgi_pass unix:/run/php/php7.4-fpm.sock;
+}
+location /.well-known/acme-challenge {
+    alias /public_html/certbot_temp/.well-known/acme-challenge;
+}

full-example/formatting/issue17-raw.conf

@@ -0,0 +1,44 @@
+	location / {
+		set $serve_URL $fullurl${uri}index.html;
+		try_files $serve_URL $uri $uri/ /index.php$is_args$args;
+
+	}
+
+	location ~* ^/xmlrpc.php$ { return 403; } # deny access to xmlrpc.php - https://kinsta.com/blog/xmlrpc-php/
+
+	location ~ \.php$ {
+
+		include snippets/fastcgi-php.conf;
+		fastcgi_param PHP_VALUE "open_basedir=$document_root:/tmp;\nerror_log=/public_html/logs/demo1-php_errors.log;";
+		fastcgi_pass unix:/run/php/php7.4-fpm.sock;
+
+	}
+
+	location /wp-content/uploads/ {
+
+		location ~ .(aspx|php|jsp|cgi)$ { return 410; }
+		#location ~ \.pdf$ { rewrite .* /custom/pdf_auth.php; }
+
+	}
+
+	location ~* \.(css|gif|ico|jpeg|jpg|js|png|woff|woff2|ttf|ttc|otf|eot)$ {
+
+		expires 30d;       # https://nginx.org/en/docs/http/ngx_http_headers_module.html
+		log_not_found off; # https://nginx.org/en/docs/http/ngx_http_core_module.html#log_not_found
+
+	}
+
+	location ~ /\.ht { deny all; } # deny access to .htaccess files
+
+	location ~ ^/(status)$ {
+		# https://www.tecmint.com/enable-monitor-php-fpm-status-in-nginx/
+		allow 127.0.0.1;
+		fastcgi_param SCRIPT_FILENAME $document_root$fastcgi_script_name;
+		fastcgi_index index.php;
+		include fastcgi_params;
+		fastcgi_pass unix:/run/php/php7.4-fpm.sock;
+	}
+
+	location /.well-known/acme-challenge {
+		alias /public_html/certbot_temp/.well-known/acme-challenge;
+	}

parser/lexer.go

@@ -75,8 +75,6 @@ reToken:
 		return s.NewToken(token.BlockEnd).Lit(string(s.read()))
 	case ch == '#':
 		return s.scanComment()
-	case ch == '$':
-		return s.scanVariable()
 	case isQuote(ch):
 		return s.scanQuotedString(ch)
 	default:
@@ -198,23 +196,12 @@ func (s *lexer) scanQuotedString(delimiter rune) token.Token {
 			panic("unexpected end of file while scanning a string, maybe an unclosed quote?")
 		}
 
-		if ch == '\\' {
-			if needsEscape(s.peek(), delimiter) {
-				switch s.read() {
-				case 'n':
-					_, _ = buf.WriteRune('\n')
-				case 'r':
-					_, _ = buf.WriteRune('\r')
-				case 't':
-					_, _ = buf.WriteRune('\t')
-				case '\\':
-					_, _ = buf.WriteRune('\\')
-				case delimiter:
-					_, _ = buf.WriteRune(delimiter)
-				}
-				continue
-			}
+		if ch == '\\' && (s.peek() == delimiter) {
+			buf.WriteRune(ch)       // the backslash
+			buf.WriteRune(s.read()) // the char needed escaping
+			continue
 		}
+
 		_, _ = buf.WriteRune(ch)
 		if ch == delimiter {
 			break
@@ -225,11 +212,33 @@ func (s *lexer) scanQuotedString(delimiter rune) token.Token {
 }
 
 func (s *lexer) scanKeyword() token.Token {
-	return s.NewToken(token.Keyword).Lit(s.readUntil(isKeywordTerminator))
-}
+	var buf bytes.Buffer
+	tok := s.NewToken(token.Keyword)
+	prev := s.read()
+	buf.WriteRune(prev)
+	for {
+		ch := s.peek()
 
-func (s *lexer) scanVariable() token.Token {
-	return s.NewToken(token.Variable).Lit(s.readUntil(isKeywordTerminator))
+		//space, ;  and file ending definitely ends the keyword.
+		if isSpace(ch) || isEOF(ch) || ch == ';' {
+			break
+		}
+
+		//the keyword could contain a variable with like ${var}
+		if ch == '{' {
+			if prev == '$' {
+				buf.WriteString(s.readUntil(func(r rune) bool {
+					return r == '}'
+				}))
+				buf.WriteRune(s.read()) //consume latest '}'
+			} else {
+				break
+			}
+		}
+		buf.WriteRune(s.read())
+	}
+
+	return tok.Lit(buf.String())
 }
 
 func (s *lexer) read() rune {
@@ -251,14 +260,6 @@ func isQuote(ch rune) bool {
 	return ch == '"' || ch == '\'' || ch == '`'
 }
 
-func isKeywordTerminator(ch rune) bool {
-	return isSpace(ch) || isEndOfLine(ch) || ch == '{' || ch == ';'
-}
-
-func needsEscape(ch, delimiter rune) bool {
-	return ch == delimiter || ch == 'n' || ch == 't' || ch == '\\' || ch == 'r'
-}
-
 func isSpace(ch rune) bool {
 	return ch == ' ' || ch == '\t' || isEndOfLine(ch)
 }

parser/lexer_test.go

@@ -11,7 +11,7 @@ import (
 
 func TestScanner_Lex(t *testing.T) {
 	t.Parallel()
-	actual := lex(`
+	conf := `
 server { # simple reverse-proxy
     listen       80;
     server_name  gonginx.com www.gonginx.com;
@@ -32,7 +32,8 @@ server { # simple reverse-proxy
   }
 include /etc/nginx/conf.d/*.conf;
 directive "with a quoted string\t \r\n \\ with some escaped thing s\" good.";
-#also cmment right before eof`).all()
+#also cmment right before eof`
+	actual := lex(conf).all()
 
 	var expect = token.Tokens{
 		{Type: token.Keyword, Literal: "server", Line: 2, Column: 1},
@@ -74,15 +75,15 @@ directive "with a quoted string\t \r\n \\ with some escaped thing s\" good.";
 		{Type: token.Semicolon, Literal: ";", Line: 16, Column: 44},
 		{Type: token.Keyword, Literal: "proxy_set_header", Line: 17, Column: 7},
 		{Type: token.Keyword, Literal: "X-Real-IP", Line: 17, Column: 26},
-		{Type: token.Variable, Literal: "$remote_addr", Line: 17, Column: 43},
+		{Type: token.Keyword, Literal: "$remote_addr", Line: 17, Column: 43},
 		{Type: token.Semicolon, Literal: ";", Line: 17, Column: 55},
 		{Type: token.BlockEnd, Literal: "}", Line: 18, Column: 5},
 		{Type: token.BlockEnd, Literal: "}", Line: 19, Column: 3},
 		{Type: token.Keyword, Literal: "include", Line: 20, Column: 1},
 		{Type: token.Keyword, Literal: "/etc/nginx/conf.d/*.conf", Line: 20, Column: 9},
 		{Type: token.Semicolon, Literal: ";", Line: 20, Column: 33},
 		{Type: token.Keyword, Literal: "directive", Line: 21, Column: 1},
-		{Type: token.QuotedString, Literal: "\"with a quoted string\t \r\n \\ with some escaped thing s\" good.\"", Line: 21, Column: 11},
+		{Type: token.QuotedString, Literal: "\"with a quoted string\\t \\r\\n \\\\ with some escaped thing s\\\" good.\"", Line: 21, Column: 11},
 		{Type: token.Semicolon, Literal: ";", Line: 21, Column: 77},
 		{Type: token.Comment, Literal: "#also cmment right before eof", Line: 22, Column: 1},
 	}
@@ -93,8 +94,8 @@ directive "with a quoted string\t \r\n \\ with some escaped thing s\" good.";
 	assert.NilError(t, err)
 
 	//assert.Assert(t, tokens, 1)
-	assert.Assert(t, actual.EqualTo(expect))
 	assert.Equal(t, string(tokenString), string(expectJSON))
+	assert.Assert(t, actual.EqualTo(expect))
 	assert.Equal(t, len(actual), len(expect))
 }
 

parser/parser_test.go

@@ -254,3 +254,54 @@ location / { proxy_pass http://big_server_com; } } }`
 			lex(fullconf)).Parse()
 	}
 }
+
+func TestParser_Issue17(t *testing.T) {
+	t.Parallel()
+	p, err := NewParser("../testdata/issues/17.conf")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	c := p.Parse()
+	s := gonginx.DumpConfig(c, gonginx.IndentedStyle)
+	assert.Equal(t, `location / {
+    set $serve_URL $fullurl${uri}index.html;
+    try_files $serve_URL $uri $uri/ /index.php$is_args$args;
+}
+# deny access to xmlrpc.php - https://kinsta.com/blog/xmlrpc-php/
+location ~* ^/xmlrpc.php$ {
+    return 403;
+}
+location ~ \.php$ {
+    include snippets/fastcgi-php.conf;
+    fastcgi_param PHP_VALUE "open_basedir=$document_root:/tmp;\nerror_log=/public_html/logs/demo1-php_errors.log;";
+    fastcgi_pass unix:/run/php/php7.4-fpm.sock;
+}
+location /wp-content/uploads/ {
+    location ~ .(aspx|php|jsp|cgi)$ {
+        return 410;
+    }
+}
+#location ~ \.pdf$ { rewrite .* /custom/pdf_auth.php; }
+location ~* \.(css|gif|ico|jpeg|jpg|js|png|woff|woff2|ttf|ttc|otf|eot)$ {
+    # https://nginx.org/en/docs/http/ngx_http_headers_module.html
+    expires 30d;
+    # https://nginx.org/en/docs/http/ngx_http_core_module.html#log_not_found
+    log_not_found off;
+}
+# deny access to .htaccess files
+location ~ /\.ht {
+    deny all;
+}
+location ~ ^/(status)$ {
+    # https://www.tecmint.com/enable-monitor-php-fpm-status-in-nginx/
+    allow 127.0.0.1;
+    fastcgi_param SCRIPT_FILENAME $document_root$fastcgi_script_name;
+    fastcgi_index index.php;
+    include fastcgi_params;
+    fastcgi_pass unix:/run/php/php7.4-fpm.sock;
+}
+location /.well-known/acme-challenge {
+    alias /public_html/certbot_temp/.well-known/acme-challenge;
+}`, s)
+}

testdata/issues/17.conf

@@ -0,0 +1,44 @@
+	location / {
+		set $serve_URL $fullurl${uri}index.html;
+		try_files $serve_URL $uri $uri/ /index.php$is_args$args;
+
+	}
+
+	location ~* ^/xmlrpc.php$ { return 403; } # deny access to xmlrpc.php - https://kinsta.com/blog/xmlrpc-php/
+
+	location ~ \.php$ {
+
+		include snippets/fastcgi-php.conf;
+		fastcgi_param PHP_VALUE "open_basedir=$document_root:/tmp;\nerror_log=/public_html/logs/demo1-php_errors.log;";
+		fastcgi_pass unix:/run/php/php7.4-fpm.sock;
+
+	}
+
+	location /wp-content/uploads/ {
+
+		location ~ .(aspx|php|jsp|cgi)$ { return 410; }
+		#location ~ \.pdf$ { rewrite .* /custom/pdf_auth.php; }
+
+	}
+
+	location ~* \.(css|gif|ico|jpeg|jpg|js|png|woff|woff2|ttf|ttc|otf|eot)$ {
+
+		expires 30d;       # https://nginx.org/en/docs/http/ngx_http_headers_module.html
+		log_not_found off; # https://nginx.org/en/docs/http/ngx_http_core_module.html#log_not_found
+
+	}
+
+	location ~ /\.ht { deny all; } # deny access to .htaccess files
+
+	location ~ ^/(status)$ {
+		# https://www.tecmint.com/enable-monitor-php-fpm-status-in-nginx/
+		allow 127.0.0.1;
+		fastcgi_param SCRIPT_FILENAME $document_root$fastcgi_script_name;
+		fastcgi_index index.php;
+		include fastcgi_params;
+		fastcgi_pass unix:/run/php/php7.4-fpm.sock;
+	}
+
+	location /.well-known/acme-challenge {
+		alias /public_html/certbot_temp/.well-known/acme-challenge;
+	}