Better handling of URLs with literal ';' and '?'

Author: markt-asf

java/org/apache/catalina/connector/CoyoteAdapter.java

@@ -650,18 +650,17 @@ protected boolean postParseRequest(org.apache.coyote.Request req, Request reques
             } else {
                 /*
                  * The URI is chars or String, and has been sent using an in-memory protocol handler. The following
-                 * assumptions are made: - req.requestURI() has been set to the 'original' non-decoded, non-normalized
-                 * URI - req.decodedURI() has been set to the decoded, normalized form of req.requestURI() -
-                 * 'suspicious' URI filtering - if required - has already been performed
+                 * assumptions are made:
+                 *
+                 * - req.requestURI() has been set to the 'original' non-decoded, non-normalized URI that includes path
+                 * parameters (if any)
+                 *
+                 * - req.decodedURI() has been set to the decoded, normalized form of req.requestURI() with any path
+                 * parameters removed
+                 *
+                 * - 'suspicious' URI filtering, if required, has already been performed
                  */
                 decodedURI.toChars();
-                // Remove all path parameters; any needed path parameter should be set
-                // using the request object rather than passing it in the URL
-                CharChunk uriCC = decodedURI.getCharChunk();
-                int semicolon = uriCC.indexOf(';');
-                if (semicolon > 0) {
-                    decodedURI.setChars(uriCC.getBuffer(), uriCC.getStart(), semicolon);
-                }
             }
         }
 

java/org/apache/catalina/valves/rewrite/RewriteValve.java

@@ -21,6 +21,7 @@
 import java.io.InputStream;
 import java.io.InputStreamReader;
 import java.io.StringReader;
+import java.net.URLDecoder;
 import java.nio.charset.Charset;
 import java.nio.charset.StandardCharsets;
 import java.util.ArrayList;
@@ -65,6 +66,24 @@
  */
 public class RewriteValve extends ValveBase {
 
+    private static final URLEncoder REWRITE_DEFAULT_ENCODER;
+    private static final URLEncoder REWRITE_QUERY_ENCODER;
+
+    static {
+        /*
+         * See the detailed explanation of encoding/decoding during URL re-writing in the invoke() method.
+         *
+         * These encoders perform the second stage of encoding, after re-writing has completed. These rewrite specific
+         * encoders treat '%' as a safe character so that URLs and query strings already processed by encodeForRewrite()
+         * do not end up with double encoding of '%' characters.
+         */
+        REWRITE_DEFAULT_ENCODER = (URLEncoder) URLEncoder.DEFAULT.clone();
+        REWRITE_DEFAULT_ENCODER.addSafeCharacter('%');
+
+        REWRITE_QUERY_ENCODER = (URLEncoder) URLEncoder.QUERY.clone();
+        REWRITE_QUERY_ENCODER.addSafeCharacter('%');
+    }
+
     /**
      * The rewrite rules that the valve will use.
      */
@@ -296,22 +315,51 @@ public void invoke(Request request, Response response) throws IOException, Servl
 
             invoked.set(Boolean.TRUE);
 
-            // As long as MB isn't a char sequence or affiliated, this has to be
-            // converted to a string
+            // As long as MB isn't a char sequence or affiliated, this has to be converted to a string
             Charset uriCharset = request.getConnector().getURICharset();
             String originalQueryStringEncoded = request.getQueryString();
             MessageBytes urlMB = context ? request.getRequestPathMB() : request.getDecodedRequestURIMB();
             urlMB.toChars();
             CharSequence urlDecoded = urlMB.getCharChunk();
+
+            /*
+             * The URL presented to the rewrite valve is the URL that is used for request mapping. That URL has been
+             * processed to: remove path parameters; remove the query string; decode; and normalize the URL. It may
+             * contain literal '%', '?' and/or ';' characters at this point.
+             *
+             * The re-write rules need to be able to process URLs with literal '?' characters and add query strings
+             * without the two becoming confused. The re-write rules also need to be able to insert literal '%'
+             * characters without them being confused with %nn encoding.
+             *
+             * The re-write rules cannot insert path parameters.
+             *
+             * To meet these requirement, the URL is processed as follows.
+             *
+             * Step 1. The URL is partially re-encoded by encodeForRewrite(). This method encodes any literal '%', ';'
+             * and/or '?' characters in the URL using the standard %nn form.
+             *
+             * Step 2. The re-write processing runs with the provided re-write rules against the partially encoded URL.
+             * If a re-write rule needs to insert a literal '%', ';' or '?', it must do so in %nn encoded form.
+             *
+             * Step 3. The URL (and query string if present) is re-encoded using the re-write specific encoders
+             * (REWRITE_DEFAULT_ENCODER and REWRITE_QUERY_ENCODER) that behave the same was as the standard encoders
+             * apart from '%' being treated as a safe character. This prevents double encoding of any '%' characters
+             * present in the URL from steps 1 or 2.
+             */
+
+            // Step 1. Encode URL for processing by the re-write rules.
+            CharSequence urlRewriteEncoded = encodeForRewrite(urlDecoded);
             CharSequence host = request.getServerName();
             boolean rewritten = false;
             boolean done = false;
             boolean qsa = false;
             boolean qsd = false;
             boolean valveSkip = false;
+
+            // Step 2. Process the URL using the re-write rules.
             for (int i = 0; i < rules.length; i++) {
                 RewriteRule rule = rules[i];
-                CharSequence test = (rule.isHost()) ? host : urlDecoded;
+                CharSequence test = (rule.isHost()) ? host : urlRewriteEncoded;
                 CharSequence newtest = rule.evaluate(test, resolver);
                 if (newtest != null && !Objects.equals(test.toString(), newtest.toString())) {
                     if (containerLog.isTraceEnabled()) {
@@ -321,7 +369,7 @@ public void invoke(Request request, Response response) throws IOException, Servl
                     if (rule.isHost()) {
                         host = newtest;
                     } else {
-                        urlDecoded = newtest;
+                        urlRewriteEncoded = newtest;
                     }
                     rewritten = true;
                 }
@@ -358,28 +406,30 @@ public void invoke(Request request, Response response) throws IOException, Servl
                 if (rule.isRedirect() && newtest != null) {
                     // Append the query string to the url if there is one and it
                     // hasn't been rewritten
-                    String urlStringDecoded = urlDecoded.toString();
-                    int index = urlStringDecoded.indexOf('?');
-                    String rewrittenQueryStringDecoded;
+                    String urlStringRewriteEncoded = urlRewriteEncoded.toString();
+                    int index = urlStringRewriteEncoded.indexOf('?');
+                    String rewrittenQueryStringRewriteEncoded;
                     if (index == -1) {
-                        rewrittenQueryStringDecoded = null;
+                        rewrittenQueryStringRewriteEncoded = null;
                     } else {
-                        rewrittenQueryStringDecoded = urlStringDecoded.substring(index + 1);
-                        urlStringDecoded = urlStringDecoded.substring(0, index);
+                        rewrittenQueryStringRewriteEncoded = urlStringRewriteEncoded.substring(index + 1);
+                        urlStringRewriteEncoded = urlStringRewriteEncoded.substring(0, index);
                     }
 
+                    // Step 3. Complete the 2nd stage to encoding.
                     StringBuilder urlStringEncoded =
-                            new StringBuilder(URLEncoder.DEFAULT.encode(urlStringDecoded, uriCharset));
+                            new StringBuilder(REWRITE_DEFAULT_ENCODER.encode(urlStringRewriteEncoded, uriCharset));
+
                     if (!qsd && originalQueryStringEncoded != null && !originalQueryStringEncoded.isEmpty()) {
-                        if (rewrittenQueryStringDecoded == null) {
+                        if (rewrittenQueryStringRewriteEncoded == null) {
                             urlStringEncoded.append('?');
                             urlStringEncoded.append(originalQueryStringEncoded);
                         } else {
                             if (qsa) {
                                 // if qsa is specified append the query
                                 urlStringEncoded.append('?');
-                                urlStringEncoded
-                                        .append(URLEncoder.QUERY.encode(rewrittenQueryStringDecoded, uriCharset));
+                                urlStringEncoded.append(
+                                        REWRITE_QUERY_ENCODER.encode(rewrittenQueryStringRewriteEncoded, uriCharset));
                                 urlStringEncoded.append('&');
                                 urlStringEncoded.append(originalQueryStringEncoded);
                             } else if (index == urlStringEncoded.length() - 1) {
@@ -388,13 +438,14 @@ public void invoke(Request request, Response response) throws IOException, Servl
                                 urlStringEncoded.deleteCharAt(index);
                             } else {
                                 urlStringEncoded.append('?');
-                                urlStringEncoded
-                                        .append(URLEncoder.QUERY.encode(rewrittenQueryStringDecoded, uriCharset));
+                                urlStringEncoded.append(
+                                        REWRITE_QUERY_ENCODER.encode(rewrittenQueryStringRewriteEncoded, uriCharset));
                             }
                         }
-                    } else if (rewrittenQueryStringDecoded != null) {
+                    } else if (rewrittenQueryStringRewriteEncoded != null) {
                         urlStringEncoded.append('?');
-                        urlStringEncoded.append(URLEncoder.QUERY.encode(rewrittenQueryStringDecoded, uriCharset));
+                        urlStringEncoded
+                                .append(REWRITE_QUERY_ENCODER.encode(rewrittenQueryStringRewriteEncoded, uriCharset));
                     }
 
                     // Insert the context if
@@ -469,12 +520,12 @@ public void invoke(Request request, Response response) throws IOException, Servl
             if (rewritten) {
                 if (!done) {
                     // See if we need to replace the query string
-                    String urlStringDecoded = urlDecoded.toString();
-                    String queryStringDecoded = null;
-                    int queryIndex = urlStringDecoded.indexOf('?');
+                    String urlStringRewriteEncoded = urlRewriteEncoded.toString();
+                    String queryStringRewriteEncoded = null;
+                    int queryIndex = urlStringRewriteEncoded.indexOf('?');
                     if (queryIndex != -1) {
-                        queryStringDecoded = urlStringDecoded.substring(queryIndex + 1);
-                        urlStringDecoded = urlStringDecoded.substring(0, queryIndex);
+                        queryStringRewriteEncoded = urlStringRewriteEncoded.substring(queryIndex + 1);
+                        urlStringRewriteEncoded = urlStringRewriteEncoded.substring(0, queryIndex);
                     }
                     // Save the current context path before re-writing starts
                     String contextPath = null;
@@ -488,22 +539,24 @@ public void invoke(Request request, Response response) throws IOException, Servl
                         // This is neither decoded nor normalized
                         chunk.append(contextPath);
                     }
-                    chunk.append(URLEncoder.DEFAULT.encode(urlStringDecoded, uriCharset));
+
+                    // Step 3. Complete the 2nd stage to encoding.
+                    chunk.append(REWRITE_DEFAULT_ENCODER.encode(urlStringRewriteEncoded, uriCharset));
                     // Decoded and normalized URI
                     // Rewriting may have denormalized the URL
-                    urlStringDecoded = RequestUtil.normalize(urlStringDecoded);
+                    urlStringRewriteEncoded = RequestUtil.normalize(urlStringRewriteEncoded);
                     request.getCoyoteRequest().decodedURI().setChars(MessageBytes.EMPTY_CHAR_ARRAY, 0, 0);
                     chunk = request.getCoyoteRequest().decodedURI().getCharChunk();
                     if (context) {
                         // This is decoded and normalized
                         chunk.append(request.getServletContext().getContextPath());
                     }
-                    chunk.append(urlStringDecoded);
+                    chunk.append(URLDecoder.decode(urlStringRewriteEncoded, uriCharset));
                     // Set the new Query if there is one
-                    if (queryStringDecoded != null) {
+                    if (queryStringRewriteEncoded != null) {
                         request.getCoyoteRequest().queryString().setChars(MessageBytes.EMPTY_CHAR_ARRAY, 0, 0);
                         chunk = request.getCoyoteRequest().queryString().getCharChunk();
-                        chunk.append(URLEncoder.QUERY.encode(queryStringDecoded, uriCharset));
+                        chunk.append(REWRITE_QUERY_ENCODER.encode(queryStringRewriteEncoded, uriCharset));
                         if (qsa && originalQueryStringEncoded != null && !originalQueryStringEncoded.isEmpty()) {
                             chunk.append('&');
                             chunk.append(originalQueryStringEncoded);
@@ -790,4 +843,31 @@ protected static void parseRuleFlag(String line, RewriteRule rule, String flag)
             throw new IllegalArgumentException(sm.getString("rewriteValve.invalidFlags", line, flag));
         }
     }
+
+
+    private CharSequence encodeForRewrite(CharSequence input) {
+        StringBuilder result = null;
+        int pos = 0;
+        int mark = 0;
+        while (pos < input.length()) {
+            char c = input.charAt(pos);
+            if (c == '%' || c == ';' || c == '?') {
+                if (result == null) {
+                    result = new StringBuilder((int) (input.length() * 1.1));
+                }
+                result.append(input.subSequence(mark, pos));
+                result.append('%');
+                result.append(Character.forDigit((c >> 4) & 0xF, 16));
+                result.append(Character.forDigit(c & 0xF, 16));
+                mark = pos + 1;
+            }
+            pos++;
+        }
+        if (result != null) {
+            result.append(input.subSequence(mark, input.length()));
+            return result;
+        } else {
+            return input;
+        }
+    }
 }